update to admin/bulk/search API, add missing HTTP actions get index, search scroll, clear scroll

This commit is contained in:
Jörg Prante 2020-05-25 18:46:10 +02:00
parent 43b6afec62
commit 5423cf0f11
106 changed files with 3509 additions and 2565 deletions

View file

@ -0,0 +1,187 @@
package org.xbib.elx.api;
import org.elasticsearch.common.settings.Settings;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
/**
* Interface for extended managing and indexing methods of an Elasticsearch client.
*/
public interface AdminClient extends BasicClient {
/**
* Build index definition from settings.
*
* @param index the index name
* @param settings the settings for the index
* @return index definition
* @throws IOException if settings/mapping URL is invalid/malformed
*/
IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException;
Map<String, ?> getMapping(String index) throws IOException;
Map<String, ?> getMapping(String index, String type) throws IOException;
/**
* Delete an index.
* @param indexDefinition the index definition
* @return this
*/
AdminClient deleteIndex(IndexDefinition indexDefinition);
/**
* Delete an index.
*
* @param index index
* @return this
*/
AdminClient deleteIndex(String index);
/**
* Update replica level.
* @param indexDefinition the index definition
* @param level the replica level
* @return this
* @throws IOException if replica setting could not be updated
*/
AdminClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException;
/**
* Update replica level.
*
* @param index index
* @param level the replica level
* @param maxWaitTime maximum wait time
* @param timeUnit time unit
* @return this
* @throws IOException if replica setting could not be updated
*/
AdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException;
/**
* Get replica level.
* @param indexDefinition the index name
* @return the replica level of the index
*/
int getReplicaLevel(IndexDefinition indexDefinition);
/**
* Get replica level.
* @param index the index name
* @return the replica level of the index
*/
int getReplicaLevel(String index);
/**
* Force segment merge of an index.
* @param indexDefinition the index definition
* @return this
*/
boolean forceMerge(IndexDefinition indexDefinition);
/**
* Force segment merge of an index.
* @param index the index
* @param maxWaitTime maximum wait time
* @param timeUnit time unit
* @return this
*/
boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit);
/**
* Resolve alias.
*
* @param alias the alias
* @return this index name behind the alias or the alias if there is no index
*/
String resolveAlias(String alias);
/**
* Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index
* name.
*
* @param alias the alias
* @return the most recent index name pointing to the alias
*/
String resolveMostRecentIndex(String alias);
/**
* Get all index aliases.
* @param index the index
* @return map of index aliases
*/
Map<String, String> getAliases(String index);
/**
* Shift from one index to another.
* @param indexDefinition the index definition
* @param additionalAliases new aliases
* @return this
*/
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases);
/**
* Shift from one index to another.
* @param indexDefinition the index definition
* @param additionalAliases new aliases
* @param indexAliasAdder method to add aliases
* @return this
*/
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases,
IndexAliasAdder indexAliasAdder);
/**
* Shift from one index to another.
* @param index the index name
* @param fullIndexName the index name with timestamp
* @param additionalAliases a list of names that should be set as index aliases
* @return this
*/
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases);
/**
* Shift from one index to another.
* @param index the index name
* @param fullIndexName the index name with timestamp
* @param additionalAliases a list of names that should be set as index aliases
* @param adder an adder method to create alias term queries
* @return this
*/
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases,
IndexAliasAdder adder);
/**
* Prune index.
* @param indexDefinition the index definition
* @return the index prune result
*/
IndexPruneResult pruneIndex(IndexDefinition indexDefinition);
/**
* Apply retention policy to prune indices. All indices before delta should be deleted,
* but the number of mintokeep indices must be kept.
*
* @param index index name
* @param fullIndexName index name with timestamp
* @param delta timestamp delta (for index timestamps)
* @param mintokeep minimum number of indices to keep
* @param perform true if pruning should be executed, false if not
* @return the index prune result
*/
IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform);
/**
* Find the timestamp of the most recently indexed document in the index.
*
* @param index the index name
* @param timestampfieldname the timestamp field name
* @return millis UTC millis of the most recent document
* @throws IOException if most rcent document can not be found
*/
Long mostRecentDocument(String index, String timestampfieldname) throws IOException;
}

View file

@ -0,0 +1,7 @@
package org.xbib.elx.api;
@FunctionalInterface
public interface AdminClientProvider<C extends AdminClient> {
C getClient();
}

View file

@ -0,0 +1,62 @@
package org.xbib.elx.api;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import java.io.Closeable;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
public interface BasicClient extends Closeable {
/**
* Set an Elasticsearch client to extend from it. May be null for TransportClient.
* @param client the Elasticsearch client
*/
void setClient(ElasticsearchClient client);
/**
* Return Elasticsearch client.
*
* @return Elasticsearch client
*/
ElasticsearchClient getClient();
/**
* Initiative the extended client, the bulk metric and bulk controller,
* creates instances and connect to cluster, if required.
*
* @param settings settings
* @throws IOException if init fails
*/
void init(Settings settings) throws IOException;
/**
* Get cluster name.
* @return the cluster name
*/
String getClusterName();
/**
* Get current health color.
*
* @param maxWaitTime maximum wait time
* @param timeUnit time unit
* @return the cluster health color
*/
String getHealthColor(long maxWaitTime, TimeUnit timeUnit);
/**
* Wait for cluster being healthy.
*
* @param healthColor cluster health color to wait for
* @param maxWaitTime time value
* @param timeUnit time unit
*/
void waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit);
void waitForShards(long maxWaitTime, TimeUnit timeUnit);
long getSearchableDocs(String index);
boolean isIndexExists(String index);
}

View file

@ -0,0 +1,227 @@
package org.xbib.elx.api;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.Flushable;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.TimeUnit;
public interface BulkClient extends BasicClient, Flushable {
/**
* Get bulk metric.
* @return the bulk metric
*/
BulkMetric getBulkMetric();
/**
* Get buulk control.
* @return the bulk control
*/
BulkController getBulkController();
/**
* Create a new index.
*
* @param index index
* @throws IOException if new index creation fails
*/
void newIndex(String index) throws IOException;
/**
* Create a new index.
* @param indexDefinition the index definition
* @throws IOException if settings/mapping is invalid or index creation fails
*/
void newIndex(IndexDefinition indexDefinition) throws IOException;
/**
* Create a new index.
*
* @param index index
* @param settings settings
* @throws IOException if settings is invalid or index creation fails
*/
void newIndex(String index, Settings settings) throws IOException;
/**
* Create a new index.
*
* @param index index
* @param settings settings
* @param mapping mapping
* @throws IOException if settings/mapping is invalid or index creation fails
*/
void newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException;
/**
* Create a new index.
*
* @param index index
* @param settings settings
* @param mapping mapping
* @throws IOException if settings/mapping is invalid or index creation fails
*/
void newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException;
/**
* Add index request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when limits are exceeded.
*
* @param index the index
* @param id the id
* @param create true if document must be created
* @param source the source
* @return this
*/
BulkClient index(String index, String id, boolean create, BytesReference source);
/**
* Index request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when limits are exceeded.
*
* @param index the index
* @param id the id
* @param create true if document is to be created, false otherwise
* @param source the source
* @return this client methods
*/
BulkClient index(String index, String id, boolean create, String source);
/**
* Index request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when bulk limits are exceeded.
*
* @param indexRequest the index request to add
* @return this
*/
BulkClient index(IndexRequest indexRequest);
/**
* Delete request.
*
* @param index the index
* @param id the id
* @return this
*/
BulkClient delete(String index, String id);
/**
* Delete request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when bulk limits are exceeded.
*
* @param deleteRequest the delete request to add
* @return this
*/
BulkClient delete(DeleteRequest deleteRequest);
/**
* Bulked update request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when bulk limits are exceeded.
* Note that updates only work correctly when all operations between nodes are synchronized.
*
* @param index the index
* @param id the id
* @param source the source
* @return this
*/
BulkClient update(String index, String id, BytesReference source);
/**
* Update document. Use with precaution! Does not work in all cases.
*
* @param index the index
* @param id the id
* @param source the source
* @return this
*/
BulkClient update(String index, String id, String source);
/**
* Bulked update request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when bulk limits are exceeded.
* Note that updates only work correctly when all operations between nodes are synchronized.
*
* @param updateRequest the update request to add
* @return this
*/
BulkClient update(UpdateRequest updateRequest);
/**
* Start bulk mode for indexes.
* @param indexDefinition index definition
* @throws IOException if bulk could not be started
*/
void startBulk(IndexDefinition indexDefinition) throws IOException;
/**
* Start bulk mode.
*
* @param index index
* @param startRefreshIntervalSeconds refresh interval before bulk
* @param stopRefreshIntervalSeconds refresh interval after bulk
* @throws IOException if bulk could not be started
*/
void startBulk(String index, long startRefreshIntervalSeconds,
long stopRefreshIntervalSeconds) throws IOException;
/**
* Stop bulk mode.
*
* @param indexDefinition index definition
* @throws IOException if bulk could not be startet
*/
void stopBulk(IndexDefinition indexDefinition) throws IOException;
/**
* Stops bulk mode.
*
* @param index index
* @param timeout maximum wait time
* @param timeUnit time unit for timeout
* @throws IOException if bulk could not be stopped
*/
void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException;
/**
* Wait for all outstanding bulk responses.
*
* @param timeout maximum wait time
* @param timeUnit unit of timeout value
* @return true if wait succeeded, false if wait timed out
*/
boolean waitForResponses(long timeout, TimeUnit timeUnit);
/**
* Update index setting.
* @param index the index
* @param key the key of the value to be updated
* @param value the new value
* @param timeout timeout
* @param timeUnit time unit
* @throws IOException if update index setting failed
*/
void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException;
/**
* Refresh the index.
*
* @param index index
*/
void refreshIndex(String index);
/**
* Flush the index. The cluster clears cache and completes indexing.
*
* @param index index
*/
void flushIndex(String index);
}

View file

@ -0,0 +1,7 @@
package org.xbib.elx.api;
@FunctionalInterface
public interface BulkClientProvider<C extends BulkClient> {
C getClient();
}

View file

@ -1,5 +1,6 @@
package org.xbib.elx.api; package org.xbib.elx.api;
import org.elasticsearch.common.settings.Settings;
import org.xbib.metrics.api.Count; import org.xbib.metrics.api.Count;
import org.xbib.metrics.api.Metered; import org.xbib.metrics.api.Metered;
@ -7,6 +8,8 @@ import java.io.Closeable;
public interface BulkMetric extends Closeable { public interface BulkMetric extends Closeable {
void init(Settings settings);
Metered getTotalIngest(); Metered getTotalIngest();
Count getTotalIngestSizeInBytes(); Count getTotalIngestSizeInBytes();

View file

@ -1,487 +0,0 @@
package org.xbib.elx.api;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.Closeable;
import java.io.Flushable;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
/**
* Interface for extended managing and indexing methods of an Elasticsearch client.
*/
public interface ExtendedClient extends Flushable, Closeable {
/**
* Set an Elasticsearch client to extend from it. May be null for TransportClient.
* @param client client
* @return this client
*/
ExtendedClient setClient(ElasticsearchClient client);
/**
* Return Elasticsearch client.
*
* @return Elasticsearch client
*/
ElasticsearchClient getClient();
/**
* Get bulk control.
* @return the bulk control
*/
BulkController getBulkController();
/**
* Initiative the extended client, the bulk metric and bulk controller,
* creates instances and connect to cluster, if required.
*
* @param settings settings
* @return this client
* @throws IOException if init fails
*/
ExtendedClient init(Settings settings) throws IOException;
/**
* Build index definition from settings.
*
* @param index the index name
* @param settings the settings for the index
* @return index definition
* @throws IOException if settings/mapping URL is invalid/malformed
*/
IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException;
/**
* Add index request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when limits are exceeded.
*
* @param index the index
* @param id the id
* @param create true if document must be created
* @param source the source
* @return this
*/
ExtendedClient index(String index, String id, boolean create, BytesReference source);
/**
* Index request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when limits are exceeded.
*
* @param index the index
* @param id the id
* @param create true if document is to be created, false otherwise
* @param source the source
* @return this client methods
*/
ExtendedClient index(String index, String id, boolean create, String source);
/**
* Index request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when bulk limits are exceeded.
*
* @param indexRequest the index request to add
* @return this
*/
ExtendedClient index(IndexRequest indexRequest);
/**
* Delete request.
*
* @param index the index
* @param id the id
* @return this
*/
ExtendedClient delete(String index, String id);
/**
* Delete request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when bulk limits are exceeded.
*
* @param deleteRequest the delete request to add
* @return this
*/
ExtendedClient delete(DeleteRequest deleteRequest);
/**
* Bulked update request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when bulk limits are exceeded.
* Note that updates only work correctly when all operations between nodes are synchronized.
*
* @param index the index
* @param id the id
* @param source the source
* @return this
* @throws IOException if update fails
*/
ExtendedClient update(String index, String id, BytesReference source) throws IOException;
/**
* Update document. Use with precaution! Does not work in all cases.
*
* @param index the index
* @param id the id
* @param source the source
* @return this
*/
ExtendedClient update(String index, String id, String source);
/**
* Bulked update request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when bulk limits are exceeded.
* Note that updates only work correctly when all operations between nodes are synchronized.
*
* @param updateRequest the update request to add
* @return this
*/
ExtendedClient update(UpdateRequest updateRequest);
/**
* Create a new index.
*
* @param index index
* @return this
* @throws IOException if new index creation fails
*/
ExtendedClient newIndex(String index) throws IOException;
/**
* Create a new index.
*
* @param index index
* @param settings settings
* @param mapping mapping
* @return this
* @throws IOException if settings/mapping is invalid or index creation fails
*/
ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException;
/**
* Create a new index.
*
* @param index index
* @param settings settings
* @return this
* @throws IOException if settings is invalid or index creation fails
*/
ExtendedClient newIndex(String index, Settings settings) throws IOException;
/**
* Create a new index.
*
* @param index index
* @param settings settings
* @param mapping mapping
* @return this
* @throws IOException if settings/mapping is invalid or index creation fails
*/
ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException;
/**
* Create a new index.
*
* @param index index
* @param settings settings
* @param mapping mapping
* @return this
* @throws IOException if settings/mapping is invalid or index creation fails
*/
ExtendedClient newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException;
/**
* Create a new index.
*
* @param index index
* @param settings settings
* @param mapping mapping
* @return this
* @throws IOException if settings/mapping is invalid or index creation fails
*/
ExtendedClient newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException;
/**
* Create a new index.
* @param indexDefinition the index definition
* @return this
* @throws IOException if settings/mapping is invalid or index creation fails
*/
ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException;
/**
* Delete an index.
* @param indexDefinition the index definition
* @return this
*/
ExtendedClient deleteIndex(IndexDefinition indexDefinition);
/**
* Delete an index.
*
* @param index index
* @return this
*/
ExtendedClient deleteIndex(String index);
/**
* Start bulk mode for indexes.
* @param indexDefinition index definition
* @return this
* @throws IOException if bulk could not be started
*/
ExtendedClient startBulk(IndexDefinition indexDefinition) throws IOException;
/**
* Start bulk mode.
*
* @param index index
* @param startRefreshIntervalSeconds refresh interval before bulk
* @param stopRefreshIntervalSeconds refresh interval after bulk
* @return this
* @throws IOException if bulk could not be started
*/
ExtendedClient startBulk(String index, long startRefreshIntervalSeconds,
long stopRefreshIntervalSeconds) throws IOException;
/**
* Stop bulk mode.
*
* @param indexDefinition index definition
* @return this
* @throws IOException if bulk could not be startet
*/
ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException;
/**
* Stops bulk mode.
*
* @param index index
* @param timeout maximum wait time
* @param timeUnit time unit for timeout
* @return this
* @throws IOException if bulk could not be stopped
*/
ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException;
/**
* Update replica level.
* @param indexDefinition the index definition
* @param level the replica level
* @return this
* @throws IOException if replica setting could not be updated
*/
ExtendedClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException;
/**
* Update replica level.
*
* @param index index
* @param level the replica level
* @param maxWaitTime maximum wait time
* @param timeUnit time unit
* @return this
* @throws IOException if replica setting could not be updated
*/
ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException;
/**
* Get replica level.
* @param indexDefinition the index name
* @return the replica level of the index
*/
int getReplicaLevel(IndexDefinition indexDefinition);
/**
* Get replica level.
* @param index the index name
* @return the replica level of the index
*/
int getReplicaLevel(String index);
/**
* Refresh the index.
*
* @param index index
* @return this
*/
ExtendedClient refreshIndex(String index);
/**
* Flush the index. The cluster clears cache and completes indexing.
*
* @param index index
* @return this
*/
ExtendedClient flushIndex(String index);
/**
* Force segment merge of an index.
* @param indexDefinition the index definition
* @return this
*/
boolean forceMerge(IndexDefinition indexDefinition);
/**
* Force segment merge of an index.
* @param index the index
* @param maxWaitTime maximum wait time
* @param timeUnit time unit
* @return this
*/
boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit);
/**
* Wait for all outstanding bulk responses.
*
* @param timeout maximum wait time
* @param timeUnit unit of timeout value
* @return true if wait succeeded, false if wait timed out
*/
boolean waitForResponses(long timeout, TimeUnit timeUnit);
/**
* Wait for cluster being healthy.
*
* @param healthColor cluster health color to wait for
* @param maxWaitTime time value
* @param timeUnit time unit
* @return true if wait succeeded, false if wait timed out
*/
boolean waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit);
/**
* Get current health color.
*
* @param maxWaitTime maximum wait time
* @param timeUnit time unit
* @return the cluster health color
*/
String getHealthColor(long maxWaitTime, TimeUnit timeUnit);
/**
* Wait for index recovery (after replica change).
*
* @param index index
* @param maxWaitTime maximum wait time
* @param timeUnit time unit
* @return true if wait succeeded, false if wait timed out
*/
boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit);
/**
* Update index setting.
* @param index the index
* @param key the key of the value to be updated
* @param value the new value
* @param timeout timeout
* @param timeUnit time unit
* @throws IOException if update index setting failed
*/
void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException;
/**
* Resolve alias.
*
* @param alias the alias
* @return this index name behind the alias or the alias if there is no index
*/
String resolveAlias(String alias);
/**
* Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index
* name.
*
* @param alias the alias
* @return the most recent index name pointing to the alias
*/
String resolveMostRecentIndex(String alias);
/**
* Get all index aliases.
* @param index the index
* @return map of index aliases
*/
Map<String, String> getAliases(String index);
/**
* Shift from one index to another.
* @param indexDefinition the index definition
* @param additionalAliases new aliases
* @return this
*/
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases);
/**
* Shift from one index to another.
* @param indexDefinition the index definition
* @param additionalAliases new aliases
* @param indexAliasAdder method to add aliases
* @return this
*/
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases,
IndexAliasAdder indexAliasAdder);
/**
* Shift from one index to another.
* @param index the index name
* @param fullIndexName the index name with timestamp
* @param additionalAliases a list of names that should be set as index aliases
* @return this
*/
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases);
/**
* Shift from one index to another.
* @param index the index name
* @param fullIndexName the index name with timestamp
* @param additionalAliases a list of names that should be set as index aliases
* @param adder an adder method to create alias term queries
* @return this
*/
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases,
IndexAliasAdder adder);
/**
* Prune index.
* @param indexDefinition the index definition
* @return the index prune result
*/
IndexPruneResult pruneIndex(IndexDefinition indexDefinition);
/**
* Apply retention policy to prune indices. All indices before delta should be deleted,
* but the number of mintokeep indices must be kept.
*
* @param index index name
* @param fullIndexName index name with timestamp
* @param delta timestamp delta (for index timestamps)
* @param mintokeep minimum number of indices to keep
* @param perform true if pruning should be executed, false if not
* @return the index prune result
*/
IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform);
/**
* Find the timestamp of the most recently indexed document in the index.
*
* @param index the index name
* @param timestampfieldname the timestamp field name
* @return millis UTC millis of the most recent document
* @throws IOException if most rcent document can not be found
*/
Long mostRecentDocument(String index, String timestampfieldname) throws IOException;
/**
* Get cluster name.
* @return the cluster name
*/
String getClusterName();
}

View file

@ -1,7 +0,0 @@
package org.xbib.elx.api;
@FunctionalInterface
public interface ExtendedClientProvider<C extends ExtendedClient> {
C getExtendedClient();
}

View file

@ -1,7 +1,5 @@
package org.xbib.elx.api; package org.xbib.elx.api;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
public interface IndexDefinition { public interface IndexDefinition {
@ -14,17 +12,13 @@ public interface IndexDefinition {
String getFullIndexName(); String getFullIndexName();
IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException; IndexDefinition setSettings(String settings);
IndexDefinition setSettingsUrl(URL settingsUrl); String getSettings();
URL getSettingsUrl(); IndexDefinition setMappings(String mappings);
IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException; String getMappings();
IndexDefinition setMappingsUrl(URL mappingsUrl);
URL getMappingsUrl();
IndexDefinition setDateTimePattern(String timeWindow); IndexDefinition setDateTimePattern(String timeWindow);

View file

@ -1,6 +1,6 @@
package org.xbib.elx.api; package org.xbib.elx.api;
import java.util.List; import java.util.Collection;
public interface IndexPruneResult { public interface IndexPruneResult {
@ -8,9 +8,9 @@ public interface IndexPruneResult {
State getState(); State getState();
List<String> getCandidateIndices(); Collection<String> getCandidateIndices();
List<String> getDeletedIndices(); Collection<String> getDeletedIndices();
boolean isAcknowledged(); boolean isAcknowledged();
} }

View file

@ -1,10 +1,10 @@
package org.xbib.elx.api; package org.xbib.elx.api;
import java.util.List; import java.util.Collection;
public interface IndexShiftResult { public interface IndexShiftResult {
List<String> getMovedAliases(); Collection<String> getMovedAliases();
List<String> getNewAliases(); Collection<String> getNewAliases();
} }

View file

@ -1,25 +0,0 @@
package org.xbib.elx.api;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.get.MultiGetRequest;
import org.elasticsearch.action.get.MultiGetResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
public interface ReadClient {
ActionFuture<GetResponse> get(GetRequest getRequest);
void get(GetRequest request, ActionListener<GetResponse> listener);
ActionFuture<MultiGetResponse> multiGet(MultiGetRequest request);
void multiGet(MultiGetRequest request, ActionListener<MultiGetResponse> listener);
ActionFuture<SearchResponse> search(SearchRequest request);
void search(SearchRequest request, ActionListener<SearchResponse> listener);
}

View file

@ -1,6 +0,0 @@
package org.xbib.elx.api;
public interface ReadClientProvider<C extends ReadClient> {
C getReadClient();
}

View file

@ -0,0 +1,27 @@
package org.xbib.elx.api;
import org.elasticsearch.action.get.GetRequestBuilder;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.get.MultiGetRequestBuilder;
import org.elasticsearch.action.get.MultiGetResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.search.SearchHit;
import java.util.Optional;
import java.util.function.Consumer;
import java.util.stream.Stream;
public interface SearchClient extends BasicClient {
Optional<GetResponse> get(Consumer<GetRequestBuilder> getRequestBuilder);
Optional<MultiGetResponse> multiGet(Consumer<MultiGetRequestBuilder> multiGetRequestBuilder);
Optional<SearchResponse> search(Consumer<SearchRequestBuilder> searchRequestBuilder);
Stream<SearchHit> search(Consumer<SearchRequestBuilder> searchRequestBuilder,
TimeValue scrollTime, int scrollSize);
Stream<String> getIds(Consumer<SearchRequestBuilder> queryBuilder);
}

View file

@ -0,0 +1,7 @@
package org.xbib.elx.api;
@FunctionalInterface
public interface SearchClientProvider<C extends SearchClient> {
C getClient();
}

View file

@ -2,14 +2,8 @@ package org.xbib.elx.common;
import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.apache.lucene.search.TotalHits;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
@ -18,13 +12,8 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.admin.indices.flush.FlushAction;
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction;
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexAction; import org.elasticsearch.action.admin.indices.get.GetIndexAction;
@ -32,36 +21,31 @@ import org.elasticsearch.action.admin.indices.get.GetIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequestBuilder;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.cluster.metadata.AliasOrIndex; import org.elasticsearch.cluster.metadata.AliasOrIndex;
import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.DeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.common.xcontent.yaml.YamlXContent;
import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHit;
@ -69,8 +53,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.search.sort.SortOrder;
import org.xbib.elx.api.BulkController; import org.xbib.elx.api.AdminClient;
import org.xbib.elx.api.ExtendedClient;
import org.xbib.elx.api.IndexAliasAdder; import org.xbib.elx.api.IndexAliasAdder;
import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.api.IndexPruneResult; import org.xbib.elx.api.IndexPruneResult;
@ -79,13 +62,15 @@ import org.xbib.elx.api.IndexShiftResult;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.nio.charset.MalformedInputException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.time.LocalDate; import java.time.LocalDate;
import java.time.ZoneId; import java.time.ZoneId;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
@ -98,22 +83,21 @@ import java.util.TreeSet;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.stream.Collectors; import java.util.stream.Collectors;
public abstract class AbstractExtendedClient implements ExtendedClient { public abstract class AbstractAdminClient extends AbstractBasicClient implements AdminClient {
private static final Logger logger = LogManager.getLogger(AbstractExtendedClient.class.getName()); private static final Logger logger = LogManager.getLogger(AbstractAdminClient.class.getName());
private ElasticsearchClient client; /**
* The one and only index type name used in the extended client.
private BulkController bulkController; * Notr that all Elasticsearch version < 6.2.0 do not allow a prepending "_".
*/
private final AtomicBoolean closed; private static final String TYPE_NAME = "doc";
private static final IndexShiftResult EMPTY_INDEX_SHIFT_RESULT = new IndexShiftResult() { private static final IndexShiftResult EMPTY_INDEX_SHIFT_RESULT = new IndexShiftResult() {
@Override @Override
@ -149,363 +133,54 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
} }
}; };
protected abstract ElasticsearchClient createClient(Settings settings) throws IOException; @Override
public Map<String, ?> getMapping(String index) throws IOException {
protected abstract void closeClient() throws IOException; return getMapping(index, TYPE_NAME);
protected AbstractExtendedClient() {
closed = new AtomicBoolean(false);
} }
@Override @Override
public AbstractExtendedClient setClient(ElasticsearchClient client) { public Map<String, ?> getMapping(String index, String mapping) throws IOException {
this.client = client; GetMappingsRequestBuilder getMappingsRequestBuilder = new GetMappingsRequestBuilder(client, GetMappingsAction.INSTANCE)
return this; .setIndices(index)
.setTypes(mapping);
GetMappingsResponse getMappingsResponse = getMappingsRequestBuilder.execute().actionGet();
logger.info("get mappings response = {}", getMappingsResponse.getMappings().get(index).get(mapping).getSourceAsMap());
return getMappingsResponse.getMappings().get(index).get(mapping).getSourceAsMap();
} }
@Override @Override
public ElasticsearchClient getClient() { public AdminClient deleteIndex(IndexDefinition indexDefinition) {
return client;
}
@Override
public BulkController getBulkController() {
return bulkController;
}
@Override
public AbstractExtendedClient init(Settings settings) throws IOException {
logger.info("initializing with settings = " + settings.toDelimitedString(','));
if (client == null) {
client = createClient(settings);
}
if (bulkController == null) {
bulkController = new DefaultBulkController(this);
bulkController.init(settings);
}
return this;
}
@Override
public void flush() throws IOException {
if (bulkController != null) {
bulkController.flush();
}
}
@Override
public void close() throws IOException {
ensureClient();
if (closed.compareAndSet(false, true)) {
if (bulkController != null) {
logger.info("closing bulk controller");
bulkController.close();
}
closeClient();
}
}
@Override
public String getClusterName() {
ensureClient();
try {
ClusterStateRequest clusterStateRequest = new ClusterStateRequest().clear();
ClusterStateResponse clusterStateResponse =
client.execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
return clusterStateResponse.getClusterName().value();
} catch (ElasticsearchTimeoutException e) {
logger.warn(e.getMessage(), e);
return "TIMEOUT";
} catch (NoNodeAvailableException e) {
logger.warn(e.getMessage(), e);
return "DISCONNECTED";
} catch (Exception e) {
logger.warn(e.getMessage(), e);
return "[" + e.getMessage() + "]";
}
}
@Override
public ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException {
ensureClient();
waitForCluster("YELLOW", 30L, TimeUnit.SECONDS);
URL indexSettings = indexDefinition.getSettingsUrl();
if (indexSettings == null) {
logger.warn("warning while creating index '{}', no settings/mappings",
indexDefinition.getFullIndexName());
newIndex(indexDefinition.getFullIndexName());
return this;
}
URL indexMappings = indexDefinition.getMappingsUrl();
if (indexMappings == null) {
logger.warn("warning while creating index '{}', no mappings",
indexDefinition.getFullIndexName());
newIndex(indexDefinition.getFullIndexName(), indexSettings.openStream(), null);
return this;
}
try (InputStream indexSettingsInput = indexSettings.openStream();
InputStream indexMappingsInput = indexMappings.openStream()) {
newIndex(indexDefinition.getFullIndexName(), indexSettingsInput, indexMappingsInput);
} catch (IOException e) {
if (indexDefinition.ignoreErrors()) {
logger.warn(e.getMessage(), e);
logger.warn("warning while creating index '{}' with settings at {} and mappings at {}",
indexDefinition.getFullIndexName(), indexSettings, indexMappings);
} else {
logger.error("error while creating index '{}' with settings at {} and mappings at {}",
indexDefinition.getFullIndexName(), indexSettings, indexMappings);
throw new IOException(e);
}
}
return this;
}
@Override
public ExtendedClient newIndex(String index) throws IOException {
return newIndex(index, Settings.EMPTY, (Map<String, Object>) null);
}
@Override
public ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException {
return newIndex(index,
Settings.builder().loadFromStream(".json", settings, true).build(),
mapping != null ? JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mapping).mapOrdered() : null);
}
@Override
public ExtendedClient newIndex(String index, Settings settings) throws IOException {
return newIndex(index, settings, (Map<String, ?>) null);
}
@Override
public ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException {
return newIndex(index, settings,
mapping != null ? JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mapping).mapOrdered() : null);
}
@Override
public ExtendedClient newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException {
ensureClient();
if (index == null) {
logger.warn("no index name given to create index");
return this;
}
CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index);
if (settings != null) {
createIndexRequest.settings(settings);
}
if (mapping != null) {
createIndexRequest.mapping("_doc", mapping);
}
CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
if (createIndexResponse.isAcknowledged()) {
return this;
}
throw new IllegalStateException("index creation not acknowledged");
}
@Override
public ExtendedClient newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException {
ensureClient();
if (index == null) {
logger.warn("no index name given to create index");
return this;
}
CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index);
if (settings != null) {
createIndexRequest.settings(settings);
}
if (mapping != null) {
createIndexRequest.mapping("_doc", mapping);
}
CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
if (createIndexResponse.isAcknowledged()) {
return this;
}
throw new IllegalStateException("index creation not acknowledged");
}
@Override
public ExtendedClient deleteIndex(IndexDefinition indexDefinition) {
return deleteIndex(indexDefinition.getFullIndexName()); return deleteIndex(indexDefinition.getFullIndexName());
} }
@Override @Override
public ExtendedClient deleteIndex(String index) { public AdminClient deleteIndex(String index) {
ensureClient(); ensureClientIsPresent();
if (index == null) { if (index == null) {
logger.warn("no index name given to delete index"); logger.warn("no index name given to delete index");
return this; return this;
} }
DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest().indices(index); DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest().indices(index);
client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet(); client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet();
waitForCluster("YELLOW", 30L, TimeUnit.SECONDS);
waitForShards(30L, TimeUnit.SECONDS);
return this; return this;
} }
@Override @Override
public ExtendedClient startBulk(IndexDefinition indexDefinition) throws IOException { public AdminClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException {
startBulk(indexDefinition.getFullIndexName(), -1, 1);
return this;
}
@Override
public ExtendedClient startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds)
throws IOException {
if (bulkController != null) {
ensureClient();
bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds);
}
return this;
}
@Override
public ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException {
if (bulkController != null) {
ensureClient();
bulkController.stopBulkMode(indexDefinition);
}
return this;
}
@Override
public ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException {
if (bulkController != null) {
ensureClient();
bulkController.stopBulkMode(index, timeout, timeUnit);
}
return this;
}
@Override
public ExtendedClient index(String index, String id, boolean create, String source) {
return index(new IndexRequest().index(index).id(id).create(create)
.source(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
}
@Override
public ExtendedClient index(String index, String id, boolean create, BytesReference source) {
return index(new IndexRequest().index(index).id(id).create(create)
.source(source, XContentType.JSON));
}
@Override
public ExtendedClient index(IndexRequest indexRequest) {
ensureClient();
bulkController.bulkIndex(indexRequest);
return this;
}
@Override
public ExtendedClient delete(String index, String id) {
return delete(new DeleteRequest().index(index).id(id));
}
@Override
public ExtendedClient delete(DeleteRequest deleteRequest) {
ensureClient();
bulkController.bulkDelete(deleteRequest);
return this;
}
@Override
public ExtendedClient update(String index, String id, BytesReference source) {
return update(new UpdateRequest().index(index).id(id)
.doc(source, XContentType.JSON));
}
@Override
public ExtendedClient update(String index, String id, String source) {
return update(new UpdateRequest().index(index).id(id)
.doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
}
@Override
public ExtendedClient update(UpdateRequest updateRequest) {
ensureClient();
bulkController.bulkUpdate(updateRequest);
return this;
}
@Override
public boolean waitForResponses(long timeout, TimeUnit timeUnit) {
ensureClient();
return bulkController.waitForBulkResponses(timeout, timeUnit);
}
@Override
public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) {
ensureClient();
ensureIndexGiven(index);
GetSettingsRequest settingsRequest = new GetSettingsRequest();
settingsRequest.indices(index);
GetSettingsResponse settingsResponse = client.execute(GetSettingsAction.INSTANCE, settingsRequest).actionGet();
int shards = settingsResponse.getIndexToSettings().get(index).getAsInt("index.number_of_shards", -1);
if (shards > 0) {
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest()
.indices(index)
.waitForActiveShards(shards)
.timeout(timeout);
ClusterHealthResponse healthResponse =
client.execute(ClusterHealthAction.INSTANCE, clusterHealthRequest).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
logger.warn("timeout waiting for recovery");
return false;
}
}
return true;
}
@Override
public boolean waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) {
ensureClient();
ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString);
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
logger.warn("timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name());
return false;
}
return true;
}
@Override
public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) {
ensureClient();
try {
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().timeout(timeout)).actionGet();
ClusterHealthStatus status = healthResponse.getStatus();
return status.name();
} catch (ElasticsearchTimeoutException e) {
logger.warn(e.getMessage(), e);
return "TIMEOUT";
} catch (NoNodeAvailableException e) {
logger.warn(e.getMessage(), e);
return "DISCONNECTED";
} catch (Exception e) {
logger.warn(e.getMessage(), e);
return "[" + e.getMessage() + "]";
}
}
@Override
public ExtendedClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException {
return updateReplicaLevel(indexDefinition.getFullIndexName(), level, return updateReplicaLevel(indexDefinition.getFullIndexName(), level,
indexDefinition.getMaxWaitTime(), indexDefinition.getMaxWaitTimeUnit()); indexDefinition.getMaxWaitTime(), indexDefinition.getMaxWaitTimeUnit());
} }
@Override @Override
public ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException { public AdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException {
waitForCluster("YELLOW", maxWaitTime, timeUnit); // let cluster settle down from critical operations if (level < 1) {
if (level > 0) { logger.warn("invalid replica level");
updateIndexSetting(index, "number_of_replicas", level, maxWaitTime, timeUnit); return this;
waitForRecovery(index, maxWaitTime, timeUnit);
} }
updateIndexSetting(index, "number_of_replicas", level, maxWaitTime, timeUnit);
waitForShards(maxWaitTime, timeUnit);
return this; return this;
} }
@ -528,30 +203,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
return replica; return replica;
} }
@Override
public ExtendedClient flushIndex(String index) {
if (index != null) {
ensureClient();
client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet();
}
return this;
}
@Override
public ExtendedClient refreshIndex(String index) {
if (index != null) {
ensureClient();
client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet();
}
return this;
}
@Override @Override
public String resolveMostRecentIndex(String alias) { public String resolveMostRecentIndex(String alias) {
ensureClientIsPresent();
if (alias == null) { if (alias == null) {
return null; return null;
} }
ensureClient();
GetAliasesRequest getAliasesRequest = new GetAliasesRequest().aliases(alias); GetAliasesRequest getAliasesRequest = new GetAliasesRequest().aliases(alias);
GetAliasesResponse getAliasesResponse = client.execute(GetAliasesAction.INSTANCE, getAliasesRequest).actionGet(); GetAliasesResponse getAliasesResponse = client.execute(GetAliasesAction.INSTANCE, getAliasesRequest).actionGet();
Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); Pattern pattern = Pattern.compile("^(.*?)(\\d+)$");
@ -576,7 +233,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
@Override @Override
public String resolveAlias(String alias) { public String resolveAlias(String alias) {
ensureClient(); ensureClientIsPresent();
ClusterStateRequest clusterStateRequest = new ClusterStateRequest(); ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
clusterStateRequest.blocks(false); clusterStateRequest.blocks(false);
clusterStateRequest.metaData(true); clusterStateRequest.metaData(true);
@ -618,7 +275,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
@Override @Override
public IndexShiftResult shiftIndex(String index, String fullIndexName, public IndexShiftResult shiftIndex(String index, String fullIndexName,
List<String> additionalAliases, IndexAliasAdder adder) { List<String> additionalAliases, IndexAliasAdder adder) {
ensureClient(); ensureClientIsPresent();
if (index == null) { if (index == null) {
return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to
} }
@ -644,9 +301,9 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
String alias = entry.getKey(); String alias = entry.getKey();
String filter = entry.getValue(); String filter = entry.getValue();
indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove() indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove()
.indices(oldIndex).alias(alias)); .index(oldIndex).alias(alias));
if (filter != null) { if (filter != null) {
indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add() indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove()
.index(fullIndexName).alias(alias).filter(filter)); .index(fullIndexName).alias(alias).filter(filter));
} else { } else {
indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add() indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add()
@ -670,7 +327,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
} else { } else {
String filter = oldAliasMap.get(additionalAlias); String filter = oldAliasMap.get(additionalAlias);
indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove() indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove()
.indices(oldIndex).alias(additionalAlias)); .index(oldIndex).alias(additionalAlias));
if (filter != null) { if (filter != null) {
indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add() indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add()
.index(fullIndexName).alias(additionalAlias).filter(filter)); .index(fullIndexName).alias(additionalAlias).filter(filter));
@ -686,8 +343,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
logger.debug("indices alias request = {}", indicesAliasesRequest.getAliasActions().toString()); logger.debug("indices alias request = {}", indicesAliasesRequest.getAliasActions().toString());
AcknowledgedResponse indicesAliasesResponse = AcknowledgedResponse indicesAliasesResponse =
client.execute(IndicesAliasesAction.INSTANCE, indicesAliasesRequest).actionGet(); client.execute(IndicesAliasesAction.INSTANCE, indicesAliasesRequest).actionGet();
logger.debug("response isAcknowledged = {} isFragment = {}", logger.debug("response isAcknowledged = {}",
indicesAliasesResponse.isAcknowledged(), indicesAliasesResponse.isFragment()); indicesAliasesResponse.isAcknowledged());
} }
return new SuccessIndexShiftResult(moveAliases, newAliases); return new SuccessIndexShiftResult(moveAliases, newAliases);
} }
@ -706,11 +363,11 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
if (index.equals(fullIndexName)) { if (index.equals(fullIndexName)) {
return EMPTY_INDEX_PRUNE_RESULT; return EMPTY_INDEX_PRUNE_RESULT;
} }
ensureClient(); ensureClientIsPresent();
GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE); GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE);
GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet(); GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet();
Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); Pattern pattern = Pattern.compile("^(.*?)(\\d+)$");
logger.info("pruneIndex: total of {} indices", getIndexResponse.getIndices().length); logger.info("{} indices", getIndexResponse.getIndices().length);
List<String> candidateIndices = new ArrayList<>(); List<String> candidateIndices = new ArrayList<>();
for (String s : getIndexResponse.getIndices()) { for (String s : getIndexResponse.getIndices()) {
Matcher m = pattern.matcher(s); Matcher m = pattern.matcher(s);
@ -746,29 +403,20 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest() DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest()
.indices(indicesToDelete.toArray(s)); .indices(indicesToDelete.toArray(s));
AcknowledgedResponse response = client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet(); AcknowledgedResponse response = client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet();
if( response.isAcknowledged()) { return new SuccessPruneResult(candidateIndices, indicesToDelete, response);
logger.log(Level.INFO, "deletion of {} acknowledged, waiting for GREEN", Arrays.asList(s));
waitForCluster("GREEN", 30L, TimeUnit.SECONDS);
return new SuccessPruneResult(candidateIndices, indicesToDelete, response);
} else {
logger.log(Level.WARN, "deletion of {} not acknowledged", Arrays.asList(s));
return new FailPruneResult(candidateIndices, indicesToDelete, response);
}
} }
@Override @Override
public Long mostRecentDocument(String index, String timestampfieldname) { public Long mostRecentDocument(String index, String timestampfieldname) {
ensureClient(); ensureClientIsPresent();
SortBuilder<?> sort = SortBuilders SortBuilder<?> sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC);
.fieldSort(timestampfieldname) SearchSourceBuilder builder = new SearchSourceBuilder();
.order(SortOrder.DESC); builder.sort(sort);
SearchSourceBuilder builder = new SearchSourceBuilder() builder.storedField(timestampfieldname);
.sort(sort) builder.size(1);
.storedField(timestampfieldname) SearchRequest searchRequest = new SearchRequest();
.size(1); searchRequest.indices(index);
SearchRequest searchRequest = new SearchRequest() searchRequest.source(builder);
.indices(index)
.source(builder);
SearchResponse searchResponse = SearchResponse searchResponse =
client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); client.execute(SearchAction.INSTANCE, searchRequest).actionGet();
if (searchResponse.getHits().getHits().length == 1) { if (searchResponse.getHits().getHits().length == 1) {
@ -813,7 +461,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
@Override @Override
public IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) public IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings)
throws IOException { throws IOException {
boolean isEnabled = settings.getAsBoolean("enabled", !(client instanceof MockExtendedClient)); boolean isEnabled = settings.getAsBoolean("enabled", false);
String indexName = settings.get("name", index); String indexName = settings.get("name", index);
String fullIndexName; String fullIndexName;
String dateTimePattern = settings.get("dateTimePattern"); String dateTimePattern = settings.get("dateTimePattern");
@ -833,8 +481,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
.setEnabled(isEnabled) .setEnabled(isEnabled)
.setIndex(indexName) .setIndex(indexName)
.setFullIndexName(fullIndexName) .setFullIndexName(fullIndexName)
.setSettingsUrl(settings.get("settings")) .setSettings(findSettingsFrom(settings.get("settings")))
.setMappingsUrl(settings.get("mapping")) .setMappings(findMappingsFrom(settings.get("mapping")))
.setDateTimePattern(dateTimePattern) .setDateTimePattern(dateTimePattern)
.setIgnoreErrors(settings.getAsBoolean("skiperrors", false)) .setIgnoreErrors(settings.getAsBoolean("skiperrors", false))
.setShift(settings.getAsBoolean("shift", true)) .setShift(settings.getAsBoolean("shift", true))
@ -847,16 +495,10 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
@Override @Override
public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException { public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException {
ensureClient(); ensureClientIsPresent();
if (index == null) { if (index == null) {
throw new IOException("no index name given"); throw new IOException("no index name given");
} }
if (key == null) {
throw new IOException("no key given");
}
if (value == null) {
throw new IOException("no value given");
}
Settings.Builder updateSettingsBuilder = Settings.builder(); Settings.Builder updateSettingsBuilder = Settings.builder();
updateSettingsBuilder.put(key, value.toString()); updateSettingsBuilder.put(key, value.toString());
UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index) UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index)
@ -864,18 +506,48 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet(); client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet();
} }
private void ensureClient() { private static String findSettingsFrom(String string) throws IOException {
if (this instanceof MockExtendedClient) { if (string == null) {
return; return null;
} }
if (client == null) { try {
throw new IllegalStateException("no client"); URL url = new URL(string);
try (InputStream inputStream = url.openStream()) {
Settings settings = Settings.builder().loadFromStream(string, inputStream, true).build();
XContentBuilder builder = JsonXContent.contentBuilder();
settings.toXContent(builder, ToXContent.EMPTY_PARAMS);
return Strings.toString(builder);
}
} catch (MalformedURLException e) {
return string;
} }
} }
private void ensureIndexGiven(String index) { private static String findMappingsFrom(String string) throws IOException {
if (index == null) { if (string == null) {
throw new IllegalArgumentException("no index given"); return null;
}
try {
URL url = new URL(string);
try (InputStream inputStream = url.openStream()) {
if (string.endsWith(".json")) {
Map<String, ?> mappings = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputStream).mapOrdered();
XContentBuilder builder = JsonXContent.contentBuilder();
builder.startObject().map(mappings).endObject();
return Strings.toString(builder);
}
if (string.endsWith(".yml") || string.endsWith(".yaml")) {
Map<String, ?> mappings = YamlXContent.yamlXContent.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputStream).mapOrdered();
XContentBuilder builder = JsonXContent.contentBuilder();
builder.startObject().map(mappings).endObject();
return Strings.toString(builder);
}
}
return string;
} catch (MalformedInputException e) {
return string;
} }
} }
@ -896,7 +568,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
} }
public void checkMapping(String index) { public void checkMapping(String index) {
ensureClient(); ensureClientIsPresent();
GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(index); GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(index);
GetMappingsResponse getMappingsResponse = client.execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); GetMappingsResponse getMappingsResponse = client.execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> map = getMappingsResponse.getMappings(); ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> map = getMappingsResponse.getMappings();
@ -912,25 +584,25 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
private void checkMapping(String index, String type, MappingMetaData mappingMetaData) { private void checkMapping(String index, String type, MappingMetaData mappingMetaData) {
try { try {
SearchSourceBuilder builder = new SearchSourceBuilder() SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE)
.query(QueryBuilders.matchAllQuery()) .setIndices(index)
.size(0) .setQuery(QueryBuilders.matchAllQuery())
.trackTotalHits(true); .setSize(0)
SearchRequest searchRequest = new SearchRequest() .setTrackTotalHits(true);
.indices(index)
.source(builder);
SearchResponse searchResponse = SearchResponse searchResponse =
client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); searchRequestBuilder.execute().actionGet();
TotalHits total = searchResponse.getHits().getTotalHits(); long total = searchResponse.getHits().getTotalHits().value;
if (total.value > 0L) { if (total > 0L) {
Map<String, Long> fields = new TreeMap<>(); Map<String, Long> fields = new TreeMap<>();
Map<String, Object> root = mappingMetaData.getSourceAsMap(); Map<String, Object> root = mappingMetaData.getSourceAsMap();
checkMapping(index, "", "", root, fields); checkMapping(index, type, "", "", root, fields);
AtomicInteger empty = new AtomicInteger(); AtomicInteger empty = new AtomicInteger();
Map<String, Long> map = sortByValue(fields); Map<String, Long> map = sortByValue(fields);
map.forEach((key, value) -> { map.forEach((key, value) -> {
logger.info("{} {} {}", logger.info("{} {} {}",
key, value, (double) value * 100 / total.value); key,
value,
(double) value * 100 / total);
if (value == 0) { if (value == 0) {
empty.incrementAndGet(); empty.incrementAndGet();
} }
@ -944,7 +616,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
private void checkMapping(String index, private void checkMapping(String index, String type,
String pathDef, String fieldName, Map<String, Object> map, String pathDef, String fieldName, Map<String, Object> map,
Map<String, Long> fields) { Map<String, Long> fields) {
String path = pathDef; String path = pathDef;
@ -969,20 +641,18 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
String fieldType = o instanceof String ? o.toString() : null; String fieldType = o instanceof String ? o.toString() : null;
// do not recurse into our custom field mapper // do not recurse into our custom field mapper
if (!"standardnumber".equals(fieldType) && !"ref".equals(fieldType)) { if (!"standardnumber".equals(fieldType) && !"ref".equals(fieldType)) {
checkMapping(index, path, key, child, fields); checkMapping(index, type, path, key, child, fields);
} }
} else if ("type".equals(key)) { } else if ("type".equals(key)) {
QueryBuilder filterBuilder = QueryBuilders.existsQuery(path); QueryBuilder filterBuilder = QueryBuilders.existsQuery(path);
QueryBuilder queryBuilder = QueryBuilders.constantScoreQuery(filterBuilder); QueryBuilder queryBuilder = QueryBuilders.constantScoreQuery(filterBuilder);
SearchSourceBuilder builder = new SearchSourceBuilder() SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE)
.query(queryBuilder) .setIndices(index)
.size(0) .setQuery(queryBuilder)
.trackTotalHits(true); .setSize(0)
SearchRequest searchRequest = new SearchRequest() .setTrackTotalHits(true);
.indices(index)
.source(builder);
SearchResponse searchResponse = SearchResponse searchResponse =
client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); searchRequestBuilder.execute().actionGet();
fields.put(path, searchResponse.getHits().getTotalHits().value); fields.put(path, searchResponse.getHits().getTotalHits().value);
} }
} }
@ -995,58 +665,38 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
return result; return result;
} }
private static TimeValue toTimeValue(long timeValue, TimeUnit timeUnit) {
switch (timeUnit) {
case DAYS:
return TimeValue.timeValueHours(24 * timeValue);
case HOURS:
return TimeValue.timeValueHours(timeValue);
case MINUTES:
return TimeValue.timeValueMinutes(timeValue);
case SECONDS:
return TimeValue.timeValueSeconds(timeValue);
case MILLISECONDS:
return TimeValue.timeValueMillis(timeValue);
case MICROSECONDS:
return TimeValue.timeValueNanos(1000 * timeValue);
case NANOSECONDS:
return TimeValue.timeValueNanos(timeValue);
default:
throw new IllegalArgumentException("unknown time unit: " + timeUnit);
}
}
private static class SuccessIndexShiftResult implements IndexShiftResult { private static class SuccessIndexShiftResult implements IndexShiftResult {
List<String> movedAliases; Collection<String> movedAliases;
List<String> newAliases; Collection<String> newAliases;
SuccessIndexShiftResult(List<String> movedAliases, List<String> newAliases) { SuccessIndexShiftResult(Collection<String> movedAliases, Collection<String> newAliases) {
this.movedAliases = movedAliases; this.movedAliases = movedAliases;
this.newAliases = newAliases; this.newAliases = newAliases;
} }
@Override @Override
public List<String> getMovedAliases() { public Collection<String> getMovedAliases() {
return movedAliases; return movedAliases;
} }
@Override @Override
public List<String> getNewAliases() { public Collection<String> getNewAliases() {
return newAliases; return newAliases;
} }
} }
private static class SuccessPruneResult implements IndexPruneResult { private static class SuccessPruneResult implements IndexPruneResult {
List<String> candidateIndices; Collection<String> candidateIndices;
List<String> indicesToDelete; Collection<String> indicesToDelete;
AcknowledgedResponse response; AcknowledgedResponse response;
SuccessPruneResult(List<String> candidateIndices, List<String> indicesToDelete, SuccessPruneResult(Collection<String> candidateIndices,
Collection<String> indicesToDelete,
AcknowledgedResponse response) { AcknowledgedResponse response) {
this.candidateIndices = candidateIndices; this.candidateIndices = candidateIndices;
this.indicesToDelete = indicesToDelete; this.indicesToDelete = indicesToDelete;
@ -1059,48 +709,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
} }
@Override @Override
public List<String> getCandidateIndices() { public Collection<String> getCandidateIndices() {
return candidateIndices; return candidateIndices;
} }
@Override @Override
public List<String> getDeletedIndices() { public Collection<String> getDeletedIndices() {
return indicesToDelete;
}
@Override
public boolean isAcknowledged() {
return response.isAcknowledged();
}
}
private static class FailPruneResult implements IndexPruneResult {
List<String> candidateIndices;
List<String> indicesToDelete;
AcknowledgedResponse response;
FailPruneResult(List<String> candidateIndices, List<String> indicesToDelete,
AcknowledgedResponse response) {
this.candidateIndices = candidateIndices;
this.indicesToDelete = indicesToDelete;
this.response = response;
}
@Override
public IndexPruneResult.State getState() {
return IndexPruneResult.State.SUCCESS;
}
@Override
public List<String> getCandidateIndices() {
return candidateIndices;
}
@Override
public List<String> getDeletedIndices() {
return indicesToDelete; return indicesToDelete;
} }
@ -1112,11 +726,11 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
private static class NothingToDoPruneResult implements IndexPruneResult { private static class NothingToDoPruneResult implements IndexPruneResult {
List<String> candidateIndices; Collection<String> candidateIndices;
List<String> indicesToDelete; Collection<String> indicesToDelete;
NothingToDoPruneResult(List<String> candidateIndices, List<String> indicesToDelete) { NothingToDoPruneResult(Collection<String> candidateIndices, List<String> indicesToDelete) {
this.candidateIndices = candidateIndices; this.candidateIndices = candidateIndices;
this.indicesToDelete = indicesToDelete; this.indicesToDelete = indicesToDelete;
} }
@ -1127,12 +741,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
} }
@Override @Override
public List<String> getCandidateIndices() { public Collection<String> getCandidateIndices() {
return candidateIndices; return candidateIndices;
} }
@Override @Override
public List<String> getDeletedIndices() { public Collection<String> getDeletedIndices() {
return indicesToDelete; return indicesToDelete;
} }

View file

@ -0,0 +1,217 @@
package org.xbib.elx.common;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.QueryBuilders;
import org.xbib.elx.api.BasicClient;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
public abstract class AbstractBasicClient implements BasicClient {
private static final Logger logger = LogManager.getLogger(AbstractBasicClient.class.getName());
protected ElasticsearchClient client;
protected Settings settings;
private final AtomicBoolean closed;
public AbstractBasicClient() {
closed = new AtomicBoolean(false);
}
@Override
public void setClient(ElasticsearchClient client) {
logger.log(Level.INFO, "setting client = " + client);
this.client = client;
}
@Override
public ElasticsearchClient getClient() {
return client;
}
@Override
public void init(Settings settings) throws IOException {
if (closed.compareAndSet(false, true)) {
logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(','));
this.settings = settings;
setClient(createClient(settings));
} else {
logger.log(Level.WARN, "not initializing");
}
}
@Override
public String getClusterName() {
ensureClientIsPresent();
try {
ClusterStateRequest clusterStateRequest = new ClusterStateRequest().clear();
ClusterStateResponse clusterStateResponse =
getClient().execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
return clusterStateResponse.getClusterName().value();
} catch (ElasticsearchTimeoutException e) {
logger.warn(e.getMessage(), e);
return "TIMEOUT";
} catch (NoNodeAvailableException e) {
logger.warn(e.getMessage(), e);
return "DISCONNECTED";
} catch (Exception e) {
logger.warn(e.getMessage(), e);
return "[" + e.getMessage() + "]";
}
}
@Override
public void waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) {
ensureClientIsPresent();
ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString);
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
String message = "timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name();
if (logger.isErrorEnabled()) {
logger.error(message);
}
throw new IllegalStateException(message);
}
}
@Override
public void waitForShards(long maxWaitTime, TimeUnit timeUnit) {
ensureClientIsPresent();
logger.info("waiting for cluster shard settling");
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest()
.waitForNoInitializingShards(true)
.waitForNoRelocatingShards(true)
.timeout(timeout);
ClusterHealthResponse healthResponse =
client.execute(ClusterHealthAction.INSTANCE, clusterHealthRequest).actionGet();
if (healthResponse.isTimedOut()) {
String message = "timeout waiting for cluster shards";
logger.error(message);
throw new IllegalStateException(message);
}
}
@Override
public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) {
ensureClientIsPresent();
try {
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().timeout(timeout)).actionGet();
ClusterHealthStatus status = healthResponse.getStatus();
return status.name();
} catch (ElasticsearchTimeoutException e) {
logger.warn(e.getMessage(), e);
return "TIMEOUT";
} catch (NoNodeAvailableException e) {
logger.warn(e.getMessage(), e);
return "DISCONNECTED";
} catch (Exception e) {
logger.warn(e.getMessage(), e);
return "[" + e.getMessage() + "]";
}
}
@Override
public long getSearchableDocs(String index) {
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE)
.setIndices(index)
.setQuery(QueryBuilders.matchAllQuery())
.setSize(0)
.setTrackTotalHits(true);
return searchRequestBuilder.execute().actionGet().getHits().getTotalHits().value;
}
@Override
public boolean isIndexExists(String index) {
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
indicesExistsRequest.indices(index);
IndicesExistsResponse indicesExistsResponse =
client.execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
return indicesExistsResponse.isExists();
}
@Override
public void close() throws IOException {
ensureClientIsPresent();
if (closed.compareAndSet(false, true)) {
closeClient(settings);
}
}
protected abstract ElasticsearchClient createClient(Settings settings) throws IOException;
protected abstract void closeClient(Settings settings) throws IOException;
protected void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException {
ensureClientIsPresent();
if (index == null) {
throw new IOException("no index name given");
}
if (key == null) {
throw new IOException("no key given");
}
if (value == null) {
throw new IOException("no value given");
}
Settings.Builder updateSettingsBuilder = Settings.builder();
updateSettingsBuilder.put(key, value.toString());
UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index)
.settings(updateSettingsBuilder).timeout(toTimeValue(timeout, timeUnit));
client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet();
}
protected void ensureClientIsPresent() {
if (client == null) {
throw new IllegalStateException("no client");
}
}
protected static TimeValue toTimeValue(long timeValue, TimeUnit timeUnit) {
switch (timeUnit) {
case DAYS:
return TimeValue.timeValueHours(24 * timeValue);
case HOURS:
return TimeValue.timeValueHours(timeValue);
case MINUTES:
return TimeValue.timeValueMinutes(timeValue);
case SECONDS:
return TimeValue.timeValueSeconds(timeValue);
case MILLISECONDS:
return TimeValue.timeValueMillis(timeValue);
case MICROSECONDS:
return TimeValue.timeValueNanos(1000 * timeValue);
case NANOSECONDS:
return TimeValue.timeValueNanos(timeValue);
default:
throw new IllegalArgumentException("unknown time unit: " + timeUnit);
}
}
}

View file

@ -0,0 +1,246 @@
package org.xbib.elx.common;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.flush.FlushAction;
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.DeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.xbib.elx.api.BulkClient;
import org.xbib.elx.api.BulkController;
import org.xbib.elx.api.BulkMetric;
import org.xbib.elx.api.IndexDefinition;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
public abstract class AbstractBulkClient extends AbstractBasicClient implements BulkClient {
private static final Logger logger = LogManager.getLogger(AbstractBulkClient.class.getName());
private BulkMetric bulkMetric;
private BulkController bulkController;
private final AtomicBoolean closed = new AtomicBoolean(true);
@Override
public void init(Settings settings) throws IOException {
if (closed.compareAndSet(true, false)) {
super.init(settings);
logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(','));
bulkMetric = new DefaultBulkMetric();
bulkMetric.init(settings);
bulkController = new DefaultBulkController(this, bulkMetric);
bulkController.init(settings);
} else {
logger.log(Level.WARN, "not initializing");
}
}
@Override
public BulkMetric getBulkMetric() {
return bulkMetric;
}
@Override
public BulkController getBulkController() {
return bulkController;
}
@Override
public void flush() throws IOException {
if (bulkController != null) {
bulkController.flush();
}
}
@Override
public void close() throws IOException {
if (closed.compareAndSet(false, true)) {
ensureClientIsPresent();
if (bulkMetric != null) {
logger.info("closing bulk metric");
bulkMetric.close();
}
if (bulkController != null) {
logger.info("closing bulk controller");
bulkController.close();
}
closeClient(settings);
}
}
@Override
public void newIndex(IndexDefinition indexDefinition) throws IOException {
Settings settings = indexDefinition.getSettings() == null ? null :
Settings.builder().loadFromSource(indexDefinition.getSettings(), XContentType.JSON).build();
Map<String, ?> mappings = indexDefinition.getMappings() == null ? null :
JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, indexDefinition.getMappings()).mapOrdered();
newIndex(indexDefinition.getFullIndexName(), settings, mappings);
}
@Override
public void newIndex(String index) throws IOException {
newIndex(index, Settings.EMPTY, (Map<String, ?>) null);
}
@Override
public void newIndex(String index, Settings settings) throws IOException {
newIndex(index, settings, (Map<String, ?>) null);
}
@Override
public void newIndex(String index, Settings settings, XContentBuilder builder) throws IOException {
String mappingString = Strings.toString(builder);
Map<String, ?> mappings = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mappingString).mapOrdered();
newIndex(index, settings, mappings);
}
@Override
public void newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException {
if (index == null) {
logger.warn("no index name given to create index");
return;
}
ensureClientIsPresent();
waitForCluster("YELLOW", 30L, TimeUnit.SECONDS);
CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index);
if (settings != null) {
createIndexRequest.settings(settings);
}
if (mapping != null) {
createIndexRequest.mapping("_doc", mapping);
}
CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
if (createIndexResponse.isAcknowledged()) {
logger.info("index {} created", index);
}
}
@Override
public void startBulk(IndexDefinition indexDefinition) throws IOException {
startBulk(indexDefinition.getFullIndexName(), -1, 1);
}
@Override
public void startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds)
throws IOException {
if (bulkController != null) {
ensureClientIsPresent();
bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds);
}
}
@Override
public void stopBulk(IndexDefinition indexDefinition) throws IOException {
if (bulkController != null) {
ensureClientIsPresent();
bulkController.stopBulkMode(indexDefinition);
}
}
@Override
public void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException {
if (bulkController != null) {
ensureClientIsPresent();
bulkController.stopBulkMode(index, timeout, timeUnit);
}
}
@Override
public BulkClient index(String index, String id, boolean create, String source) {
return index(index, id, create, new BytesArray(source.getBytes(StandardCharsets.UTF_8)));
}
@Override
public BulkClient index(String index, String id, boolean create, BytesReference source) {
return index(new IndexRequest().index(index).id(id).create(create)
.source(source, XContentType.JSON));
}
@Override
public BulkClient index(IndexRequest indexRequest) {
ensureClientIsPresent();
bulkController.bulkIndex(indexRequest);
return this;
}
@Override
public BulkClient delete(String index, String id) {
return delete(new DeleteRequest().index(index).id(id));
}
@Override
public BulkClient delete(DeleteRequest deleteRequest) {
ensureClientIsPresent();
bulkController.bulkDelete(deleteRequest);
return this;
}
@Override
public BulkClient update(String index, String id, BytesReference source) {
return update(new UpdateRequest().index(index).id(id)
.doc(source, XContentType.JSON));
}
@Override
public BulkClient update(String index, String id, String source) {
return update(new UpdateRequest().index(index).id(id)
.doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
}
@Override
public BulkClient update(UpdateRequest updateRequest) {
ensureClientIsPresent();
bulkController.bulkUpdate(updateRequest);
return this;
}
@Override
public boolean waitForResponses(long timeout, TimeUnit timeUnit) {
ensureClientIsPresent();
return bulkController.waitForBulkResponses(timeout, timeUnit);
}
@Override
public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException {
super.updateIndexSetting(index, key, value, timeout, timeUnit);
}
@Override
public void flushIndex(String index) {
if (index != null) {
ensureClientIsPresent();
client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet();
}
}
@Override
public void refreshIndex(String index) {
if (index != null) {
ensureClientIsPresent();
client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet();
}
}
}

View file

@ -0,0 +1,158 @@
package org.xbib.elx.common;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.get.GetAction;
import org.elasticsearch.action.get.GetRequestBuilder;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.get.MultiGetAction;
import org.elasticsearch.action.get.MultiGetRequestBuilder;
import org.elasticsearch.action.get.MultiGetResponse;
import org.elasticsearch.action.search.ClearScrollAction;
import org.elasticsearch.action.search.ClearScrollRequestBuilder;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollAction;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.action.search.ShardSearchFailure;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.search.SearchHit;
import org.xbib.elx.api.SearchClient;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Optional;
import java.util.Spliterator;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
public abstract class AbstractSearchClient extends AbstractBasicClient implements SearchClient {
@Override
public Optional<GetResponse> get(Consumer<GetRequestBuilder> getRequestBuilderConsumer) {
GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE);
getRequestBuilderConsumer.accept(getRequestBuilder);
GetResponse getResponse = getRequestBuilder.execute().actionGet();
return getResponse.isExists() ? Optional.of(getResponse) : Optional.empty();
}
@Override
public Optional<MultiGetResponse> multiGet(Consumer<MultiGetRequestBuilder> multiGetRequestBuilderConsumer) {
MultiGetRequestBuilder multiGetRequestBuilder = new MultiGetRequestBuilder(client, MultiGetAction.INSTANCE);
multiGetRequestBuilderConsumer.accept(multiGetRequestBuilder);
MultiGetResponse multiGetItemResponse = multiGetRequestBuilder.execute().actionGet();
return multiGetItemResponse.getResponses().length == 0 ? Optional.empty() : Optional.of(multiGetItemResponse);
}
@Override
public Optional<SearchResponse> search(Consumer<SearchRequestBuilder> queryBuilder) {
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
queryBuilder.accept(searchRequestBuilder);
SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
if (searchResponse.getFailedShards() > 0) {
StringBuilder sb = new StringBuilder("Search failed:");
for (ShardSearchFailure failure : searchResponse.getShardFailures()) {
sb.append("\n").append(failure.reason());
}
throw new ElasticsearchException(sb.toString());
}
return searchResponse.getHits().getHits().length == 0 ? Optional.empty() : Optional.of(searchResponse);
}
@Override
public Stream<SearchHit> search(Consumer<SearchRequestBuilder> queryBuilder,
TimeValue scrollTime, int scrollSize) {
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
queryBuilder.accept(searchRequestBuilder);
searchRequestBuilder.setScroll(scrollTime).setSize(scrollSize);
SearchResponse originalSearchResponse = searchRequestBuilder.execute().actionGet();
Stream<SearchResponse> infiniteResponses = Stream.iterate(originalSearchResponse,
searchResponse -> new SearchScrollRequestBuilder(client, SearchScrollAction.INSTANCE)
.setScrollId(searchResponse.getScrollId())
.setScroll(scrollTime)
.execute().actionGet());
Predicate<SearchResponse> condition = searchResponse -> searchResponse.getHits().getHits().length > 0;
Consumer<SearchResponse> lastAction = searchResponse -> {
ClearScrollRequestBuilder clearScrollRequestBuilder =
new ClearScrollRequestBuilder(client, ClearScrollAction.INSTANCE)
.addScrollId(searchResponse.getScrollId());
clearScrollRequestBuilder.execute().actionGet();
};
return StreamSupport.stream(TakeWhileSpliterator.over(infiniteResponses.spliterator(),
condition, lastAction), false)
.onClose(infiniteResponses::close)
.flatMap(searchResponse -> Arrays.stream(searchResponse.getHits().getHits()));
}
@Override
public Stream<String> getIds(Consumer<SearchRequestBuilder> queryBuilder) {
return search(queryBuilder, TimeValue.timeValueMinutes(1), 1000).map(SearchHit::getId);
}
static class TakeWhileSpliterator<T> implements Spliterator<T> {
private final Spliterator<T> source;
private final Predicate<T> condition;
private final Consumer<T> lastElement;
private final boolean inclusive;
private final AtomicBoolean checked;
static <T> TakeWhileSpliterator<T> over(Spliterator<T> source,
Predicate<T> condition,
Consumer<T> lastElement) {
return new TakeWhileSpliterator<>(source, condition, lastElement, true);
}
private TakeWhileSpliterator(Spliterator<T> source,
Predicate<T> condition,
Consumer<T> lastElement,
boolean inclusive) {
this.source = source;
this.condition = condition;
this.lastElement = lastElement;
this.inclusive = inclusive;
this.checked = new AtomicBoolean(true);
}
@Override
public boolean tryAdvance(Consumer<? super T> action) {
return checked.get() && source.tryAdvance(e -> {
if (condition.test(e)) {
action.accept(e);
} else {
if (inclusive && checked.get()) {
action.accept(e);
}
lastElement.accept(e);
checked.set(false);
}
});
}
@Override
public Spliterator<T> trySplit() {
return null;
}
@Override
public long estimateSize() {
return checked.get() ? source.estimateSize() : 0;
}
@Override
public int characteristics() {
return source.characteristics() &~ Spliterator.SIZED;
}
@Override
public Comparator<? super T> getComparator() {
return source.getComparator();
}
}
}

View file

@ -8,11 +8,12 @@ import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.unit.TimeValue;
import org.xbib.elx.api.ExtendedClient; import org.xbib.elx.api.AdminClientProvider;
import org.xbib.elx.api.ExtendedClientProvider; import org.xbib.elx.api.BulkClientProvider;
import org.xbib.elx.api.BasicClient;
import org.xbib.elx.api.SearchClientProvider;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.ServiceLoader; import java.util.ServiceLoader;
@ -23,11 +24,15 @@ public class ClientBuilder {
private final ElasticsearchClient client; private final ElasticsearchClient client;
private final ClassLoader classLoader;
private final Settings.Builder settingsBuilder; private final Settings.Builder settingsBuilder;
private Map<Class<? extends ExtendedClientProvider>, ExtendedClientProvider> providerMap; private Class<? extends AdminClientProvider> adminClientProvider;
private Class<? extends ExtendedClientProvider> provider; private Class<? extends BulkClientProvider> bulkClientProvider;
private Class<? extends SearchClientProvider> searchClientProvider;
public ClientBuilder() { public ClientBuilder() {
this(null); this(null);
@ -39,14 +44,9 @@ public class ClientBuilder {
public ClientBuilder(ElasticsearchClient client, ClassLoader classLoader) { public ClientBuilder(ElasticsearchClient client, ClassLoader classLoader) {
this.client = client; this.client = client;
this.classLoader = classLoader;
this.settingsBuilder = Settings.builder(); this.settingsBuilder = Settings.builder();
settingsBuilder.put("node.name", "elx-client-" + Version.CURRENT); settingsBuilder.put("node.name", "elx-client-" + Version.CURRENT);
this.providerMap = new HashMap<>();
ServiceLoader<ExtendedClientProvider> serviceLoader = ServiceLoader.load(ExtendedClientProvider.class,
classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader());
for (ExtendedClientProvider provider : serviceLoader) {
providerMap.put(provider.getClass(), provider);
}
} }
public static ClientBuilder builder() { public static ClientBuilder builder() {
@ -57,8 +57,18 @@ public class ClientBuilder {
return new ClientBuilder(client); return new ClientBuilder(client);
} }
public ClientBuilder provider(Class<? extends ExtendedClientProvider> provider) { public ClientBuilder setAdminClientProvider(Class<? extends AdminClientProvider> adminClientProvider) {
this.provider = provider; this.adminClientProvider = adminClientProvider;
return this;
}
public ClientBuilder setBulkClientProvider(Class<? extends BulkClientProvider> bulkClientProvider) {
this.bulkClientProvider = bulkClientProvider;
return this;
}
public ClientBuilder setSearchClientProvider(Class<? extends SearchClientProvider> searchClientProvider) {
this.searchClientProvider = searchClientProvider;
return this; return this;
} }
@ -97,15 +107,59 @@ public class ClientBuilder {
return this; return this;
} }
@SuppressWarnings("unchecked") public ClientBuilder put(Map<String, ?> map) {
public <C extends ExtendedClient> C build() throws IOException { for (Map.Entry<String, ?> entry : map.entrySet()) {
if (provider == null) { if (entry.getValue() == null) {
throw new IllegalArgumentException("no provider"); continue;
}
if (entry.getValue() instanceof String ||
entry.getValue() instanceof Integer ||
entry.getValue() instanceof Long ||
entry.getValue() instanceof Float ||
entry.getValue() instanceof TimeValue) {
settingsBuilder.put(entry.getKey(), entry.getValue().toString());
} else {
logger.log(Level.WARN, "skipping " + entry.getValue() +
" because invalid class type " + entry.getValue().getClass().getName());
}
} }
return this;
}
@SuppressWarnings("unchecked")
public <C extends BasicClient> C build() throws IOException {
Settings settings = settingsBuilder.build(); Settings settings = settingsBuilder.build();
logger.log(Level.INFO, "settings = " + settings.toDelimitedString(',')); logger.log(Level.INFO, "settings = " + settings.toDelimitedString(','));
return (C) providerMap.get(provider).getExtendedClient() if (adminClientProvider != null) {
.setClient(client) for (AdminClientProvider provider : ServiceLoader.load(AdminClientProvider.class, classLoader)) {
.init(settings); if (provider.getClass().isAssignableFrom(adminClientProvider)) {
C c = (C) provider.getClient();
c.setClient(client);
c.init(settings);
return c;
}
}
}
if (bulkClientProvider != null) {
for (BulkClientProvider provider : ServiceLoader.load(BulkClientProvider.class, classLoader)) {
if (provider.getClass().isAssignableFrom(bulkClientProvider)) {
C c = (C) provider.getClient();
c.setClient(client);
c.init(settings);
return c;
}
}
}
if (searchClientProvider != null) {
for (SearchClientProvider provider : ServiceLoader.load(SearchClientProvider.class, classLoader)) {
if (provider.getClass().isAssignableFrom(searchClientProvider)) {
C c = (C) provider.getClient();
c.setClient(client);
c.init(settings);
return c;
}
}
}
throw new IllegalArgumentException("no provider");
} }
} }

View file

@ -8,11 +8,11 @@ import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.unit.TimeValue;
import org.xbib.elx.api.BulkClient;
import org.xbib.elx.api.BulkController; import org.xbib.elx.api.BulkController;
import org.xbib.elx.api.BulkListener; import org.xbib.elx.api.BulkListener;
import org.xbib.elx.api.BulkMetric; import org.xbib.elx.api.BulkMetric;
import org.xbib.elx.api.BulkProcessor; import org.xbib.elx.api.BulkProcessor;
import org.xbib.elx.api.ExtendedClient;
import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.api.IndexDefinition;
import java.io.IOException; import java.io.IOException;
@ -27,7 +27,7 @@ public class DefaultBulkController implements BulkController {
private static final Logger logger = LogManager.getLogger(DefaultBulkController.class); private static final Logger logger = LogManager.getLogger(DefaultBulkController.class);
private final ExtendedClient client; private final BulkClient bulkClient;
private final BulkMetric bulkMetric; private final BulkMetric bulkMetric;
@ -45,9 +45,9 @@ public class DefaultBulkController implements BulkController {
private final AtomicBoolean active; private final AtomicBoolean active;
public DefaultBulkController(ExtendedClient client) { public DefaultBulkController(BulkClient bulkClient, BulkMetric bulkMetric) {
this.client = client; this.bulkClient = bulkClient;
this.bulkMetric = new DefaultBulkMetric(); this.bulkMetric = bulkMetric;
this.indexNames = new ArrayList<>(); this.indexNames = new ArrayList<>();
this.active = new AtomicBoolean(false); this.active = new AtomicBoolean(false);
this.startBulkRefreshIntervals = new HashMap<>(); this.startBulkRefreshIntervals = new HashMap<>();
@ -80,7 +80,7 @@ public class DefaultBulkController implements BulkController {
boolean enableBulkLogging = settings.getAsBoolean(Parameters.ENABLE_BULK_LOGGING.name(), boolean enableBulkLogging = settings.getAsBoolean(Parameters.ENABLE_BULK_LOGGING.name(),
Parameters.ENABLE_BULK_LOGGING.getValue()); Parameters.ENABLE_BULK_LOGGING.getValue());
BulkListener bulkListener = new DefaultBulkListener(this, bulkMetric, enableBulkLogging); BulkListener bulkListener = new DefaultBulkListener(this, bulkMetric, enableBulkLogging);
this.bulkProcessor = DefaultBulkProcessor.builder(client.getClient(), bulkListener) this.bulkProcessor = DefaultBulkProcessor.builder(bulkClient.getClient(), bulkListener)
.setBulkActions(maxActionsPerRequest) .setBulkActions(maxActionsPerRequest)
.setConcurrentRequests(maxConcurrentRequests) .setConcurrentRequests(maxConcurrentRequests)
.setFlushInterval(flushIngestInterval) .setFlushInterval(flushIngestInterval)
@ -115,7 +115,7 @@ public class DefaultBulkController implements BulkController {
startBulkRefreshIntervals.put(indexName, startRefreshIntervalInSeconds); startBulkRefreshIntervals.put(indexName, startRefreshIntervalInSeconds);
stopBulkRefreshIntervals.put(indexName, stopRefreshIntervalInSeconds); stopBulkRefreshIntervals.put(indexName, stopRefreshIntervalInSeconds);
if (startRefreshIntervalInSeconds != 0L) { if (startRefreshIntervalInSeconds != 0L) {
client.updateIndexSetting(indexName, "refresh_interval", startRefreshIntervalInSeconds + "s", bulkClient.updateIndexSetting(indexName, "refresh_interval", startRefreshIntervalInSeconds + "s",
30L, TimeUnit.SECONDS); 30L, TimeUnit.SECONDS);
} }
} }
@ -187,7 +187,7 @@ public class DefaultBulkController implements BulkController {
if (indexNames.contains(index)) { if (indexNames.contains(index)) {
Long secs = stopBulkRefreshIntervals.get(index); Long secs = stopBulkRefreshIntervals.get(index);
if (secs != null && secs != 0L) { if (secs != null && secs != 0L) {
client.updateIndexSetting(index, "refresh_interval", secs + "s", bulkClient.updateIndexSetting(index, "refresh_interval", secs + "s",
30L, TimeUnit.SECONDS); 30L, TimeUnit.SECONDS);
} }
indexNames.remove(index); indexNames.remove(index);
@ -206,11 +206,11 @@ public class DefaultBulkController implements BulkController {
public void close() throws IOException { public void close() throws IOException {
flush(); flush();
bulkMetric.close(); bulkMetric.close();
if (client.waitForResponses(maxWaitTime, maxWaitTimeUnit)) { if (bulkClient.waitForResponses(maxWaitTime, maxWaitTimeUnit)) {
for (String index : indexNames) { for (String index : indexNames) {
Long secs = stopBulkRefreshIntervals.get(index); Long secs = stopBulkRefreshIntervals.get(index);
if (secs != null && secs != 0L) if (secs != null && secs != 0L)
client.updateIndexSetting(index, "refresh_interval", secs + "s", bulkClient.updateIndexSetting(index, "refresh_interval", secs + "s",
30L, TimeUnit.SECONDS); 30L, TimeUnit.SECONDS);
} }
indexNames.clear(); indexNames.clear();

View file

@ -1,5 +1,6 @@
package org.xbib.elx.common; package org.xbib.elx.common;
import org.elasticsearch.common.settings.Settings;
import org.xbib.elx.api.BulkMetric; import org.xbib.elx.api.BulkMetric;
import org.xbib.metrics.api.Count; import org.xbib.metrics.api.Count;
import org.xbib.metrics.api.Metered; import org.xbib.metrics.api.Metered;
@ -36,6 +37,10 @@ public class DefaultBulkMetric implements BulkMetric {
submitted = new CountMetric(); submitted = new CountMetric();
succeeded = new CountMetric(); succeeded = new CountMetric();
failed = new CountMetric(); failed = new CountMetric();
}
@Override
public void init(Settings settings) {
start(); start();
} }

View file

@ -3,8 +3,6 @@ package org.xbib.elx.common;
import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.api.IndexRetention; import org.xbib.elx.api.IndexRetention;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
public class DefaultIndexDefinition implements IndexDefinition { public class DefaultIndexDefinition implements IndexDefinition {
@ -15,9 +13,9 @@ public class DefaultIndexDefinition implements IndexDefinition {
private String dateTimePattern; private String dateTimePattern;
private URL settingsUrl; private String settings;
private URL mappingsUrl; private String mappings;
private boolean enabled; private boolean enabled;
@ -62,37 +60,25 @@ public class DefaultIndexDefinition implements IndexDefinition {
} }
@Override @Override
public IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException { public IndexDefinition setSettings(String settings) {
this.settingsUrl = settingsUrlString != null ? new URL(settingsUrlString) : null; this.settings = settings;
return this; return this;
} }
@Override @Override
public IndexDefinition setSettingsUrl(URL settingsUrl) { public String getSettings() {
this.settingsUrl = settingsUrl; return settings;
}
@Override
public IndexDefinition setMappings(String mappings) {
this.mappings = mappings;
return this; return this;
} }
@Override @Override
public URL getSettingsUrl() { public String getMappings() {
return settingsUrl; return mappings;
}
@Override
public IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException {
this.mappingsUrl = mappingsUrlString != null ? new URL(mappingsUrlString) : null;
return this;
}
@Override
public IndexDefinition setMappingsUrl(URL mappingsUrl) {
this.mappingsUrl = mappingsUrl;
return this;
}
@Override
public URL getMappingsUrl() {
return mappingsUrl;
} }
@Override @Override

View file

@ -0,0 +1,59 @@
package org.xbib.elx.common;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import java.util.concurrent.TimeUnit;
/**
* A mocked client, it does not perform any actions on a cluster. Useful for testing.
*/
public class MockAdminClient extends AbstractAdminClient {
@Override
public ElasticsearchClient getClient() {
return null;
}
@Override
public void init(Settings settings) {
}
@Override
protected ElasticsearchClient createClient(Settings settings) {
return null;
}
@Override
protected void closeClient(Settings settings) {
}
@Override
public MockAdminClient deleteIndex(String index) {
return this;
}
@Override
public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) {
return true;
}
@Override
public void waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) {
}
@Override
public void waitForShards(long maxWaitTime, TimeUnit timeUnit) {
}
@Override
public MockAdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) {
return this;
}
@Override
public void close() {
// nothing to do
}
}

View file

@ -0,0 +1,10 @@
package org.xbib.elx.common;
import org.xbib.elx.api.AdminClientProvider;
public class MockAdminClientProvider implements AdminClientProvider<MockAdminClient> {
@Override
public MockAdminClient getClient() {
return new MockAdminClient();
}
}

View file

@ -0,0 +1,103 @@
package org.xbib.elx.common;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import java.util.concurrent.TimeUnit;
/**
* A mocked client, it does not perform any actions on a cluster. Useful for testing.
*/
public class MockBulkClient extends AbstractBulkClient {
@Override
public ElasticsearchClient getClient() {
return null;
}
@Override
public void init(Settings settings) {
}
@Override
public String getClusterName() {
return null;
}
@Override
public void waitForShards(long maxWaitTime, TimeUnit timeUnit) {
}
@Override
protected ElasticsearchClient createClient(Settings settings) {
return null;
}
@Override
protected void closeClient(Settings settings) {
}
@Override
public MockBulkClient index(String index, String id, boolean create, String source) {
return this;
}
@Override
public MockBulkClient delete(String index, String id) {
return this;
}
@Override
public MockBulkClient update(String index, String id, String source) {
return this;
}
@Override
public MockBulkClient index(IndexRequest indexRequest) {
return this;
}
@Override
public MockBulkClient delete(DeleteRequest deleteRequest) {
return this;
}
@Override
public MockBulkClient update(UpdateRequest updateRequest) {
return this;
}
@Override
public void startBulk(String index, long startRefreshInterval, long stopRefreshIterval) {
}
@Override
public void stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) {
}
@Override
public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) {
return true;
}
@Override
public void refreshIndex(String index) {
}
@Override
public void flushIndex(String index) {
}
@Override
public void flush() {
// nothing to do
}
@Override
public void close() {
// nothing to do
}
}

View file

@ -0,0 +1,11 @@
package org.xbib.elx.common;
import org.xbib.elx.api.BulkClientProvider;
public class MockBulkClientProvider implements BulkClientProvider<MockBulkClient> {
@Override
public MockBulkClient getClient() {
return new MockBulkClient();
}
}

View file

@ -1,129 +0,0 @@
package org.xbib.elx.common;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import java.util.concurrent.TimeUnit;
/**
* A mocked client, it does not perform any actions on a cluster. Useful for testing.
*/
public class MockExtendedClient extends AbstractExtendedClient {
@Override
public ElasticsearchClient getClient() {
return null;
}
@Override
public MockExtendedClient init(Settings settings) {
return this;
}
@Override
protected ElasticsearchClient createClient(Settings settings) {
return null;
}
@Override
protected void closeClient() {
}
@Override
public MockExtendedClient index(String index, String id, boolean create, String source) {
return this;
}
@Override
public MockExtendedClient delete(String index, String id) {
return this;
}
@Override
public MockExtendedClient update(String index, String id, String source) {
return this;
}
@Override
public MockExtendedClient index(IndexRequest indexRequest) {
return this;
}
@Override
public MockExtendedClient delete(DeleteRequest deleteRequest) {
return this;
}
@Override
public MockExtendedClient update(UpdateRequest updateRequest) {
return this;
}
@Override
public MockExtendedClient startBulk(String index, long startRefreshInterval, long stopRefreshIterval) {
return this;
}
@Override
public MockExtendedClient stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) {
return this;
}
@Override
public MockExtendedClient newIndex(String index) {
return this;
}
@Override
public MockExtendedClient deleteIndex(String index) {
return this;
}
@Override
public MockExtendedClient refreshIndex(String index) {
return this;
}
@Override
public MockExtendedClient flushIndex(String index) {
return this;
}
@Override
public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) {
return true;
}
@Override
public boolean waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) {
return true;
}
@Override
public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) {
return true;
}
@Override
public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) {
return true;
}
@Override
public MockExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) {
return this;
}
@Override
public void flush() {
// nothing to do
}
@Override
public void close() {
// nothing to do
}
}

View file

@ -1,10 +0,0 @@
package org.xbib.elx.common;
import org.xbib.elx.api.ExtendedClientProvider;
public class MockExtendedClientProvider implements ExtendedClientProvider<MockExtendedClient> {
@Override
public MockExtendedClient getExtendedClient() {
return new MockExtendedClient();
}
}

View file

@ -0,0 +1,44 @@
package org.xbib.elx.common;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import java.util.concurrent.TimeUnit;
/**
* A mocked client, it does not perform any actions on a cluster. Useful for testing.
*/
public class MockSearchClient extends AbstractSearchClient {
@Override
public ElasticsearchClient getClient() {
return null;
}
@Override
public void init(Settings settings) {
}
@Override
public String getClusterName() {
return null;
}
@Override
public void waitForShards(long maxWaitTime, TimeUnit timeUnit) {
}
@Override
protected ElasticsearchClient createClient(Settings settings) {
return null;
}
@Override
protected void closeClient(Settings settings) {
}
@Override
public void close() {
// nothing to do
}
}

View file

@ -0,0 +1,11 @@
package org.xbib.elx.common;
import org.xbib.elx.api.SearchClientProvider;
public class MockSearchClientProvider implements SearchClientProvider<MockSearchClient> {
@Override
public MockSearchClient getClient() {
return new MockSearchClient();
}
}

View file

@ -0,0 +1 @@
org.xbib.elx.common.MockAdminClientProvider

View file

@ -0,0 +1 @@
org.xbib.elx.common.MockBulkClientProvider

View file

@ -1 +0,0 @@
org.xbib.elx.common.MockExtendedClientProvider

View file

@ -0,0 +1 @@
org.xbib.elx.common.MockSearchClientProvider

View file

@ -0,0 +1,41 @@
package org.xbib.elx.common.test;
import org.junit.jupiter.api.Test;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.MockAdminClient;
import org.xbib.elx.common.MockAdminClientProvider;
import org.xbib.elx.common.MockBulkClient;
import org.xbib.elx.common.MockBulkClientProvider;
import org.xbib.elx.common.MockSearchClient;
import org.xbib.elx.common.MockSearchClientProvider;
import java.io.IOException;
import static org.junit.jupiter.api.Assertions.assertNotNull;
class MockClientProviderTest {
@Test
void testMockAdminProvider() throws IOException {
MockAdminClient client = ClientBuilder.builder()
.setAdminClientProvider(MockAdminClientProvider.class)
.build();
assertNotNull(client);
}
@Test
void testMockBulkProvider() throws IOException {
MockBulkClient client = ClientBuilder.builder()
.setBulkClientProvider(MockBulkClientProvider.class)
.build();
assertNotNull(client);
}
@Test
void testMockSearchProvider() throws IOException {
MockSearchClient client = ClientBuilder.builder()
.setSearchClientProvider(MockSearchClientProvider.class)
.build();
assertNotNull(client);
}
}

View file

@ -1,19 +0,0 @@
package org.xbib.elx.common.test;
import org.junit.jupiter.api.Test;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.MockExtendedClient;
import org.xbib.elx.common.MockExtendedClientProvider;
import java.io.IOException;
import static org.junit.jupiter.api.Assertions.assertNotNull;
class MockExtendedClientProviderTest {
@Test
void testMockExtendedProvider() throws IOException {
MockExtendedClient client = ClientBuilder.builder().provider(MockExtendedClientProvider.class).build();
assertNotNull(client);
}
}

View file

@ -1,12 +0,0 @@
package org.xbib.elx.http;
import org.xbib.elx.api.ExtendedClientProvider;
import java.util.Collections;
public class ExtendedHttpClientProvider implements ExtendedClientProvider<ExtendedHttpClient> {
@Override
public ExtendedHttpClient getExtendedClient() {
return new ExtendedHttpClient(Collections.emptyList(), Thread.currentThread().getContextClassLoader());
}
}

View file

@ -13,7 +13,7 @@ import org.xbib.netty.http.common.HttpResponse;
*/ */
public class HttpActionContext<R extends ActionRequest, T extends ActionResponse> { public class HttpActionContext<R extends ActionRequest, T extends ActionResponse> {
private final ExtendedHttpClient extendedHttpClient; private final HttpClientHelper extendedHttpClient;
private final R request; private final R request;
@ -23,13 +23,13 @@ public class HttpActionContext<R extends ActionRequest, T extends ActionResponse
private HttpResponse httpResponse; private HttpResponse httpResponse;
HttpActionContext(ExtendedHttpClient extendedHttpClient, R request, String url) { HttpActionContext(HttpClientHelper extendedHttpClient, R request, String url) {
this.extendedHttpClient = extendedHttpClient; this.extendedHttpClient = extendedHttpClient;
this.request = request; this.request = request;
this.url = url; this.url = url;
} }
public ExtendedHttpClient getExtendedHttpClient() { public HttpClientHelper getExtendedHttpClient() {
return extendedHttpClient; return extendedHttpClient;
} }

View file

@ -0,0 +1,55 @@
package org.xbib.elx.http;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.ActionType;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.threadpool.ThreadPool;
import org.xbib.elx.common.AbstractAdminClient;
import java.io.IOException;
/**
* Elasticsearch HTTP admin client.
*/
public class HttpAdminClient extends AbstractAdminClient implements ElasticsearchClient {
private final HttpClientHelper helper;
public HttpAdminClient() {
this.helper = new HttpClientHelper();
}
@Override
public void init(Settings settings) throws IOException {
super.init(settings);
helper.init(settings);
}
@Override
protected ElasticsearchClient createClient(Settings settings) throws IOException {
return this;
}
@Override
protected void closeClient(Settings settings) throws IOException {
helper.closeClient(settings);
}
@Override
public <Request extends ActionRequest, Response extends ActionResponse> ActionFuture<Response> execute(ActionType<Response> action, Request request) {
return helper.execute(action, request);
}
@Override
public <Request extends ActionRequest, Response extends ActionResponse> void execute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
helper.execute(action, request, listener);
}
@Override
public ThreadPool threadPool() {
return helper.threadPool();
}
}

View file

@ -0,0 +1,11 @@
package org.xbib.elx.http;
import org.xbib.elx.api.AdminClientProvider;
public class HttpAdminClientProvider implements AdminClientProvider<HttpAdminClient> {
@Override
public HttpAdminClient getClient() {
return new HttpAdminClient();
}
}

View file

@ -0,0 +1,55 @@
package org.xbib.elx.http;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.ActionType;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.threadpool.ThreadPool;
import org.xbib.elx.common.AbstractBulkClient;
import java.io.IOException;
/**
* Elasticsearch HTTP bulk client.
*/
public class HttpBulkClient extends AbstractBulkClient implements ElasticsearchClient {
private final HttpClientHelper helper;
public HttpBulkClient() {
this.helper = new HttpClientHelper();
}
@Override
public void init(Settings settings) throws IOException {
super.init(settings);
helper.init(settings);
}
@Override
protected ElasticsearchClient createClient(Settings settings) throws IOException {
return this;
}
@Override
protected void closeClient(Settings settings) throws IOException {
helper.closeClient(settings);
}
@Override
public <Request extends ActionRequest, Response extends ActionResponse> ActionFuture<Response> execute(ActionType<Response> action, Request request) {
return helper.execute(action, request);
}
@Override
public <Request extends ActionRequest, Response extends ActionResponse> void execute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
helper.execute(action, request, listener);
}
@Override
public ThreadPool threadPool() {
return helper.threadPool();
}
}

View file

@ -0,0 +1,11 @@
package org.xbib.elx.http;
import org.xbib.elx.api.BulkClientProvider;
public class HttpBulkClientProvider implements BulkClientProvider<HttpBulkClient> {
@Override
public HttpBulkClient getClient() {
return new HttpBulkClient();
}
}

View file

@ -10,16 +10,15 @@ import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.ActionType; import org.elasticsearch.action.ActionType;
import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import org.xbib.elx.common.AbstractExtendedClient;
import org.xbib.net.URL; import org.xbib.net.URL;
import org.xbib.netty.http.client.Client; import org.xbib.netty.http.client.Client;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -31,9 +30,9 @@ import java.util.stream.Stream;
/** /**
* Elasticsearch HTTP client. * Elasticsearch HTTP client.
*/ */
public class ExtendedHttpClient extends AbstractExtendedClient implements ElasticsearchClient { public class HttpClientHelper {
private static final Logger logger = LogManager.getLogger(ExtendedHttpClient.class); private static final Logger logger = LogManager.getLogger(HttpClientHelper.class);
private Client nettyHttpClient; private Client nettyHttpClient;
@ -46,17 +45,20 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
private String url; private String url;
public ExtendedHttpClient(List<NamedXContentRegistry.Entry> namedXContentEntries, ClassLoader classLoader) { public HttpClientHelper() {
this(Collections.emptyList(), Thread.currentThread().getContextClassLoader());
}
public HttpClientHelper(List<NamedXContentRegistry.Entry> namedXContentEntries,
ClassLoader classLoader) {
this.registry = new NamedXContentRegistry(Stream.of(getNamedXContents().stream(), this.registry = new NamedXContentRegistry(Stream.of(getNamedXContents().stream(),
namedXContentEntries.stream()).flatMap(Function.identity()).collect(Collectors.toList())); namedXContentEntries.stream()).flatMap(Function.identity()).collect(Collectors.toList()));
this.classLoader = classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader(); this.classLoader = classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader();
this.actionMap = new HashMap<>(); this.actionMap = new HashMap<>();
} }
@Override
@SuppressWarnings({"unchecked", "rawtypes"}) @SuppressWarnings({"unchecked", "rawtypes"})
public ExtendedHttpClient init(Settings settings) throws IOException { public HttpClientHelper init(Settings settings) throws IOException {
super.init(settings);
if (settings.hasValue("url")) { if (settings.hasValue("url")) {
this.url = settings.get("url"); this.url = settings.get("url");
} else if (settings.hasValue("host")) { } else if (settings.hasValue("host")) {
@ -92,22 +94,10 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
return nettyHttpClient; return nettyHttpClient;
} }
@Override protected void closeClient(Settings settings) throws IOException {
public ElasticsearchClient getClient() {
return this;
}
@Override
protected ElasticsearchClient createClient(Settings settings) {
return this;
}
@Override
protected void closeClient() throws IOException {
nettyHttpClient.shutdownGracefully(); nettyHttpClient.shutdownGracefully();
} }
@Override
public <Request extends ActionRequest, Response extends ActionResponse> public <Request extends ActionRequest, Response extends ActionResponse>
ActionFuture<Response> execute(ActionType<Response> action, Request request) { ActionFuture<Response> execute(ActionType<Response> action, Request request) {
PlainActionFuture<Response> actionFuture = PlainActionFuture.newFuture(); PlainActionFuture<Response> actionFuture = PlainActionFuture.newFuture();
@ -115,13 +105,11 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
return actionFuture; return actionFuture;
} }
@Override
public <Request extends ActionRequest, Response extends ActionResponse> public <Request extends ActionRequest, Response extends ActionResponse>
void execute(ActionType<Response> action, Request request, ActionListener<Response> listener) { void execute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
doExecute(action, request, listener); doExecute(action, request, listener);
} }
@Override
public ThreadPool threadPool() { public ThreadPool threadPool() {
logger.log(Level.DEBUG, "returning null for threadPool() request"); logger.log(Level.DEBUG, "returning null for threadPool() request");
return null; return null;

View file

@ -0,0 +1,55 @@
package org.xbib.elx.http;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.ActionType;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.threadpool.ThreadPool;
import org.xbib.elx.common.AbstractSearchClient;
import java.io.IOException;
/**
* Elasticsearch HTTP search client.
*/
public class HttpSearchClient extends AbstractSearchClient implements ElasticsearchClient {
private final HttpClientHelper helper;
public HttpSearchClient() {
this.helper = new HttpClientHelper();
}
@Override
public void init(Settings settings) throws IOException {
super.init(settings);
helper.init(settings);
}
@Override
protected ElasticsearchClient createClient(Settings settings) throws IOException {
return this;
}
@Override
protected void closeClient(Settings settings) throws IOException {
helper.closeClient(settings);
}
@Override
public <Request extends ActionRequest, Response extends ActionResponse> ActionFuture<Response> execute(ActionType<Response> action, Request request) {
return helper.execute(action, request);
}
@Override
public <Request extends ActionRequest, Response extends ActionResponse> void execute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
helper.execute(action, request, listener);
}
@Override
public ThreadPool threadPool() {
return helper.threadPool();
}
}

View file

@ -0,0 +1,11 @@
package org.xbib.elx.http;
import org.xbib.elx.api.SearchClientProvider;
public class HttpSearchClientProvider implements SearchClientProvider<HttpSearchClient> {
@Override
public HttpSearchClient getClient() {
return new HttpSearchClient();
}
}

View file

@ -0,0 +1,35 @@
package org.xbib.elx.http.action.admin.indices.get;
import org.elasticsearch.action.admin.indices.get.GetIndexAction;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.xcontent.XContentParser;
import org.xbib.elx.http.HttpAction;
import org.xbib.netty.http.client.api.Request;
import org.xbib.netty.http.common.HttpResponse;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
public class HtppGetIndexAction extends HttpAction<GetIndexRequest, GetIndexResponse> {
@Override
public GetIndexAction getActionInstance() {
return GetIndexAction.INSTANCE;
}
@Override
protected Request.Builder createHttpRequest(String url, GetIndexRequest getIndexRequest) throws IOException {
List<String> list = getIndexRequest.indices().length == 0 ?
List.of("*") : Arrays.asList(getIndexRequest.indices());
String command = "/" + String.join(",", list);
logger.info("command = " + command);
return newGetRequest(url, command);
}
@Override
protected CheckedFunction<XContentParser, GetIndexResponse, IOException> entityParser(HttpResponse httpResponse) {
return GetIndexResponse::fromXContent;
}
}

View file

@ -0,0 +1,35 @@
package org.xbib.elx.http.action.search;
import org.elasticsearch.action.search.ClearScrollAction;
import org.elasticsearch.action.search.ClearScrollRequest;
import org.elasticsearch.action.search.ClearScrollResponse;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.xbib.elx.http.HttpAction;
import org.xbib.netty.http.client.api.Request;
import org.xbib.netty.http.common.HttpResponse;
import java.io.IOException;
public class HttpClearScrollAction extends HttpAction<ClearScrollRequest, ClearScrollResponse> {
@Override
public ClearScrollAction getActionInstance() {
return ClearScrollAction.INSTANCE;
}
@Override
protected Request.Builder createHttpRequest(String baseUrl, ClearScrollRequest request) throws IOException {
XContentBuilder builder = JsonXContent.contentBuilder();
request.toXContent(builder, ToXContent.EMPTY_PARAMS);
return newDeleteRequest(baseUrl, "_search/scroll", BytesReference.bytes(builder));
}
@Override
protected CheckedFunction<XContentParser, ClearScrollResponse, IOException> entityParser(HttpResponse httpResponse) {
return ClearScrollResponse::fromXContent;
}
}

View file

@ -5,6 +5,7 @@ import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.search.Scroll;
import org.xbib.elx.http.HttpAction; import org.xbib.elx.http.HttpAction;
import org.xbib.netty.http.client.api.Request; import org.xbib.netty.http.client.api.Request;
import org.xbib.netty.http.common.HttpResponse; import org.xbib.netty.http.common.HttpResponse;
@ -20,9 +21,10 @@ public class HttpSearchAction extends HttpAction<SearchRequest, SearchResponse>
@Override @Override
protected Request.Builder createHttpRequest(String url, SearchRequest request) { protected Request.Builder createHttpRequest(String url, SearchRequest request) {
// request.indices() always empty array Scroll scroll = request.scroll();
String params = scroll != null ? "?scroll=" + scroll.keepAlive() : "";
String index = request.indices() != null ? String.join(",", request.indices()) + "/" : ""; String index = request.indices() != null ? String.join(",", request.indices()) + "/" : "";
return newPostRequest(url, index + "_search", request.source().toString()); return newPostRequest(url, index + "_search" + params, request.source().toString());
} }
@Override @Override

View file

@ -0,0 +1,35 @@
package org.xbib.elx.http.action.search;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollAction;
import org.elasticsearch.action.search.SearchScrollRequest;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.xbib.elx.http.HttpAction;
import org.xbib.netty.http.client.api.Request;
import org.xbib.netty.http.common.HttpResponse;
import java.io.IOException;
public class HttpSearchScrollAction extends HttpAction<SearchScrollRequest, SearchResponse> {
@Override
public SearchScrollAction getActionInstance() {
return SearchScrollAction.INSTANCE;
}
@Override
protected Request.Builder createHttpRequest(String baseUrl, SearchScrollRequest request) throws IOException {
XContentBuilder builder = JsonXContent.contentBuilder();
request.toXContent(builder, ToXContent.EMPTY_PARAMS);
return newPostRequest(baseUrl, "_search/scroll", BytesReference.bytes(builder));
}
@Override
protected CheckedFunction<XContentParser, SearchResponse, IOException> entityParser(HttpResponse httpResponse) {
return SearchResponse::fromXContent;
}
}

View file

@ -0,0 +1 @@
org.xbib.elx.http.HttpAdminClientProvider

View file

@ -0,0 +1 @@
org.xbib.elx.http.HttpBulkClientProvider

View file

@ -1 +0,0 @@
org.xbib.elx.http.ExtendedHttpClientProvider

View file

@ -0,0 +1 @@
org.xbib.elx.http.HttpSearchClientProvider

View file

@ -8,12 +8,15 @@ org.xbib.elx.http.action.admin.indices.alias.get.HttpGetAliasAction
org.xbib.elx.http.action.admin.indices.create.HttpCreateIndexAction org.xbib.elx.http.action.admin.indices.create.HttpCreateIndexAction
org.xbib.elx.http.action.admin.indices.delete.HttpDeleteIndexAction org.xbib.elx.http.action.admin.indices.delete.HttpDeleteIndexAction
org.xbib.elx.http.action.admin.indices.exists.indices.HttpIndicesExistsAction org.xbib.elx.http.action.admin.indices.exists.indices.HttpIndicesExistsAction
org.xbib.elx.http.action.admin.indices.get.HtppGetIndexAction
org.xbib.elx.http.action.admin.indices.refresh.HttpRefreshIndexAction org.xbib.elx.http.action.admin.indices.refresh.HttpRefreshIndexAction
org.xbib.elx.http.action.admin.indices.settings.get.HttpGetSettingsAction org.xbib.elx.http.action.admin.indices.settings.get.HttpGetSettingsAction
org.xbib.elx.http.action.admin.indices.settings.put.HttpUpdateSettingsAction org.xbib.elx.http.action.admin.indices.settings.put.HttpUpdateSettingsAction
org.xbib.elx.http.action.bulk.HttpBulkAction org.xbib.elx.http.action.bulk.HttpBulkAction
org.xbib.elx.http.action.index.HttpIndexAction org.xbib.elx.http.action.index.HttpIndexAction
org.xbib.elx.http.action.search.HttpClearScrollAction
org.xbib.elx.http.action.search.HttpSearchAction org.xbib.elx.http.action.search.HttpSearchAction
org.xbib.elx.http.action.search.HttpSearchScrollAction
org.xbib.elx.http.action.main.HttpMainAction org.xbib.elx.http.action.main.HttpMainAction
org.xbib.elx.http.action.get.HttpExistsAction org.xbib.elx.http.action.get.HttpExistsAction
org.xbib.elx.http.action.get.HttpGetAction org.xbib.elx.http.action.get.HttpGetAction

View file

@ -0,0 +1,182 @@
package org.xbib.elx.http.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.http.HttpAdminClient;
import org.xbib.elx.http.HttpAdminClientProvider;
import org.xbib.elx.http.HttpBulkClient;
import org.xbib.elx.http.HttpBulkClientProvider;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ExtendWith(TestExtension.class)
class BulkClientTest {
private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getSimpleName());
private static final Long ACTIONS = 100L;
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
private final TestExtension.Helper helper;
BulkClientTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
void testSingleDoc() throws Exception {
final HttpBulkClient client = ClientBuilder.builder()
.setBulkClientProvider(HttpBulkClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
.build();
try {
client.newIndex("test");
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
} finally {
assertEquals(1, client.getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
client.close();
}
}
@Test
void testNewIndex() throws Exception {
final HttpBulkClient client = ClientBuilder.builder()
.setBulkClientProvider(HttpBulkClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
client.newIndex("test");
client.close();
}
@Test
void testMapping() throws Exception {
try (HttpAdminClient adminClient = ClientBuilder.builder()
.setAdminClientProvider(HttpAdminClientProvider.class)
.put(helper.getHttpSettings())
.build();
HttpBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(HttpBulkClientProvider.class)
.put(helper.getHttpSettings())
.build()) {
XContentBuilder builder = JsonXContent.contentBuilder()
.startObject()
.startObject("properties")
.startObject("location")
.field("type", "geo_point")
.endObject()
.endObject()
.endObject();
bulkClient.newIndex("test", Settings.EMPTY, builder);
assertTrue(adminClient.getMapping("test", "_doc").containsKey("properties"));
}
}
@Test
void testRandomDocs() throws Exception {
long numactions = ACTIONS;
final HttpBulkClient client = ClientBuilder.builder()
.setBulkClientProvider(HttpBulkClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.build();
try {
client.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
client.index("test", null, false, "{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
} finally {
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
client.refreshIndex("test");
assertEquals(numactions, client.getSearchableDocs("test"));
client.close();
}
}
@Test
void testThreadedRandomDocs() throws Exception {
int maxthreads = Runtime.getRuntime().availableProcessors();
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
final long actions = ACTIONS;
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
final HttpBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(HttpBulkClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.build();
try {
Settings settings = Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
bulkClient.newIndex("test", settings);
bulkClient.startBulk("test", 0, 1000);
logger.info("index created");
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
final CountDownLatch latch = new CountDownLatch(maxthreads);
for (int i = 0; i < maxthreads; i++) {
executorService.execute(() -> {
for (int i1 = 0; i1 < actions; i1++) {
bulkClient.index("test", null, false,"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
latch.countDown();
});
}
logger.info("waiting for latch...");
if (latch.await(60L, TimeUnit.SECONDS)) {
logger.info("flush...");
bulkClient.flush();
bulkClient.waitForResponses(60L, TimeUnit.SECONDS);
logger.info("got all responses, executor service shutdown...");
executorService.shutdown();
executorService.awaitTermination(60L, TimeUnit.SECONDS);
logger.info("pool is shut down");
} else {
logger.warn("latch timeout");
}
bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS);
assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount());
} finally {
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(bulkClient.getBulkController().getLastBulkError());
bulkClient.refreshIndex("test");
assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test"));
bulkClient.close();
}
}
}

View file

@ -1,244 +0,0 @@
package org.xbib.elx.http.test;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.http.ExtendedHttpClient;
import org.xbib.elx.http.ExtendedHttpClientProvider;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ExtendWith(TestExtension.class)
class ClientTest {
private static final Logger logger = LogManager.getLogger(ClientTest.class.getSimpleName());
private static final Long ACTIONS = 100L;
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
private final TestExtension.Helper helper;
ClientTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
void testSingleDoc() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
.build();
try {
client.newIndex("test");
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
client.close();
}
}
@Test
void testNewIndex() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
client.newIndex("test");
client.close();
}
@Test
void testNewIndexWithSettings() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
client.newIndex("test", settings);
GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
.indices("test");
GetSettingsResponse getSettingsResponse =
client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet();
logger.log(Level.INFO, "settings=" + getSettingsResponse.getSetting("test", "index.number_of_shards"));
assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards"));
client.close();
}
@Test
void testNewIndexWithSettingsAndMappings() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
XContentBuilder builder = JsonXContent.contentBuilder()
.startObject()
.field("date_detection", false)
.startObject("properties")
.startObject("pos")
.field("type", "geo_point")
.endObject()
.endObject()
.endObject();
client.newIndex("test", settings, builder);
GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
.indices("test");
GetSettingsResponse getSettingsResponse =
client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet();
assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards"));
GetMappingsRequest getMappingsRequest = new GetMappingsRequest()
.indices("test");
GetMappingsResponse getMappingsResponse =
client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
assertTrue(getMappingsResponse.getMappings().get("test").containsKey("_doc"));
client.close();
}
@Test
void testRandomDocs() throws Exception {
long numactions = ACTIONS;
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.build();
try {
client.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
client.index("test", null, false, "{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
client.refreshIndex("test");
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
.setQuery(QueryBuilders.matchAllQuery()).setSize(0);
assertEquals(numactions, searchRequestBuilder.execute().actionGet().getHits().getTotalHits().value);
client.close();
}
}
@Test
void testThreadedRandomDocs() throws Exception {
int maxthreads = Runtime.getRuntime().availableProcessors();
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
final long actions = ACTIONS;
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.build();
try {
Settings settings = Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
client.newIndex("test", settings, (String)null)
.startBulk("test", 0, 1000);
logger.info("index created");
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
final CountDownLatch latch = new CountDownLatch(maxthreads);
for (int i = 0; i < maxthreads; i++) {
executorService.execute(() -> {
for (int i1 = 0; i1 < actions; i1++) {
client.index("test", null, false,"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
latch.countDown();
});
}
logger.info("waiting for latch...");
if (latch.await(60L, TimeUnit.SECONDS)) {
logger.info("flush...");
client.flush();
client.waitForResponses(60L, TimeUnit.SECONDS);
logger.info("got all responses, executor service shutdown...");
executorService.shutdown();
executorService.awaitTermination(60L, TimeUnit.SECONDS);
logger.info("pool is shut down");
} else {
logger.warn("latch timeout");
}
client.stopBulk("test", 30L, TimeUnit.SECONDS);
assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
logger.log(Level.INFO, "refreshing index test");
client.refreshIndex("test");
SearchSourceBuilder builder = new SearchSourceBuilder()
.query(QueryBuilders.matchAllQuery())
.size(0)
.trackTotalHits(true);
SearchRequest searchRequest = new SearchRequest()
.indices("test")
.source(builder);
SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
assertEquals(maxthreads * actions, searchResponse.getHits().getTotalHits().value);
client.close();
client.close();
}
}
}

View file

@ -2,18 +2,12 @@ package org.xbib.elx.http.test;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters; import org.xbib.elx.common.Parameters;
import org.xbib.elx.http.ExtendedHttpClient; import org.xbib.elx.http.HttpBulkClient;
import org.xbib.elx.http.ExtendedHttpClientProvider; import org.xbib.elx.http.HttpBulkClientProvider;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -26,10 +20,10 @@ class DuplicateIDTest {
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getSimpleName()); private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getSimpleName());
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
private static final Long ACTIONS = 100L; private static final Long ACTIONS = 100L;
private static final Long MAX_ACTIONS_PER_REQUEST = 5L;
private final TestExtension.Helper helper; private final TestExtension.Helper helper;
DuplicateIDTest(TestExtension.Helper helper) { DuplicateIDTest(TestExtension.Helper helper) {
@ -39,8 +33,8 @@ class DuplicateIDTest {
@Test @Test
void testDuplicateDocIDs() throws Exception { void testDuplicateDocIDs() throws Exception {
long numactions = ACTIONS; long numactions = ACTIONS;
final ExtendedHttpClient client = ClientBuilder.builder() final HttpBulkClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class) .setBulkClientProvider(HttpBulkClientProvider.class)
.put(helper.getHttpSettings()) .put(helper.getHttpSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.build(); .build();
@ -53,23 +47,11 @@ class DuplicateIDTest {
client.flush(); client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS); client.waitForResponses(30L, TimeUnit.SECONDS);
client.refreshIndex("test"); client.refreshIndex("test");
SearchSourceBuilder builder = new SearchSourceBuilder() long hits = client.getSearchableDocs("test");
.query(QueryBuilders.matchAllQuery())
.size(0)
.trackTotalHits(true);
SearchRequest searchRequest = new SearchRequest()
.indices("test")
.source(builder);
SearchResponse searchResponse =
helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet();
long hits = searchResponse.getHits().getTotalHits().value;
logger.info("hits = {}", hits);
assertTrue(hits < ACTIONS); assertTrue(hits < ACTIONS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally { } finally {
client.close(); client.close();
assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) { if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError()); logger.error("error", client.getBulkController().getLastBulkError());
} }

View file

@ -2,17 +2,15 @@ package org.xbib.elx.http.test;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexPruneResult; import org.xbib.elx.api.IndexPruneResult;
import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.http.ExtendedHttpClient; import org.xbib.elx.http.HttpAdminClient;
import org.xbib.elx.http.ExtendedHttpClientProvider; import org.xbib.elx.http.HttpAdminClientProvider;
import org.xbib.elx.http.HttpBulkClient;
import org.xbib.elx.http.HttpBulkClientProvider;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
@ -37,50 +35,49 @@ class IndexPruneTest {
@Test @Test
void testPrune() throws IOException { void testPrune() throws IOException {
final ExtendedHttpClient client = ClientBuilder.builder(helper.client("1")) final HttpAdminClient adminClient = ClientBuilder.builder()
.setAdminClientProvider(HttpAdminClientProvider.class)
.put(helper.getHttpSettings())
.build();
final HttpBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(HttpBulkClientProvider.class)
.put(helper.getHttpSettings()) .put(helper.getHttpSettings())
.provider(ExtendedHttpClientProvider.class)
.build(); .build();
try { try {
Settings settings = Settings.builder() Settings settings = Settings.builder()
.put("index.number_of_shards", 1) .put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0) .put("index.number_of_replicas", 0)
.build(); .build();
client.newIndex("test1", settings); bulkClient.newIndex("test1", settings);
client.shiftIndex("test", "test1", Collections.emptyList()); adminClient.shiftIndex("test", "test1", Collections.emptyList());
client.newIndex("test2", settings); bulkClient.newIndex("test2", settings);
client.shiftIndex("test", "test2", Collections.emptyList()); adminClient.shiftIndex("test", "test2", Collections.emptyList());
client.newIndex("test3", settings); bulkClient.newIndex("test3", settings);
client.shiftIndex("test", "test3", Collections.emptyList()); adminClient.shiftIndex("test", "test3", Collections.emptyList());
client.newIndex("test4", settings); bulkClient.newIndex("test4", settings);
client.shiftIndex("test", "test4", Collections.emptyList()); adminClient.shiftIndex("test", "test4", Collections.emptyList());
IndexPruneResult indexPruneResult = IndexPruneResult indexPruneResult =
client.pruneIndex("test", "test4", 2, 2, true); adminClient.pruneIndex("test", "test4", 2, 2, true);
assertTrue(indexPruneResult.getDeletedIndices().contains("test1")); assertTrue(indexPruneResult.getDeletedIndices().contains("test1"));
assertTrue(indexPruneResult.getDeletedIndices().contains("test2")); assertTrue(indexPruneResult.getDeletedIndices().contains("test2"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test3")); assertFalse(indexPruneResult.getDeletedIndices().contains("test3"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test4")); assertFalse(indexPruneResult.getDeletedIndices().contains("test4"));
List<Boolean> list = new ArrayList<>(); List<Boolean> list = new ArrayList<>();
for (String index : Arrays.asList("test1", "test2", "test3", "test4")) { for (String index : Arrays.asList("test1", "test2", "test3", "test4")) {
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(); list.add(bulkClient.isIndexExists(index));
indicesExistsRequest.indices(index);
IndicesExistsResponse indicesExistsResponse =
client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
logger.info("indices exists response for {} is {}", index, indicesExistsResponse.isExists());
list.add(indicesExistsResponse.isExists());
} }
logger.info(list);
assertFalse(list.get(0)); assertFalse(list.get(0));
assertFalse(list.get(1)); assertFalse(list.get(1));
assertTrue(list.get(2)); assertTrue(list.get(2));
assertTrue(list.get(3)); assertTrue(list.get(3));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally { } finally {
client.close(); bulkClient.close();
if (client.getBulkController().getLastBulkError() != null) { if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError()); logger.error("error", bulkClient.getBulkController().getLastBulkError());
} }
assertNull(client.getBulkController().getLastBulkError()); assertNull(bulkClient.getBulkController().getLastBulkError());
adminClient.close();
} }
} }
} }

View file

@ -4,16 +4,16 @@ import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryBuilders;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.api.IndexShiftResult;
import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.http.ExtendedHttpClient; import org.xbib.elx.http.HttpAdminClient;
import org.xbib.elx.http.ExtendedHttpClientProvider; import org.xbib.elx.http.HttpAdminClientProvider;
import org.xbib.elx.http.HttpBulkClient;
import org.xbib.elx.http.HttpBulkClientProvider;
import java.util.Arrays; import java.util.Arrays;
import java.util.Map; import java.util.Map;
@ -35,56 +35,54 @@ class IndexShiftTest {
@Test @Test
void testIndexShift() throws Exception { void testIndexShift() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder() final HttpAdminClient adminClient = ClientBuilder.builder()
.setAdminClientProvider(HttpAdminClientProvider.class)
.put(helper.getHttpSettings())
.build();
final HttpBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(HttpBulkClientProvider.class)
.put(helper.getHttpSettings()) .put(helper.getHttpSettings())
.provider(ExtendedHttpClientProvider.class)
.build(); .build();
try { try {
Settings settings = Settings.builder() Settings settings = Settings.builder()
.put("index.number_of_shards", 1) .put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0) .put("index.number_of_replicas", 0)
.build(); .build();
client.newIndex("test1234", settings); bulkClient.newIndex("test1234", settings);
for (int i = 0; i < 1; i++) { for (int i = 0; i < 1; i++) {
client.index("test1234", helper.randomString(1), false, bulkClient.index("test1234", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}"); "{ \"name\" : \"" + helper.randomString(32) + "\"}");
} }
client.flush(); bulkClient.flush();
client.waitForResponses(30L, TimeUnit.SECONDS); bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
IndexShiftResult indexShiftResult = IndexShiftResult indexShiftResult =
client.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c")); adminClient.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c"));
assertTrue(indexShiftResult.getNewAliases().contains("a")); assertTrue(indexShiftResult.getNewAliases().contains("a"));
assertTrue(indexShiftResult.getNewAliases().contains("b")); assertTrue(indexShiftResult.getNewAliases().contains("b"));
assertTrue(indexShiftResult.getNewAliases().contains("c")); assertTrue(indexShiftResult.getNewAliases().contains("c"));
assertTrue(indexShiftResult.getMovedAliases().isEmpty()); assertTrue(indexShiftResult.getMovedAliases().isEmpty());
Map<String, String> aliases = adminClient.getAliases("test1234");
Map<String, String> aliases = client.getAliases("test1234");
logger.log(Level.DEBUG, "aliases = " + aliases); logger.log(Level.DEBUG, "aliases = " + aliases);
assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("test")); assertTrue(aliases.containsKey("test"));
String resolved = adminClient.resolveAlias("test");
String resolved = client.resolveAlias("test");
logger.log(Level.DEBUG, "resolved = " + resolved); logger.log(Level.DEBUG, "resolved = " + resolved);
aliases = client.getAliases(resolved); aliases = adminClient.getAliases(resolved);
logger.log(Level.DEBUG, "aliases = " + aliases); logger.log(Level.DEBUG, "aliases = " + aliases);
assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("test")); assertTrue(aliases.containsKey("test"));
bulkClient.newIndex("test5678", settings);
client.newIndex("test5678", settings);
for (int i = 0; i < 1; i++) { for (int i = 0; i < 1; i++) {
client.index("test5678", helper.randomString(1), false, bulkClient.index("test5678", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}"); "{ \"name\" : \"" + helper.randomString(32) + "\"}");
} }
client.flush(); bulkClient.flush();
client.waitForResponses(30L, TimeUnit.SECONDS); bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
indexShiftResult = adminClient.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"),
indexShiftResult = client.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"),
(request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add() (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
.index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias))) .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
); );
@ -94,31 +92,28 @@ class IndexShiftTest {
assertTrue(indexShiftResult.getMovedAliases().contains("a")); assertTrue(indexShiftResult.getMovedAliases().contains("a"));
assertTrue(indexShiftResult.getMovedAliases().contains("b")); assertTrue(indexShiftResult.getMovedAliases().contains("b"));
assertTrue(indexShiftResult.getMovedAliases().contains("c")); assertTrue(indexShiftResult.getMovedAliases().contains("c"));
aliases = adminClient.getAliases("test5678");
aliases = client.getAliases("test5678");
assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("d"));
assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("e"));
assertTrue(aliases.containsKey("f")); assertTrue(aliases.containsKey("f"));
resolved = adminClient.resolveAlias("test");
resolved = client.resolveAlias("test"); aliases = adminClient.getAliases(resolved);
aliases = client.getAliases(resolved);
assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("d"));
assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("e"));
assertTrue(aliases.containsKey("f")); assertTrue(aliases.containsKey("f"));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally { } finally {
client.close(); bulkClient.close();
if (client.getBulkController().getLastBulkError() != null) { if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError()); logger.error("error", bulkClient.getBulkController().getLastBulkError());
} }
assertNull(client.getBulkController().getLastBulkError()); assertNull(bulkClient.getBulkController().getLastBulkError());
adminClient.close();
} }
} }
} }

View file

@ -0,0 +1,82 @@
package org.xbib.elx.http.test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.http.HttpBulkClient;
import org.xbib.elx.http.HttpBulkClientProvider;
import org.xbib.elx.http.HttpSearchClient;
import org.xbib.elx.http.HttpSearchClientProvider;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;
@ExtendWith(TestExtension.class)
class SearchTest {
private static final Logger logger = LogManager.getLogger(SearchTest.class.getName());
private static final Long ACTIONS = 100L;
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
private final TestExtension.Helper helper;
SearchTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
void testDocStream() throws Exception {
long numactions = ACTIONS;
final HttpBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(HttpBulkClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.build();
try (bulkClient) {
bulkClient.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
bulkClient.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
bulkClient.refreshIndex("test");
assertEquals(numactions, bulkClient.getSearchableDocs("test"));
}
assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(bulkClient.getBulkController().getLastBulkError());
try (HttpSearchClient searchClient = ClientBuilder.builder()
.setSearchClientProvider(HttpSearchClientProvider.class)
.put(helper.getHttpSettings())
.build()) {
Stream<SearchHit> stream = searchClient.search(qb -> qb
.setIndices("test")
.setQuery(QueryBuilders.matchAllQuery()),
TimeValue.timeValueMinutes(1), 10);
long count = stream.count();
assertEquals(numactions, count);
Stream<String> ids = searchClient.getIds(qb -> qb
.setIndices("test")
.setQuery(QueryBuilders.matchAllQuery()));
final AtomicInteger idcount = new AtomicInteger();
ids.forEach(id -> {
logger.info(id);
idcount.incrementAndGet();
});
assertEquals(numactions, idcount.get());
}
}
}

View file

@ -2,14 +2,15 @@ package org.xbib.elx.http.test;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.http.ExtendedHttpClient; import org.xbib.elx.http.HttpAdminClient;
import org.xbib.elx.http.ExtendedHttpClientProvider; import org.xbib.elx.http.HttpAdminClientProvider;
import org.xbib.elx.http.HttpBulkClient;
import org.xbib.elx.http.HttpBulkClientProvider;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -29,47 +30,46 @@ class SmokeTest {
@Test @Test
void smokeTest() throws Exception { void smokeTest() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder() try (HttpAdminClient adminClient = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class) .setAdminClientProvider(HttpAdminClientProvider.class)
.put(helper.getHttpSettings()) .put(helper.getHttpSettings())
.build(); .build();
try { HttpBulkClient bulkClient = ClientBuilder.builder()
assertEquals(helper.getClusterName(), client.getClusterName()); .setBulkClientProvider(HttpBulkClientProvider.class)
client.newIndex("test_smoke"); .put(helper.getHttpSettings())
client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest .build()) {
client.flush(); IndexDefinition indexDefinition =
client.waitForResponses(30, TimeUnit.SECONDS); adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.EMPTY);
client.checkMapping("test_smoke");
client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
client.delete("test_smoke", "1");
client.flush();
client.waitForResponses(30, TimeUnit.SECONDS);
client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS);
client.delete("test_smoke", "1");
client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
client.flush();
client.deleteIndex("test_smoke");
IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder()
.build());
assertEquals(0, indexDefinition.getReplicaLevel()); assertEquals(0, indexDefinition.getReplicaLevel());
client.newIndex(indexDefinition); assertEquals(helper.getClusterName(), adminClient.getClusterName());
client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); bulkClient.newIndex("test_smoke");
client.flush(); bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
client.waitForResponses(30, TimeUnit.SECONDS); bulkClient.flush();
client.updateReplicaLevel(indexDefinition, 2); bulkClient.waitForResponses(30, TimeUnit.SECONDS);
int replica = client.getReplicaLevel(indexDefinition); adminClient.checkMapping("test_smoke");
bulkClient.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
bulkClient.delete("test_smoke", "1");
bulkClient.flush();
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
bulkClient.delete("test_smoke", "1");
bulkClient.flush();
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
adminClient.deleteIndex("test_smoke");
bulkClient.newIndex(indexDefinition);
bulkClient.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
bulkClient.flush();
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
adminClient.updateReplicaLevel(indexDefinition, 2);
int replica = adminClient.getReplicaLevel(indexDefinition);
assertEquals(2, replica); assertEquals(2, replica);
client.deleteIndex(indexDefinition); assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount());
assertEquals(0, client.getBulkController().getBulkMetric().getFailed().getCount()); assertEquals(6, bulkClient.getBulkMetric().getSucceeded().getCount());
assertEquals(6, client.getBulkController().getBulkMetric().getSucceeded().getCount()); if (bulkClient.getBulkController().getLastBulkError() != null) {
} catch (NoNodeAvailableException e) { logger.error("error", bulkClient.getBulkController().getLastBulkError());
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
} }
assertNull(client.getBulkController().getLastBulkError()); assertNull(bulkClient.getBulkController().getLastBulkError());
adminClient.deleteIndex(indexDefinition);
} }
} }
} }

View file

@ -1,69 +0,0 @@
package org.xbib.elx.node;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.env.Environment;
import org.elasticsearch.node.Node;
import org.elasticsearch.plugins.Plugin;
import org.xbib.elx.common.AbstractExtendedClient;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
public class ExtendedNodeClient extends AbstractExtendedClient {
private static final Logger logger = LogManager.getLogger(ExtendedNodeClient.class.getName());
private Node node;
@Override
protected ElasticsearchClient createClient(Settings settings) throws IOException {
if (settings == null) {
return null;
}
String version = System.getProperty("os.name")
+ " " + System.getProperty("java.vm.name")
+ " " + System.getProperty("java.vm.vendor")
+ " " + System.getProperty("java.runtime.version")
+ " " + System.getProperty("java.vm.version");
Settings effectiveSettings = Settings.builder().put(settings)
.put("node.client", true)
.put("node.master", false)
.put("node.data", false)
.build();
XContentBuilder builder = XContentFactory.jsonBuilder();
effectiveSettings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true")));
logger.info("creating node client on {} with effective settings {}",
version, Strings.toString(builder));
Collection<Class<? extends Plugin>> plugins = Collections.emptyList();
this.node = new BulkNode(new Environment(effectiveSettings, null), plugins);
try {
node.start();
} catch (Exception e) {
throw new IOException(e);
}
return node.client();
}
@Override
protected void closeClient() throws IOException {
if (node != null) {
logger.debug("closing node client");
node.close();
}
}
private static class BulkNode extends Node {
BulkNode(Environment env, Collection<Class<? extends Plugin>> classpathPlugins) {
super(env, classpathPlugins, false);
}
}
}

View file

@ -1,10 +0,0 @@
package org.xbib.elx.node;
import org.xbib.elx.api.ExtendedClientProvider;
public class ExtendedNodeClientProvider implements ExtendedClientProvider<ExtendedNodeClient> {
@Override
public ExtendedNodeClient getExtendedClient() {
return new ExtendedNodeClient();
}
}

View file

@ -0,0 +1,25 @@
package org.xbib.elx.node;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import org.xbib.elx.common.AbstractAdminClient;
import java.io.IOException;
public class NodeAdminClient extends AbstractAdminClient {
private final NodeClientHelper helper;
public NodeAdminClient() {
this.helper = new NodeClientHelper();
}
@Override
public ElasticsearchClient createClient(Settings settings) {
return helper.createClient(settings, null);
}
@Override
public void closeClient(Settings settings) throws IOException {
helper.closeClient(settings);
}
}

View file

@ -0,0 +1,11 @@
package org.xbib.elx.node;
import org.xbib.elx.api.AdminClientProvider;
public class NodeAdminClientProvider implements AdminClientProvider<NodeAdminClient> {
@Override
public NodeAdminClient getClient() {
return new NodeAdminClient();
}
}

View file

@ -0,0 +1,25 @@
package org.xbib.elx.node;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import org.xbib.elx.common.AbstractBulkClient;
import java.io.IOException;
public class NodeBulkClient extends AbstractBulkClient {
private final NodeClientHelper helper;
public NodeBulkClient() {
this.helper = new NodeClientHelper();
}
@Override
public ElasticsearchClient createClient(Settings settings) {
return helper.createClient(settings, null);
}
@Override
public void closeClient(Settings settings) throws IOException {
helper.closeClient(settings);
}
}

View file

@ -0,0 +1,11 @@
package org.xbib.elx.node;
import org.xbib.elx.api.BulkClientProvider;
public class NodeBulkClientProvider implements BulkClientProvider<NodeBulkClient> {
@Override
public NodeBulkClient getClient() {
return new NodeBulkClient();
}
}

View file

@ -0,0 +1,98 @@
package org.xbib.elx.node;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.env.Environment;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.Netty4Plugin;
import org.xbib.elx.common.Parameters;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
public class NodeClientHelper {
private static final Logger logger = LogManager.getLogger(NodeClientHelper.class.getName());
private static Object configurationObject;
private static Node node;
private static final Map<String, ElasticsearchClient> clientMap = new HashMap<>();
public ElasticsearchClient createClient(Settings settings, Object object) {
if (configurationObject == null) {
configurationObject = object;
}
if (configurationObject instanceof ElasticsearchClient) {
return (ElasticsearchClient) configurationObject;
}
return clientMap.computeIfAbsent(settings.get("cluster.name"),
key -> innerCreateClient(settings));
}
public void closeClient(Settings settings) throws IOException {
ElasticsearchClient client = clientMap.remove(settings.get("cluster.name"));
if (client != null) {
logger.debug("closing node...");
node.close();
node = null;
}
}
private ElasticsearchClient innerCreateClient(Settings settings) {
String version = System.getProperty("os.name")
+ " " + System.getProperty("java.vm.name")
+ " " + System.getProperty("java.vm.vendor")
+ " " + System.getProperty("java.runtime.version")
+ " " + System.getProperty("java.vm.version");
Settings effectiveSettings = Settings.builder()
.put(settings.filter(key -> !isPrivateSettings(key)))
.put("node.master", false)
.put("node.data", false)
// "node.processors"
.put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(),
settings.get(EsExecutors.NODE_PROCESSORS_SETTING.getKey(),
String.valueOf(Runtime.getRuntime().availableProcessors())))
// "transport.type"
.put(NetworkModule.TRANSPORT_TYPE_KEY,
Netty4Plugin.NETTY_TRANSPORT_NAME)
.build();
logger.info("creating node client on {} with effective settings {}",
version, effectiveSettings.toDelimitedString(','));
Collection<Class<? extends Plugin>> plugins =
Collections.singletonList(Netty4Plugin.class);
node = new BulkNode(new Environment(effectiveSettings, null), plugins);
try {
node.start();
return node.client();
} catch (NodeValidationException e) {
logger.log(Level.ERROR, e.getMessage(), e);
}
return null;
}
private static boolean isPrivateSettings(String key) {
return key.equals(Parameters.MAX_ACTIONS_PER_REQUEST.name()) ||
key.equals(Parameters.MAX_CONCURRENT_REQUESTS.name()) ||
key.equals(Parameters.MAX_VOLUME_PER_REQUEST.name()) ||
key.equals(Parameters.FLUSH_INTERVAL.name());
}
private static class BulkNode extends Node {
BulkNode(Environment env, Collection<Class<? extends Plugin>> classpathPlugins) {
super(env, classpathPlugins, false);
}
}
}

View file

@ -0,0 +1,25 @@
package org.xbib.elx.node;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import org.xbib.elx.common.AbstractSearchClient;
import java.io.IOException;
public class NodeSearchClient extends AbstractSearchClient {
private final NodeClientHelper helper;
public NodeSearchClient() {
this.helper = new NodeClientHelper();
}
@Override
public ElasticsearchClient createClient(Settings settings) {
return helper.createClient(settings, null);
}
@Override
public void closeClient(Settings settings) throws IOException {
helper.closeClient(settings);
}
}

View file

@ -0,0 +1,11 @@
package org.xbib.elx.node;
import org.xbib.elx.api.SearchClientProvider;
public class NodeSearchClientProvider implements SearchClientProvider<NodeSearchClient> {
@Override
public NodeSearchClient getClient() {
return new NodeSearchClient();
}
}

View file

@ -0,0 +1 @@
org.xbib.elx.node.NodeAdminClientProvider

View file

@ -0,0 +1 @@
org.xbib.elx.node.NodeBulkClientProvider

View file

@ -1 +0,0 @@
org.xbib.elx.node.ExtendedNodeClientProvider

View file

@ -0,0 +1 @@
org.xbib.elx.node.NodeSearchClientProvider

View file

@ -0,0 +1,180 @@
package org.xbib.elx.node.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.node.NodeAdminClient;
import org.xbib.elx.node.NodeAdminClientProvider;
import org.xbib.elx.node.NodeBulkClient;
import org.xbib.elx.node.NodeBulkClientProvider;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ExtendWith(TestExtension.class)
class BulkClientTest {
private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getName());
private static final Long ACTIONS = 10000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
private final TestExtension.Helper helper;
BulkClientTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
void testSingleDoc() throws Exception {
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
.setBulkClientProvider(NodeBulkClientProvider.class)
.put(helper.getNodeSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
.build();
try {
bulkClient.newIndex("test");
bulkClient.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
} finally {
assertEquals(1, bulkClient.getBulkMetric().getSucceeded().getCount());
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(bulkClient.getBulkController().getLastBulkError());
bulkClient.close();
}
}
@Test
void testNewIndex() throws Exception {
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
.setBulkClientProvider(NodeBulkClientProvider.class)
.put(helper.getNodeSettings())
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
bulkClient.newIndex("test");
bulkClient.close();
}
@Test
void testMapping() throws Exception {
try (NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
.setAdminClientProvider(NodeAdminClientProvider.class)
.put(helper.getNodeSettings())
.build();
NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
.setBulkClientProvider(NodeBulkClientProvider.class)
.put(helper.getNodeSettings())
.build()) {
XContentBuilder builder = JsonXContent.contentBuilder()
.startObject()
.startObject("properties")
.startObject("location")
.field("type", "geo_point")
.endObject()
.endObject()
.endObject();
bulkClient.newIndex("test", Settings.EMPTY, builder);
assertTrue(adminClient.getMapping("test", "_doc").containsKey("properties"));
}
}
@Test
void testRandomDocs() throws Exception {
long numactions = ACTIONS;
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
.setBulkClientProvider(NodeBulkClientProvider.class)
.put(helper.getNodeSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.build();
try {
bulkClient.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
bulkClient.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
} finally {
assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(bulkClient.getBulkController().getLastBulkError());
bulkClient.refreshIndex("test");
assertEquals(numactions, bulkClient.getSearchableDocs("test"));
bulkClient.close();
}
}
@Test
void testThreadedRandomDocs() throws Exception {
int maxthreads = Runtime.getRuntime().availableProcessors();
final long actions = ACTIONS;
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
.setBulkClientProvider(NodeBulkClientProvider.class)
.put(helper.getNodeSettings())
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.build();
try {
Settings settings = Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
bulkClient.newIndex("test", settings);
bulkClient.startBulk("test", 0, 1000);
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
final CountDownLatch latch = new CountDownLatch(maxthreads);
for (int i = 0; i < maxthreads; i++) {
executorService.execute(() -> {
for (int i1 = 0; i1 < actions; i1++) {
bulkClient.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
latch.countDown();
});
}
logger.info("waiting for latch...");
if (latch.await(30L, TimeUnit.SECONDS)) {
logger.info("flush...");
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
logger.info("got all responses, executor service shutdown...");
executorService.shutdown();
executorService.awaitTermination(30L, TimeUnit.SECONDS);
logger.info("pool is shut down");
} else {
logger.warn("latch timeout");
}
bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS);
assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount());
} finally {
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(bulkClient.getBulkController().getLastBulkError());
bulkClient.refreshIndex("test");
assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test"));
bulkClient.close();
}
}
}

View file

@ -1,236 +0,0 @@
package org.xbib.elx.node.test;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.node.ExtendedNodeClient;
import org.xbib.elx.node.ExtendedNodeClientProvider;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ExtendWith(TestExtension.class)
class ClientTest {
private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
private static final Long ACTIONS = 10000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
private final TestExtension.Helper helper;
ClientTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
void testSingleDoc() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
.build();
try {
client.newIndex("test");
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
client.close();
}
}
@Test
void testNewIndex() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
client.newIndex("test");
client.close();
}
@Test
void testNewIndexWithSettings() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
client.newIndex("test", settings);
GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
.indices("test");
GetSettingsResponse getSettingsResponse =
client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet();
logger.log(Level.INFO, "settings=" + getSettingsResponse.getSetting("test", "index.number_of_shards"));
assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards"));
client.close();
}
@Test
void testNewIndexWithSettingsAndMappings() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
XContentBuilder builder = JsonXContent.contentBuilder()
.startObject()
.field("date_detection", false)
.startObject("properties")
.startObject("location")
.field("type", "geo_point")
.endObject()
.endObject()
.endObject();
client.newIndex("test", settings, builder);
GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
.indices("test");
GetSettingsResponse getSettingsResponse =
client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet();
logger.log(Level.INFO, "settings=" + getSettingsResponse.getSetting("test", "index.number_of_shards"));
assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards"));
GetMappingsRequest getMappingsRequest = new GetMappingsRequest()
.indices("test");
GetMappingsResponse getMappingsResponse =
client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
logger.info("mappings={}", getMappingsResponse.getMappings());
assertTrue(getMappingsResponse.getMappings().get("test").containsKey("_doc"));
client.close();
}
@Test
void testRandomDocs() throws Exception {
long numactions = ACTIONS;
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.build();
try {
client.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
client.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
client.refreshIndex("test");
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
.setQuery(QueryBuilders.matchAllQuery()).setSize(0);
assertEquals(numactions, searchRequestBuilder.execute().actionGet().getHits().getTotalHits().value);
client.close();
}
}
@Test
void testThreadedRandomDocs() throws Exception {
int maxthreads = Runtime.getRuntime().availableProcessors();
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
final long actions = ACTIONS;
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.build();
try {
Settings settings = Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
client.newIndex("test", settings, (String)null)
.startBulk("test", 0, 1000);
logger.info("index created");
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
final CountDownLatch latch = new CountDownLatch(maxthreads);
for (int i = 0; i < maxthreads; i++) {
executorService.execute(() -> {
for (int i1 = 0; i1 < actions; i1++) {
client.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
latch.countDown();
});
}
logger.info("waiting for latch...");
if (latch.await(30L, TimeUnit.SECONDS)) {
logger.info("flush...");
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
logger.info("got all responses, executor service shutdown...");
executorService.shutdown();
executorService.awaitTermination(30L, TimeUnit.SECONDS);
logger.info("pool is shut down");
} else {
logger.warn("latch timeout");
}
client.stopBulk("test", 30L, TimeUnit.SECONDS);
assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
logger.log(Level.INFO, "refreshing index test");
client.refreshIndex("test");
SearchSourceBuilder builder = new SearchSourceBuilder()
.query(QueryBuilders.matchAllQuery())
.size(0)
.trackTotalHits(true);
SearchRequest searchRequest = new SearchRequest()
.indices("test")
.source(builder);
SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
assertEquals(maxthreads * actions, searchResponse.getHits().getTotalHits().value);
client.close();
}
}
}

View file

@ -2,18 +2,12 @@ package org.xbib.elx.node.test;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters; import org.xbib.elx.common.Parameters;
import org.xbib.elx.node.ExtendedNodeClient; import org.xbib.elx.node.NodeBulkClient;
import org.xbib.elx.node.ExtendedNodeClientProvider; import org.xbib.elx.node.NodeBulkClientProvider;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -26,10 +20,10 @@ class DuplicateIDTest {
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName()); private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName());
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
private static final Long ACTIONS = 100L; private static final Long ACTIONS = 100L;
private static final Long MAX_ACTIONS_PER_REQUEST = 5L;
private final TestExtension.Helper helper; private final TestExtension.Helper helper;
DuplicateIDTest(TestExtension.Helper helper) { DuplicateIDTest(TestExtension.Helper helper) {
@ -39,40 +33,28 @@ class DuplicateIDTest {
@Test @Test
void testDuplicateDocIDs() throws Exception { void testDuplicateDocIDs() throws Exception {
long numactions = ACTIONS; long numactions = ACTIONS;
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class) .setBulkClientProvider(NodeBulkClientProvider.class)
.put(helper.getNodeSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.build(); .build();
try { try {
client.newIndex("test"); bulkClient.newIndex("test");
for (int i = 0; i < ACTIONS; i++) { for (int i = 0; i < ACTIONS; i++) {
client.index("test", helper.randomString(1), false, bulkClient.index("test", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}"); "{ \"name\" : \"" + helper.randomString(32) + "\"}");
} }
client.flush(); bulkClient.flush();
client.waitForResponses(30L, TimeUnit.SECONDS); bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
client.refreshIndex("test"); bulkClient.refreshIndex("test");
SearchSourceBuilder builder = new SearchSourceBuilder() assertTrue(bulkClient.getSearchableDocs("test") < ACTIONS);
.query(QueryBuilders.matchAllQuery())
.size(0)
.trackTotalHits(true);
SearchRequest searchRequest = new SearchRequest()
.indices("test")
.source(builder);
SearchResponse searchResponse =
helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet();
long hits = searchResponse.getHits().getTotalHits().value;
logger.info("hits = {}", hits);
assertTrue(hits < ACTIONS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally { } finally {
client.close(); bulkClient.close();
assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) { if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError()); logger.error("error", bulkClient.getBulkController().getLastBulkError());
} }
assertNull(client.getBulkController().getLastBulkError()); assertNull(bulkClient.getBulkController().getLastBulkError());
} }
} }
} }

View file

@ -2,17 +2,15 @@ package org.xbib.elx.node.test;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexPruneResult; import org.xbib.elx.api.IndexPruneResult;
import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.node.ExtendedNodeClient; import org.xbib.elx.node.NodeAdminClient;
import org.xbib.elx.node.ExtendedNodeClientProvider; import org.xbib.elx.node.NodeAdminClientProvider;
import org.xbib.elx.node.NodeBulkClient;
import org.xbib.elx.node.NodeBulkClientProvider;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
@ -37,49 +35,49 @@ class IndexPruneTest {
@Test @Test
void testPrune() throws IOException { void testPrune() throws IOException {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class) .setAdminClientProvider(NodeAdminClientProvider.class)
.put(helper.getNodeSettings())
.build();
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
.setBulkClientProvider(NodeBulkClientProvider.class)
.put(helper.getNodeSettings())
.build(); .build();
try { try {
Settings settings = Settings.builder() Settings settings = Settings.builder()
.put("index.number_of_shards", 1) .put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0) .put("index.number_of_replicas", 0)
.build(); .build();
client.newIndex("test_prune1", settings); bulkClient.newIndex("test_prune1", settings);
client.shiftIndex("test_prune", "test_prune1", Collections.emptyList()); adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
client.newIndex("test_prune2", settings); bulkClient.newIndex("test_prune2", settings);
client.shiftIndex("test_prune", "test_prune2", Collections.emptyList()); adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
client.newIndex("test_prune3", settings); bulkClient.newIndex("test_prune3", settings);
client.shiftIndex("test_prune", "test_prune3", Collections.emptyList()); adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
client.newIndex("test_prune4", settings); bulkClient.newIndex("test_prune4", settings);
client.shiftIndex("test_prune", "test_prune4", Collections.emptyList()); adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
IndexPruneResult indexPruneResult = IndexPruneResult indexPruneResult =
client.pruneIndex("test_prune", "test_prune4", 2, 2, true); adminClient.pruneIndex("test_prune", "test_prune4", 2, 2, true);
assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1")); assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1"));
assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2")); assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3")); assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune4")); assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune4"));
List<Boolean> list = new ArrayList<>(); List<Boolean> list = new ArrayList<>();
for (String index : Arrays.asList("test_prune1", "test_prune2", "test_prune3", "test_prune4")) { for (String index : Arrays.asList("test_prune1", "test_prune2", "test_prune3", "test_prune4")) {
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(); list.add(bulkClient.isIndexExists(index));
indicesExistsRequest.indices(index);
IndicesExistsResponse indicesExistsResponse =
client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
list.add(indicesExistsResponse.isExists());
} }
logger.info(list); logger.info(list);
assertFalse(list.get(0)); assertFalse(list.get(0));
assertFalse(list.get(1)); assertFalse(list.get(1));
assertTrue(list.get(2)); assertTrue(list.get(2));
assertTrue(list.get(3)); assertTrue(list.get(3));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally { } finally {
client.close(); adminClient.close();
if (client.getBulkController().getLastBulkError() != null) { bulkClient.close();
logger.error("error", client.getBulkController().getLastBulkError()); if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
} }
assertNull(client.getBulkController().getLastBulkError()); assertNull(bulkClient.getBulkController().getLastBulkError());
} }
} }
} }

View file

@ -3,15 +3,16 @@ package org.xbib.elx.node.test;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryBuilders;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.api.IndexShiftResult;
import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.node.ExtendedNodeClient; import org.xbib.elx.node.NodeAdminClient;
import org.xbib.elx.node.ExtendedNodeClientProvider; import org.xbib.elx.node.NodeAdminClientProvider;
import org.xbib.elx.node.NodeBulkClient;
import org.xbib.elx.node.NodeBulkClientProvider;
import java.util.Arrays; import java.util.Arrays;
import java.util.Map; import java.util.Map;
@ -33,50 +34,51 @@ class IndexShiftTest {
@Test @Test
void testIndexShift() throws Exception { void testIndexShift() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class) .setAdminClientProvider(NodeAdminClientProvider.class)
.put(helper.getNodeSettings())
.build();
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
.setBulkClientProvider(NodeBulkClientProvider.class)
.put(helper.getNodeSettings())
.build(); .build();
try { try {
Settings settings = Settings.builder() Settings settings = Settings.builder()
.put("index.number_of_shards", 1) .put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0) .put("index.number_of_replicas", 0)
.build(); .build();
client.newIndex("test_shift", settings); bulkClient.newIndex("test_shift", settings);
for (int i = 0; i < 1; i++) { for (int i = 0; i < 1; i++) {
client.index("test_shift", helper.randomString(1), false, bulkClient.index("test_shift", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}"); "{ \"name\" : \"" + helper.randomString(32) + "\"}");
} }
client.flush(); bulkClient.flush();
client.waitForResponses(30L, TimeUnit.SECONDS); bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
IndexShiftResult indexShiftResult = IndexShiftResult indexShiftResult =
client.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c")); adminClient.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c"));
assertTrue(indexShiftResult.getNewAliases().contains("a")); assertTrue(indexShiftResult.getNewAliases().contains("a"));
assertTrue(indexShiftResult.getNewAliases().contains("b")); assertTrue(indexShiftResult.getNewAliases().contains("b"));
assertTrue(indexShiftResult.getNewAliases().contains("c")); assertTrue(indexShiftResult.getNewAliases().contains("c"));
assertTrue(indexShiftResult.getMovedAliases().isEmpty()); assertTrue(indexShiftResult.getMovedAliases().isEmpty());
Map<String, String> aliases = adminClient.getAliases("test_shift");
Map<String, String> aliases = client.getAliases("test_shift");
assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("test")); assertTrue(aliases.containsKey("test"));
String resolved = adminClient.resolveAlias("test");
String resolved = client.resolveAlias("test"); aliases = adminClient.getAliases(resolved);
aliases = client.getAliases(resolved);
assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("test")); assertTrue(aliases.containsKey("test"));
bulkClient.newIndex("test_shift2", settings);
client.newIndex("test_shift2", settings);
for (int i = 0; i < 1; i++) { for (int i = 0; i < 1; i++) {
client.index("test_shift2", helper.randomString(1), false, bulkClient.index("test_shift2", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}"); "{ \"name\" : \"" + helper.randomString(32) + "\"}");
} }
client.flush(); bulkClient.flush();
client.waitForResponses(30L, TimeUnit.SECONDS); bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
indexShiftResult = adminClient.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"),
indexShiftResult = client.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"),
(request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add() (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
.index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias))) .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
); );
@ -86,32 +88,28 @@ class IndexShiftTest {
assertTrue(indexShiftResult.getMovedAliases().contains("a")); assertTrue(indexShiftResult.getMovedAliases().contains("a"));
assertTrue(indexShiftResult.getMovedAliases().contains("b")); assertTrue(indexShiftResult.getMovedAliases().contains("b"));
assertTrue(indexShiftResult.getMovedAliases().contains("c")); assertTrue(indexShiftResult.getMovedAliases().contains("c"));
aliases = adminClient.getAliases("test_shift2");
aliases = client.getAliases("test_shift2");
assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("d"));
assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("e"));
assertTrue(aliases.containsKey("f")); assertTrue(aliases.containsKey("f"));
resolved = adminClient.resolveAlias("test");
resolved = client.resolveAlias("test"); aliases = adminClient.getAliases(resolved);
aliases = client.getAliases(resolved);
assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("d"));
assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("e"));
assertTrue(aliases.containsKey("f")); assertTrue(aliases.containsKey("f"));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally { } finally {
client.close(); adminClient.close();
if (client.getBulkController().getLastBulkError() != null) { bulkClient.close();
logger.error("error", client.getBulkController().getLastBulkError()); if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
} }
assertNull(client.getBulkController().getLastBulkError()); assertNull(bulkClient.getBulkController().getLastBulkError());
} }
} }
} }

View file

@ -0,0 +1,82 @@
package org.xbib.elx.node.test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.node.NodeBulkClient;
import org.xbib.elx.node.NodeBulkClientProvider;
import org.xbib.elx.node.NodeSearchClient;
import org.xbib.elx.node.NodeSearchClientProvider;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;
@ExtendWith(TestExtension.class)
class SearchTest {
private static final Logger logger = LogManager.getLogger(SearchTest.class.getName());
private static final Long ACTIONS = 100L;
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
private final TestExtension.Helper helper;
SearchTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
void testDocStream() throws Exception {
long numactions = ACTIONS;
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
.setBulkClientProvider(NodeBulkClientProvider.class)
.put(helper.getNodeSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.build();
try (bulkClient) {
bulkClient.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
bulkClient.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
bulkClient.refreshIndex("test");
assertEquals(numactions, bulkClient.getSearchableDocs("test"));
}
assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(bulkClient.getBulkController().getLastBulkError());
try (NodeSearchClient searchClient = ClientBuilder.builder(helper.client("1"))
.setSearchClientProvider(NodeSearchClientProvider.class)
.put(helper.getNodeSettings())
.build()) {
Stream<SearchHit> stream = searchClient.search(qb -> qb
.setIndices("test")
.setQuery(QueryBuilders.matchAllQuery()),
TimeValue.timeValueMinutes(1), 10);
long count = stream.count();
assertEquals(numactions, count);
Stream<String> ids = searchClient.getIds(qb -> qb
.setIndices("test")
.setQuery(QueryBuilders.matchAllQuery()));
final AtomicInteger idcount = new AtomicInteger();
ids.forEach(id -> {
logger.info(id);
idcount.incrementAndGet();
});
assertEquals(numactions, idcount.get());
}
}
}

View file

@ -2,14 +2,15 @@ package org.xbib.elx.node.test;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.node.ExtendedNodeClient; import org.xbib.elx.node.NodeAdminClient;
import org.xbib.elx.node.ExtendedNodeClientProvider; import org.xbib.elx.node.NodeAdminClientProvider;
import org.xbib.elx.node.NodeBulkClient;
import org.xbib.elx.node.NodeBulkClientProvider;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -29,46 +30,46 @@ class SmokeTest {
@Test @Test
void smokeTest() throws Exception { void smokeTest() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) try (NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class) .setAdminClientProvider(NodeAdminClientProvider.class)
.put(helper.getNodeSettings())
.build(); .build();
try { NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
assertEquals(helper.getClusterName(), client.getClusterName()); .setBulkClientProvider(NodeBulkClientProvider.class)
client.newIndex("test_smoke"); .put(helper.getNodeSettings())
client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest .build()) {
client.flush(); IndexDefinition indexDefinition =
client.waitForResponses(30, TimeUnit.SECONDS); adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.EMPTY);
client.checkMapping("test_smoke");
client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
client.delete("test_smoke", "1");
client.flush();
client.waitForResponses(30, TimeUnit.SECONDS);
client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS);
client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
client.delete("test_smoke", "1");
client.flush();
client.deleteIndex("test_smoke");
IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder()
.build());
assertEquals(0, indexDefinition.getReplicaLevel()); assertEquals(0, indexDefinition.getReplicaLevel());
client.newIndex(indexDefinition); assertEquals(helper.getClusterName(), adminClient.getClusterName());
client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); bulkClient.newIndex("test_smoke");
client.flush(); bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
client.waitForResponses(30, TimeUnit.SECONDS); bulkClient.flush();
client.updateReplicaLevel(indexDefinition, 2); bulkClient.waitForResponses(30, TimeUnit.SECONDS);
int replica = client.getReplicaLevel(indexDefinition); adminClient.checkMapping("test_smoke");
bulkClient.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
bulkClient.delete("test_smoke", "1");
bulkClient.flush();
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
bulkClient.delete("test_smoke", "1");
bulkClient.flush();
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
adminClient.deleteIndex("test_smoke");
bulkClient.newIndex(indexDefinition);
bulkClient.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
bulkClient.flush();
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
adminClient.updateReplicaLevel(indexDefinition, 2);
int replica = adminClient.getReplicaLevel(indexDefinition);
assertEquals(2, replica); assertEquals(2, replica);
client.deleteIndex(indexDefinition); assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount());
assertEquals(0, client.getBulkController().getBulkMetric().getFailed().getCount()); assertEquals(6, bulkClient.getBulkMetric().getSucceeded().getCount());
assertEquals(6, client.getBulkController().getBulkMetric().getSucceeded().getCount()); if (bulkClient.getBulkController().getLastBulkError() != null) {
} catch (NoNodeAvailableException e) { logger.error("error", bulkClient.getBulkController().getLastBulkError());
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
} }
assertNull(client.getBulkController().getLastBulkError()); assertNull(bulkClient.getBulkController().getLastBulkError());
adminClient.deleteIndex(indexDefinition);
} }
} }
} }

View file

@ -193,6 +193,7 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft
.put("path.home", getHome()) .put("path.home", getHome())
.put("cluster.initial_master_nodes", "1") .put("cluster.initial_master_nodes", "1")
.put("discovery.seed_hosts", "127.0.0.1:9300") .put("discovery.seed_hosts", "127.0.0.1:9300")
.put("node.max_local_storage_nodes", "2")
.build(); .build();
} }

View file

@ -1,11 +0,0 @@
package org.xbib.elx.transport;
import org.xbib.elx.api.ExtendedClientProvider;
public class ExtendedTransportClientProvider implements ExtendedClientProvider<ExtendedTransportClient> {
@Override
public ExtendedTransportClient getExtendedClient() {
return new ExtendedTransportClient();
}
}

View file

@ -0,0 +1,36 @@
package org.xbib.elx.transport;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.xbib.elx.common.AbstractAdminClient;
import java.io.IOException;
/**
* Transport admin client.
*/
public class TransportAdminClient extends AbstractAdminClient {
private final TransportClientHelper helper;
public TransportAdminClient() {
this.helper = new TransportClientHelper();
}
@Override
public ElasticsearchClient createClient(Settings settings) throws IOException {
return helper.createClient(settings);
}
@Override
public void init(Settings settings) throws IOException {
super.init(settings);
helper.init((TransportClient) getClient(), settings);
}
@Override
public void closeClient(Settings settings) {
helper.closeClient(settings);
}
}

View file

@ -0,0 +1,11 @@
package org.xbib.elx.transport;
import org.xbib.elx.api.AdminClientProvider;
public class TransportAdminClientProvider implements AdminClientProvider<TransportAdminClient> {
@Override
public TransportAdminClient getClient() {
return new TransportAdminClient();
}
}

View file

@ -0,0 +1,35 @@
package org.xbib.elx.transport;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.xbib.elx.common.AbstractBulkClient;
import java.io.IOException;
/**
* Transport search client with additional methods.
*/
public class TransportBulkClient extends AbstractBulkClient {
private final TransportClientHelper helper;
public TransportBulkClient() {
this.helper = new TransportClientHelper();
}
@Override
public ElasticsearchClient createClient(Settings settings) throws IOException {
return helper.createClient(settings);
}
@Override
public void init(Settings settings) throws IOException {
super.init(settings);
helper.init((TransportClient) getClient(), settings);
}
@Override
public void closeClient(Settings settings) {
helper.closeClient(settings);
}
}

View file

@ -0,0 +1,11 @@
package org.xbib.elx.transport;
import org.xbib.elx.api.BulkClientProvider;
public class TransportBulkClientProvider implements BulkClientProvider<TransportBulkClient> {
@Override
public TransportBulkClient getClient() {
return new TransportBulkClient();
}
}

View file

@ -7,6 +7,7 @@ import org.elasticsearch.Version;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.client.transport.TransportClient;
@ -23,7 +24,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.Netty4Plugin; import org.elasticsearch.transport.Netty4Plugin;
import org.xbib.elx.common.AbstractExtendedClient;
import org.xbib.elx.common.util.NetworkUtils; import org.xbib.elx.common.util.NetworkUtils;
import java.io.IOException; import java.io.IOException;
@ -31,16 +31,19 @@ import java.net.InetAddress;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
/** /**
* Transport client with additional methods using the BulkProcessor. * Transport client with additional methods using the BulkProcessor.
*/ */
public class ExtendedTransportClient extends AbstractExtendedClient { public class TransportClientHelper {
private static final Logger logger = LogManager.getLogger(ExtendedTransportClient.class.getName()); private static final Logger logger = LogManager.getLogger(TransportClientHelper.class.getName());
private static final Map<String, ElasticsearchClient> clientMap = new HashMap<>();
@Override
protected ElasticsearchClient createClient(Settings settings) throws IOException { protected ElasticsearchClient createClient(Settings settings) throws IOException {
if (settings != null) { if (settings != null) {
String systemIdentifier = System.getProperty("os.name") String systemIdentifier = System.getProperty("os.name")
@ -49,11 +52,9 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
+ " " + System.getProperty("java.vm.version") + " " + System.getProperty("java.vm.version")
+ " Elasticsearch " + Version.CURRENT.toString(); + " Elasticsearch " + Version.CURRENT.toString();
Settings transportClientSettings = getTransportClientSettings(settings); Settings transportClientSettings = getTransportClientSettings(settings);
//XContentBuilder settingsBuilder = XContentFactory.jsonBuilder().startObject();
XContentBuilder effectiveSettingsBuilder = XContentFactory.jsonBuilder().startObject(); XContentBuilder effectiveSettingsBuilder = XContentFactory.jsonBuilder().startObject();
logger.log(Level.INFO, "creating transport client on {} with settings {}", logger.log(Level.INFO, "creating transport client on {} with settings {}",
systemIdentifier, systemIdentifier,
//Strings.toString(settings.toXContent(settingsBuilder, ToXContent.EMPTY_PARAMS).endObject()),
Strings.toString(transportClientSettings.toXContent(effectiveSettingsBuilder, Strings.toString(transportClientSettings.toXContent(effectiveSettingsBuilder,
ToXContent.EMPTY_PARAMS).endObject())); ToXContent.EMPTY_PARAMS).endObject()));
return new MyTransportClient(transportClientSettings, Collections.singletonList(Netty4Plugin.class)); return new MyTransportClient(transportClientSettings, Collections.singletonList(Netty4Plugin.class));
@ -61,29 +62,22 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
return null; return null;
} }
@Override public void closeClient(Settings settings) {
protected void closeClient() { ElasticsearchClient client = clientMap.remove(settings.get("cluster.name"));
if (getClient() != null) { if (client != null) {
TransportClient client = (TransportClient) getClient(); if (client instanceof Client) {
client.close(); ((Client) client).close();
client.threadPool().shutdown(); }
client.threadPool().shutdownNow();
} }
} }
@Override public void init(TransportClient transportClient, Settings settings) throws IOException {
public ExtendedTransportClient init(Settings settings) throws IOException { Collection<TransportAddress> addrs = findAddresses(settings);
super.init(settings); if (!connect(transportClient, addrs, settings.getAsBoolean("autodiscover", false))) {
// additional auto-connect throw new NoNodeAvailableException("no cluster nodes available, check settings "
try { + settings.toString());
Collection<TransportAddress> addrs = findAddresses(settings);
if (!connect(addrs, settings.getAsBoolean("autodiscover", false))) {
throw new NoNodeAvailableException("no cluster nodes available, check settings "
+ settings.toString());
}
} catch (IOException e) {
logger.error(e.getMessage(), e);
} }
return this;
} }
private Collection<TransportAddress> findAddresses(Settings settings) throws IOException { private Collection<TransportAddress> findAddresses(Settings settings) throws IOException {
@ -112,11 +106,7 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
return addresses; return addresses;
} }
private boolean connect(Collection<TransportAddress> addresses, boolean autodiscover) { private boolean connect(TransportClient transportClient, Collection<TransportAddress> addresses, boolean autodiscover) {
if (getClient() == null) {
throw new IllegalStateException("no client present");
}
TransportClient transportClient = (TransportClient) getClient();
for (TransportAddress address : addresses) { for (TransportAddress address : addresses) {
transportClient.addTransportAddresses(address); transportClient.addTransportAddresses(address);
} }
@ -126,7 +116,7 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
if (autodiscover) { if (autodiscover) {
logger.debug("trying to discover all nodes..."); logger.debug("trying to discover all nodes...");
ClusterStateRequestBuilder clusterStateRequestBuilder = ClusterStateRequestBuilder clusterStateRequestBuilder =
new ClusterStateRequestBuilder(getClient(), ClusterStateAction.INSTANCE); new ClusterStateRequestBuilder(transportClient, ClusterStateAction.INSTANCE);
ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet();
DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes(); DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes();
for (DiscoveryNode discoveryNode : discoveryNodes) { for (DiscoveryNode discoveryNode : discoveryNodes) {

View file

@ -0,0 +1,35 @@
package org.xbib.elx.transport;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.xbib.elx.common.AbstractSearchClient;
import java.io.IOException;
/**
* Transport search client with additional methods.
*/
public class TransportSearchClient extends AbstractSearchClient {
private final TransportClientHelper helper;
public TransportSearchClient() {
this.helper = new TransportClientHelper();
}
@Override
public ElasticsearchClient createClient(Settings settings) throws IOException {
return helper.createClient(settings);
}
@Override
public void init(Settings settings) throws IOException {
super.init(settings);
helper.init((TransportClient) getClient(), settings);
}
@Override
public void closeClient(Settings settings) {
helper.closeClient(settings);
}
}

View file

@ -0,0 +1,11 @@
package org.xbib.elx.transport;
import org.xbib.elx.api.SearchClientProvider;
public class TransportSearchClientProvider implements SearchClientProvider<TransportSearchClient> {
@Override
public TransportSearchClient getClient() {
return new TransportSearchClient();
}
}

View file

@ -0,0 +1 @@
org.xbib.elx.transport.TransportAdminClientProvider

View file

@ -0,0 +1 @@
org.xbib.elx.transport.TransportBulkClientProvider

View file

@ -1 +0,0 @@
org.xbib.elx.transport.ExtendedTransportClientProvider

View file

@ -0,0 +1 @@
org.xbib.elx.transport.TransportSearchClientProvider

View file

@ -0,0 +1,190 @@
package org.xbib.elx.transport.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.transport.TransportAdminClient;
import org.xbib.elx.transport.TransportAdminClientProvider;
import org.xbib.elx.transport.TransportBulkClient;
import org.xbib.elx.transport.TransportBulkClientProvider;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ExtendWith(TestExtension.class)
class BulkClientTest {
private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getName());
private static final Long ACTIONS = 10000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
private final TestExtension.Helper helper;
BulkClientTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
void testSingleDoc() throws Exception {
final TransportBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(TransportBulkClientProvider.class)
.put(helper.getTransportSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
.build();
try {
bulkClient.newIndex("test");
bulkClient.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
} finally {
assertEquals(1, bulkClient.getBulkMetric().getSucceeded().getCount());
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(bulkClient.getBulkController().getLastBulkError());
bulkClient.close();
}
}
@Test
void testNewIndex() throws Exception {
final TransportBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(TransportBulkClientProvider.class)
.put(helper.getTransportSettings())
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
bulkClient.newIndex("test");
bulkClient.close();
}
@Test
void testMapping() throws Exception {
try (TransportAdminClient adminClient = ClientBuilder.builder()
.setAdminClientProvider(TransportAdminClientProvider.class)
.put(helper.getTransportSettings())
.build();
TransportBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(TransportBulkClientProvider.class)
.put(helper.getTransportSettings())
.build()) {
XContentBuilder builder = JsonXContent.contentBuilder()
.startObject()
.startObject("properties")
.startObject("location")
.field("type", "geo_point")
.endObject()
.endObject()
.endObject();
bulkClient.newIndex("test", Settings.EMPTY, builder);
assertTrue(adminClient.getMapping("test", "_doc").containsKey("properties"));
}
}
@Test
void testRandomDocs() throws Exception {
long numactions = ACTIONS;
final TransportBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(TransportBulkClientProvider.class)
.put(helper.getTransportSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.build();
try {
bulkClient.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
bulkClient.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(bulkClient.getBulkController().getLastBulkError());
bulkClient.refreshIndex("test");
assertEquals(numactions, bulkClient.getSearchableDocs("test"));
bulkClient.close();
}
}
@Test
void testThreadedRandomDocs() throws Exception {
int maxthreads = Runtime.getRuntime().availableProcessors();
final long actions = ACTIONS;
final TransportBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(TransportBulkClientProvider.class)
.put(helper.getTransportSettings())
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.put(Parameters.ENABLE_BULK_LOGGING.name(), "true")
.build();
try {
Settings settings = Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
bulkClient.newIndex("test", settings);
bulkClient.startBulk("test", 0, 1000);
logger.info("index created");
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
final CountDownLatch latch = new CountDownLatch(maxthreads);
for (int i = 0; i < maxthreads; i++) {
executorService.execute(() -> {
for (int i1 = 0; i1 < actions; i1++) {
bulkClient.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
latch.countDown();
});
}
logger.info("waiting for latch...");
if (latch.await(30L, TimeUnit.SECONDS)) {
logger.info("flush...");
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
logger.info("got all responses, executor service shutdown...");
executorService.shutdown();
executorService.awaitTermination(30L, TimeUnit.SECONDS);
logger.info("pool is shut down");
} else {
logger.warn("latch timeout");
}
bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS);
assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount());
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(bulkClient.getBulkController().getLastBulkError());
bulkClient.refreshIndex("test");
assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test"));
bulkClient.close();
}
}
}

View file

@ -1,247 +0,0 @@
package org.xbib.elx.transport.test;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.transport.ExtendedTransportClient;
import org.xbib.elx.transport.ExtendedTransportClientProvider;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ExtendWith(TestExtension.class)
class ClientTest {
private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
private static final Long ACTIONS = 10000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
private final TestExtension.Helper helper;
ClientTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
void testSingleDoc() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(helper.getTransportSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
.build();
try {
client.newIndex("test");
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
client.close();
}
}
@Test
void testNewIndex() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(helper.getTransportSettings())
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
client.newIndex("test");
client.close();
}
@Test
void testNewIndexWithSettings() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(helper.getTransportSettings())
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
client.newIndex("test", settings);
GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
.indices("test");
GetSettingsResponse getSettingsResponse =
client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet();
logger.log(Level.INFO, "settings=" + getSettingsResponse.getSetting("test", "index.number_of_shards"));
assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards"));
client.close();
}
@Test
void testNewIndexWithSettingsAndMapping() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(helper.getTransportSettings())
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
XContentBuilder builder = JsonXContent.contentBuilder()
.startObject()
.field("date_detection", false)
.startObject("properties")
.startObject("location")
.field("type", "geo_point")
.endObject()
.endObject()
.endObject();
client.newIndex("test", settings, builder);
GetMappingsRequest getMappingsRequest = new GetMappingsRequest()
.indices("test");
GetMappingsResponse getMappingsResponse = client.getClient()
.execute(GetMappingsAction.INSTANCE, getMappingsRequest)
.actionGet();
logger.info("mappings={}", getMappingsResponse.getMappings());
assertTrue(getMappingsResponse.getMappings().get("test").containsKey("_doc"));
client.close();
}
@Test
void testRandomDocs() throws Exception {
long numactions = ACTIONS;
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(helper.getTransportSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.build();
try {
client.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
client.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
client.refreshIndex("test");
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.query(QueryBuilders.matchAllQuery());
builder.size(0);
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices("test");
searchRequest.source(builder);
SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
logger.log(Level.INFO, searchResponse.toString());
assertEquals(numactions, searchResponse.getHits().getTotalHits().value);
client.close();
}
}
@Test
void testThreadedRandomDocs() throws Exception {
int maxthreads = Runtime.getRuntime().availableProcessors();
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
final long actions = ACTIONS;
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(helper.getTransportSettings())
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.put(Parameters.ENABLE_BULK_LOGGING.name(), "true")
.build();
try {
Settings settings = Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
client.newIndex("test", settings, (String)null)
.startBulk("test", 0, 1000);
logger.info("index created");
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
final CountDownLatch latch = new CountDownLatch(maxthreads);
for (int i = 0; i < maxthreads; i++) {
executorService.execute(() -> {
for (int i1 = 0; i1 < actions; i1++) {
client.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
latch.countDown();
});
}
logger.info("waiting for latch...");
if (latch.await(30L, TimeUnit.SECONDS)) {
logger.info("flush...");
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
logger.info("got all responses, executor service shutdown...");
executorService.shutdown();
executorService.awaitTermination(30L, TimeUnit.SECONDS);
logger.info("pool is shut down");
} else {
logger.warn("latch timeout");
}
client.stopBulk("test", 30L, TimeUnit.SECONDS);
assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
logger.log(Level.INFO, "refreshing index test");
client.refreshIndex("test");
SearchSourceBuilder builder = new SearchSourceBuilder()
.query(QueryBuilders.matchAllQuery())
.size(0)
.trackTotalHits(true);
SearchRequest searchRequest = new SearchRequest()
.indices("test")
.source(builder);
SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
assertEquals(maxthreads * actions, searchResponse.getHits().getTotalHits().value);
client.close();
}
}
}

Some files were not shown because too many files have changed in this diff Show more