update to admin/bulk/search API, add missing HTTP actions get index, search scroll, clear scroll
This commit is contained in:
parent
43b6afec62
commit
5423cf0f11
106 changed files with 3509 additions and 2565 deletions
187
elx-api/src/main/java/org/xbib/elx/api/AdminClient.java
Normal file
187
elx-api/src/main/java/org/xbib/elx/api/AdminClient.java
Normal file
|
@ -0,0 +1,187 @@
|
|||
package org.xbib.elx.api;
|
||||
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* Interface for extended managing and indexing methods of an Elasticsearch client.
|
||||
*/
|
||||
public interface AdminClient extends BasicClient {
|
||||
|
||||
/**
|
||||
* Build index definition from settings.
|
||||
*
|
||||
* @param index the index name
|
||||
* @param settings the settings for the index
|
||||
* @return index definition
|
||||
* @throws IOException if settings/mapping URL is invalid/malformed
|
||||
*/
|
||||
IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException;
|
||||
|
||||
Map<String, ?> getMapping(String index) throws IOException;
|
||||
|
||||
Map<String, ?> getMapping(String index, String type) throws IOException;
|
||||
|
||||
/**
|
||||
* Delete an index.
|
||||
* @param indexDefinition the index definition
|
||||
* @return this
|
||||
*/
|
||||
AdminClient deleteIndex(IndexDefinition indexDefinition);
|
||||
|
||||
/**
|
||||
* Delete an index.
|
||||
*
|
||||
* @param index index
|
||||
* @return this
|
||||
*/
|
||||
AdminClient deleteIndex(String index);
|
||||
|
||||
|
||||
/**
|
||||
* Update replica level.
|
||||
* @param indexDefinition the index definition
|
||||
* @param level the replica level
|
||||
* @return this
|
||||
* @throws IOException if replica setting could not be updated
|
||||
*/
|
||||
AdminClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException;
|
||||
|
||||
/**
|
||||
* Update replica level.
|
||||
*
|
||||
* @param index index
|
||||
* @param level the replica level
|
||||
* @param maxWaitTime maximum wait time
|
||||
* @param timeUnit time unit
|
||||
* @return this
|
||||
* @throws IOException if replica setting could not be updated
|
||||
*/
|
||||
AdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException;
|
||||
|
||||
/**
|
||||
* Get replica level.
|
||||
* @param indexDefinition the index name
|
||||
* @return the replica level of the index
|
||||
*/
|
||||
int getReplicaLevel(IndexDefinition indexDefinition);
|
||||
|
||||
/**
|
||||
* Get replica level.
|
||||
* @param index the index name
|
||||
* @return the replica level of the index
|
||||
*/
|
||||
int getReplicaLevel(String index);
|
||||
|
||||
/**
|
||||
* Force segment merge of an index.
|
||||
* @param indexDefinition the index definition
|
||||
* @return this
|
||||
*/
|
||||
boolean forceMerge(IndexDefinition indexDefinition);
|
||||
|
||||
/**
|
||||
* Force segment merge of an index.
|
||||
* @param index the index
|
||||
* @param maxWaitTime maximum wait time
|
||||
* @param timeUnit time unit
|
||||
* @return this
|
||||
*/
|
||||
boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit);
|
||||
|
||||
/**
|
||||
* Resolve alias.
|
||||
*
|
||||
* @param alias the alias
|
||||
* @return this index name behind the alias or the alias if there is no index
|
||||
*/
|
||||
String resolveAlias(String alias);
|
||||
|
||||
/**
|
||||
* Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index
|
||||
* name.
|
||||
*
|
||||
* @param alias the alias
|
||||
* @return the most recent index name pointing to the alias
|
||||
*/
|
||||
String resolveMostRecentIndex(String alias);
|
||||
|
||||
/**
|
||||
* Get all index aliases.
|
||||
* @param index the index
|
||||
* @return map of index aliases
|
||||
*/
|
||||
Map<String, String> getAliases(String index);
|
||||
|
||||
/**
|
||||
* Shift from one index to another.
|
||||
* @param indexDefinition the index definition
|
||||
* @param additionalAliases new aliases
|
||||
* @return this
|
||||
*/
|
||||
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases);
|
||||
|
||||
/**
|
||||
* Shift from one index to another.
|
||||
* @param indexDefinition the index definition
|
||||
* @param additionalAliases new aliases
|
||||
* @param indexAliasAdder method to add aliases
|
||||
* @return this
|
||||
*/
|
||||
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases,
|
||||
IndexAliasAdder indexAliasAdder);
|
||||
|
||||
/**
|
||||
* Shift from one index to another.
|
||||
* @param index the index name
|
||||
* @param fullIndexName the index name with timestamp
|
||||
* @param additionalAliases a list of names that should be set as index aliases
|
||||
* @return this
|
||||
*/
|
||||
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases);
|
||||
|
||||
/**
|
||||
* Shift from one index to another.
|
||||
* @param index the index name
|
||||
* @param fullIndexName the index name with timestamp
|
||||
* @param additionalAliases a list of names that should be set as index aliases
|
||||
* @param adder an adder method to create alias term queries
|
||||
* @return this
|
||||
*/
|
||||
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases,
|
||||
IndexAliasAdder adder);
|
||||
|
||||
/**
|
||||
* Prune index.
|
||||
* @param indexDefinition the index definition
|
||||
* @return the index prune result
|
||||
*/
|
||||
IndexPruneResult pruneIndex(IndexDefinition indexDefinition);
|
||||
|
||||
/**
|
||||
* Apply retention policy to prune indices. All indices before delta should be deleted,
|
||||
* but the number of mintokeep indices must be kept.
|
||||
*
|
||||
* @param index index name
|
||||
* @param fullIndexName index name with timestamp
|
||||
* @param delta timestamp delta (for index timestamps)
|
||||
* @param mintokeep minimum number of indices to keep
|
||||
* @param perform true if pruning should be executed, false if not
|
||||
* @return the index prune result
|
||||
*/
|
||||
IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform);
|
||||
|
||||
/**
|
||||
* Find the timestamp of the most recently indexed document in the index.
|
||||
*
|
||||
* @param index the index name
|
||||
* @param timestampfieldname the timestamp field name
|
||||
* @return millis UTC millis of the most recent document
|
||||
* @throws IOException if most rcent document can not be found
|
||||
*/
|
||||
Long mostRecentDocument(String index, String timestampfieldname) throws IOException;
|
||||
}
|
|
@ -0,0 +1,7 @@
|
|||
package org.xbib.elx.api;
|
||||
|
||||
@FunctionalInterface
|
||||
public interface AdminClientProvider<C extends AdminClient> {
|
||||
|
||||
C getClient();
|
||||
}
|
62
elx-api/src/main/java/org/xbib/elx/api/BasicClient.java
Normal file
62
elx-api/src/main/java/org/xbib/elx/api/BasicClient.java
Normal file
|
@ -0,0 +1,62 @@
|
|||
package org.xbib.elx.api;
|
||||
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public interface BasicClient extends Closeable {
|
||||
|
||||
/**
|
||||
* Set an Elasticsearch client to extend from it. May be null for TransportClient.
|
||||
* @param client the Elasticsearch client
|
||||
*/
|
||||
void setClient(ElasticsearchClient client);
|
||||
|
||||
/**
|
||||
* Return Elasticsearch client.
|
||||
*
|
||||
* @return Elasticsearch client
|
||||
*/
|
||||
ElasticsearchClient getClient();
|
||||
|
||||
/**
|
||||
* Initiative the extended client, the bulk metric and bulk controller,
|
||||
* creates instances and connect to cluster, if required.
|
||||
*
|
||||
* @param settings settings
|
||||
* @throws IOException if init fails
|
||||
*/
|
||||
void init(Settings settings) throws IOException;
|
||||
|
||||
/**
|
||||
* Get cluster name.
|
||||
* @return the cluster name
|
||||
*/
|
||||
String getClusterName();
|
||||
|
||||
/**
|
||||
* Get current health color.
|
||||
*
|
||||
* @param maxWaitTime maximum wait time
|
||||
* @param timeUnit time unit
|
||||
* @return the cluster health color
|
||||
*/
|
||||
String getHealthColor(long maxWaitTime, TimeUnit timeUnit);
|
||||
|
||||
/**
|
||||
* Wait for cluster being healthy.
|
||||
*
|
||||
* @param healthColor cluster health color to wait for
|
||||
* @param maxWaitTime time value
|
||||
* @param timeUnit time unit
|
||||
*/
|
||||
void waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit);
|
||||
|
||||
void waitForShards(long maxWaitTime, TimeUnit timeUnit);
|
||||
|
||||
long getSearchableDocs(String index);
|
||||
|
||||
boolean isIndexExists(String index);
|
||||
}
|
227
elx-api/src/main/java/org/xbib/elx/api/BulkClient.java
Normal file
227
elx-api/src/main/java/org/xbib/elx/api/BulkClient.java
Normal file
|
@ -0,0 +1,227 @@
|
|||
package org.xbib.elx.api;
|
||||
|
||||
import org.elasticsearch.action.delete.DeleteRequest;
|
||||
import org.elasticsearch.action.index.IndexRequest;
|
||||
import org.elasticsearch.action.update.UpdateRequest;
|
||||
import org.elasticsearch.common.bytes.BytesReference;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import java.io.Flushable;
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public interface BulkClient extends BasicClient, Flushable {
|
||||
|
||||
/**
|
||||
* Get bulk metric.
|
||||
* @return the bulk metric
|
||||
*/
|
||||
BulkMetric getBulkMetric();
|
||||
|
||||
/**
|
||||
* Get buulk control.
|
||||
* @return the bulk control
|
||||
*/
|
||||
BulkController getBulkController();
|
||||
|
||||
|
||||
/**
|
||||
* Create a new index.
|
||||
*
|
||||
* @param index index
|
||||
* @throws IOException if new index creation fails
|
||||
*/
|
||||
void newIndex(String index) throws IOException;
|
||||
|
||||
/**
|
||||
* Create a new index.
|
||||
* @param indexDefinition the index definition
|
||||
* @throws IOException if settings/mapping is invalid or index creation fails
|
||||
*/
|
||||
void newIndex(IndexDefinition indexDefinition) throws IOException;
|
||||
|
||||
/**
|
||||
* Create a new index.
|
||||
*
|
||||
* @param index index
|
||||
* @param settings settings
|
||||
* @throws IOException if settings is invalid or index creation fails
|
||||
*/
|
||||
void newIndex(String index, Settings settings) throws IOException;
|
||||
|
||||
/**
|
||||
* Create a new index.
|
||||
*
|
||||
* @param index index
|
||||
* @param settings settings
|
||||
* @param mapping mapping
|
||||
* @throws IOException if settings/mapping is invalid or index creation fails
|
||||
*/
|
||||
void newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException;
|
||||
|
||||
/**
|
||||
* Create a new index.
|
||||
*
|
||||
* @param index index
|
||||
* @param settings settings
|
||||
* @param mapping mapping
|
||||
* @throws IOException if settings/mapping is invalid or index creation fails
|
||||
*/
|
||||
void newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException;
|
||||
|
||||
/**
|
||||
* Add index request. Each request will be added to a queue for bulking requests.
|
||||
* Submitting request will be done when limits are exceeded.
|
||||
*
|
||||
* @param index the index
|
||||
* @param id the id
|
||||
* @param create true if document must be created
|
||||
* @param source the source
|
||||
* @return this
|
||||
*/
|
||||
BulkClient index(String index, String id, boolean create, BytesReference source);
|
||||
|
||||
/**
|
||||
* Index request. Each request will be added to a queue for bulking requests.
|
||||
* Submitting request will be done when limits are exceeded.
|
||||
*
|
||||
* @param index the index
|
||||
* @param id the id
|
||||
* @param create true if document is to be created, false otherwise
|
||||
* @param source the source
|
||||
* @return this client methods
|
||||
*/
|
||||
BulkClient index(String index, String id, boolean create, String source);
|
||||
|
||||
/**
|
||||
* Index request. Each request will be added to a queue for bulking requests.
|
||||
* Submitting request will be done when bulk limits are exceeded.
|
||||
*
|
||||
* @param indexRequest the index request to add
|
||||
* @return this
|
||||
*/
|
||||
BulkClient index(IndexRequest indexRequest);
|
||||
|
||||
/**
|
||||
* Delete request.
|
||||
*
|
||||
* @param index the index
|
||||
* @param id the id
|
||||
* @return this
|
||||
*/
|
||||
BulkClient delete(String index, String id);
|
||||
|
||||
/**
|
||||
* Delete request. Each request will be added to a queue for bulking requests.
|
||||
* Submitting request will be done when bulk limits are exceeded.
|
||||
*
|
||||
* @param deleteRequest the delete request to add
|
||||
* @return this
|
||||
*/
|
||||
BulkClient delete(DeleteRequest deleteRequest);
|
||||
|
||||
/**
|
||||
* Bulked update request. Each request will be added to a queue for bulking requests.
|
||||
* Submitting request will be done when bulk limits are exceeded.
|
||||
* Note that updates only work correctly when all operations between nodes are synchronized.
|
||||
*
|
||||
* @param index the index
|
||||
* @param id the id
|
||||
* @param source the source
|
||||
* @return this
|
||||
*/
|
||||
BulkClient update(String index, String id, BytesReference source);
|
||||
|
||||
/**
|
||||
* Update document. Use with precaution! Does not work in all cases.
|
||||
*
|
||||
* @param index the index
|
||||
* @param id the id
|
||||
* @param source the source
|
||||
* @return this
|
||||
*/
|
||||
BulkClient update(String index, String id, String source);
|
||||
|
||||
/**
|
||||
* Bulked update request. Each request will be added to a queue for bulking requests.
|
||||
* Submitting request will be done when bulk limits are exceeded.
|
||||
* Note that updates only work correctly when all operations between nodes are synchronized.
|
||||
*
|
||||
* @param updateRequest the update request to add
|
||||
* @return this
|
||||
*/
|
||||
BulkClient update(UpdateRequest updateRequest);
|
||||
|
||||
/**
|
||||
* Start bulk mode for indexes.
|
||||
* @param indexDefinition index definition
|
||||
* @throws IOException if bulk could not be started
|
||||
*/
|
||||
void startBulk(IndexDefinition indexDefinition) throws IOException;
|
||||
|
||||
/**
|
||||
* Start bulk mode.
|
||||
*
|
||||
* @param index index
|
||||
* @param startRefreshIntervalSeconds refresh interval before bulk
|
||||
* @param stopRefreshIntervalSeconds refresh interval after bulk
|
||||
* @throws IOException if bulk could not be started
|
||||
*/
|
||||
void startBulk(String index, long startRefreshIntervalSeconds,
|
||||
long stopRefreshIntervalSeconds) throws IOException;
|
||||
|
||||
|
||||
/**
|
||||
* Stop bulk mode.
|
||||
*
|
||||
* @param indexDefinition index definition
|
||||
* @throws IOException if bulk could not be startet
|
||||
*/
|
||||
void stopBulk(IndexDefinition indexDefinition) throws IOException;
|
||||
|
||||
/**
|
||||
* Stops bulk mode.
|
||||
*
|
||||
* @param index index
|
||||
* @param timeout maximum wait time
|
||||
* @param timeUnit time unit for timeout
|
||||
* @throws IOException if bulk could not be stopped
|
||||
*/
|
||||
void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException;
|
||||
|
||||
/**
|
||||
* Wait for all outstanding bulk responses.
|
||||
*
|
||||
* @param timeout maximum wait time
|
||||
* @param timeUnit unit of timeout value
|
||||
* @return true if wait succeeded, false if wait timed out
|
||||
*/
|
||||
boolean waitForResponses(long timeout, TimeUnit timeUnit);
|
||||
|
||||
/**
|
||||
* Update index setting.
|
||||
* @param index the index
|
||||
* @param key the key of the value to be updated
|
||||
* @param value the new value
|
||||
* @param timeout timeout
|
||||
* @param timeUnit time unit
|
||||
* @throws IOException if update index setting failed
|
||||
*/
|
||||
void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException;
|
||||
|
||||
/**
|
||||
* Refresh the index.
|
||||
*
|
||||
* @param index index
|
||||
*/
|
||||
void refreshIndex(String index);
|
||||
|
||||
/**
|
||||
* Flush the index. The cluster clears cache and completes indexing.
|
||||
*
|
||||
* @param index index
|
||||
*/
|
||||
void flushIndex(String index);
|
||||
|
||||
}
|
|
@ -0,0 +1,7 @@
|
|||
package org.xbib.elx.api;
|
||||
|
||||
@FunctionalInterface
|
||||
public interface BulkClientProvider<C extends BulkClient> {
|
||||
|
||||
C getClient();
|
||||
}
|
|
@ -1,5 +1,6 @@
|
|||
package org.xbib.elx.api;
|
||||
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.xbib.metrics.api.Count;
|
||||
import org.xbib.metrics.api.Metered;
|
||||
|
||||
|
@ -7,6 +8,8 @@ import java.io.Closeable;
|
|||
|
||||
public interface BulkMetric extends Closeable {
|
||||
|
||||
void init(Settings settings);
|
||||
|
||||
Metered getTotalIngest();
|
||||
|
||||
Count getTotalIngestSizeInBytes();
|
||||
|
|
|
@ -1,487 +0,0 @@
|
|||
package org.xbib.elx.api;
|
||||
|
||||
import org.elasticsearch.action.delete.DeleteRequest;
|
||||
import org.elasticsearch.action.index.IndexRequest;
|
||||
import org.elasticsearch.action.update.UpdateRequest;
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.common.bytes.BytesReference;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.Flushable;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* Interface for extended managing and indexing methods of an Elasticsearch client.
|
||||
*/
|
||||
public interface ExtendedClient extends Flushable, Closeable {
|
||||
|
||||
/**
|
||||
* Set an Elasticsearch client to extend from it. May be null for TransportClient.
|
||||
* @param client client
|
||||
* @return this client
|
||||
*/
|
||||
ExtendedClient setClient(ElasticsearchClient client);
|
||||
|
||||
/**
|
||||
* Return Elasticsearch client.
|
||||
*
|
||||
* @return Elasticsearch client
|
||||
*/
|
||||
ElasticsearchClient getClient();
|
||||
|
||||
/**
|
||||
* Get bulk control.
|
||||
* @return the bulk control
|
||||
*/
|
||||
BulkController getBulkController();
|
||||
|
||||
/**
|
||||
* Initiative the extended client, the bulk metric and bulk controller,
|
||||
* creates instances and connect to cluster, if required.
|
||||
*
|
||||
* @param settings settings
|
||||
* @return this client
|
||||
* @throws IOException if init fails
|
||||
*/
|
||||
ExtendedClient init(Settings settings) throws IOException;
|
||||
|
||||
/**
|
||||
* Build index definition from settings.
|
||||
*
|
||||
* @param index the index name
|
||||
* @param settings the settings for the index
|
||||
* @return index definition
|
||||
* @throws IOException if settings/mapping URL is invalid/malformed
|
||||
*/
|
||||
IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException;
|
||||
|
||||
/**
|
||||
* Add index request. Each request will be added to a queue for bulking requests.
|
||||
* Submitting request will be done when limits are exceeded.
|
||||
*
|
||||
* @param index the index
|
||||
* @param id the id
|
||||
* @param create true if document must be created
|
||||
* @param source the source
|
||||
* @return this
|
||||
*/
|
||||
ExtendedClient index(String index, String id, boolean create, BytesReference source);
|
||||
|
||||
/**
|
||||
* Index request. Each request will be added to a queue for bulking requests.
|
||||
* Submitting request will be done when limits are exceeded.
|
||||
*
|
||||
* @param index the index
|
||||
* @param id the id
|
||||
* @param create true if document is to be created, false otherwise
|
||||
* @param source the source
|
||||
* @return this client methods
|
||||
*/
|
||||
ExtendedClient index(String index, String id, boolean create, String source);
|
||||
|
||||
/**
|
||||
* Index request. Each request will be added to a queue for bulking requests.
|
||||
* Submitting request will be done when bulk limits are exceeded.
|
||||
*
|
||||
* @param indexRequest the index request to add
|
||||
* @return this
|
||||
*/
|
||||
ExtendedClient index(IndexRequest indexRequest);
|
||||
|
||||
/**
|
||||
* Delete request.
|
||||
*
|
||||
* @param index the index
|
||||
* @param id the id
|
||||
* @return this
|
||||
*/
|
||||
ExtendedClient delete(String index, String id);
|
||||
|
||||
/**
|
||||
* Delete request. Each request will be added to a queue for bulking requests.
|
||||
* Submitting request will be done when bulk limits are exceeded.
|
||||
*
|
||||
* @param deleteRequest the delete request to add
|
||||
* @return this
|
||||
*/
|
||||
ExtendedClient delete(DeleteRequest deleteRequest);
|
||||
|
||||
/**
|
||||
* Bulked update request. Each request will be added to a queue for bulking requests.
|
||||
* Submitting request will be done when bulk limits are exceeded.
|
||||
* Note that updates only work correctly when all operations between nodes are synchronized.
|
||||
*
|
||||
* @param index the index
|
||||
* @param id the id
|
||||
* @param source the source
|
||||
* @return this
|
||||
* @throws IOException if update fails
|
||||
*/
|
||||
ExtendedClient update(String index, String id, BytesReference source) throws IOException;
|
||||
|
||||
/**
|
||||
* Update document. Use with precaution! Does not work in all cases.
|
||||
*
|
||||
* @param index the index
|
||||
* @param id the id
|
||||
* @param source the source
|
||||
* @return this
|
||||
*/
|
||||
ExtendedClient update(String index, String id, String source);
|
||||
|
||||
/**
|
||||
* Bulked update request. Each request will be added to a queue for bulking requests.
|
||||
* Submitting request will be done when bulk limits are exceeded.
|
||||
* Note that updates only work correctly when all operations between nodes are synchronized.
|
||||
*
|
||||
* @param updateRequest the update request to add
|
||||
* @return this
|
||||
*/
|
||||
ExtendedClient update(UpdateRequest updateRequest);
|
||||
|
||||
/**
|
||||
* Create a new index.
|
||||
*
|
||||
* @param index index
|
||||
* @return this
|
||||
* @throws IOException if new index creation fails
|
||||
*/
|
||||
ExtendedClient newIndex(String index) throws IOException;
|
||||
|
||||
/**
|
||||
* Create a new index.
|
||||
*
|
||||
* @param index index
|
||||
* @param settings settings
|
||||
* @param mapping mapping
|
||||
* @return this
|
||||
* @throws IOException if settings/mapping is invalid or index creation fails
|
||||
*/
|
||||
ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException;
|
||||
|
||||
/**
|
||||
* Create a new index.
|
||||
*
|
||||
* @param index index
|
||||
* @param settings settings
|
||||
* @return this
|
||||
* @throws IOException if settings is invalid or index creation fails
|
||||
*/
|
||||
ExtendedClient newIndex(String index, Settings settings) throws IOException;
|
||||
|
||||
/**
|
||||
* Create a new index.
|
||||
*
|
||||
* @param index index
|
||||
* @param settings settings
|
||||
* @param mapping mapping
|
||||
* @return this
|
||||
* @throws IOException if settings/mapping is invalid or index creation fails
|
||||
*/
|
||||
ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException;
|
||||
|
||||
/**
|
||||
* Create a new index.
|
||||
*
|
||||
* @param index index
|
||||
* @param settings settings
|
||||
* @param mapping mapping
|
||||
* @return this
|
||||
* @throws IOException if settings/mapping is invalid or index creation fails
|
||||
*/
|
||||
ExtendedClient newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException;
|
||||
|
||||
/**
|
||||
* Create a new index.
|
||||
*
|
||||
* @param index index
|
||||
* @param settings settings
|
||||
* @param mapping mapping
|
||||
* @return this
|
||||
* @throws IOException if settings/mapping is invalid or index creation fails
|
||||
*/
|
||||
ExtendedClient newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException;
|
||||
|
||||
/**
|
||||
* Create a new index.
|
||||
* @param indexDefinition the index definition
|
||||
* @return this
|
||||
* @throws IOException if settings/mapping is invalid or index creation fails
|
||||
*/
|
||||
ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException;
|
||||
|
||||
/**
|
||||
* Delete an index.
|
||||
* @param indexDefinition the index definition
|
||||
* @return this
|
||||
*/
|
||||
ExtendedClient deleteIndex(IndexDefinition indexDefinition);
|
||||
|
||||
/**
|
||||
* Delete an index.
|
||||
*
|
||||
* @param index index
|
||||
* @return this
|
||||
*/
|
||||
ExtendedClient deleteIndex(String index);
|
||||
|
||||
/**
|
||||
* Start bulk mode for indexes.
|
||||
* @param indexDefinition index definition
|
||||
* @return this
|
||||
* @throws IOException if bulk could not be started
|
||||
*/
|
||||
ExtendedClient startBulk(IndexDefinition indexDefinition) throws IOException;
|
||||
|
||||
/**
|
||||
* Start bulk mode.
|
||||
*
|
||||
* @param index index
|
||||
* @param startRefreshIntervalSeconds refresh interval before bulk
|
||||
* @param stopRefreshIntervalSeconds refresh interval after bulk
|
||||
* @return this
|
||||
* @throws IOException if bulk could not be started
|
||||
*/
|
||||
ExtendedClient startBulk(String index, long startRefreshIntervalSeconds,
|
||||
long stopRefreshIntervalSeconds) throws IOException;
|
||||
|
||||
/**
|
||||
* Stop bulk mode.
|
||||
*
|
||||
* @param indexDefinition index definition
|
||||
* @return this
|
||||
* @throws IOException if bulk could not be startet
|
||||
*/
|
||||
ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException;
|
||||
|
||||
/**
|
||||
* Stops bulk mode.
|
||||
*
|
||||
* @param index index
|
||||
* @param timeout maximum wait time
|
||||
* @param timeUnit time unit for timeout
|
||||
* @return this
|
||||
* @throws IOException if bulk could not be stopped
|
||||
*/
|
||||
ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException;
|
||||
|
||||
/**
|
||||
* Update replica level.
|
||||
* @param indexDefinition the index definition
|
||||
* @param level the replica level
|
||||
* @return this
|
||||
* @throws IOException if replica setting could not be updated
|
||||
*/
|
||||
ExtendedClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException;
|
||||
|
||||
/**
|
||||
* Update replica level.
|
||||
*
|
||||
* @param index index
|
||||
* @param level the replica level
|
||||
* @param maxWaitTime maximum wait time
|
||||
* @param timeUnit time unit
|
||||
* @return this
|
||||
* @throws IOException if replica setting could not be updated
|
||||
*/
|
||||
ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException;
|
||||
|
||||
/**
|
||||
* Get replica level.
|
||||
* @param indexDefinition the index name
|
||||
* @return the replica level of the index
|
||||
*/
|
||||
int getReplicaLevel(IndexDefinition indexDefinition);
|
||||
|
||||
/**
|
||||
* Get replica level.
|
||||
* @param index the index name
|
||||
* @return the replica level of the index
|
||||
*/
|
||||
int getReplicaLevel(String index);
|
||||
|
||||
/**
|
||||
* Refresh the index.
|
||||
*
|
||||
* @param index index
|
||||
* @return this
|
||||
*/
|
||||
ExtendedClient refreshIndex(String index);
|
||||
|
||||
/**
|
||||
* Flush the index. The cluster clears cache and completes indexing.
|
||||
*
|
||||
* @param index index
|
||||
* @return this
|
||||
*/
|
||||
ExtendedClient flushIndex(String index);
|
||||
|
||||
/**
|
||||
* Force segment merge of an index.
|
||||
* @param indexDefinition the index definition
|
||||
* @return this
|
||||
*/
|
||||
boolean forceMerge(IndexDefinition indexDefinition);
|
||||
|
||||
/**
|
||||
* Force segment merge of an index.
|
||||
* @param index the index
|
||||
* @param maxWaitTime maximum wait time
|
||||
* @param timeUnit time unit
|
||||
* @return this
|
||||
*/
|
||||
boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit);
|
||||
|
||||
/**
|
||||
* Wait for all outstanding bulk responses.
|
||||
*
|
||||
* @param timeout maximum wait time
|
||||
* @param timeUnit unit of timeout value
|
||||
* @return true if wait succeeded, false if wait timed out
|
||||
*/
|
||||
boolean waitForResponses(long timeout, TimeUnit timeUnit);
|
||||
|
||||
/**
|
||||
* Wait for cluster being healthy.
|
||||
*
|
||||
* @param healthColor cluster health color to wait for
|
||||
* @param maxWaitTime time value
|
||||
* @param timeUnit time unit
|
||||
* @return true if wait succeeded, false if wait timed out
|
||||
*/
|
||||
boolean waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit);
|
||||
|
||||
/**
|
||||
* Get current health color.
|
||||
*
|
||||
* @param maxWaitTime maximum wait time
|
||||
* @param timeUnit time unit
|
||||
* @return the cluster health color
|
||||
*/
|
||||
String getHealthColor(long maxWaitTime, TimeUnit timeUnit);
|
||||
|
||||
/**
|
||||
* Wait for index recovery (after replica change).
|
||||
*
|
||||
* @param index index
|
||||
* @param maxWaitTime maximum wait time
|
||||
* @param timeUnit time unit
|
||||
* @return true if wait succeeded, false if wait timed out
|
||||
*/
|
||||
boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit);
|
||||
|
||||
/**
|
||||
* Update index setting.
|
||||
* @param index the index
|
||||
* @param key the key of the value to be updated
|
||||
* @param value the new value
|
||||
* @param timeout timeout
|
||||
* @param timeUnit time unit
|
||||
* @throws IOException if update index setting failed
|
||||
*/
|
||||
void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException;
|
||||
|
||||
/**
|
||||
* Resolve alias.
|
||||
*
|
||||
* @param alias the alias
|
||||
* @return this index name behind the alias or the alias if there is no index
|
||||
*/
|
||||
String resolveAlias(String alias);
|
||||
|
||||
/**
|
||||
* Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index
|
||||
* name.
|
||||
*
|
||||
* @param alias the alias
|
||||
* @return the most recent index name pointing to the alias
|
||||
*/
|
||||
String resolveMostRecentIndex(String alias);
|
||||
|
||||
/**
|
||||
* Get all index aliases.
|
||||
* @param index the index
|
||||
* @return map of index aliases
|
||||
*/
|
||||
Map<String, String> getAliases(String index);
|
||||
|
||||
/**
|
||||
* Shift from one index to another.
|
||||
* @param indexDefinition the index definition
|
||||
* @param additionalAliases new aliases
|
||||
* @return this
|
||||
*/
|
||||
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases);
|
||||
|
||||
/**
|
||||
* Shift from one index to another.
|
||||
* @param indexDefinition the index definition
|
||||
* @param additionalAliases new aliases
|
||||
* @param indexAliasAdder method to add aliases
|
||||
* @return this
|
||||
*/
|
||||
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases,
|
||||
IndexAliasAdder indexAliasAdder);
|
||||
|
||||
/**
|
||||
* Shift from one index to another.
|
||||
* @param index the index name
|
||||
* @param fullIndexName the index name with timestamp
|
||||
* @param additionalAliases a list of names that should be set as index aliases
|
||||
* @return this
|
||||
*/
|
||||
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases);
|
||||
|
||||
/**
|
||||
* Shift from one index to another.
|
||||
* @param index the index name
|
||||
* @param fullIndexName the index name with timestamp
|
||||
* @param additionalAliases a list of names that should be set as index aliases
|
||||
* @param adder an adder method to create alias term queries
|
||||
* @return this
|
||||
*/
|
||||
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases,
|
||||
IndexAliasAdder adder);
|
||||
|
||||
/**
|
||||
* Prune index.
|
||||
* @param indexDefinition the index definition
|
||||
* @return the index prune result
|
||||
*/
|
||||
IndexPruneResult pruneIndex(IndexDefinition indexDefinition);
|
||||
|
||||
/**
|
||||
* Apply retention policy to prune indices. All indices before delta should be deleted,
|
||||
* but the number of mintokeep indices must be kept.
|
||||
*
|
||||
* @param index index name
|
||||
* @param fullIndexName index name with timestamp
|
||||
* @param delta timestamp delta (for index timestamps)
|
||||
* @param mintokeep minimum number of indices to keep
|
||||
* @param perform true if pruning should be executed, false if not
|
||||
* @return the index prune result
|
||||
*/
|
||||
IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform);
|
||||
|
||||
/**
|
||||
* Find the timestamp of the most recently indexed document in the index.
|
||||
*
|
||||
* @param index the index name
|
||||
* @param timestampfieldname the timestamp field name
|
||||
* @return millis UTC millis of the most recent document
|
||||
* @throws IOException if most rcent document can not be found
|
||||
*/
|
||||
Long mostRecentDocument(String index, String timestampfieldname) throws IOException;
|
||||
|
||||
/**
|
||||
* Get cluster name.
|
||||
* @return the cluster name
|
||||
*/
|
||||
String getClusterName();
|
||||
}
|
|
@ -1,7 +0,0 @@
|
|||
package org.xbib.elx.api;
|
||||
|
||||
@FunctionalInterface
|
||||
public interface ExtendedClientProvider<C extends ExtendedClient> {
|
||||
|
||||
C getExtendedClient();
|
||||
}
|
|
@ -1,7 +1,5 @@
|
|||
package org.xbib.elx.api;
|
||||
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public interface IndexDefinition {
|
||||
|
@ -14,17 +12,13 @@ public interface IndexDefinition {
|
|||
|
||||
String getFullIndexName();
|
||||
|
||||
IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException;
|
||||
IndexDefinition setSettings(String settings);
|
||||
|
||||
IndexDefinition setSettingsUrl(URL settingsUrl);
|
||||
String getSettings();
|
||||
|
||||
URL getSettingsUrl();
|
||||
IndexDefinition setMappings(String mappings);
|
||||
|
||||
IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException;
|
||||
|
||||
IndexDefinition setMappingsUrl(URL mappingsUrl);
|
||||
|
||||
URL getMappingsUrl();
|
||||
String getMappings();
|
||||
|
||||
IndexDefinition setDateTimePattern(String timeWindow);
|
||||
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
package org.xbib.elx.api;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Collection;
|
||||
|
||||
public interface IndexPruneResult {
|
||||
|
||||
|
@ -8,9 +8,9 @@ public interface IndexPruneResult {
|
|||
|
||||
State getState();
|
||||
|
||||
List<String> getCandidateIndices();
|
||||
Collection<String> getCandidateIndices();
|
||||
|
||||
List<String> getDeletedIndices();
|
||||
Collection<String> getDeletedIndices();
|
||||
|
||||
boolean isAcknowledged();
|
||||
}
|
||||
|
|
|
@ -1,10 +1,10 @@
|
|||
package org.xbib.elx.api;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Collection;
|
||||
|
||||
public interface IndexShiftResult {
|
||||
|
||||
List<String> getMovedAliases();
|
||||
Collection<String> getMovedAliases();
|
||||
|
||||
List<String> getNewAliases();
|
||||
Collection<String> getNewAliases();
|
||||
}
|
||||
|
|
|
@ -1,25 +0,0 @@
|
|||
package org.xbib.elx.api;
|
||||
|
||||
import org.elasticsearch.action.ActionFuture;
|
||||
import org.elasticsearch.action.ActionListener;
|
||||
import org.elasticsearch.action.get.GetRequest;
|
||||
import org.elasticsearch.action.get.GetResponse;
|
||||
import org.elasticsearch.action.get.MultiGetRequest;
|
||||
import org.elasticsearch.action.get.MultiGetResponse;
|
||||
import org.elasticsearch.action.search.SearchRequest;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
|
||||
public interface ReadClient {
|
||||
|
||||
ActionFuture<GetResponse> get(GetRequest getRequest);
|
||||
|
||||
void get(GetRequest request, ActionListener<GetResponse> listener);
|
||||
|
||||
ActionFuture<MultiGetResponse> multiGet(MultiGetRequest request);
|
||||
|
||||
void multiGet(MultiGetRequest request, ActionListener<MultiGetResponse> listener);
|
||||
|
||||
ActionFuture<SearchResponse> search(SearchRequest request);
|
||||
|
||||
void search(SearchRequest request, ActionListener<SearchResponse> listener);
|
||||
}
|
|
@ -1,6 +0,0 @@
|
|||
package org.xbib.elx.api;
|
||||
|
||||
public interface ReadClientProvider<C extends ReadClient> {
|
||||
|
||||
C getReadClient();
|
||||
}
|
27
elx-api/src/main/java/org/xbib/elx/api/SearchClient.java
Normal file
27
elx-api/src/main/java/org/xbib/elx/api/SearchClient.java
Normal file
|
@ -0,0 +1,27 @@
|
|||
package org.xbib.elx.api;
|
||||
|
||||
import org.elasticsearch.action.get.GetRequestBuilder;
|
||||
import org.elasticsearch.action.get.GetResponse;
|
||||
import org.elasticsearch.action.get.MultiGetRequestBuilder;
|
||||
import org.elasticsearch.action.get.MultiGetResponse;
|
||||
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.search.SearchHit;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
public interface SearchClient extends BasicClient {
|
||||
|
||||
Optional<GetResponse> get(Consumer<GetRequestBuilder> getRequestBuilder);
|
||||
|
||||
Optional<MultiGetResponse> multiGet(Consumer<MultiGetRequestBuilder> multiGetRequestBuilder);
|
||||
|
||||
Optional<SearchResponse> search(Consumer<SearchRequestBuilder> searchRequestBuilder);
|
||||
|
||||
Stream<SearchHit> search(Consumer<SearchRequestBuilder> searchRequestBuilder,
|
||||
TimeValue scrollTime, int scrollSize);
|
||||
|
||||
Stream<String> getIds(Consumer<SearchRequestBuilder> queryBuilder);
|
||||
}
|
|
@ -0,0 +1,7 @@
|
|||
package org.xbib.elx.api;
|
||||
|
||||
@FunctionalInterface
|
||||
public interface SearchClientProvider<C extends SearchClient> {
|
||||
|
||||
C getClient();
|
||||
}
|
|
@ -2,14 +2,8 @@ package org.xbib.elx.common;
|
|||
|
||||
import com.carrotsearch.hppc.cursors.ObjectCursor;
|
||||
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.lucene.search.TotalHits;
|
||||
import org.elasticsearch.ElasticsearchTimeoutException;
|
||||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
|
||||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
|
||||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
|
||||
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
|
||||
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
|
||||
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
|
||||
|
@ -18,13 +12,8 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
|
|||
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction;
|
||||
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
|
||||
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
|
||||
import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
|
||||
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
|
||||
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
|
||||
import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
|
||||
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
|
||||
import org.elasticsearch.action.admin.indices.flush.FlushAction;
|
||||
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
|
||||
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction;
|
||||
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
|
||||
import org.elasticsearch.action.admin.indices.get.GetIndexAction;
|
||||
|
@ -32,36 +21,31 @@ import org.elasticsearch.action.admin.indices.get.GetIndexRequestBuilder;
|
|||
import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequestBuilder;
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
|
||||
import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
|
||||
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
|
||||
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
|
||||
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
|
||||
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
|
||||
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
|
||||
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
|
||||
import org.elasticsearch.action.delete.DeleteRequest;
|
||||
import org.elasticsearch.action.index.IndexRequest;
|
||||
import org.elasticsearch.action.search.SearchAction;
|
||||
import org.elasticsearch.action.search.SearchRequest;
|
||||
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.action.support.master.AcknowledgedResponse;
|
||||
import org.elasticsearch.action.update.UpdateRequest;
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.elasticsearch.cluster.health.ClusterHealthStatus;
|
||||
import org.elasticsearch.cluster.metadata.AliasMetaData;
|
||||
import org.elasticsearch.cluster.metadata.AliasOrIndex;
|
||||
import org.elasticsearch.cluster.metadata.MappingMetaData;
|
||||
import org.elasticsearch.common.bytes.BytesReference;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.collect.ImmutableOpenMap;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.common.xcontent.DeprecationHandler;
|
||||
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
|
||||
import org.elasticsearch.common.xcontent.ToXContent;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.XContentType;
|
||||
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||
import org.elasticsearch.common.xcontent.yaml.YamlXContent;
|
||||
import org.elasticsearch.index.query.QueryBuilder;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.search.SearchHit;
|
||||
|
@ -69,8 +53,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
|
|||
import org.elasticsearch.search.sort.SortBuilder;
|
||||
import org.elasticsearch.search.sort.SortBuilders;
|
||||
import org.elasticsearch.search.sort.SortOrder;
|
||||
import org.xbib.elx.api.BulkController;
|
||||
import org.xbib.elx.api.ExtendedClient;
|
||||
import org.xbib.elx.api.AdminClient;
|
||||
import org.xbib.elx.api.IndexAliasAdder;
|
||||
import org.xbib.elx.api.IndexDefinition;
|
||||
import org.xbib.elx.api.IndexPruneResult;
|
||||
|
@ -79,13 +62,15 @@ import org.xbib.elx.api.IndexShiftResult;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.MalformedInputException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.LocalDate;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
|
@ -98,22 +83,21 @@ import java.util.TreeSet;
|
|||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public abstract class AbstractExtendedClient implements ExtendedClient {
|
||||
public abstract class AbstractAdminClient extends AbstractBasicClient implements AdminClient {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(AbstractExtendedClient.class.getName());
|
||||
private static final Logger logger = LogManager.getLogger(AbstractAdminClient.class.getName());
|
||||
|
||||
private ElasticsearchClient client;
|
||||
|
||||
private BulkController bulkController;
|
||||
|
||||
private final AtomicBoolean closed;
|
||||
/**
|
||||
* The one and only index type name used in the extended client.
|
||||
* Notr that all Elasticsearch version < 6.2.0 do not allow a prepending "_".
|
||||
*/
|
||||
private static final String TYPE_NAME = "doc";
|
||||
|
||||
private static final IndexShiftResult EMPTY_INDEX_SHIFT_RESULT = new IndexShiftResult() {
|
||||
@Override
|
||||
|
@ -149,363 +133,54 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
}
|
||||
};
|
||||
|
||||
protected abstract ElasticsearchClient createClient(Settings settings) throws IOException;
|
||||
|
||||
protected abstract void closeClient() throws IOException;
|
||||
|
||||
protected AbstractExtendedClient() {
|
||||
closed = new AtomicBoolean(false);
|
||||
@Override
|
||||
public Map<String, ?> getMapping(String index) throws IOException {
|
||||
return getMapping(index, TYPE_NAME);
|
||||
}
|
||||
|
||||
@Override
|
||||
public AbstractExtendedClient setClient(ElasticsearchClient client) {
|
||||
this.client = client;
|
||||
return this;
|
||||
public Map<String, ?> getMapping(String index, String mapping) throws IOException {
|
||||
GetMappingsRequestBuilder getMappingsRequestBuilder = new GetMappingsRequestBuilder(client, GetMappingsAction.INSTANCE)
|
||||
.setIndices(index)
|
||||
.setTypes(mapping);
|
||||
GetMappingsResponse getMappingsResponse = getMappingsRequestBuilder.execute().actionGet();
|
||||
logger.info("get mappings response = {}", getMappingsResponse.getMappings().get(index).get(mapping).getSourceAsMap());
|
||||
return getMappingsResponse.getMappings().get(index).get(mapping).getSourceAsMap();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ElasticsearchClient getClient() {
|
||||
return client;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BulkController getBulkController() {
|
||||
return bulkController;
|
||||
}
|
||||
|
||||
@Override
|
||||
public AbstractExtendedClient init(Settings settings) throws IOException {
|
||||
logger.info("initializing with settings = " + settings.toDelimitedString(','));
|
||||
if (client == null) {
|
||||
client = createClient(settings);
|
||||
}
|
||||
if (bulkController == null) {
|
||||
bulkController = new DefaultBulkController(this);
|
||||
bulkController.init(settings);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void flush() throws IOException {
|
||||
if (bulkController != null) {
|
||||
bulkController.flush();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
ensureClient();
|
||||
if (closed.compareAndSet(false, true)) {
|
||||
if (bulkController != null) {
|
||||
logger.info("closing bulk controller");
|
||||
bulkController.close();
|
||||
}
|
||||
closeClient();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getClusterName() {
|
||||
ensureClient();
|
||||
try {
|
||||
ClusterStateRequest clusterStateRequest = new ClusterStateRequest().clear();
|
||||
ClusterStateResponse clusterStateResponse =
|
||||
client.execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
|
||||
return clusterStateResponse.getClusterName().value();
|
||||
} catch (ElasticsearchTimeoutException e) {
|
||||
logger.warn(e.getMessage(), e);
|
||||
return "TIMEOUT";
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn(e.getMessage(), e);
|
||||
return "DISCONNECTED";
|
||||
} catch (Exception e) {
|
||||
logger.warn(e.getMessage(), e);
|
||||
return "[" + e.getMessage() + "]";
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException {
|
||||
ensureClient();
|
||||
waitForCluster("YELLOW", 30L, TimeUnit.SECONDS);
|
||||
URL indexSettings = indexDefinition.getSettingsUrl();
|
||||
if (indexSettings == null) {
|
||||
logger.warn("warning while creating index '{}', no settings/mappings",
|
||||
indexDefinition.getFullIndexName());
|
||||
newIndex(indexDefinition.getFullIndexName());
|
||||
return this;
|
||||
}
|
||||
URL indexMappings = indexDefinition.getMappingsUrl();
|
||||
if (indexMappings == null) {
|
||||
logger.warn("warning while creating index '{}', no mappings",
|
||||
indexDefinition.getFullIndexName());
|
||||
newIndex(indexDefinition.getFullIndexName(), indexSettings.openStream(), null);
|
||||
return this;
|
||||
}
|
||||
try (InputStream indexSettingsInput = indexSettings.openStream();
|
||||
InputStream indexMappingsInput = indexMappings.openStream()) {
|
||||
newIndex(indexDefinition.getFullIndexName(), indexSettingsInput, indexMappingsInput);
|
||||
} catch (IOException e) {
|
||||
if (indexDefinition.ignoreErrors()) {
|
||||
logger.warn(e.getMessage(), e);
|
||||
logger.warn("warning while creating index '{}' with settings at {} and mappings at {}",
|
||||
indexDefinition.getFullIndexName(), indexSettings, indexMappings);
|
||||
} else {
|
||||
logger.error("error while creating index '{}' with settings at {} and mappings at {}",
|
||||
indexDefinition.getFullIndexName(), indexSettings, indexMappings);
|
||||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient newIndex(String index) throws IOException {
|
||||
return newIndex(index, Settings.EMPTY, (Map<String, Object>) null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException {
|
||||
return newIndex(index,
|
||||
Settings.builder().loadFromStream(".json", settings, true).build(),
|
||||
mapping != null ? JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
|
||||
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mapping).mapOrdered() : null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient newIndex(String index, Settings settings) throws IOException {
|
||||
return newIndex(index, settings, (Map<String, ?>) null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException {
|
||||
return newIndex(index, settings,
|
||||
mapping != null ? JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
|
||||
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mapping).mapOrdered() : null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException {
|
||||
ensureClient();
|
||||
if (index == null) {
|
||||
logger.warn("no index name given to create index");
|
||||
return this;
|
||||
}
|
||||
CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index);
|
||||
if (settings != null) {
|
||||
createIndexRequest.settings(settings);
|
||||
}
|
||||
if (mapping != null) {
|
||||
createIndexRequest.mapping("_doc", mapping);
|
||||
}
|
||||
CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
|
||||
if (createIndexResponse.isAcknowledged()) {
|
||||
return this;
|
||||
}
|
||||
throw new IllegalStateException("index creation not acknowledged");
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException {
|
||||
ensureClient();
|
||||
if (index == null) {
|
||||
logger.warn("no index name given to create index");
|
||||
return this;
|
||||
}
|
||||
CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index);
|
||||
if (settings != null) {
|
||||
createIndexRequest.settings(settings);
|
||||
}
|
||||
if (mapping != null) {
|
||||
createIndexRequest.mapping("_doc", mapping);
|
||||
}
|
||||
CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
|
||||
if (createIndexResponse.isAcknowledged()) {
|
||||
return this;
|
||||
}
|
||||
throw new IllegalStateException("index creation not acknowledged");
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient deleteIndex(IndexDefinition indexDefinition) {
|
||||
public AdminClient deleteIndex(IndexDefinition indexDefinition) {
|
||||
return deleteIndex(indexDefinition.getFullIndexName());
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient deleteIndex(String index) {
|
||||
ensureClient();
|
||||
public AdminClient deleteIndex(String index) {
|
||||
ensureClientIsPresent();
|
||||
if (index == null) {
|
||||
logger.warn("no index name given to delete index");
|
||||
return this;
|
||||
}
|
||||
DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest().indices(index);
|
||||
client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet();
|
||||
waitForCluster("YELLOW", 30L, TimeUnit.SECONDS);
|
||||
waitForShards(30L, TimeUnit.SECONDS);
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient startBulk(IndexDefinition indexDefinition) throws IOException {
|
||||
startBulk(indexDefinition.getFullIndexName(), -1, 1);
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds)
|
||||
throws IOException {
|
||||
if (bulkController != null) {
|
||||
ensureClient();
|
||||
bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException {
|
||||
if (bulkController != null) {
|
||||
ensureClient();
|
||||
bulkController.stopBulkMode(indexDefinition);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException {
|
||||
if (bulkController != null) {
|
||||
ensureClient();
|
||||
bulkController.stopBulkMode(index, timeout, timeUnit);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient index(String index, String id, boolean create, String source) {
|
||||
return index(new IndexRequest().index(index).id(id).create(create)
|
||||
.source(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient index(String index, String id, boolean create, BytesReference source) {
|
||||
return index(new IndexRequest().index(index).id(id).create(create)
|
||||
.source(source, XContentType.JSON));
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient index(IndexRequest indexRequest) {
|
||||
ensureClient();
|
||||
bulkController.bulkIndex(indexRequest);
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient delete(String index, String id) {
|
||||
return delete(new DeleteRequest().index(index).id(id));
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient delete(DeleteRequest deleteRequest) {
|
||||
ensureClient();
|
||||
bulkController.bulkDelete(deleteRequest);
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient update(String index, String id, BytesReference source) {
|
||||
return update(new UpdateRequest().index(index).id(id)
|
||||
.doc(source, XContentType.JSON));
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient update(String index, String id, String source) {
|
||||
return update(new UpdateRequest().index(index).id(id)
|
||||
.doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient update(UpdateRequest updateRequest) {
|
||||
ensureClient();
|
||||
bulkController.bulkUpdate(updateRequest);
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean waitForResponses(long timeout, TimeUnit timeUnit) {
|
||||
ensureClient();
|
||||
return bulkController.waitForBulkResponses(timeout, timeUnit);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) {
|
||||
ensureClient();
|
||||
ensureIndexGiven(index);
|
||||
GetSettingsRequest settingsRequest = new GetSettingsRequest();
|
||||
settingsRequest.indices(index);
|
||||
GetSettingsResponse settingsResponse = client.execute(GetSettingsAction.INSTANCE, settingsRequest).actionGet();
|
||||
int shards = settingsResponse.getIndexToSettings().get(index).getAsInt("index.number_of_shards", -1);
|
||||
if (shards > 0) {
|
||||
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
|
||||
ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest()
|
||||
.indices(index)
|
||||
.waitForActiveShards(shards)
|
||||
.timeout(timeout);
|
||||
ClusterHealthResponse healthResponse =
|
||||
client.execute(ClusterHealthAction.INSTANCE, clusterHealthRequest).actionGet();
|
||||
if (healthResponse != null && healthResponse.isTimedOut()) {
|
||||
logger.warn("timeout waiting for recovery");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) {
|
||||
ensureClient();
|
||||
ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString);
|
||||
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
|
||||
ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
|
||||
new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet();
|
||||
if (healthResponse != null && healthResponse.isTimedOut()) {
|
||||
logger.warn("timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name());
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) {
|
||||
ensureClient();
|
||||
try {
|
||||
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
|
||||
ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
|
||||
new ClusterHealthRequest().timeout(timeout)).actionGet();
|
||||
ClusterHealthStatus status = healthResponse.getStatus();
|
||||
return status.name();
|
||||
} catch (ElasticsearchTimeoutException e) {
|
||||
logger.warn(e.getMessage(), e);
|
||||
return "TIMEOUT";
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn(e.getMessage(), e);
|
||||
return "DISCONNECTED";
|
||||
} catch (Exception e) {
|
||||
logger.warn(e.getMessage(), e);
|
||||
return "[" + e.getMessage() + "]";
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException {
|
||||
public AdminClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException {
|
||||
return updateReplicaLevel(indexDefinition.getFullIndexName(), level,
|
||||
indexDefinition.getMaxWaitTime(), indexDefinition.getMaxWaitTimeUnit());
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException {
|
||||
waitForCluster("YELLOW", maxWaitTime, timeUnit); // let cluster settle down from critical operations
|
||||
if (level > 0) {
|
||||
updateIndexSetting(index, "number_of_replicas", level, maxWaitTime, timeUnit);
|
||||
waitForRecovery(index, maxWaitTime, timeUnit);
|
||||
public AdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException {
|
||||
if (level < 1) {
|
||||
logger.warn("invalid replica level");
|
||||
return this;
|
||||
}
|
||||
updateIndexSetting(index, "number_of_replicas", level, maxWaitTime, timeUnit);
|
||||
waitForShards(maxWaitTime, timeUnit);
|
||||
return this;
|
||||
}
|
||||
|
||||
|
@ -528,30 +203,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
return replica;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient flushIndex(String index) {
|
||||
if (index != null) {
|
||||
ensureClient();
|
||||
client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet();
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedClient refreshIndex(String index) {
|
||||
if (index != null) {
|
||||
ensureClient();
|
||||
client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet();
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String resolveMostRecentIndex(String alias) {
|
||||
ensureClientIsPresent();
|
||||
if (alias == null) {
|
||||
return null;
|
||||
}
|
||||
ensureClient();
|
||||
GetAliasesRequest getAliasesRequest = new GetAliasesRequest().aliases(alias);
|
||||
GetAliasesResponse getAliasesResponse = client.execute(GetAliasesAction.INSTANCE, getAliasesRequest).actionGet();
|
||||
Pattern pattern = Pattern.compile("^(.*?)(\\d+)$");
|
||||
|
@ -576,7 +233,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
|
||||
@Override
|
||||
public String resolveAlias(String alias) {
|
||||
ensureClient();
|
||||
ensureClientIsPresent();
|
||||
ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
|
||||
clusterStateRequest.blocks(false);
|
||||
clusterStateRequest.metaData(true);
|
||||
|
@ -618,7 +275,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
@Override
|
||||
public IndexShiftResult shiftIndex(String index, String fullIndexName,
|
||||
List<String> additionalAliases, IndexAliasAdder adder) {
|
||||
ensureClient();
|
||||
ensureClientIsPresent();
|
||||
if (index == null) {
|
||||
return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to
|
||||
}
|
||||
|
@ -644,9 +301,9 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
String alias = entry.getKey();
|
||||
String filter = entry.getValue();
|
||||
indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove()
|
||||
.indices(oldIndex).alias(alias));
|
||||
.index(oldIndex).alias(alias));
|
||||
if (filter != null) {
|
||||
indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add()
|
||||
indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove()
|
||||
.index(fullIndexName).alias(alias).filter(filter));
|
||||
} else {
|
||||
indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add()
|
||||
|
@ -670,7 +327,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
} else {
|
||||
String filter = oldAliasMap.get(additionalAlias);
|
||||
indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove()
|
||||
.indices(oldIndex).alias(additionalAlias));
|
||||
.index(oldIndex).alias(additionalAlias));
|
||||
if (filter != null) {
|
||||
indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add()
|
||||
.index(fullIndexName).alias(additionalAlias).filter(filter));
|
||||
|
@ -686,8 +343,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
logger.debug("indices alias request = {}", indicesAliasesRequest.getAliasActions().toString());
|
||||
AcknowledgedResponse indicesAliasesResponse =
|
||||
client.execute(IndicesAliasesAction.INSTANCE, indicesAliasesRequest).actionGet();
|
||||
logger.debug("response isAcknowledged = {} isFragment = {}",
|
||||
indicesAliasesResponse.isAcknowledged(), indicesAliasesResponse.isFragment());
|
||||
logger.debug("response isAcknowledged = {}",
|
||||
indicesAliasesResponse.isAcknowledged());
|
||||
}
|
||||
return new SuccessIndexShiftResult(moveAliases, newAliases);
|
||||
}
|
||||
|
@ -706,11 +363,11 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
if (index.equals(fullIndexName)) {
|
||||
return EMPTY_INDEX_PRUNE_RESULT;
|
||||
}
|
||||
ensureClient();
|
||||
ensureClientIsPresent();
|
||||
GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE);
|
||||
GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet();
|
||||
Pattern pattern = Pattern.compile("^(.*?)(\\d+)$");
|
||||
logger.info("pruneIndex: total of {} indices", getIndexResponse.getIndices().length);
|
||||
logger.info("{} indices", getIndexResponse.getIndices().length);
|
||||
List<String> candidateIndices = new ArrayList<>();
|
||||
for (String s : getIndexResponse.getIndices()) {
|
||||
Matcher m = pattern.matcher(s);
|
||||
|
@ -746,29 +403,20 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest()
|
||||
.indices(indicesToDelete.toArray(s));
|
||||
AcknowledgedResponse response = client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet();
|
||||
if( response.isAcknowledged()) {
|
||||
logger.log(Level.INFO, "deletion of {} acknowledged, waiting for GREEN", Arrays.asList(s));
|
||||
waitForCluster("GREEN", 30L, TimeUnit.SECONDS);
|
||||
return new SuccessPruneResult(candidateIndices, indicesToDelete, response);
|
||||
} else {
|
||||
logger.log(Level.WARN, "deletion of {} not acknowledged", Arrays.asList(s));
|
||||
return new FailPruneResult(candidateIndices, indicesToDelete, response);
|
||||
}
|
||||
return new SuccessPruneResult(candidateIndices, indicesToDelete, response);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long mostRecentDocument(String index, String timestampfieldname) {
|
||||
ensureClient();
|
||||
SortBuilder<?> sort = SortBuilders
|
||||
.fieldSort(timestampfieldname)
|
||||
.order(SortOrder.DESC);
|
||||
SearchSourceBuilder builder = new SearchSourceBuilder()
|
||||
.sort(sort)
|
||||
.storedField(timestampfieldname)
|
||||
.size(1);
|
||||
SearchRequest searchRequest = new SearchRequest()
|
||||
.indices(index)
|
||||
.source(builder);
|
||||
ensureClientIsPresent();
|
||||
SortBuilder<?> sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC);
|
||||
SearchSourceBuilder builder = new SearchSourceBuilder();
|
||||
builder.sort(sort);
|
||||
builder.storedField(timestampfieldname);
|
||||
builder.size(1);
|
||||
SearchRequest searchRequest = new SearchRequest();
|
||||
searchRequest.indices(index);
|
||||
searchRequest.source(builder);
|
||||
SearchResponse searchResponse =
|
||||
client.execute(SearchAction.INSTANCE, searchRequest).actionGet();
|
||||
if (searchResponse.getHits().getHits().length == 1) {
|
||||
|
@ -813,7 +461,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
@Override
|
||||
public IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings)
|
||||
throws IOException {
|
||||
boolean isEnabled = settings.getAsBoolean("enabled", !(client instanceof MockExtendedClient));
|
||||
boolean isEnabled = settings.getAsBoolean("enabled", false);
|
||||
String indexName = settings.get("name", index);
|
||||
String fullIndexName;
|
||||
String dateTimePattern = settings.get("dateTimePattern");
|
||||
|
@ -833,8 +481,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
.setEnabled(isEnabled)
|
||||
.setIndex(indexName)
|
||||
.setFullIndexName(fullIndexName)
|
||||
.setSettingsUrl(settings.get("settings"))
|
||||
.setMappingsUrl(settings.get("mapping"))
|
||||
.setSettings(findSettingsFrom(settings.get("settings")))
|
||||
.setMappings(findMappingsFrom(settings.get("mapping")))
|
||||
.setDateTimePattern(dateTimePattern)
|
||||
.setIgnoreErrors(settings.getAsBoolean("skiperrors", false))
|
||||
.setShift(settings.getAsBoolean("shift", true))
|
||||
|
@ -847,16 +495,10 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
|
||||
@Override
|
||||
public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException {
|
||||
ensureClient();
|
||||
ensureClientIsPresent();
|
||||
if (index == null) {
|
||||
throw new IOException("no index name given");
|
||||
}
|
||||
if (key == null) {
|
||||
throw new IOException("no key given");
|
||||
}
|
||||
if (value == null) {
|
||||
throw new IOException("no value given");
|
||||
}
|
||||
Settings.Builder updateSettingsBuilder = Settings.builder();
|
||||
updateSettingsBuilder.put(key, value.toString());
|
||||
UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index)
|
||||
|
@ -864,18 +506,48 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet();
|
||||
}
|
||||
|
||||
private void ensureClient() {
|
||||
if (this instanceof MockExtendedClient) {
|
||||
return;
|
||||
private static String findSettingsFrom(String string) throws IOException {
|
||||
if (string == null) {
|
||||
return null;
|
||||
}
|
||||
if (client == null) {
|
||||
throw new IllegalStateException("no client");
|
||||
try {
|
||||
URL url = new URL(string);
|
||||
try (InputStream inputStream = url.openStream()) {
|
||||
Settings settings = Settings.builder().loadFromStream(string, inputStream, true).build();
|
||||
XContentBuilder builder = JsonXContent.contentBuilder();
|
||||
settings.toXContent(builder, ToXContent.EMPTY_PARAMS);
|
||||
return Strings.toString(builder);
|
||||
}
|
||||
} catch (MalformedURLException e) {
|
||||
return string;
|
||||
}
|
||||
}
|
||||
|
||||
private void ensureIndexGiven(String index) {
|
||||
if (index == null) {
|
||||
throw new IllegalArgumentException("no index given");
|
||||
private static String findMappingsFrom(String string) throws IOException {
|
||||
if (string == null) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
URL url = new URL(string);
|
||||
try (InputStream inputStream = url.openStream()) {
|
||||
if (string.endsWith(".json")) {
|
||||
Map<String, ?> mappings = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
|
||||
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputStream).mapOrdered();
|
||||
XContentBuilder builder = JsonXContent.contentBuilder();
|
||||
builder.startObject().map(mappings).endObject();
|
||||
return Strings.toString(builder);
|
||||
}
|
||||
if (string.endsWith(".yml") || string.endsWith(".yaml")) {
|
||||
Map<String, ?> mappings = YamlXContent.yamlXContent.createParser(NamedXContentRegistry.EMPTY,
|
||||
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputStream).mapOrdered();
|
||||
XContentBuilder builder = JsonXContent.contentBuilder();
|
||||
builder.startObject().map(mappings).endObject();
|
||||
return Strings.toString(builder);
|
||||
}
|
||||
}
|
||||
return string;
|
||||
} catch (MalformedInputException e) {
|
||||
return string;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -896,7 +568,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
}
|
||||
|
||||
public void checkMapping(String index) {
|
||||
ensureClient();
|
||||
ensureClientIsPresent();
|
||||
GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(index);
|
||||
GetMappingsResponse getMappingsResponse = client.execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
|
||||
ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> map = getMappingsResponse.getMappings();
|
||||
|
@ -912,25 +584,25 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
|
||||
private void checkMapping(String index, String type, MappingMetaData mappingMetaData) {
|
||||
try {
|
||||
SearchSourceBuilder builder = new SearchSourceBuilder()
|
||||
.query(QueryBuilders.matchAllQuery())
|
||||
.size(0)
|
||||
.trackTotalHits(true);
|
||||
SearchRequest searchRequest = new SearchRequest()
|
||||
.indices(index)
|
||||
.source(builder);
|
||||
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE)
|
||||
.setIndices(index)
|
||||
.setQuery(QueryBuilders.matchAllQuery())
|
||||
.setSize(0)
|
||||
.setTrackTotalHits(true);
|
||||
SearchResponse searchResponse =
|
||||
client.execute(SearchAction.INSTANCE, searchRequest).actionGet();
|
||||
TotalHits total = searchResponse.getHits().getTotalHits();
|
||||
if (total.value > 0L) {
|
||||
searchRequestBuilder.execute().actionGet();
|
||||
long total = searchResponse.getHits().getTotalHits().value;
|
||||
if (total > 0L) {
|
||||
Map<String, Long> fields = new TreeMap<>();
|
||||
Map<String, Object> root = mappingMetaData.getSourceAsMap();
|
||||
checkMapping(index, "", "", root, fields);
|
||||
checkMapping(index, type, "", "", root, fields);
|
||||
AtomicInteger empty = new AtomicInteger();
|
||||
Map<String, Long> map = sortByValue(fields);
|
||||
map.forEach((key, value) -> {
|
||||
logger.info("{} {} {}",
|
||||
key, value, (double) value * 100 / total.value);
|
||||
key,
|
||||
value,
|
||||
(double) value * 100 / total);
|
||||
if (value == 0) {
|
||||
empty.incrementAndGet();
|
||||
}
|
||||
|
@ -944,7 +616,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private void checkMapping(String index,
|
||||
private void checkMapping(String index, String type,
|
||||
String pathDef, String fieldName, Map<String, Object> map,
|
||||
Map<String, Long> fields) {
|
||||
String path = pathDef;
|
||||
|
@ -969,20 +641,18 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
String fieldType = o instanceof String ? o.toString() : null;
|
||||
// do not recurse into our custom field mapper
|
||||
if (!"standardnumber".equals(fieldType) && !"ref".equals(fieldType)) {
|
||||
checkMapping(index, path, key, child, fields);
|
||||
checkMapping(index, type, path, key, child, fields);
|
||||
}
|
||||
} else if ("type".equals(key)) {
|
||||
QueryBuilder filterBuilder = QueryBuilders.existsQuery(path);
|
||||
QueryBuilder queryBuilder = QueryBuilders.constantScoreQuery(filterBuilder);
|
||||
SearchSourceBuilder builder = new SearchSourceBuilder()
|
||||
.query(queryBuilder)
|
||||
.size(0)
|
||||
.trackTotalHits(true);
|
||||
SearchRequest searchRequest = new SearchRequest()
|
||||
.indices(index)
|
||||
.source(builder);
|
||||
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE)
|
||||
.setIndices(index)
|
||||
.setQuery(queryBuilder)
|
||||
.setSize(0)
|
||||
.setTrackTotalHits(true);
|
||||
SearchResponse searchResponse =
|
||||
client.execute(SearchAction.INSTANCE, searchRequest).actionGet();
|
||||
searchRequestBuilder.execute().actionGet();
|
||||
fields.put(path, searchResponse.getHits().getTotalHits().value);
|
||||
}
|
||||
}
|
||||
|
@ -995,58 +665,38 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
return result;
|
||||
}
|
||||
|
||||
private static TimeValue toTimeValue(long timeValue, TimeUnit timeUnit) {
|
||||
switch (timeUnit) {
|
||||
case DAYS:
|
||||
return TimeValue.timeValueHours(24 * timeValue);
|
||||
case HOURS:
|
||||
return TimeValue.timeValueHours(timeValue);
|
||||
case MINUTES:
|
||||
return TimeValue.timeValueMinutes(timeValue);
|
||||
case SECONDS:
|
||||
return TimeValue.timeValueSeconds(timeValue);
|
||||
case MILLISECONDS:
|
||||
return TimeValue.timeValueMillis(timeValue);
|
||||
case MICROSECONDS:
|
||||
return TimeValue.timeValueNanos(1000 * timeValue);
|
||||
case NANOSECONDS:
|
||||
return TimeValue.timeValueNanos(timeValue);
|
||||
default:
|
||||
throw new IllegalArgumentException("unknown time unit: " + timeUnit);
|
||||
}
|
||||
}
|
||||
|
||||
private static class SuccessIndexShiftResult implements IndexShiftResult {
|
||||
|
||||
List<String> movedAliases;
|
||||
Collection<String> movedAliases;
|
||||
|
||||
List<String> newAliases;
|
||||
Collection<String> newAliases;
|
||||
|
||||
SuccessIndexShiftResult(List<String> movedAliases, List<String> newAliases) {
|
||||
SuccessIndexShiftResult(Collection<String> movedAliases, Collection<String> newAliases) {
|
||||
this.movedAliases = movedAliases;
|
||||
this.newAliases = newAliases;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getMovedAliases() {
|
||||
public Collection<String> getMovedAliases() {
|
||||
return movedAliases;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getNewAliases() {
|
||||
public Collection<String> getNewAliases() {
|
||||
return newAliases;
|
||||
}
|
||||
}
|
||||
|
||||
private static class SuccessPruneResult implements IndexPruneResult {
|
||||
|
||||
List<String> candidateIndices;
|
||||
Collection<String> candidateIndices;
|
||||
|
||||
List<String> indicesToDelete;
|
||||
Collection<String> indicesToDelete;
|
||||
|
||||
AcknowledgedResponse response;
|
||||
|
||||
SuccessPruneResult(List<String> candidateIndices, List<String> indicesToDelete,
|
||||
SuccessPruneResult(Collection<String> candidateIndices,
|
||||
Collection<String> indicesToDelete,
|
||||
AcknowledgedResponse response) {
|
||||
this.candidateIndices = candidateIndices;
|
||||
this.indicesToDelete = indicesToDelete;
|
||||
|
@ -1059,48 +709,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<String> getCandidateIndices() {
|
||||
public Collection<String> getCandidateIndices() {
|
||||
return candidateIndices;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getDeletedIndices() {
|
||||
return indicesToDelete;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isAcknowledged() {
|
||||
return response.isAcknowledged();
|
||||
}
|
||||
}
|
||||
|
||||
private static class FailPruneResult implements IndexPruneResult {
|
||||
|
||||
List<String> candidateIndices;
|
||||
|
||||
List<String> indicesToDelete;
|
||||
|
||||
AcknowledgedResponse response;
|
||||
|
||||
FailPruneResult(List<String> candidateIndices, List<String> indicesToDelete,
|
||||
AcknowledgedResponse response) {
|
||||
this.candidateIndices = candidateIndices;
|
||||
this.indicesToDelete = indicesToDelete;
|
||||
this.response = response;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexPruneResult.State getState() {
|
||||
return IndexPruneResult.State.SUCCESS;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getCandidateIndices() {
|
||||
return candidateIndices;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getDeletedIndices() {
|
||||
public Collection<String> getDeletedIndices() {
|
||||
return indicesToDelete;
|
||||
}
|
||||
|
||||
|
@ -1112,11 +726,11 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
|
||||
private static class NothingToDoPruneResult implements IndexPruneResult {
|
||||
|
||||
List<String> candidateIndices;
|
||||
Collection<String> candidateIndices;
|
||||
|
||||
List<String> indicesToDelete;
|
||||
Collection<String> indicesToDelete;
|
||||
|
||||
NothingToDoPruneResult(List<String> candidateIndices, List<String> indicesToDelete) {
|
||||
NothingToDoPruneResult(Collection<String> candidateIndices, List<String> indicesToDelete) {
|
||||
this.candidateIndices = candidateIndices;
|
||||
this.indicesToDelete = indicesToDelete;
|
||||
}
|
||||
|
@ -1127,12 +741,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<String> getCandidateIndices() {
|
||||
public Collection<String> getCandidateIndices() {
|
||||
return candidateIndices;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getDeletedIndices() {
|
||||
public Collection<String> getDeletedIndices() {
|
||||
return indicesToDelete;
|
||||
}
|
||||
|
|
@ -0,0 +1,217 @@
|
|||
package org.xbib.elx.common;
|
||||
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.ElasticsearchTimeoutException;
|
||||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
|
||||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
|
||||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
|
||||
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
|
||||
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
|
||||
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
|
||||
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
|
||||
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
|
||||
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
|
||||
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
|
||||
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
|
||||
import org.elasticsearch.action.search.SearchAction;
|
||||
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.elasticsearch.cluster.health.ClusterHealthStatus;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.xbib.elx.api.BasicClient;
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
public abstract class AbstractBasicClient implements BasicClient {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(AbstractBasicClient.class.getName());
|
||||
|
||||
protected ElasticsearchClient client;
|
||||
|
||||
protected Settings settings;
|
||||
|
||||
private final AtomicBoolean closed;
|
||||
|
||||
public AbstractBasicClient() {
|
||||
closed = new AtomicBoolean(false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setClient(ElasticsearchClient client) {
|
||||
logger.log(Level.INFO, "setting client = " + client);
|
||||
this.client = client;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ElasticsearchClient getClient() {
|
||||
return client;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(Settings settings) throws IOException {
|
||||
if (closed.compareAndSet(false, true)) {
|
||||
logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(','));
|
||||
this.settings = settings;
|
||||
setClient(createClient(settings));
|
||||
} else {
|
||||
logger.log(Level.WARN, "not initializing");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getClusterName() {
|
||||
ensureClientIsPresent();
|
||||
try {
|
||||
ClusterStateRequest clusterStateRequest = new ClusterStateRequest().clear();
|
||||
ClusterStateResponse clusterStateResponse =
|
||||
getClient().execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
|
||||
return clusterStateResponse.getClusterName().value();
|
||||
} catch (ElasticsearchTimeoutException e) {
|
||||
logger.warn(e.getMessage(), e);
|
||||
return "TIMEOUT";
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn(e.getMessage(), e);
|
||||
return "DISCONNECTED";
|
||||
} catch (Exception e) {
|
||||
logger.warn(e.getMessage(), e);
|
||||
return "[" + e.getMessage() + "]";
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) {
|
||||
ensureClientIsPresent();
|
||||
ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString);
|
||||
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
|
||||
ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
|
||||
new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet();
|
||||
if (healthResponse != null && healthResponse.isTimedOut()) {
|
||||
String message = "timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name();
|
||||
if (logger.isErrorEnabled()) {
|
||||
logger.error(message);
|
||||
}
|
||||
throw new IllegalStateException(message);
|
||||
}
|
||||
}
|
||||
@Override
|
||||
public void waitForShards(long maxWaitTime, TimeUnit timeUnit) {
|
||||
ensureClientIsPresent();
|
||||
logger.info("waiting for cluster shard settling");
|
||||
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
|
||||
ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest()
|
||||
.waitForNoInitializingShards(true)
|
||||
.waitForNoRelocatingShards(true)
|
||||
.timeout(timeout);
|
||||
ClusterHealthResponse healthResponse =
|
||||
client.execute(ClusterHealthAction.INSTANCE, clusterHealthRequest).actionGet();
|
||||
if (healthResponse.isTimedOut()) {
|
||||
String message = "timeout waiting for cluster shards";
|
||||
logger.error(message);
|
||||
throw new IllegalStateException(message);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) {
|
||||
ensureClientIsPresent();
|
||||
try {
|
||||
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
|
||||
ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
|
||||
new ClusterHealthRequest().timeout(timeout)).actionGet();
|
||||
ClusterHealthStatus status = healthResponse.getStatus();
|
||||
return status.name();
|
||||
} catch (ElasticsearchTimeoutException e) {
|
||||
logger.warn(e.getMessage(), e);
|
||||
return "TIMEOUT";
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn(e.getMessage(), e);
|
||||
return "DISCONNECTED";
|
||||
} catch (Exception e) {
|
||||
logger.warn(e.getMessage(), e);
|
||||
return "[" + e.getMessage() + "]";
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getSearchableDocs(String index) {
|
||||
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE)
|
||||
.setIndices(index)
|
||||
.setQuery(QueryBuilders.matchAllQuery())
|
||||
.setSize(0)
|
||||
.setTrackTotalHits(true);
|
||||
return searchRequestBuilder.execute().actionGet().getHits().getTotalHits().value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isIndexExists(String index) {
|
||||
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
|
||||
indicesExistsRequest.indices(index);
|
||||
IndicesExistsResponse indicesExistsResponse =
|
||||
client.execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
|
||||
return indicesExistsResponse.isExists();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
ensureClientIsPresent();
|
||||
if (closed.compareAndSet(false, true)) {
|
||||
closeClient(settings);
|
||||
}
|
||||
}
|
||||
|
||||
protected abstract ElasticsearchClient createClient(Settings settings) throws IOException;
|
||||
|
||||
protected abstract void closeClient(Settings settings) throws IOException;
|
||||
|
||||
protected void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException {
|
||||
ensureClientIsPresent();
|
||||
if (index == null) {
|
||||
throw new IOException("no index name given");
|
||||
}
|
||||
if (key == null) {
|
||||
throw new IOException("no key given");
|
||||
}
|
||||
if (value == null) {
|
||||
throw new IOException("no value given");
|
||||
}
|
||||
Settings.Builder updateSettingsBuilder = Settings.builder();
|
||||
updateSettingsBuilder.put(key, value.toString());
|
||||
UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index)
|
||||
.settings(updateSettingsBuilder).timeout(toTimeValue(timeout, timeUnit));
|
||||
client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet();
|
||||
}
|
||||
|
||||
protected void ensureClientIsPresent() {
|
||||
if (client == null) {
|
||||
throw new IllegalStateException("no client");
|
||||
}
|
||||
}
|
||||
|
||||
protected static TimeValue toTimeValue(long timeValue, TimeUnit timeUnit) {
|
||||
switch (timeUnit) {
|
||||
case DAYS:
|
||||
return TimeValue.timeValueHours(24 * timeValue);
|
||||
case HOURS:
|
||||
return TimeValue.timeValueHours(timeValue);
|
||||
case MINUTES:
|
||||
return TimeValue.timeValueMinutes(timeValue);
|
||||
case SECONDS:
|
||||
return TimeValue.timeValueSeconds(timeValue);
|
||||
case MILLISECONDS:
|
||||
return TimeValue.timeValueMillis(timeValue);
|
||||
case MICROSECONDS:
|
||||
return TimeValue.timeValueNanos(1000 * timeValue);
|
||||
case NANOSECONDS:
|
||||
return TimeValue.timeValueNanos(timeValue);
|
||||
default:
|
||||
throw new IllegalArgumentException("unknown time unit: " + timeUnit);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,246 @@
|
|||
package org.xbib.elx.common;
|
||||
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
|
||||
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
|
||||
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
|
||||
import org.elasticsearch.action.admin.indices.flush.FlushAction;
|
||||
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
|
||||
import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
|
||||
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
|
||||
import org.elasticsearch.action.delete.DeleteRequest;
|
||||
import org.elasticsearch.action.index.IndexRequest;
|
||||
import org.elasticsearch.action.update.UpdateRequest;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.bytes.BytesArray;
|
||||
import org.elasticsearch.common.bytes.BytesReference;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.DeprecationHandler;
|
||||
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.XContentType;
|
||||
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||
import org.xbib.elx.api.BulkClient;
|
||||
import org.xbib.elx.api.BulkController;
|
||||
import org.xbib.elx.api.BulkMetric;
|
||||
import org.xbib.elx.api.IndexDefinition;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
public abstract class AbstractBulkClient extends AbstractBasicClient implements BulkClient {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(AbstractBulkClient.class.getName());
|
||||
|
||||
private BulkMetric bulkMetric;
|
||||
|
||||
private BulkController bulkController;
|
||||
|
||||
private final AtomicBoolean closed = new AtomicBoolean(true);
|
||||
|
||||
@Override
|
||||
public void init(Settings settings) throws IOException {
|
||||
if (closed.compareAndSet(true, false)) {
|
||||
super.init(settings);
|
||||
logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(','));
|
||||
bulkMetric = new DefaultBulkMetric();
|
||||
bulkMetric.init(settings);
|
||||
bulkController = new DefaultBulkController(this, bulkMetric);
|
||||
bulkController.init(settings);
|
||||
} else {
|
||||
logger.log(Level.WARN, "not initializing");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public BulkMetric getBulkMetric() {
|
||||
return bulkMetric;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BulkController getBulkController() {
|
||||
return bulkController;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void flush() throws IOException {
|
||||
if (bulkController != null) {
|
||||
bulkController.flush();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
if (closed.compareAndSet(false, true)) {
|
||||
ensureClientIsPresent();
|
||||
if (bulkMetric != null) {
|
||||
logger.info("closing bulk metric");
|
||||
bulkMetric.close();
|
||||
}
|
||||
if (bulkController != null) {
|
||||
logger.info("closing bulk controller");
|
||||
bulkController.close();
|
||||
}
|
||||
closeClient(settings);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void newIndex(IndexDefinition indexDefinition) throws IOException {
|
||||
Settings settings = indexDefinition.getSettings() == null ? null :
|
||||
Settings.builder().loadFromSource(indexDefinition.getSettings(), XContentType.JSON).build();
|
||||
Map<String, ?> mappings = indexDefinition.getMappings() == null ? null :
|
||||
JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
|
||||
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, indexDefinition.getMappings()).mapOrdered();
|
||||
newIndex(indexDefinition.getFullIndexName(), settings, mappings);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void newIndex(String index) throws IOException {
|
||||
newIndex(index, Settings.EMPTY, (Map<String, ?>) null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void newIndex(String index, Settings settings) throws IOException {
|
||||
newIndex(index, settings, (Map<String, ?>) null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void newIndex(String index, Settings settings, XContentBuilder builder) throws IOException {
|
||||
String mappingString = Strings.toString(builder);
|
||||
Map<String, ?> mappings = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
|
||||
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mappingString).mapOrdered();
|
||||
newIndex(index, settings, mappings);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException {
|
||||
if (index == null) {
|
||||
logger.warn("no index name given to create index");
|
||||
return;
|
||||
}
|
||||
ensureClientIsPresent();
|
||||
waitForCluster("YELLOW", 30L, TimeUnit.SECONDS);
|
||||
CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index);
|
||||
if (settings != null) {
|
||||
createIndexRequest.settings(settings);
|
||||
}
|
||||
if (mapping != null) {
|
||||
createIndexRequest.mapping("_doc", mapping);
|
||||
}
|
||||
CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
|
||||
if (createIndexResponse.isAcknowledged()) {
|
||||
logger.info("index {} created", index);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startBulk(IndexDefinition indexDefinition) throws IOException {
|
||||
startBulk(indexDefinition.getFullIndexName(), -1, 1);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds)
|
||||
throws IOException {
|
||||
if (bulkController != null) {
|
||||
ensureClientIsPresent();
|
||||
bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stopBulk(IndexDefinition indexDefinition) throws IOException {
|
||||
if (bulkController != null) {
|
||||
ensureClientIsPresent();
|
||||
bulkController.stopBulkMode(indexDefinition);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException {
|
||||
if (bulkController != null) {
|
||||
ensureClientIsPresent();
|
||||
bulkController.stopBulkMode(index, timeout, timeUnit);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public BulkClient index(String index, String id, boolean create, String source) {
|
||||
return index(index, id, create, new BytesArray(source.getBytes(StandardCharsets.UTF_8)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public BulkClient index(String index, String id, boolean create, BytesReference source) {
|
||||
return index(new IndexRequest().index(index).id(id).create(create)
|
||||
.source(source, XContentType.JSON));
|
||||
}
|
||||
|
||||
@Override
|
||||
public BulkClient index(IndexRequest indexRequest) {
|
||||
ensureClientIsPresent();
|
||||
bulkController.bulkIndex(indexRequest);
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BulkClient delete(String index, String id) {
|
||||
return delete(new DeleteRequest().index(index).id(id));
|
||||
}
|
||||
|
||||
@Override
|
||||
public BulkClient delete(DeleteRequest deleteRequest) {
|
||||
ensureClientIsPresent();
|
||||
bulkController.bulkDelete(deleteRequest);
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BulkClient update(String index, String id, BytesReference source) {
|
||||
return update(new UpdateRequest().index(index).id(id)
|
||||
.doc(source, XContentType.JSON));
|
||||
}
|
||||
|
||||
@Override
|
||||
public BulkClient update(String index, String id, String source) {
|
||||
return update(new UpdateRequest().index(index).id(id)
|
||||
.doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
|
||||
}
|
||||
|
||||
@Override
|
||||
public BulkClient update(UpdateRequest updateRequest) {
|
||||
ensureClientIsPresent();
|
||||
bulkController.bulkUpdate(updateRequest);
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean waitForResponses(long timeout, TimeUnit timeUnit) {
|
||||
ensureClientIsPresent();
|
||||
return bulkController.waitForBulkResponses(timeout, timeUnit);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException {
|
||||
super.updateIndexSetting(index, key, value, timeout, timeUnit);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void flushIndex(String index) {
|
||||
if (index != null) {
|
||||
ensureClientIsPresent();
|
||||
client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void refreshIndex(String index) {
|
||||
if (index != null) {
|
||||
ensureClientIsPresent();
|
||||
client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet();
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,158 @@
|
|||
package org.xbib.elx.common;
|
||||
|
||||
import org.elasticsearch.ElasticsearchException;
|
||||
import org.elasticsearch.action.get.GetAction;
|
||||
import org.elasticsearch.action.get.GetRequestBuilder;
|
||||
import org.elasticsearch.action.get.GetResponse;
|
||||
import org.elasticsearch.action.get.MultiGetAction;
|
||||
import org.elasticsearch.action.get.MultiGetRequestBuilder;
|
||||
import org.elasticsearch.action.get.MultiGetResponse;
|
||||
import org.elasticsearch.action.search.ClearScrollAction;
|
||||
import org.elasticsearch.action.search.ClearScrollRequestBuilder;
|
||||
import org.elasticsearch.action.search.SearchAction;
|
||||
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.action.search.SearchScrollAction;
|
||||
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
|
||||
import org.elasticsearch.action.search.ShardSearchFailure;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.search.SearchHit;
|
||||
import org.xbib.elx.api.SearchClient;
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.util.Optional;
|
||||
import java.util.Spliterator;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.stream.Stream;
|
||||
import java.util.stream.StreamSupport;
|
||||
|
||||
public abstract class AbstractSearchClient extends AbstractBasicClient implements SearchClient {
|
||||
|
||||
@Override
|
||||
public Optional<GetResponse> get(Consumer<GetRequestBuilder> getRequestBuilderConsumer) {
|
||||
GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE);
|
||||
getRequestBuilderConsumer.accept(getRequestBuilder);
|
||||
GetResponse getResponse = getRequestBuilder.execute().actionGet();
|
||||
return getResponse.isExists() ? Optional.of(getResponse) : Optional.empty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<MultiGetResponse> multiGet(Consumer<MultiGetRequestBuilder> multiGetRequestBuilderConsumer) {
|
||||
MultiGetRequestBuilder multiGetRequestBuilder = new MultiGetRequestBuilder(client, MultiGetAction.INSTANCE);
|
||||
multiGetRequestBuilderConsumer.accept(multiGetRequestBuilder);
|
||||
MultiGetResponse multiGetItemResponse = multiGetRequestBuilder.execute().actionGet();
|
||||
return multiGetItemResponse.getResponses().length == 0 ? Optional.empty() : Optional.of(multiGetItemResponse);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<SearchResponse> search(Consumer<SearchRequestBuilder> queryBuilder) {
|
||||
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
|
||||
queryBuilder.accept(searchRequestBuilder);
|
||||
SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
|
||||
if (searchResponse.getFailedShards() > 0) {
|
||||
StringBuilder sb = new StringBuilder("Search failed:");
|
||||
for (ShardSearchFailure failure : searchResponse.getShardFailures()) {
|
||||
sb.append("\n").append(failure.reason());
|
||||
}
|
||||
throw new ElasticsearchException(sb.toString());
|
||||
}
|
||||
return searchResponse.getHits().getHits().length == 0 ? Optional.empty() : Optional.of(searchResponse);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Stream<SearchHit> search(Consumer<SearchRequestBuilder> queryBuilder,
|
||||
TimeValue scrollTime, int scrollSize) {
|
||||
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
|
||||
queryBuilder.accept(searchRequestBuilder);
|
||||
searchRequestBuilder.setScroll(scrollTime).setSize(scrollSize);
|
||||
SearchResponse originalSearchResponse = searchRequestBuilder.execute().actionGet();
|
||||
Stream<SearchResponse> infiniteResponses = Stream.iterate(originalSearchResponse,
|
||||
searchResponse -> new SearchScrollRequestBuilder(client, SearchScrollAction.INSTANCE)
|
||||
.setScrollId(searchResponse.getScrollId())
|
||||
.setScroll(scrollTime)
|
||||
.execute().actionGet());
|
||||
Predicate<SearchResponse> condition = searchResponse -> searchResponse.getHits().getHits().length > 0;
|
||||
Consumer<SearchResponse> lastAction = searchResponse -> {
|
||||
ClearScrollRequestBuilder clearScrollRequestBuilder =
|
||||
new ClearScrollRequestBuilder(client, ClearScrollAction.INSTANCE)
|
||||
.addScrollId(searchResponse.getScrollId());
|
||||
clearScrollRequestBuilder.execute().actionGet();
|
||||
};
|
||||
return StreamSupport.stream(TakeWhileSpliterator.over(infiniteResponses.spliterator(),
|
||||
condition, lastAction), false)
|
||||
.onClose(infiniteResponses::close)
|
||||
.flatMap(searchResponse -> Arrays.stream(searchResponse.getHits().getHits()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Stream<String> getIds(Consumer<SearchRequestBuilder> queryBuilder) {
|
||||
return search(queryBuilder, TimeValue.timeValueMinutes(1), 1000).map(SearchHit::getId);
|
||||
}
|
||||
|
||||
static class TakeWhileSpliterator<T> implements Spliterator<T> {
|
||||
|
||||
private final Spliterator<T> source;
|
||||
|
||||
private final Predicate<T> condition;
|
||||
|
||||
private final Consumer<T> lastElement;
|
||||
|
||||
private final boolean inclusive;
|
||||
|
||||
private final AtomicBoolean checked;
|
||||
|
||||
static <T> TakeWhileSpliterator<T> over(Spliterator<T> source,
|
||||
Predicate<T> condition,
|
||||
Consumer<T> lastElement) {
|
||||
return new TakeWhileSpliterator<>(source, condition, lastElement, true);
|
||||
}
|
||||
|
||||
private TakeWhileSpliterator(Spliterator<T> source,
|
||||
Predicate<T> condition,
|
||||
Consumer<T> lastElement,
|
||||
boolean inclusive) {
|
||||
this.source = source;
|
||||
this.condition = condition;
|
||||
this.lastElement = lastElement;
|
||||
this.inclusive = inclusive;
|
||||
this.checked = new AtomicBoolean(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean tryAdvance(Consumer<? super T> action) {
|
||||
return checked.get() && source.tryAdvance(e -> {
|
||||
if (condition.test(e)) {
|
||||
action.accept(e);
|
||||
} else {
|
||||
if (inclusive && checked.get()) {
|
||||
action.accept(e);
|
||||
}
|
||||
lastElement.accept(e);
|
||||
checked.set(false);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public Spliterator<T> trySplit() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long estimateSize() {
|
||||
return checked.get() ? source.estimateSize() : 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int characteristics() {
|
||||
return source.characteristics() &~ Spliterator.SIZED;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Comparator<? super T> getComparator() {
|
||||
return source.getComparator();
|
||||
}
|
||||
}
|
||||
}
|
|
@ -8,11 +8,12 @@ import org.elasticsearch.client.ElasticsearchClient;
|
|||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.xbib.elx.api.ExtendedClient;
|
||||
import org.xbib.elx.api.ExtendedClientProvider;
|
||||
import org.xbib.elx.api.AdminClientProvider;
|
||||
import org.xbib.elx.api.BulkClientProvider;
|
||||
import org.xbib.elx.api.BasicClient;
|
||||
import org.xbib.elx.api.SearchClientProvider;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.ServiceLoader;
|
||||
|
||||
|
@ -23,11 +24,15 @@ public class ClientBuilder {
|
|||
|
||||
private final ElasticsearchClient client;
|
||||
|
||||
private final ClassLoader classLoader;
|
||||
|
||||
private final Settings.Builder settingsBuilder;
|
||||
|
||||
private Map<Class<? extends ExtendedClientProvider>, ExtendedClientProvider> providerMap;
|
||||
private Class<? extends AdminClientProvider> adminClientProvider;
|
||||
|
||||
private Class<? extends ExtendedClientProvider> provider;
|
||||
private Class<? extends BulkClientProvider> bulkClientProvider;
|
||||
|
||||
private Class<? extends SearchClientProvider> searchClientProvider;
|
||||
|
||||
public ClientBuilder() {
|
||||
this(null);
|
||||
|
@ -39,14 +44,9 @@ public class ClientBuilder {
|
|||
|
||||
public ClientBuilder(ElasticsearchClient client, ClassLoader classLoader) {
|
||||
this.client = client;
|
||||
this.classLoader = classLoader;
|
||||
this.settingsBuilder = Settings.builder();
|
||||
settingsBuilder.put("node.name", "elx-client-" + Version.CURRENT);
|
||||
this.providerMap = new HashMap<>();
|
||||
ServiceLoader<ExtendedClientProvider> serviceLoader = ServiceLoader.load(ExtendedClientProvider.class,
|
||||
classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader());
|
||||
for (ExtendedClientProvider provider : serviceLoader) {
|
||||
providerMap.put(provider.getClass(), provider);
|
||||
}
|
||||
}
|
||||
|
||||
public static ClientBuilder builder() {
|
||||
|
@ -57,8 +57,18 @@ public class ClientBuilder {
|
|||
return new ClientBuilder(client);
|
||||
}
|
||||
|
||||
public ClientBuilder provider(Class<? extends ExtendedClientProvider> provider) {
|
||||
this.provider = provider;
|
||||
public ClientBuilder setAdminClientProvider(Class<? extends AdminClientProvider> adminClientProvider) {
|
||||
this.adminClientProvider = adminClientProvider;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ClientBuilder setBulkClientProvider(Class<? extends BulkClientProvider> bulkClientProvider) {
|
||||
this.bulkClientProvider = bulkClientProvider;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ClientBuilder setSearchClientProvider(Class<? extends SearchClientProvider> searchClientProvider) {
|
||||
this.searchClientProvider = searchClientProvider;
|
||||
return this;
|
||||
}
|
||||
|
||||
|
@ -97,15 +107,59 @@ public class ClientBuilder {
|
|||
return this;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public <C extends ExtendedClient> C build() throws IOException {
|
||||
if (provider == null) {
|
||||
throw new IllegalArgumentException("no provider");
|
||||
public ClientBuilder put(Map<String, ?> map) {
|
||||
for (Map.Entry<String, ?> entry : map.entrySet()) {
|
||||
if (entry.getValue() == null) {
|
||||
continue;
|
||||
}
|
||||
if (entry.getValue() instanceof String ||
|
||||
entry.getValue() instanceof Integer ||
|
||||
entry.getValue() instanceof Long ||
|
||||
entry.getValue() instanceof Float ||
|
||||
entry.getValue() instanceof TimeValue) {
|
||||
settingsBuilder.put(entry.getKey(), entry.getValue().toString());
|
||||
} else {
|
||||
logger.log(Level.WARN, "skipping " + entry.getValue() +
|
||||
" because invalid class type " + entry.getValue().getClass().getName());
|
||||
}
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public <C extends BasicClient> C build() throws IOException {
|
||||
Settings settings = settingsBuilder.build();
|
||||
logger.log(Level.INFO, "settings = " + settings.toDelimitedString(','));
|
||||
return (C) providerMap.get(provider).getExtendedClient()
|
||||
.setClient(client)
|
||||
.init(settings);
|
||||
if (adminClientProvider != null) {
|
||||
for (AdminClientProvider provider : ServiceLoader.load(AdminClientProvider.class, classLoader)) {
|
||||
if (provider.getClass().isAssignableFrom(adminClientProvider)) {
|
||||
C c = (C) provider.getClient();
|
||||
c.setClient(client);
|
||||
c.init(settings);
|
||||
return c;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (bulkClientProvider != null) {
|
||||
for (BulkClientProvider provider : ServiceLoader.load(BulkClientProvider.class, classLoader)) {
|
||||
if (provider.getClass().isAssignableFrom(bulkClientProvider)) {
|
||||
C c = (C) provider.getClient();
|
||||
c.setClient(client);
|
||||
c.init(settings);
|
||||
return c;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (searchClientProvider != null) {
|
||||
for (SearchClientProvider provider : ServiceLoader.load(SearchClientProvider.class, classLoader)) {
|
||||
if (provider.getClass().isAssignableFrom(searchClientProvider)) {
|
||||
C c = (C) provider.getClient();
|
||||
c.setClient(client);
|
||||
c.init(settings);
|
||||
return c;
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new IllegalArgumentException("no provider");
|
||||
}
|
||||
}
|
||||
|
|
|
@ -8,11 +8,11 @@ import org.elasticsearch.action.update.UpdateRequest;
|
|||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.xbib.elx.api.BulkClient;
|
||||
import org.xbib.elx.api.BulkController;
|
||||
import org.xbib.elx.api.BulkListener;
|
||||
import org.xbib.elx.api.BulkMetric;
|
||||
import org.xbib.elx.api.BulkProcessor;
|
||||
import org.xbib.elx.api.ExtendedClient;
|
||||
import org.xbib.elx.api.IndexDefinition;
|
||||
|
||||
import java.io.IOException;
|
||||
|
@ -27,7 +27,7 @@ public class DefaultBulkController implements BulkController {
|
|||
|
||||
private static final Logger logger = LogManager.getLogger(DefaultBulkController.class);
|
||||
|
||||
private final ExtendedClient client;
|
||||
private final BulkClient bulkClient;
|
||||
|
||||
private final BulkMetric bulkMetric;
|
||||
|
||||
|
@ -45,9 +45,9 @@ public class DefaultBulkController implements BulkController {
|
|||
|
||||
private final AtomicBoolean active;
|
||||
|
||||
public DefaultBulkController(ExtendedClient client) {
|
||||
this.client = client;
|
||||
this.bulkMetric = new DefaultBulkMetric();
|
||||
public DefaultBulkController(BulkClient bulkClient, BulkMetric bulkMetric) {
|
||||
this.bulkClient = bulkClient;
|
||||
this.bulkMetric = bulkMetric;
|
||||
this.indexNames = new ArrayList<>();
|
||||
this.active = new AtomicBoolean(false);
|
||||
this.startBulkRefreshIntervals = new HashMap<>();
|
||||
|
@ -80,7 +80,7 @@ public class DefaultBulkController implements BulkController {
|
|||
boolean enableBulkLogging = settings.getAsBoolean(Parameters.ENABLE_BULK_LOGGING.name(),
|
||||
Parameters.ENABLE_BULK_LOGGING.getValue());
|
||||
BulkListener bulkListener = new DefaultBulkListener(this, bulkMetric, enableBulkLogging);
|
||||
this.bulkProcessor = DefaultBulkProcessor.builder(client.getClient(), bulkListener)
|
||||
this.bulkProcessor = DefaultBulkProcessor.builder(bulkClient.getClient(), bulkListener)
|
||||
.setBulkActions(maxActionsPerRequest)
|
||||
.setConcurrentRequests(maxConcurrentRequests)
|
||||
.setFlushInterval(flushIngestInterval)
|
||||
|
@ -115,7 +115,7 @@ public class DefaultBulkController implements BulkController {
|
|||
startBulkRefreshIntervals.put(indexName, startRefreshIntervalInSeconds);
|
||||
stopBulkRefreshIntervals.put(indexName, stopRefreshIntervalInSeconds);
|
||||
if (startRefreshIntervalInSeconds != 0L) {
|
||||
client.updateIndexSetting(indexName, "refresh_interval", startRefreshIntervalInSeconds + "s",
|
||||
bulkClient.updateIndexSetting(indexName, "refresh_interval", startRefreshIntervalInSeconds + "s",
|
||||
30L, TimeUnit.SECONDS);
|
||||
}
|
||||
}
|
||||
|
@ -187,7 +187,7 @@ public class DefaultBulkController implements BulkController {
|
|||
if (indexNames.contains(index)) {
|
||||
Long secs = stopBulkRefreshIntervals.get(index);
|
||||
if (secs != null && secs != 0L) {
|
||||
client.updateIndexSetting(index, "refresh_interval", secs + "s",
|
||||
bulkClient.updateIndexSetting(index, "refresh_interval", secs + "s",
|
||||
30L, TimeUnit.SECONDS);
|
||||
}
|
||||
indexNames.remove(index);
|
||||
|
@ -206,11 +206,11 @@ public class DefaultBulkController implements BulkController {
|
|||
public void close() throws IOException {
|
||||
flush();
|
||||
bulkMetric.close();
|
||||
if (client.waitForResponses(maxWaitTime, maxWaitTimeUnit)) {
|
||||
if (bulkClient.waitForResponses(maxWaitTime, maxWaitTimeUnit)) {
|
||||
for (String index : indexNames) {
|
||||
Long secs = stopBulkRefreshIntervals.get(index);
|
||||
if (secs != null && secs != 0L)
|
||||
client.updateIndexSetting(index, "refresh_interval", secs + "s",
|
||||
bulkClient.updateIndexSetting(index, "refresh_interval", secs + "s",
|
||||
30L, TimeUnit.SECONDS);
|
||||
}
|
||||
indexNames.clear();
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
package org.xbib.elx.common;
|
||||
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.xbib.elx.api.BulkMetric;
|
||||
import org.xbib.metrics.api.Count;
|
||||
import org.xbib.metrics.api.Metered;
|
||||
|
@ -36,6 +37,10 @@ public class DefaultBulkMetric implements BulkMetric {
|
|||
submitted = new CountMetric();
|
||||
succeeded = new CountMetric();
|
||||
failed = new CountMetric();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(Settings settings) {
|
||||
start();
|
||||
}
|
||||
|
||||
|
|
|
@ -3,8 +3,6 @@ package org.xbib.elx.common;
|
|||
import org.xbib.elx.api.IndexDefinition;
|
||||
import org.xbib.elx.api.IndexRetention;
|
||||
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class DefaultIndexDefinition implements IndexDefinition {
|
||||
|
@ -15,9 +13,9 @@ public class DefaultIndexDefinition implements IndexDefinition {
|
|||
|
||||
private String dateTimePattern;
|
||||
|
||||
private URL settingsUrl;
|
||||
private String settings;
|
||||
|
||||
private URL mappingsUrl;
|
||||
private String mappings;
|
||||
|
||||
private boolean enabled;
|
||||
|
||||
|
@ -62,37 +60,25 @@ public class DefaultIndexDefinition implements IndexDefinition {
|
|||
}
|
||||
|
||||
@Override
|
||||
public IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException {
|
||||
this.settingsUrl = settingsUrlString != null ? new URL(settingsUrlString) : null;
|
||||
public IndexDefinition setSettings(String settings) {
|
||||
this.settings = settings;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexDefinition setSettingsUrl(URL settingsUrl) {
|
||||
this.settingsUrl = settingsUrl;
|
||||
public String getSettings() {
|
||||
return settings;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexDefinition setMappings(String mappings) {
|
||||
this.mappings = mappings;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public URL getSettingsUrl() {
|
||||
return settingsUrl;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException {
|
||||
this.mappingsUrl = mappingsUrlString != null ? new URL(mappingsUrlString) : null;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexDefinition setMappingsUrl(URL mappingsUrl) {
|
||||
this.mappingsUrl = mappingsUrl;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public URL getMappingsUrl() {
|
||||
return mappingsUrl;
|
||||
public String getMappings() {
|
||||
return mappings;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -0,0 +1,59 @@
|
|||
package org.xbib.elx.common;
|
||||
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* A mocked client, it does not perform any actions on a cluster. Useful for testing.
|
||||
*/
|
||||
public class MockAdminClient extends AbstractAdminClient {
|
||||
|
||||
@Override
|
||||
public ElasticsearchClient getClient() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(Settings settings) {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ElasticsearchClient createClient(Settings settings) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void closeClient(Settings settings) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockAdminClient deleteIndex(String index) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void waitForShards(long maxWaitTime, TimeUnit timeUnit) {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockAdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
// nothing to do
|
||||
}
|
||||
}
|
|
@ -0,0 +1,10 @@
|
|||
package org.xbib.elx.common;
|
||||
|
||||
import org.xbib.elx.api.AdminClientProvider;
|
||||
|
||||
public class MockAdminClientProvider implements AdminClientProvider<MockAdminClient> {
|
||||
@Override
|
||||
public MockAdminClient getClient() {
|
||||
return new MockAdminClient();
|
||||
}
|
||||
}
|
103
elx-common/src/main/java/org/xbib/elx/common/MockBulkClient.java
Normal file
103
elx-common/src/main/java/org/xbib/elx/common/MockBulkClient.java
Normal file
|
@ -0,0 +1,103 @@
|
|||
package org.xbib.elx.common;
|
||||
|
||||
import org.elasticsearch.action.delete.DeleteRequest;
|
||||
import org.elasticsearch.action.index.IndexRequest;
|
||||
import org.elasticsearch.action.update.UpdateRequest;
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* A mocked client, it does not perform any actions on a cluster. Useful for testing.
|
||||
*/
|
||||
public class MockBulkClient extends AbstractBulkClient {
|
||||
|
||||
@Override
|
||||
public ElasticsearchClient getClient() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(Settings settings) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getClusterName() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void waitForShards(long maxWaitTime, TimeUnit timeUnit) {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ElasticsearchClient createClient(Settings settings) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void closeClient(Settings settings) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockBulkClient index(String index, String id, boolean create, String source) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockBulkClient delete(String index, String id) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockBulkClient update(String index, String id, String source) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockBulkClient index(IndexRequest indexRequest) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockBulkClient delete(DeleteRequest deleteRequest) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockBulkClient update(UpdateRequest updateRequest) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startBulk(String index, long startRefreshInterval, long stopRefreshIterval) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void refreshIndex(String index) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void flushIndex(String index) {
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void flush() {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
// nothing to do
|
||||
}
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
package org.xbib.elx.common;
|
||||
|
||||
import org.xbib.elx.api.BulkClientProvider;
|
||||
|
||||
public class MockBulkClientProvider implements BulkClientProvider<MockBulkClient> {
|
||||
|
||||
@Override
|
||||
public MockBulkClient getClient() {
|
||||
return new MockBulkClient();
|
||||
}
|
||||
}
|
|
@ -1,129 +0,0 @@
|
|||
package org.xbib.elx.common;
|
||||
|
||||
import org.elasticsearch.action.delete.DeleteRequest;
|
||||
import org.elasticsearch.action.index.IndexRequest;
|
||||
import org.elasticsearch.action.update.UpdateRequest;
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* A mocked client, it does not perform any actions on a cluster. Useful for testing.
|
||||
*/
|
||||
public class MockExtendedClient extends AbstractExtendedClient {
|
||||
|
||||
@Override
|
||||
public ElasticsearchClient getClient() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient init(Settings settings) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ElasticsearchClient createClient(Settings settings) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void closeClient() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient index(String index, String id, boolean create, String source) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient delete(String index, String id) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient update(String index, String id, String source) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient index(IndexRequest indexRequest) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient delete(DeleteRequest deleteRequest) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient update(UpdateRequest updateRequest) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient startBulk(String index, long startRefreshInterval, long stopRefreshIterval) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient newIndex(String index) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient deleteIndex(String index) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient refreshIndex(String index) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient flushIndex(String index) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void flush() {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
// nothing to do
|
||||
}
|
||||
}
|
|
@ -1,10 +0,0 @@
|
|||
package org.xbib.elx.common;
|
||||
|
||||
import org.xbib.elx.api.ExtendedClientProvider;
|
||||
|
||||
public class MockExtendedClientProvider implements ExtendedClientProvider<MockExtendedClient> {
|
||||
@Override
|
||||
public MockExtendedClient getExtendedClient() {
|
||||
return new MockExtendedClient();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,44 @@
|
|||
package org.xbib.elx.common;
|
||||
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* A mocked client, it does not perform any actions on a cluster. Useful for testing.
|
||||
*/
|
||||
public class MockSearchClient extends AbstractSearchClient {
|
||||
|
||||
@Override
|
||||
public ElasticsearchClient getClient() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(Settings settings) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getClusterName() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void waitForShards(long maxWaitTime, TimeUnit timeUnit) {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ElasticsearchClient createClient(Settings settings) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void closeClient(Settings settings) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
// nothing to do
|
||||
}
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
package org.xbib.elx.common;
|
||||
|
||||
import org.xbib.elx.api.SearchClientProvider;
|
||||
|
||||
public class MockSearchClientProvider implements SearchClientProvider<MockSearchClient> {
|
||||
|
||||
@Override
|
||||
public MockSearchClient getClient() {
|
||||
return new MockSearchClient();
|
||||
}
|
||||
}
|
|
@ -0,0 +1 @@
|
|||
org.xbib.elx.common.MockAdminClientProvider
|
|
@ -0,0 +1 @@
|
|||
org.xbib.elx.common.MockBulkClientProvider
|
|
@ -1 +0,0 @@
|
|||
org.xbib.elx.common.MockExtendedClientProvider
|
|
@ -0,0 +1 @@
|
|||
org.xbib.elx.common.MockSearchClientProvider
|
|
@ -0,0 +1,41 @@
|
|||
package org.xbib.elx.common.test;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
import org.xbib.elx.common.MockAdminClient;
|
||||
import org.xbib.elx.common.MockAdminClientProvider;
|
||||
import org.xbib.elx.common.MockBulkClient;
|
||||
import org.xbib.elx.common.MockBulkClientProvider;
|
||||
import org.xbib.elx.common.MockSearchClient;
|
||||
import org.xbib.elx.common.MockSearchClientProvider;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
|
||||
class MockClientProviderTest {
|
||||
|
||||
@Test
|
||||
void testMockAdminProvider() throws IOException {
|
||||
MockAdminClient client = ClientBuilder.builder()
|
||||
.setAdminClientProvider(MockAdminClientProvider.class)
|
||||
.build();
|
||||
assertNotNull(client);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testMockBulkProvider() throws IOException {
|
||||
MockBulkClient client = ClientBuilder.builder()
|
||||
.setBulkClientProvider(MockBulkClientProvider.class)
|
||||
.build();
|
||||
assertNotNull(client);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testMockSearchProvider() throws IOException {
|
||||
MockSearchClient client = ClientBuilder.builder()
|
||||
.setSearchClientProvider(MockSearchClientProvider.class)
|
||||
.build();
|
||||
assertNotNull(client);
|
||||
}
|
||||
}
|
|
@ -1,19 +0,0 @@
|
|||
package org.xbib.elx.common.test;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
import org.xbib.elx.common.MockExtendedClient;
|
||||
import org.xbib.elx.common.MockExtendedClientProvider;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
|
||||
class MockExtendedClientProviderTest {
|
||||
|
||||
@Test
|
||||
void testMockExtendedProvider() throws IOException {
|
||||
MockExtendedClient client = ClientBuilder.builder().provider(MockExtendedClientProvider.class).build();
|
||||
assertNotNull(client);
|
||||
}
|
||||
}
|
|
@ -1,12 +0,0 @@
|
|||
package org.xbib.elx.http;
|
||||
|
||||
import org.xbib.elx.api.ExtendedClientProvider;
|
||||
|
||||
import java.util.Collections;
|
||||
|
||||
public class ExtendedHttpClientProvider implements ExtendedClientProvider<ExtendedHttpClient> {
|
||||
@Override
|
||||
public ExtendedHttpClient getExtendedClient() {
|
||||
return new ExtendedHttpClient(Collections.emptyList(), Thread.currentThread().getContextClassLoader());
|
||||
}
|
||||
}
|
|
@ -13,7 +13,7 @@ import org.xbib.netty.http.common.HttpResponse;
|
|||
*/
|
||||
public class HttpActionContext<R extends ActionRequest, T extends ActionResponse> {
|
||||
|
||||
private final ExtendedHttpClient extendedHttpClient;
|
||||
private final HttpClientHelper extendedHttpClient;
|
||||
|
||||
private final R request;
|
||||
|
||||
|
@ -23,13 +23,13 @@ public class HttpActionContext<R extends ActionRequest, T extends ActionResponse
|
|||
|
||||
private HttpResponse httpResponse;
|
||||
|
||||
HttpActionContext(ExtendedHttpClient extendedHttpClient, R request, String url) {
|
||||
HttpActionContext(HttpClientHelper extendedHttpClient, R request, String url) {
|
||||
this.extendedHttpClient = extendedHttpClient;
|
||||
this.request = request;
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
public ExtendedHttpClient getExtendedHttpClient() {
|
||||
public HttpClientHelper getExtendedHttpClient() {
|
||||
return extendedHttpClient;
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,55 @@
|
|||
package org.xbib.elx.http;
|
||||
|
||||
import org.elasticsearch.action.ActionFuture;
|
||||
import org.elasticsearch.action.ActionListener;
|
||||
import org.elasticsearch.action.ActionRequest;
|
||||
import org.elasticsearch.action.ActionResponse;
|
||||
import org.elasticsearch.action.ActionType;
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.threadpool.ThreadPool;
|
||||
import org.xbib.elx.common.AbstractAdminClient;
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Elasticsearch HTTP admin client.
|
||||
*/
|
||||
public class HttpAdminClient extends AbstractAdminClient implements ElasticsearchClient {
|
||||
|
||||
private final HttpClientHelper helper;
|
||||
|
||||
public HttpAdminClient() {
|
||||
this.helper = new HttpClientHelper();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(Settings settings) throws IOException {
|
||||
super.init(settings);
|
||||
helper.init(settings);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void closeClient(Settings settings) throws IOException {
|
||||
helper.closeClient(settings);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <Request extends ActionRequest, Response extends ActionResponse> ActionFuture<Response> execute(ActionType<Response> action, Request request) {
|
||||
return helper.execute(action, request);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <Request extends ActionRequest, Response extends ActionResponse> void execute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
|
||||
helper.execute(action, request, listener);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ThreadPool threadPool() {
|
||||
return helper.threadPool();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
package org.xbib.elx.http;
|
||||
|
||||
import org.xbib.elx.api.AdminClientProvider;
|
||||
|
||||
public class HttpAdminClientProvider implements AdminClientProvider<HttpAdminClient> {
|
||||
|
||||
@Override
|
||||
public HttpAdminClient getClient() {
|
||||
return new HttpAdminClient();
|
||||
}
|
||||
}
|
55
elx-http/src/main/java/org/xbib/elx/http/HttpBulkClient.java
Normal file
55
elx-http/src/main/java/org/xbib/elx/http/HttpBulkClient.java
Normal file
|
@ -0,0 +1,55 @@
|
|||
package org.xbib.elx.http;
|
||||
|
||||
import org.elasticsearch.action.ActionFuture;
|
||||
import org.elasticsearch.action.ActionListener;
|
||||
import org.elasticsearch.action.ActionRequest;
|
||||
import org.elasticsearch.action.ActionResponse;
|
||||
import org.elasticsearch.action.ActionType;
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.threadpool.ThreadPool;
|
||||
import org.xbib.elx.common.AbstractBulkClient;
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Elasticsearch HTTP bulk client.
|
||||
*/
|
||||
public class HttpBulkClient extends AbstractBulkClient implements ElasticsearchClient {
|
||||
|
||||
private final HttpClientHelper helper;
|
||||
|
||||
public HttpBulkClient() {
|
||||
this.helper = new HttpClientHelper();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(Settings settings) throws IOException {
|
||||
super.init(settings);
|
||||
helper.init(settings);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void closeClient(Settings settings) throws IOException {
|
||||
helper.closeClient(settings);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <Request extends ActionRequest, Response extends ActionResponse> ActionFuture<Response> execute(ActionType<Response> action, Request request) {
|
||||
return helper.execute(action, request);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <Request extends ActionRequest, Response extends ActionResponse> void execute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
|
||||
helper.execute(action, request, listener);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ThreadPool threadPool() {
|
||||
return helper.threadPool();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
package org.xbib.elx.http;
|
||||
|
||||
import org.xbib.elx.api.BulkClientProvider;
|
||||
|
||||
public class HttpBulkClientProvider implements BulkClientProvider<HttpBulkClient> {
|
||||
|
||||
@Override
|
||||
public HttpBulkClient getClient() {
|
||||
return new HttpBulkClient();
|
||||
}
|
||||
}
|
|
@ -10,16 +10,15 @@ import org.elasticsearch.action.ActionRequestBuilder;
|
|||
import org.elasticsearch.action.ActionResponse;
|
||||
import org.elasticsearch.action.ActionType;
|
||||
import org.elasticsearch.action.support.PlainActionFuture;
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
|
||||
import org.elasticsearch.threadpool.ThreadPool;
|
||||
import org.xbib.elx.common.AbstractExtendedClient;
|
||||
import org.xbib.net.URL;
|
||||
import org.xbib.netty.http.client.Client;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
@ -31,9 +30,9 @@ import java.util.stream.Stream;
|
|||
/**
|
||||
* Elasticsearch HTTP client.
|
||||
*/
|
||||
public class ExtendedHttpClient extends AbstractExtendedClient implements ElasticsearchClient {
|
||||
public class HttpClientHelper {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(ExtendedHttpClient.class);
|
||||
private static final Logger logger = LogManager.getLogger(HttpClientHelper.class);
|
||||
|
||||
private Client nettyHttpClient;
|
||||
|
||||
|
@ -46,17 +45,20 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
|
|||
|
||||
private String url;
|
||||
|
||||
public ExtendedHttpClient(List<NamedXContentRegistry.Entry> namedXContentEntries, ClassLoader classLoader) {
|
||||
public HttpClientHelper() {
|
||||
this(Collections.emptyList(), Thread.currentThread().getContextClassLoader());
|
||||
}
|
||||
|
||||
public HttpClientHelper(List<NamedXContentRegistry.Entry> namedXContentEntries,
|
||||
ClassLoader classLoader) {
|
||||
this.registry = new NamedXContentRegistry(Stream.of(getNamedXContents().stream(),
|
||||
namedXContentEntries.stream()).flatMap(Function.identity()).collect(Collectors.toList()));
|
||||
this.classLoader = classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader();
|
||||
this.actionMap = new HashMap<>();
|
||||
}
|
||||
|
||||
@Override
|
||||
@SuppressWarnings({"unchecked", "rawtypes"})
|
||||
public ExtendedHttpClient init(Settings settings) throws IOException {
|
||||
super.init(settings);
|
||||
public HttpClientHelper init(Settings settings) throws IOException {
|
||||
if (settings.hasValue("url")) {
|
||||
this.url = settings.get("url");
|
||||
} else if (settings.hasValue("host")) {
|
||||
|
@ -92,22 +94,10 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
|
|||
return nettyHttpClient;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ElasticsearchClient getClient() {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ElasticsearchClient createClient(Settings settings) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void closeClient() throws IOException {
|
||||
protected void closeClient(Settings settings) throws IOException {
|
||||
nettyHttpClient.shutdownGracefully();
|
||||
}
|
||||
|
||||
@Override
|
||||
public <Request extends ActionRequest, Response extends ActionResponse>
|
||||
ActionFuture<Response> execute(ActionType<Response> action, Request request) {
|
||||
PlainActionFuture<Response> actionFuture = PlainActionFuture.newFuture();
|
||||
|
@ -115,13 +105,11 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
|
|||
return actionFuture;
|
||||
}
|
||||
|
||||
@Override
|
||||
public <Request extends ActionRequest, Response extends ActionResponse>
|
||||
void execute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
|
||||
doExecute(action, request, listener);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ThreadPool threadPool() {
|
||||
logger.log(Level.DEBUG, "returning null for threadPool() request");
|
||||
return null;
|
|
@ -0,0 +1,55 @@
|
|||
package org.xbib.elx.http;
|
||||
|
||||
import org.elasticsearch.action.ActionFuture;
|
||||
import org.elasticsearch.action.ActionListener;
|
||||
import org.elasticsearch.action.ActionRequest;
|
||||
import org.elasticsearch.action.ActionResponse;
|
||||
import org.elasticsearch.action.ActionType;
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.threadpool.ThreadPool;
|
||||
import org.xbib.elx.common.AbstractSearchClient;
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Elasticsearch HTTP search client.
|
||||
*/
|
||||
public class HttpSearchClient extends AbstractSearchClient implements ElasticsearchClient {
|
||||
|
||||
private final HttpClientHelper helper;
|
||||
|
||||
public HttpSearchClient() {
|
||||
this.helper = new HttpClientHelper();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(Settings settings) throws IOException {
|
||||
super.init(settings);
|
||||
helper.init(settings);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void closeClient(Settings settings) throws IOException {
|
||||
helper.closeClient(settings);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <Request extends ActionRequest, Response extends ActionResponse> ActionFuture<Response> execute(ActionType<Response> action, Request request) {
|
||||
return helper.execute(action, request);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <Request extends ActionRequest, Response extends ActionResponse> void execute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
|
||||
helper.execute(action, request, listener);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ThreadPool threadPool() {
|
||||
return helper.threadPool();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
package org.xbib.elx.http;
|
||||
|
||||
import org.xbib.elx.api.SearchClientProvider;
|
||||
|
||||
public class HttpSearchClientProvider implements SearchClientProvider<HttpSearchClient> {
|
||||
|
||||
@Override
|
||||
public HttpSearchClient getClient() {
|
||||
return new HttpSearchClient();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,35 @@
|
|||
package org.xbib.elx.http.action.admin.indices.get;
|
||||
|
||||
import org.elasticsearch.action.admin.indices.get.GetIndexAction;
|
||||
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
|
||||
import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
|
||||
import org.elasticsearch.common.CheckedFunction;
|
||||
import org.elasticsearch.common.xcontent.XContentParser;
|
||||
import org.xbib.elx.http.HttpAction;
|
||||
import org.xbib.netty.http.client.api.Request;
|
||||
import org.xbib.netty.http.common.HttpResponse;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
public class HtppGetIndexAction extends HttpAction<GetIndexRequest, GetIndexResponse> {
|
||||
|
||||
@Override
|
||||
public GetIndexAction getActionInstance() {
|
||||
return GetIndexAction.INSTANCE;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Request.Builder createHttpRequest(String url, GetIndexRequest getIndexRequest) throws IOException {
|
||||
List<String> list = getIndexRequest.indices().length == 0 ?
|
||||
List.of("*") : Arrays.asList(getIndexRequest.indices());
|
||||
String command = "/" + String.join(",", list);
|
||||
logger.info("command = " + command);
|
||||
return newGetRequest(url, command);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected CheckedFunction<XContentParser, GetIndexResponse, IOException> entityParser(HttpResponse httpResponse) {
|
||||
return GetIndexResponse::fromXContent;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,35 @@
|
|||
package org.xbib.elx.http.action.search;
|
||||
|
||||
import org.elasticsearch.action.search.ClearScrollAction;
|
||||
import org.elasticsearch.action.search.ClearScrollRequest;
|
||||
import org.elasticsearch.action.search.ClearScrollResponse;
|
||||
import org.elasticsearch.common.CheckedFunction;
|
||||
import org.elasticsearch.common.bytes.BytesReference;
|
||||
import org.elasticsearch.common.xcontent.ToXContent;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.XContentParser;
|
||||
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||
import org.xbib.elx.http.HttpAction;
|
||||
import org.xbib.netty.http.client.api.Request;
|
||||
import org.xbib.netty.http.common.HttpResponse;
|
||||
import java.io.IOException;
|
||||
|
||||
public class HttpClearScrollAction extends HttpAction<ClearScrollRequest, ClearScrollResponse> {
|
||||
|
||||
@Override
|
||||
public ClearScrollAction getActionInstance() {
|
||||
return ClearScrollAction.INSTANCE;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Request.Builder createHttpRequest(String baseUrl, ClearScrollRequest request) throws IOException {
|
||||
XContentBuilder builder = JsonXContent.contentBuilder();
|
||||
request.toXContent(builder, ToXContent.EMPTY_PARAMS);
|
||||
return newDeleteRequest(baseUrl, "_search/scroll", BytesReference.bytes(builder));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected CheckedFunction<XContentParser, ClearScrollResponse, IOException> entityParser(HttpResponse httpResponse) {
|
||||
return ClearScrollResponse::fromXContent;
|
||||
}
|
||||
}
|
|
@ -5,6 +5,7 @@ import org.elasticsearch.action.search.SearchRequest;
|
|||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.common.CheckedFunction;
|
||||
import org.elasticsearch.common.xcontent.XContentParser;
|
||||
import org.elasticsearch.search.Scroll;
|
||||
import org.xbib.elx.http.HttpAction;
|
||||
import org.xbib.netty.http.client.api.Request;
|
||||
import org.xbib.netty.http.common.HttpResponse;
|
||||
|
@ -20,9 +21,10 @@ public class HttpSearchAction extends HttpAction<SearchRequest, SearchResponse>
|
|||
|
||||
@Override
|
||||
protected Request.Builder createHttpRequest(String url, SearchRequest request) {
|
||||
// request.indices() always empty array
|
||||
Scroll scroll = request.scroll();
|
||||
String params = scroll != null ? "?scroll=" + scroll.keepAlive() : "";
|
||||
String index = request.indices() != null ? String.join(",", request.indices()) + "/" : "";
|
||||
return newPostRequest(url, index + "_search", request.source().toString());
|
||||
return newPostRequest(url, index + "_search" + params, request.source().toString());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -0,0 +1,35 @@
|
|||
package org.xbib.elx.http.action.search;
|
||||
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.action.search.SearchScrollAction;
|
||||
import org.elasticsearch.action.search.SearchScrollRequest;
|
||||
import org.elasticsearch.common.CheckedFunction;
|
||||
import org.elasticsearch.common.bytes.BytesReference;
|
||||
import org.elasticsearch.common.xcontent.ToXContent;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.XContentParser;
|
||||
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||
import org.xbib.elx.http.HttpAction;
|
||||
import org.xbib.netty.http.client.api.Request;
|
||||
import org.xbib.netty.http.common.HttpResponse;
|
||||
import java.io.IOException;
|
||||
|
||||
public class HttpSearchScrollAction extends HttpAction<SearchScrollRequest, SearchResponse> {
|
||||
|
||||
@Override
|
||||
public SearchScrollAction getActionInstance() {
|
||||
return SearchScrollAction.INSTANCE;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Request.Builder createHttpRequest(String baseUrl, SearchScrollRequest request) throws IOException {
|
||||
XContentBuilder builder = JsonXContent.contentBuilder();
|
||||
request.toXContent(builder, ToXContent.EMPTY_PARAMS);
|
||||
return newPostRequest(baseUrl, "_search/scroll", BytesReference.bytes(builder));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected CheckedFunction<XContentParser, SearchResponse, IOException> entityParser(HttpResponse httpResponse) {
|
||||
return SearchResponse::fromXContent;
|
||||
}
|
||||
}
|
|
@ -0,0 +1 @@
|
|||
org.xbib.elx.http.HttpAdminClientProvider
|
|
@ -0,0 +1 @@
|
|||
org.xbib.elx.http.HttpBulkClientProvider
|
|
@ -1 +0,0 @@
|
|||
org.xbib.elx.http.ExtendedHttpClientProvider
|
|
@ -0,0 +1 @@
|
|||
org.xbib.elx.http.HttpSearchClientProvider
|
|
@ -8,12 +8,15 @@ org.xbib.elx.http.action.admin.indices.alias.get.HttpGetAliasAction
|
|||
org.xbib.elx.http.action.admin.indices.create.HttpCreateIndexAction
|
||||
org.xbib.elx.http.action.admin.indices.delete.HttpDeleteIndexAction
|
||||
org.xbib.elx.http.action.admin.indices.exists.indices.HttpIndicesExistsAction
|
||||
org.xbib.elx.http.action.admin.indices.get.HtppGetIndexAction
|
||||
org.xbib.elx.http.action.admin.indices.refresh.HttpRefreshIndexAction
|
||||
org.xbib.elx.http.action.admin.indices.settings.get.HttpGetSettingsAction
|
||||
org.xbib.elx.http.action.admin.indices.settings.put.HttpUpdateSettingsAction
|
||||
org.xbib.elx.http.action.bulk.HttpBulkAction
|
||||
org.xbib.elx.http.action.index.HttpIndexAction
|
||||
org.xbib.elx.http.action.search.HttpClearScrollAction
|
||||
org.xbib.elx.http.action.search.HttpSearchAction
|
||||
org.xbib.elx.http.action.search.HttpSearchScrollAction
|
||||
org.xbib.elx.http.action.main.HttpMainAction
|
||||
org.xbib.elx.http.action.get.HttpExistsAction
|
||||
org.xbib.elx.http.action.get.HttpGetAction
|
||||
|
|
|
@ -0,0 +1,182 @@
|
|||
package org.xbib.elx.http.test;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
import org.xbib.elx.common.Parameters;
|
||||
import org.xbib.elx.http.HttpAdminClient;
|
||||
import org.xbib.elx.http.HttpAdminClientProvider;
|
||||
import org.xbib.elx.http.HttpBulkClient;
|
||||
import org.xbib.elx.http.HttpBulkClientProvider;
|
||||
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@ExtendWith(TestExtension.class)
|
||||
class BulkClientTest {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getSimpleName());
|
||||
|
||||
private static final Long ACTIONS = 100L;
|
||||
|
||||
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
|
||||
|
||||
private final TestExtension.Helper helper;
|
||||
|
||||
BulkClientTest(TestExtension.Helper helper) {
|
||||
this.helper = helper;
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSingleDoc() throws Exception {
|
||||
final HttpBulkClient client = ClientBuilder.builder()
|
||||
.setBulkClientProvider(HttpBulkClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
|
||||
.build();
|
||||
try {
|
||||
client.newIndex("test");
|
||||
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
||||
client.flush();
|
||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
} finally {
|
||||
assertEquals(1, client.getBulkMetric().getSucceeded().getCount());
|
||||
if (client.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", client.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(client.getBulkController().getLastBulkError());
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testNewIndex() throws Exception {
|
||||
final HttpBulkClient client = ClientBuilder.builder()
|
||||
.setBulkClientProvider(HttpBulkClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
||||
.build();
|
||||
client.newIndex("test");
|
||||
client.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testMapping() throws Exception {
|
||||
try (HttpAdminClient adminClient = ClientBuilder.builder()
|
||||
.setAdminClientProvider(HttpAdminClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.build();
|
||||
HttpBulkClient bulkClient = ClientBuilder.builder()
|
||||
.setBulkClientProvider(HttpBulkClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.build()) {
|
||||
XContentBuilder builder = JsonXContent.contentBuilder()
|
||||
.startObject()
|
||||
.startObject("properties")
|
||||
.startObject("location")
|
||||
.field("type", "geo_point")
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject();
|
||||
bulkClient.newIndex("test", Settings.EMPTY, builder);
|
||||
assertTrue(adminClient.getMapping("test", "_doc").containsKey("properties"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRandomDocs() throws Exception {
|
||||
long numactions = ACTIONS;
|
||||
final HttpBulkClient client = ClientBuilder.builder()
|
||||
.setBulkClientProvider(HttpBulkClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
||||
.build();
|
||||
try {
|
||||
client.newIndex("test");
|
||||
for (int i = 0; i < ACTIONS; i++) {
|
||||
client.index("test", null, false, "{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||
}
|
||||
client.flush();
|
||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
} finally {
|
||||
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
|
||||
if (client.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", client.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(client.getBulkController().getLastBulkError());
|
||||
client.refreshIndex("test");
|
||||
assertEquals(numactions, client.getSearchableDocs("test"));
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testThreadedRandomDocs() throws Exception {
|
||||
int maxthreads = Runtime.getRuntime().availableProcessors();
|
||||
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
|
||||
final long actions = ACTIONS;
|
||||
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
|
||||
final HttpBulkClient bulkClient = ClientBuilder.builder()
|
||||
.setBulkClientProvider(HttpBulkClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
||||
.build();
|
||||
try {
|
||||
Settings settings = Settings.builder()
|
||||
.put("index.number_of_shards", 1)
|
||||
.put("index.number_of_replicas", 0)
|
||||
.build();
|
||||
bulkClient.newIndex("test", settings);
|
||||
bulkClient.startBulk("test", 0, 1000);
|
||||
logger.info("index created");
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
|
||||
final CountDownLatch latch = new CountDownLatch(maxthreads);
|
||||
for (int i = 0; i < maxthreads; i++) {
|
||||
executorService.execute(() -> {
|
||||
for (int i1 = 0; i1 < actions; i1++) {
|
||||
bulkClient.index("test", null, false,"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||
}
|
||||
latch.countDown();
|
||||
});
|
||||
}
|
||||
logger.info("waiting for latch...");
|
||||
if (latch.await(60L, TimeUnit.SECONDS)) {
|
||||
logger.info("flush...");
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(60L, TimeUnit.SECONDS);
|
||||
logger.info("got all responses, executor service shutdown...");
|
||||
executorService.shutdown();
|
||||
executorService.awaitTermination(60L, TimeUnit.SECONDS);
|
||||
logger.info("pool is shut down");
|
||||
} else {
|
||||
logger.warn("latch timeout");
|
||||
}
|
||||
bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS);
|
||||
assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||
} finally {
|
||||
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||
bulkClient.refreshIndex("test");
|
||||
assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test"));
|
||||
bulkClient.close();
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,244 +0,0 @@
|
|||
package org.xbib.elx.http.test;
|
||||
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
|
||||
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
|
||||
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
|
||||
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
|
||||
import org.elasticsearch.action.search.SearchAction;
|
||||
import org.elasticsearch.action.search.SearchRequest;
|
||||
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
import org.xbib.elx.common.Parameters;
|
||||
import org.xbib.elx.http.ExtendedHttpClient;
|
||||
import org.xbib.elx.http.ExtendedHttpClientProvider;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@ExtendWith(TestExtension.class)
|
||||
class ClientTest {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(ClientTest.class.getSimpleName());
|
||||
|
||||
private static final Long ACTIONS = 100L;
|
||||
|
||||
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
|
||||
|
||||
private final TestExtension.Helper helper;
|
||||
|
||||
ClientTest(TestExtension.Helper helper) {
|
||||
this.helper = helper;
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSingleDoc() throws Exception {
|
||||
final ExtendedHttpClient client = ClientBuilder.builder()
|
||||
.provider(ExtendedHttpClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
|
||||
.build();
|
||||
try {
|
||||
client.newIndex("test");
|
||||
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
||||
client.flush();
|
||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} finally {
|
||||
assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount());
|
||||
if (client.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", client.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(client.getBulkController().getLastBulkError());
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testNewIndex() throws Exception {
|
||||
final ExtendedHttpClient client = ClientBuilder.builder()
|
||||
.provider(ExtendedHttpClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
||||
.build();
|
||||
client.newIndex("test");
|
||||
client.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testNewIndexWithSettings() throws Exception {
|
||||
final ExtendedHttpClient client = ClientBuilder.builder()
|
||||
.provider(ExtendedHttpClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
||||
.build();
|
||||
Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
|
||||
client.newIndex("test", settings);
|
||||
GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
|
||||
.indices("test");
|
||||
GetSettingsResponse getSettingsResponse =
|
||||
client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet();
|
||||
logger.log(Level.INFO, "settings=" + getSettingsResponse.getSetting("test", "index.number_of_shards"));
|
||||
assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards"));
|
||||
client.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testNewIndexWithSettingsAndMappings() throws Exception {
|
||||
final ExtendedHttpClient client = ClientBuilder.builder()
|
||||
.provider(ExtendedHttpClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
||||
.build();
|
||||
Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
|
||||
XContentBuilder builder = JsonXContent.contentBuilder()
|
||||
.startObject()
|
||||
.field("date_detection", false)
|
||||
.startObject("properties")
|
||||
.startObject("pos")
|
||||
.field("type", "geo_point")
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject();
|
||||
client.newIndex("test", settings, builder);
|
||||
GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
|
||||
.indices("test");
|
||||
GetSettingsResponse getSettingsResponse =
|
||||
client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet();
|
||||
assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards"));
|
||||
GetMappingsRequest getMappingsRequest = new GetMappingsRequest()
|
||||
.indices("test");
|
||||
GetMappingsResponse getMappingsResponse =
|
||||
client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
|
||||
assertTrue(getMappingsResponse.getMappings().get("test").containsKey("_doc"));
|
||||
client.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRandomDocs() throws Exception {
|
||||
long numactions = ACTIONS;
|
||||
final ExtendedHttpClient client = ClientBuilder.builder()
|
||||
.provider(ExtendedHttpClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
||||
.build();
|
||||
try {
|
||||
client.newIndex("test");
|
||||
for (int i = 0; i < ACTIONS; i++) {
|
||||
client.index("test", null, false, "{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||
}
|
||||
client.flush();
|
||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
} finally {
|
||||
assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
|
||||
if (client.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", client.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(client.getBulkController().getLastBulkError());
|
||||
client.refreshIndex("test");
|
||||
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
|
||||
.setQuery(QueryBuilders.matchAllQuery()).setSize(0);
|
||||
assertEquals(numactions, searchRequestBuilder.execute().actionGet().getHits().getTotalHits().value);
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testThreadedRandomDocs() throws Exception {
|
||||
int maxthreads = Runtime.getRuntime().availableProcessors();
|
||||
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
|
||||
final long actions = ACTIONS;
|
||||
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
|
||||
final ExtendedHttpClient client = ClientBuilder.builder()
|
||||
.provider(ExtendedHttpClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
||||
.build();
|
||||
try {
|
||||
Settings settings = Settings.builder()
|
||||
.put("index.number_of_shards", 1)
|
||||
.put("index.number_of_replicas", 0)
|
||||
.build();
|
||||
client.newIndex("test", settings, (String)null)
|
||||
.startBulk("test", 0, 1000);
|
||||
logger.info("index created");
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
|
||||
final CountDownLatch latch = new CountDownLatch(maxthreads);
|
||||
for (int i = 0; i < maxthreads; i++) {
|
||||
executorService.execute(() -> {
|
||||
for (int i1 = 0; i1 < actions; i1++) {
|
||||
client.index("test", null, false,"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||
}
|
||||
latch.countDown();
|
||||
});
|
||||
}
|
||||
logger.info("waiting for latch...");
|
||||
if (latch.await(60L, TimeUnit.SECONDS)) {
|
||||
logger.info("flush...");
|
||||
client.flush();
|
||||
client.waitForResponses(60L, TimeUnit.SECONDS);
|
||||
logger.info("got all responses, executor service shutdown...");
|
||||
executorService.shutdown();
|
||||
executorService.awaitTermination(60L, TimeUnit.SECONDS);
|
||||
logger.info("pool is shut down");
|
||||
} else {
|
||||
logger.warn("latch timeout");
|
||||
}
|
||||
client.stopBulk("test", 30L, TimeUnit.SECONDS);
|
||||
assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
} finally {
|
||||
if (client.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", client.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(client.getBulkController().getLastBulkError());
|
||||
assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
|
||||
logger.log(Level.INFO, "refreshing index test");
|
||||
client.refreshIndex("test");
|
||||
SearchSourceBuilder builder = new SearchSourceBuilder()
|
||||
.query(QueryBuilders.matchAllQuery())
|
||||
.size(0)
|
||||
.trackTotalHits(true);
|
||||
SearchRequest searchRequest = new SearchRequest()
|
||||
.indices("test")
|
||||
.source(builder);
|
||||
SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
|
||||
assertEquals(maxthreads * actions, searchResponse.getHits().getTotalHits().value);
|
||||
client.close();
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
}
|
|
@ -2,18 +2,12 @@ package org.xbib.elx.http.test;
|
|||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.action.search.SearchAction;
|
||||
import org.elasticsearch.action.search.SearchRequest;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
import org.xbib.elx.common.Parameters;
|
||||
import org.xbib.elx.http.ExtendedHttpClient;
|
||||
import org.xbib.elx.http.ExtendedHttpClientProvider;
|
||||
import org.xbib.elx.http.HttpBulkClient;
|
||||
import org.xbib.elx.http.HttpBulkClientProvider;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
|
@ -26,10 +20,10 @@ class DuplicateIDTest {
|
|||
|
||||
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getSimpleName());
|
||||
|
||||
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
|
||||
|
||||
private static final Long ACTIONS = 100L;
|
||||
|
||||
private static final Long MAX_ACTIONS_PER_REQUEST = 5L;
|
||||
|
||||
private final TestExtension.Helper helper;
|
||||
|
||||
DuplicateIDTest(TestExtension.Helper helper) {
|
||||
|
@ -39,8 +33,8 @@ class DuplicateIDTest {
|
|||
@Test
|
||||
void testDuplicateDocIDs() throws Exception {
|
||||
long numactions = ACTIONS;
|
||||
final ExtendedHttpClient client = ClientBuilder.builder()
|
||||
.provider(ExtendedHttpClientProvider.class)
|
||||
final HttpBulkClient client = ClientBuilder.builder()
|
||||
.setBulkClientProvider(HttpBulkClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||
.build();
|
||||
|
@ -53,23 +47,11 @@ class DuplicateIDTest {
|
|||
client.flush();
|
||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
client.refreshIndex("test");
|
||||
SearchSourceBuilder builder = new SearchSourceBuilder()
|
||||
.query(QueryBuilders.matchAllQuery())
|
||||
.size(0)
|
||||
.trackTotalHits(true);
|
||||
SearchRequest searchRequest = new SearchRequest()
|
||||
.indices("test")
|
||||
.source(builder);
|
||||
SearchResponse searchResponse =
|
||||
helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet();
|
||||
long hits = searchResponse.getHits().getTotalHits().value;
|
||||
logger.info("hits = {}", hits);
|
||||
long hits = client.getSearchableDocs("test");
|
||||
assertTrue(hits < ACTIONS);
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} finally {
|
||||
client.close();
|
||||
assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
|
||||
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
|
||||
if (client.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", client.getBulkController().getLastBulkError());
|
||||
}
|
||||
|
|
|
@ -2,17 +2,15 @@ package org.xbib.elx.http.test;
|
|||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
|
||||
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
|
||||
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.xbib.elx.api.IndexPruneResult;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
import org.xbib.elx.http.ExtendedHttpClient;
|
||||
import org.xbib.elx.http.ExtendedHttpClientProvider;
|
||||
import org.xbib.elx.http.HttpAdminClient;
|
||||
import org.xbib.elx.http.HttpAdminClientProvider;
|
||||
import org.xbib.elx.http.HttpBulkClient;
|
||||
import org.xbib.elx.http.HttpBulkClientProvider;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
|
@ -37,50 +35,49 @@ class IndexPruneTest {
|
|||
|
||||
@Test
|
||||
void testPrune() throws IOException {
|
||||
final ExtendedHttpClient client = ClientBuilder.builder(helper.client("1"))
|
||||
final HttpAdminClient adminClient = ClientBuilder.builder()
|
||||
.setAdminClientProvider(HttpAdminClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.build();
|
||||
final HttpBulkClient bulkClient = ClientBuilder.builder()
|
||||
.setBulkClientProvider(HttpBulkClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.provider(ExtendedHttpClientProvider.class)
|
||||
.build();
|
||||
try {
|
||||
Settings settings = Settings.builder()
|
||||
.put("index.number_of_shards", 1)
|
||||
.put("index.number_of_replicas", 0)
|
||||
.build();
|
||||
client.newIndex("test1", settings);
|
||||
client.shiftIndex("test", "test1", Collections.emptyList());
|
||||
client.newIndex("test2", settings);
|
||||
client.shiftIndex("test", "test2", Collections.emptyList());
|
||||
client.newIndex("test3", settings);
|
||||
client.shiftIndex("test", "test3", Collections.emptyList());
|
||||
client.newIndex("test4", settings);
|
||||
client.shiftIndex("test", "test4", Collections.emptyList());
|
||||
bulkClient.newIndex("test1", settings);
|
||||
adminClient.shiftIndex("test", "test1", Collections.emptyList());
|
||||
bulkClient.newIndex("test2", settings);
|
||||
adminClient.shiftIndex("test", "test2", Collections.emptyList());
|
||||
bulkClient.newIndex("test3", settings);
|
||||
adminClient.shiftIndex("test", "test3", Collections.emptyList());
|
||||
bulkClient.newIndex("test4", settings);
|
||||
adminClient.shiftIndex("test", "test4", Collections.emptyList());
|
||||
IndexPruneResult indexPruneResult =
|
||||
client.pruneIndex("test", "test4", 2, 2, true);
|
||||
adminClient.pruneIndex("test", "test4", 2, 2, true);
|
||||
assertTrue(indexPruneResult.getDeletedIndices().contains("test1"));
|
||||
assertTrue(indexPruneResult.getDeletedIndices().contains("test2"));
|
||||
assertFalse(indexPruneResult.getDeletedIndices().contains("test3"));
|
||||
assertFalse(indexPruneResult.getDeletedIndices().contains("test4"));
|
||||
List<Boolean> list = new ArrayList<>();
|
||||
for (String index : Arrays.asList("test1", "test2", "test3", "test4")) {
|
||||
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
|
||||
indicesExistsRequest.indices(index);
|
||||
IndicesExistsResponse indicesExistsResponse =
|
||||
client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
|
||||
logger.info("indices exists response for {} is {}", index, indicesExistsResponse.isExists());
|
||||
list.add(indicesExistsResponse.isExists());
|
||||
list.add(bulkClient.isIndexExists(index));
|
||||
}
|
||||
logger.info(list);
|
||||
assertFalse(list.get(0));
|
||||
assertFalse(list.get(1));
|
||||
assertTrue(list.get(2));
|
||||
assertTrue(list.get(3));
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} finally {
|
||||
client.close();
|
||||
if (client.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", client.getBulkController().getLastBulkError());
|
||||
bulkClient.close();
|
||||
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(client.getBulkController().getLastBulkError());
|
||||
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||
adminClient.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,16 +4,16 @@ import org.apache.logging.log4j.Level;
|
|||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.xbib.elx.api.IndexShiftResult;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
import org.xbib.elx.http.ExtendedHttpClient;
|
||||
import org.xbib.elx.http.ExtendedHttpClientProvider;
|
||||
import org.xbib.elx.http.HttpAdminClient;
|
||||
import org.xbib.elx.http.HttpAdminClientProvider;
|
||||
import org.xbib.elx.http.HttpBulkClient;
|
||||
import org.xbib.elx.http.HttpBulkClientProvider;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Map;
|
||||
|
@ -35,56 +35,54 @@ class IndexShiftTest {
|
|||
|
||||
@Test
|
||||
void testIndexShift() throws Exception {
|
||||
final ExtendedHttpClient client = ClientBuilder.builder()
|
||||
final HttpAdminClient adminClient = ClientBuilder.builder()
|
||||
.setAdminClientProvider(HttpAdminClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.build();
|
||||
final HttpBulkClient bulkClient = ClientBuilder.builder()
|
||||
.setBulkClientProvider(HttpBulkClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.provider(ExtendedHttpClientProvider.class)
|
||||
.build();
|
||||
try {
|
||||
Settings settings = Settings.builder()
|
||||
.put("index.number_of_shards", 1)
|
||||
.put("index.number_of_replicas", 0)
|
||||
.build();
|
||||
client.newIndex("test1234", settings);
|
||||
bulkClient.newIndex("test1234", settings);
|
||||
for (int i = 0; i < 1; i++) {
|
||||
client.index("test1234", helper.randomString(1), false,
|
||||
bulkClient.index("test1234", helper.randomString(1), false,
|
||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||
}
|
||||
client.flush();
|
||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
IndexShiftResult indexShiftResult =
|
||||
client.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c"));
|
||||
|
||||
adminClient.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c"));
|
||||
assertTrue(indexShiftResult.getNewAliases().contains("a"));
|
||||
assertTrue(indexShiftResult.getNewAliases().contains("b"));
|
||||
assertTrue(indexShiftResult.getNewAliases().contains("c"));
|
||||
assertTrue(indexShiftResult.getMovedAliases().isEmpty());
|
||||
|
||||
Map<String, String> aliases = client.getAliases("test1234");
|
||||
Map<String, String> aliases = adminClient.getAliases("test1234");
|
||||
logger.log(Level.DEBUG, "aliases = " + aliases);
|
||||
assertTrue(aliases.containsKey("a"));
|
||||
assertTrue(aliases.containsKey("b"));
|
||||
assertTrue(aliases.containsKey("c"));
|
||||
assertTrue(aliases.containsKey("test"));
|
||||
|
||||
String resolved = client.resolveAlias("test");
|
||||
String resolved = adminClient.resolveAlias("test");
|
||||
logger.log(Level.DEBUG, "resolved = " + resolved);
|
||||
aliases = client.getAliases(resolved);
|
||||
aliases = adminClient.getAliases(resolved);
|
||||
logger.log(Level.DEBUG, "aliases = " + aliases);
|
||||
assertTrue(aliases.containsKey("a"));
|
||||
assertTrue(aliases.containsKey("b"));
|
||||
assertTrue(aliases.containsKey("c"));
|
||||
assertTrue(aliases.containsKey("test"));
|
||||
|
||||
client.newIndex("test5678", settings);
|
||||
bulkClient.newIndex("test5678", settings);
|
||||
for (int i = 0; i < 1; i++) {
|
||||
client.index("test5678", helper.randomString(1), false,
|
||||
bulkClient.index("test5678", helper.randomString(1), false,
|
||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||
}
|
||||
client.flush();
|
||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
|
||||
indexShiftResult = client.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"),
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
indexShiftResult = adminClient.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"),
|
||||
(request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
|
||||
.index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
|
||||
);
|
||||
|
@ -94,31 +92,28 @@ class IndexShiftTest {
|
|||
assertTrue(indexShiftResult.getMovedAliases().contains("a"));
|
||||
assertTrue(indexShiftResult.getMovedAliases().contains("b"));
|
||||
assertTrue(indexShiftResult.getMovedAliases().contains("c"));
|
||||
|
||||
aliases = client.getAliases("test5678");
|
||||
aliases = adminClient.getAliases("test5678");
|
||||
assertTrue(aliases.containsKey("a"));
|
||||
assertTrue(aliases.containsKey("b"));
|
||||
assertTrue(aliases.containsKey("c"));
|
||||
assertTrue(aliases.containsKey("d"));
|
||||
assertTrue(aliases.containsKey("e"));
|
||||
assertTrue(aliases.containsKey("f"));
|
||||
|
||||
resolved = client.resolveAlias("test");
|
||||
aliases = client.getAliases(resolved);
|
||||
resolved = adminClient.resolveAlias("test");
|
||||
aliases = adminClient.getAliases(resolved);
|
||||
assertTrue(aliases.containsKey("a"));
|
||||
assertTrue(aliases.containsKey("b"));
|
||||
assertTrue(aliases.containsKey("c"));
|
||||
assertTrue(aliases.containsKey("d"));
|
||||
assertTrue(aliases.containsKey("e"));
|
||||
assertTrue(aliases.containsKey("f"));
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} finally {
|
||||
client.close();
|
||||
if (client.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", client.getBulkController().getLastBulkError());
|
||||
bulkClient.close();
|
||||
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(client.getBulkController().getLastBulkError());
|
||||
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||
adminClient.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,82 @@
|
|||
package org.xbib.elx.http.test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.search.SearchHit;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
import org.xbib.elx.common.Parameters;
|
||||
import org.xbib.elx.http.HttpBulkClient;
|
||||
import org.xbib.elx.http.HttpBulkClientProvider;
|
||||
import org.xbib.elx.http.HttpSearchClient;
|
||||
import org.xbib.elx.http.HttpSearchClientProvider;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@ExtendWith(TestExtension.class)
|
||||
class SearchTest {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(SearchTest.class.getName());
|
||||
|
||||
private static final Long ACTIONS = 100L;
|
||||
|
||||
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
|
||||
|
||||
private final TestExtension.Helper helper;
|
||||
|
||||
SearchTest(TestExtension.Helper helper) {
|
||||
this.helper = helper;
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDocStream() throws Exception {
|
||||
long numactions = ACTIONS;
|
||||
final HttpBulkClient bulkClient = ClientBuilder.builder()
|
||||
.setBulkClientProvider(HttpBulkClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||
.build();
|
||||
try (bulkClient) {
|
||||
bulkClient.newIndex("test");
|
||||
for (int i = 0; i < ACTIONS; i++) {
|
||||
bulkClient.index("test", null, false,
|
||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||
}
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
bulkClient.refreshIndex("test");
|
||||
assertEquals(numactions, bulkClient.getSearchableDocs("test"));
|
||||
}
|
||||
assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||
try (HttpSearchClient searchClient = ClientBuilder.builder()
|
||||
.setSearchClientProvider(HttpSearchClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.build()) {
|
||||
Stream<SearchHit> stream = searchClient.search(qb -> qb
|
||||
.setIndices("test")
|
||||
.setQuery(QueryBuilders.matchAllQuery()),
|
||||
TimeValue.timeValueMinutes(1), 10);
|
||||
long count = stream.count();
|
||||
assertEquals(numactions, count);
|
||||
Stream<String> ids = searchClient.getIds(qb -> qb
|
||||
.setIndices("test")
|
||||
.setQuery(QueryBuilders.matchAllQuery()));
|
||||
final AtomicInteger idcount = new AtomicInteger();
|
||||
ids.forEach(id -> {
|
||||
logger.info(id);
|
||||
idcount.incrementAndGet();
|
||||
});
|
||||
assertEquals(numactions, idcount.get());
|
||||
}
|
||||
}
|
||||
}
|
|
@ -2,14 +2,15 @@ package org.xbib.elx.http.test;
|
|||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.xbib.elx.api.IndexDefinition;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
import org.xbib.elx.http.ExtendedHttpClient;
|
||||
import org.xbib.elx.http.ExtendedHttpClientProvider;
|
||||
import org.xbib.elx.http.HttpAdminClient;
|
||||
import org.xbib.elx.http.HttpAdminClientProvider;
|
||||
import org.xbib.elx.http.HttpBulkClient;
|
||||
import org.xbib.elx.http.HttpBulkClientProvider;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
|
@ -29,47 +30,46 @@ class SmokeTest {
|
|||
|
||||
@Test
|
||||
void smokeTest() throws Exception {
|
||||
final ExtendedHttpClient client = ClientBuilder.builder()
|
||||
.provider(ExtendedHttpClientProvider.class)
|
||||
try (HttpAdminClient adminClient = ClientBuilder.builder()
|
||||
.setAdminClientProvider(HttpAdminClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.build();
|
||||
try {
|
||||
assertEquals(helper.getClusterName(), client.getClusterName());
|
||||
client.newIndex("test_smoke");
|
||||
client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
||||
client.flush();
|
||||
client.waitForResponses(30, TimeUnit.SECONDS);
|
||||
client.checkMapping("test_smoke");
|
||||
client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
|
||||
client.delete("test_smoke", "1");
|
||||
client.flush();
|
||||
client.waitForResponses(30, TimeUnit.SECONDS);
|
||||
client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS);
|
||||
client.delete("test_smoke", "1");
|
||||
client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
|
||||
client.flush();
|
||||
client.deleteIndex("test_smoke");
|
||||
IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder()
|
||||
.build());
|
||||
HttpBulkClient bulkClient = ClientBuilder.builder()
|
||||
.setBulkClientProvider(HttpBulkClientProvider.class)
|
||||
.put(helper.getHttpSettings())
|
||||
.build()) {
|
||||
IndexDefinition indexDefinition =
|
||||
adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.EMPTY);
|
||||
assertEquals(0, indexDefinition.getReplicaLevel());
|
||||
client.newIndex(indexDefinition);
|
||||
client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
|
||||
client.flush();
|
||||
client.waitForResponses(30, TimeUnit.SECONDS);
|
||||
client.updateReplicaLevel(indexDefinition, 2);
|
||||
int replica = client.getReplicaLevel(indexDefinition);
|
||||
assertEquals(helper.getClusterName(), adminClient.getClusterName());
|
||||
bulkClient.newIndex("test_smoke");
|
||||
bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
|
||||
adminClient.checkMapping("test_smoke");
|
||||
bulkClient.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
|
||||
bulkClient.delete("test_smoke", "1");
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
|
||||
bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
|
||||
bulkClient.delete("test_smoke", "1");
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
|
||||
adminClient.deleteIndex("test_smoke");
|
||||
bulkClient.newIndex(indexDefinition);
|
||||
bulkClient.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
|
||||
adminClient.updateReplicaLevel(indexDefinition, 2);
|
||||
int replica = adminClient.getReplicaLevel(indexDefinition);
|
||||
assertEquals(2, replica);
|
||||
client.deleteIndex(indexDefinition);
|
||||
assertEquals(0, client.getBulkController().getBulkMetric().getFailed().getCount());
|
||||
assertEquals(6, client.getBulkController().getBulkMetric().getSucceeded().getCount());
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} finally {
|
||||
client.close();
|
||||
if (client.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", client.getBulkController().getLastBulkError());
|
||||
assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount());
|
||||
assertEquals(6, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(client.getBulkController().getLastBulkError());
|
||||
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||
adminClient.deleteIndex(indexDefinition);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,69 +0,0 @@
|
|||
package org.xbib.elx.node;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.ToXContent;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.env.Environment;
|
||||
import org.elasticsearch.node.Node;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.xbib.elx.common.AbstractExtendedClient;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
|
||||
public class ExtendedNodeClient extends AbstractExtendedClient {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(ExtendedNodeClient.class.getName());
|
||||
|
||||
private Node node;
|
||||
|
||||
@Override
|
||||
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||
if (settings == null) {
|
||||
return null;
|
||||
}
|
||||
String version = System.getProperty("os.name")
|
||||
+ " " + System.getProperty("java.vm.name")
|
||||
+ " " + System.getProperty("java.vm.vendor")
|
||||
+ " " + System.getProperty("java.runtime.version")
|
||||
+ " " + System.getProperty("java.vm.version");
|
||||
Settings effectiveSettings = Settings.builder().put(settings)
|
||||
.put("node.client", true)
|
||||
.put("node.master", false)
|
||||
.put("node.data", false)
|
||||
.build();
|
||||
XContentBuilder builder = XContentFactory.jsonBuilder();
|
||||
effectiveSettings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true")));
|
||||
logger.info("creating node client on {} with effective settings {}",
|
||||
version, Strings.toString(builder));
|
||||
Collection<Class<? extends Plugin>> plugins = Collections.emptyList();
|
||||
this.node = new BulkNode(new Environment(effectiveSettings, null), plugins);
|
||||
try {
|
||||
node.start();
|
||||
} catch (Exception e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
return node.client();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void closeClient() throws IOException {
|
||||
if (node != null) {
|
||||
logger.debug("closing node client");
|
||||
node.close();
|
||||
}
|
||||
}
|
||||
|
||||
private static class BulkNode extends Node {
|
||||
|
||||
BulkNode(Environment env, Collection<Class<? extends Plugin>> classpathPlugins) {
|
||||
super(env, classpathPlugins, false);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,10 +0,0 @@
|
|||
package org.xbib.elx.node;
|
||||
|
||||
import org.xbib.elx.api.ExtendedClientProvider;
|
||||
|
||||
public class ExtendedNodeClientProvider implements ExtendedClientProvider<ExtendedNodeClient> {
|
||||
@Override
|
||||
public ExtendedNodeClient getExtendedClient() {
|
||||
return new ExtendedNodeClient();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,25 @@
|
|||
package org.xbib.elx.node;
|
||||
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.xbib.elx.common.AbstractAdminClient;
|
||||
import java.io.IOException;
|
||||
|
||||
public class NodeAdminClient extends AbstractAdminClient {
|
||||
|
||||
private final NodeClientHelper helper;
|
||||
|
||||
public NodeAdminClient() {
|
||||
this.helper = new NodeClientHelper();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ElasticsearchClient createClient(Settings settings) {
|
||||
return helper.createClient(settings, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void closeClient(Settings settings) throws IOException {
|
||||
helper.closeClient(settings);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
package org.xbib.elx.node;
|
||||
|
||||
import org.xbib.elx.api.AdminClientProvider;
|
||||
|
||||
public class NodeAdminClientProvider implements AdminClientProvider<NodeAdminClient> {
|
||||
|
||||
@Override
|
||||
public NodeAdminClient getClient() {
|
||||
return new NodeAdminClient();
|
||||
}
|
||||
}
|
25
elx-node/src/main/java/org/xbib/elx/node/NodeBulkClient.java
Normal file
25
elx-node/src/main/java/org/xbib/elx/node/NodeBulkClient.java
Normal file
|
@ -0,0 +1,25 @@
|
|||
package org.xbib.elx.node;
|
||||
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.xbib.elx.common.AbstractBulkClient;
|
||||
import java.io.IOException;
|
||||
|
||||
public class NodeBulkClient extends AbstractBulkClient {
|
||||
|
||||
private final NodeClientHelper helper;
|
||||
|
||||
public NodeBulkClient() {
|
||||
this.helper = new NodeClientHelper();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ElasticsearchClient createClient(Settings settings) {
|
||||
return helper.createClient(settings, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void closeClient(Settings settings) throws IOException {
|
||||
helper.closeClient(settings);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
package org.xbib.elx.node;
|
||||
|
||||
import org.xbib.elx.api.BulkClientProvider;
|
||||
|
||||
public class NodeBulkClientProvider implements BulkClientProvider<NodeBulkClient> {
|
||||
|
||||
@Override
|
||||
public NodeBulkClient getClient() {
|
||||
return new NodeBulkClient();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,98 @@
|
|||
package org.xbib.elx.node;
|
||||
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.cluster.ClusterName;
|
||||
import org.elasticsearch.common.network.NetworkModule;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.util.concurrent.EsExecutors;
|
||||
import org.elasticsearch.env.Environment;
|
||||
import org.elasticsearch.node.Node;
|
||||
import org.elasticsearch.node.NodeValidationException;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.transport.Netty4Plugin;
|
||||
import org.xbib.elx.common.Parameters;
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class NodeClientHelper {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(NodeClientHelper.class.getName());
|
||||
|
||||
private static Object configurationObject;
|
||||
|
||||
private static Node node;
|
||||
|
||||
private static final Map<String, ElasticsearchClient> clientMap = new HashMap<>();
|
||||
|
||||
public ElasticsearchClient createClient(Settings settings, Object object) {
|
||||
if (configurationObject == null) {
|
||||
configurationObject = object;
|
||||
}
|
||||
if (configurationObject instanceof ElasticsearchClient) {
|
||||
return (ElasticsearchClient) configurationObject;
|
||||
}
|
||||
return clientMap.computeIfAbsent(settings.get("cluster.name"),
|
||||
key -> innerCreateClient(settings));
|
||||
}
|
||||
|
||||
public void closeClient(Settings settings) throws IOException {
|
||||
ElasticsearchClient client = clientMap.remove(settings.get("cluster.name"));
|
||||
if (client != null) {
|
||||
logger.debug("closing node...");
|
||||
node.close();
|
||||
node = null;
|
||||
}
|
||||
}
|
||||
|
||||
private ElasticsearchClient innerCreateClient(Settings settings) {
|
||||
String version = System.getProperty("os.name")
|
||||
+ " " + System.getProperty("java.vm.name")
|
||||
+ " " + System.getProperty("java.vm.vendor")
|
||||
+ " " + System.getProperty("java.runtime.version")
|
||||
+ " " + System.getProperty("java.vm.version");
|
||||
Settings effectiveSettings = Settings.builder()
|
||||
.put(settings.filter(key -> !isPrivateSettings(key)))
|
||||
.put("node.master", false)
|
||||
.put("node.data", false)
|
||||
// "node.processors"
|
||||
.put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(),
|
||||
settings.get(EsExecutors.NODE_PROCESSORS_SETTING.getKey(),
|
||||
String.valueOf(Runtime.getRuntime().availableProcessors())))
|
||||
// "transport.type"
|
||||
.put(NetworkModule.TRANSPORT_TYPE_KEY,
|
||||
Netty4Plugin.NETTY_TRANSPORT_NAME)
|
||||
.build();
|
||||
logger.info("creating node client on {} with effective settings {}",
|
||||
version, effectiveSettings.toDelimitedString(','));
|
||||
Collection<Class<? extends Plugin>> plugins =
|
||||
Collections.singletonList(Netty4Plugin.class);
|
||||
node = new BulkNode(new Environment(effectiveSettings, null), plugins);
|
||||
try {
|
||||
node.start();
|
||||
return node.client();
|
||||
} catch (NodeValidationException e) {
|
||||
logger.log(Level.ERROR, e.getMessage(), e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static boolean isPrivateSettings(String key) {
|
||||
return key.equals(Parameters.MAX_ACTIONS_PER_REQUEST.name()) ||
|
||||
key.equals(Parameters.MAX_CONCURRENT_REQUESTS.name()) ||
|
||||
key.equals(Parameters.MAX_VOLUME_PER_REQUEST.name()) ||
|
||||
key.equals(Parameters.FLUSH_INTERVAL.name());
|
||||
}
|
||||
|
||||
private static class BulkNode extends Node {
|
||||
|
||||
BulkNode(Environment env, Collection<Class<? extends Plugin>> classpathPlugins) {
|
||||
super(env, classpathPlugins, false);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,25 @@
|
|||
package org.xbib.elx.node;
|
||||
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.xbib.elx.common.AbstractSearchClient;
|
||||
import java.io.IOException;
|
||||
|
||||
public class NodeSearchClient extends AbstractSearchClient {
|
||||
|
||||
private final NodeClientHelper helper;
|
||||
|
||||
public NodeSearchClient() {
|
||||
this.helper = new NodeClientHelper();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ElasticsearchClient createClient(Settings settings) {
|
||||
return helper.createClient(settings, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void closeClient(Settings settings) throws IOException {
|
||||
helper.closeClient(settings);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
package org.xbib.elx.node;
|
||||
|
||||
import org.xbib.elx.api.SearchClientProvider;
|
||||
|
||||
public class NodeSearchClientProvider implements SearchClientProvider<NodeSearchClient> {
|
||||
|
||||
@Override
|
||||
public NodeSearchClient getClient() {
|
||||
return new NodeSearchClient();
|
||||
}
|
||||
}
|
|
@ -0,0 +1 @@
|
|||
org.xbib.elx.node.NodeAdminClientProvider
|
|
@ -0,0 +1 @@
|
|||
org.xbib.elx.node.NodeBulkClientProvider
|
|
@ -1 +0,0 @@
|
|||
org.xbib.elx.node.ExtendedNodeClientProvider
|
|
@ -0,0 +1 @@
|
|||
org.xbib.elx.node.NodeSearchClientProvider
|
|
@ -0,0 +1,180 @@
|
|||
package org.xbib.elx.node.test;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
import org.xbib.elx.common.Parameters;
|
||||
import org.xbib.elx.node.NodeAdminClient;
|
||||
import org.xbib.elx.node.NodeAdminClientProvider;
|
||||
import org.xbib.elx.node.NodeBulkClient;
|
||||
import org.xbib.elx.node.NodeBulkClientProvider;
|
||||
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@ExtendWith(TestExtension.class)
|
||||
class BulkClientTest {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getName());
|
||||
|
||||
private static final Long ACTIONS = 10000L;
|
||||
|
||||
private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
|
||||
|
||||
private final TestExtension.Helper helper;
|
||||
|
||||
BulkClientTest(TestExtension.Helper helper) {
|
||||
this.helper = helper;
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSingleDoc() throws Exception {
|
||||
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||
.put(helper.getNodeSettings())
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
|
||||
.build();
|
||||
try {
|
||||
bulkClient.newIndex("test");
|
||||
bulkClient.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
} finally {
|
||||
assertEquals(1, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||
bulkClient.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testNewIndex() throws Exception {
|
||||
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||
.put(helper.getNodeSettings())
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
||||
.build();
|
||||
bulkClient.newIndex("test");
|
||||
bulkClient.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testMapping() throws Exception {
|
||||
try (NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
|
||||
.setAdminClientProvider(NodeAdminClientProvider.class)
|
||||
.put(helper.getNodeSettings())
|
||||
.build();
|
||||
NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||
.put(helper.getNodeSettings())
|
||||
.build()) {
|
||||
XContentBuilder builder = JsonXContent.contentBuilder()
|
||||
.startObject()
|
||||
.startObject("properties")
|
||||
.startObject("location")
|
||||
.field("type", "geo_point")
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject();
|
||||
bulkClient.newIndex("test", Settings.EMPTY, builder);
|
||||
assertTrue(adminClient.getMapping("test", "_doc").containsKey("properties"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRandomDocs() throws Exception {
|
||||
long numactions = ACTIONS;
|
||||
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||
.put(helper.getNodeSettings())
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
||||
.build();
|
||||
try {
|
||||
bulkClient.newIndex("test");
|
||||
for (int i = 0; i < ACTIONS; i++) {
|
||||
bulkClient.index("test", null, false,
|
||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||
}
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
} finally {
|
||||
assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||
bulkClient.refreshIndex("test");
|
||||
assertEquals(numactions, bulkClient.getSearchableDocs("test"));
|
||||
bulkClient.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testThreadedRandomDocs() throws Exception {
|
||||
int maxthreads = Runtime.getRuntime().availableProcessors();
|
||||
final long actions = ACTIONS;
|
||||
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||
.put(helper.getNodeSettings())
|
||||
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads)
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||
.build();
|
||||
try {
|
||||
Settings settings = Settings.builder()
|
||||
.put("index.number_of_shards", 1)
|
||||
.put("index.number_of_replicas", 0)
|
||||
.build();
|
||||
bulkClient.newIndex("test", settings);
|
||||
bulkClient.startBulk("test", 0, 1000);
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
|
||||
final CountDownLatch latch = new CountDownLatch(maxthreads);
|
||||
for (int i = 0; i < maxthreads; i++) {
|
||||
executorService.execute(() -> {
|
||||
for (int i1 = 0; i1 < actions; i1++) {
|
||||
bulkClient.index("test", null, false,
|
||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||
}
|
||||
latch.countDown();
|
||||
});
|
||||
}
|
||||
logger.info("waiting for latch...");
|
||||
if (latch.await(30L, TimeUnit.SECONDS)) {
|
||||
logger.info("flush...");
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
logger.info("got all responses, executor service shutdown...");
|
||||
executorService.shutdown();
|
||||
executorService.awaitTermination(30L, TimeUnit.SECONDS);
|
||||
logger.info("pool is shut down");
|
||||
} else {
|
||||
logger.warn("latch timeout");
|
||||
}
|
||||
bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS);
|
||||
assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||
} finally {
|
||||
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||
bulkClient.refreshIndex("test");
|
||||
assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test"));
|
||||
bulkClient.close();
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,236 +0,0 @@
|
|||
package org.xbib.elx.node.test;
|
||||
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
|
||||
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
|
||||
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
|
||||
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
|
||||
import org.elasticsearch.action.search.SearchAction;
|
||||
import org.elasticsearch.action.search.SearchRequest;
|
||||
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
import org.xbib.elx.common.Parameters;
|
||||
import org.xbib.elx.node.ExtendedNodeClient;
|
||||
import org.xbib.elx.node.ExtendedNodeClientProvider;
|
||||
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@ExtendWith(TestExtension.class)
|
||||
class ClientTest {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
|
||||
|
||||
private static final Long ACTIONS = 10000L;
|
||||
|
||||
private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
|
||||
|
||||
private final TestExtension.Helper helper;
|
||||
|
||||
ClientTest(TestExtension.Helper helper) {
|
||||
this.helper = helper;
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSingleDoc() throws Exception {
|
||||
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
|
||||
.provider(ExtendedNodeClientProvider.class)
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
|
||||
.build();
|
||||
try {
|
||||
client.newIndex("test");
|
||||
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
||||
client.flush();
|
||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} finally {
|
||||
assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount());
|
||||
if (client.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", client.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(client.getBulkController().getLastBulkError());
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testNewIndex() throws Exception {
|
||||
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
|
||||
.provider(ExtendedNodeClientProvider.class)
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
||||
.build();
|
||||
client.newIndex("test");
|
||||
client.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testNewIndexWithSettings() throws Exception {
|
||||
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
|
||||
.provider(ExtendedNodeClientProvider.class)
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
||||
.build();
|
||||
Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
|
||||
client.newIndex("test", settings);
|
||||
GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
|
||||
.indices("test");
|
||||
GetSettingsResponse getSettingsResponse =
|
||||
client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet();
|
||||
logger.log(Level.INFO, "settings=" + getSettingsResponse.getSetting("test", "index.number_of_shards"));
|
||||
assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards"));
|
||||
client.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testNewIndexWithSettingsAndMappings() throws Exception {
|
||||
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
|
||||
.provider(ExtendedNodeClientProvider.class)
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
||||
.build();
|
||||
Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
|
||||
XContentBuilder builder = JsonXContent.contentBuilder()
|
||||
.startObject()
|
||||
.field("date_detection", false)
|
||||
.startObject("properties")
|
||||
.startObject("location")
|
||||
.field("type", "geo_point")
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject();
|
||||
client.newIndex("test", settings, builder);
|
||||
GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
|
||||
.indices("test");
|
||||
GetSettingsResponse getSettingsResponse =
|
||||
client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet();
|
||||
logger.log(Level.INFO, "settings=" + getSettingsResponse.getSetting("test", "index.number_of_shards"));
|
||||
assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards"));
|
||||
GetMappingsRequest getMappingsRequest = new GetMappingsRequest()
|
||||
.indices("test");
|
||||
GetMappingsResponse getMappingsResponse =
|
||||
client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
|
||||
logger.info("mappings={}", getMappingsResponse.getMappings());
|
||||
assertTrue(getMappingsResponse.getMappings().get("test").containsKey("_doc"));
|
||||
client.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRandomDocs() throws Exception {
|
||||
long numactions = ACTIONS;
|
||||
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
|
||||
.provider(ExtendedNodeClientProvider.class)
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
||||
.build();
|
||||
try {
|
||||
client.newIndex("test");
|
||||
for (int i = 0; i < ACTIONS; i++) {
|
||||
client.index("test", null, false,
|
||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||
}
|
||||
client.flush();
|
||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} finally {
|
||||
assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
|
||||
if (client.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", client.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(client.getBulkController().getLastBulkError());
|
||||
client.refreshIndex("test");
|
||||
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
|
||||
.setQuery(QueryBuilders.matchAllQuery()).setSize(0);
|
||||
assertEquals(numactions, searchRequestBuilder.execute().actionGet().getHits().getTotalHits().value);
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testThreadedRandomDocs() throws Exception {
|
||||
int maxthreads = Runtime.getRuntime().availableProcessors();
|
||||
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
|
||||
final long actions = ACTIONS;
|
||||
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
|
||||
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
|
||||
.provider(ExtendedNodeClientProvider.class)
|
||||
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads)
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
||||
.build();
|
||||
try {
|
||||
Settings settings = Settings.builder()
|
||||
.put("index.number_of_shards", 1)
|
||||
.put("index.number_of_replicas", 0)
|
||||
.build();
|
||||
client.newIndex("test", settings, (String)null)
|
||||
.startBulk("test", 0, 1000);
|
||||
logger.info("index created");
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
|
||||
final CountDownLatch latch = new CountDownLatch(maxthreads);
|
||||
for (int i = 0; i < maxthreads; i++) {
|
||||
executorService.execute(() -> {
|
||||
for (int i1 = 0; i1 < actions; i1++) {
|
||||
client.index("test", null, false,
|
||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||
}
|
||||
latch.countDown();
|
||||
});
|
||||
}
|
||||
logger.info("waiting for latch...");
|
||||
if (latch.await(30L, TimeUnit.SECONDS)) {
|
||||
logger.info("flush...");
|
||||
client.flush();
|
||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
logger.info("got all responses, executor service shutdown...");
|
||||
executorService.shutdown();
|
||||
executorService.awaitTermination(30L, TimeUnit.SECONDS);
|
||||
logger.info("pool is shut down");
|
||||
} else {
|
||||
logger.warn("latch timeout");
|
||||
}
|
||||
client.stopBulk("test", 30L, TimeUnit.SECONDS);
|
||||
assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} finally {
|
||||
if (client.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", client.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(client.getBulkController().getLastBulkError());
|
||||
assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
|
||||
logger.log(Level.INFO, "refreshing index test");
|
||||
client.refreshIndex("test");
|
||||
SearchSourceBuilder builder = new SearchSourceBuilder()
|
||||
.query(QueryBuilders.matchAllQuery())
|
||||
.size(0)
|
||||
.trackTotalHits(true);
|
||||
SearchRequest searchRequest = new SearchRequest()
|
||||
.indices("test")
|
||||
.source(builder);
|
||||
SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
|
||||
assertEquals(maxthreads * actions, searchResponse.getHits().getTotalHits().value);
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
}
|
|
@ -2,18 +2,12 @@ package org.xbib.elx.node.test;
|
|||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.action.search.SearchAction;
|
||||
import org.elasticsearch.action.search.SearchRequest;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
import org.xbib.elx.common.Parameters;
|
||||
import org.xbib.elx.node.ExtendedNodeClient;
|
||||
import org.xbib.elx.node.ExtendedNodeClientProvider;
|
||||
import org.xbib.elx.node.NodeBulkClient;
|
||||
import org.xbib.elx.node.NodeBulkClientProvider;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
|
@ -26,10 +20,10 @@ class DuplicateIDTest {
|
|||
|
||||
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName());
|
||||
|
||||
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
|
||||
|
||||
private static final Long ACTIONS = 100L;
|
||||
|
||||
private static final Long MAX_ACTIONS_PER_REQUEST = 5L;
|
||||
|
||||
private final TestExtension.Helper helper;
|
||||
|
||||
DuplicateIDTest(TestExtension.Helper helper) {
|
||||
|
@ -39,40 +33,28 @@ class DuplicateIDTest {
|
|||
@Test
|
||||
void testDuplicateDocIDs() throws Exception {
|
||||
long numactions = ACTIONS;
|
||||
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
|
||||
.provider(ExtendedNodeClientProvider.class)
|
||||
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||
.put(helper.getNodeSettings())
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||
.build();
|
||||
try {
|
||||
client.newIndex("test");
|
||||
bulkClient.newIndex("test");
|
||||
for (int i = 0; i < ACTIONS; i++) {
|
||||
client.index("test", helper.randomString(1), false,
|
||||
bulkClient.index("test", helper.randomString(1), false,
|
||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||
}
|
||||
client.flush();
|
||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
client.refreshIndex("test");
|
||||
SearchSourceBuilder builder = new SearchSourceBuilder()
|
||||
.query(QueryBuilders.matchAllQuery())
|
||||
.size(0)
|
||||
.trackTotalHits(true);
|
||||
SearchRequest searchRequest = new SearchRequest()
|
||||
.indices("test")
|
||||
.source(builder);
|
||||
SearchResponse searchResponse =
|
||||
helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet();
|
||||
long hits = searchResponse.getHits().getTotalHits().value;
|
||||
logger.info("hits = {}", hits);
|
||||
assertTrue(hits < ACTIONS);
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
bulkClient.refreshIndex("test");
|
||||
assertTrue(bulkClient.getSearchableDocs("test") < ACTIONS);
|
||||
} finally {
|
||||
client.close();
|
||||
assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
|
||||
if (client.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", client.getBulkController().getLastBulkError());
|
||||
bulkClient.close();
|
||||
assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(client.getBulkController().getLastBulkError());
|
||||
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -2,17 +2,15 @@ package org.xbib.elx.node.test;
|
|||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
|
||||
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
|
||||
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.xbib.elx.api.IndexPruneResult;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
import org.xbib.elx.node.ExtendedNodeClient;
|
||||
import org.xbib.elx.node.ExtendedNodeClientProvider;
|
||||
import org.xbib.elx.node.NodeAdminClient;
|
||||
import org.xbib.elx.node.NodeAdminClientProvider;
|
||||
import org.xbib.elx.node.NodeBulkClient;
|
||||
import org.xbib.elx.node.NodeBulkClientProvider;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
|
@ -37,49 +35,49 @@ class IndexPruneTest {
|
|||
|
||||
@Test
|
||||
void testPrune() throws IOException {
|
||||
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
|
||||
.provider(ExtendedNodeClientProvider.class)
|
||||
final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
|
||||
.setAdminClientProvider(NodeAdminClientProvider.class)
|
||||
.put(helper.getNodeSettings())
|
||||
.build();
|
||||
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||
.put(helper.getNodeSettings())
|
||||
.build();
|
||||
try {
|
||||
Settings settings = Settings.builder()
|
||||
.put("index.number_of_shards", 1)
|
||||
.put("index.number_of_replicas", 0)
|
||||
.build();
|
||||
client.newIndex("test_prune1", settings);
|
||||
client.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
|
||||
client.newIndex("test_prune2", settings);
|
||||
client.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
|
||||
client.newIndex("test_prune3", settings);
|
||||
client.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
|
||||
client.newIndex("test_prune4", settings);
|
||||
client.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
|
||||
bulkClient.newIndex("test_prune1", settings);
|
||||
adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
|
||||
bulkClient.newIndex("test_prune2", settings);
|
||||
adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
|
||||
bulkClient.newIndex("test_prune3", settings);
|
||||
adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
|
||||
bulkClient.newIndex("test_prune4", settings);
|
||||
adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
|
||||
IndexPruneResult indexPruneResult =
|
||||
client.pruneIndex("test_prune", "test_prune4", 2, 2, true);
|
||||
adminClient.pruneIndex("test_prune", "test_prune4", 2, 2, true);
|
||||
assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1"));
|
||||
assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2"));
|
||||
assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3"));
|
||||
assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune4"));
|
||||
List<Boolean> list = new ArrayList<>();
|
||||
for (String index : Arrays.asList("test_prune1", "test_prune2", "test_prune3", "test_prune4")) {
|
||||
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
|
||||
indicesExistsRequest.indices(index);
|
||||
IndicesExistsResponse indicesExistsResponse =
|
||||
client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
|
||||
list.add(indicesExistsResponse.isExists());
|
||||
list.add(bulkClient.isIndexExists(index));
|
||||
}
|
||||
logger.info(list);
|
||||
assertFalse(list.get(0));
|
||||
assertFalse(list.get(1));
|
||||
assertTrue(list.get(2));
|
||||
assertTrue(list.get(3));
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} finally {
|
||||
client.close();
|
||||
if (client.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", client.getBulkController().getLastBulkError());
|
||||
adminClient.close();
|
||||
bulkClient.close();
|
||||
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(client.getBulkController().getLastBulkError());
|
||||
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -3,15 +3,16 @@ package org.xbib.elx.node.test;
|
|||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.xbib.elx.api.IndexShiftResult;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
import org.xbib.elx.node.ExtendedNodeClient;
|
||||
import org.xbib.elx.node.ExtendedNodeClientProvider;
|
||||
import org.xbib.elx.node.NodeAdminClient;
|
||||
import org.xbib.elx.node.NodeAdminClientProvider;
|
||||
import org.xbib.elx.node.NodeBulkClient;
|
||||
import org.xbib.elx.node.NodeBulkClientProvider;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Map;
|
||||
|
@ -33,50 +34,51 @@ class IndexShiftTest {
|
|||
|
||||
@Test
|
||||
void testIndexShift() throws Exception {
|
||||
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
|
||||
.provider(ExtendedNodeClientProvider.class)
|
||||
final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
|
||||
.setAdminClientProvider(NodeAdminClientProvider.class)
|
||||
.put(helper.getNodeSettings())
|
||||
.build();
|
||||
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||
.put(helper.getNodeSettings())
|
||||
.build();
|
||||
try {
|
||||
Settings settings = Settings.builder()
|
||||
.put("index.number_of_shards", 1)
|
||||
.put("index.number_of_replicas", 0)
|
||||
.build();
|
||||
client.newIndex("test_shift", settings);
|
||||
bulkClient.newIndex("test_shift", settings);
|
||||
for (int i = 0; i < 1; i++) {
|
||||
client.index("test_shift", helper.randomString(1), false,
|
||||
bulkClient.index("test_shift", helper.randomString(1), false,
|
||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||
}
|
||||
client.flush();
|
||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
IndexShiftResult indexShiftResult =
|
||||
client.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c"));
|
||||
adminClient.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c"));
|
||||
assertTrue(indexShiftResult.getNewAliases().contains("a"));
|
||||
assertTrue(indexShiftResult.getNewAliases().contains("b"));
|
||||
assertTrue(indexShiftResult.getNewAliases().contains("c"));
|
||||
assertTrue(indexShiftResult.getMovedAliases().isEmpty());
|
||||
|
||||
Map<String, String> aliases = client.getAliases("test_shift");
|
||||
Map<String, String> aliases = adminClient.getAliases("test_shift");
|
||||
assertTrue(aliases.containsKey("a"));
|
||||
assertTrue(aliases.containsKey("b"));
|
||||
assertTrue(aliases.containsKey("c"));
|
||||
assertTrue(aliases.containsKey("test"));
|
||||
|
||||
String resolved = client.resolveAlias("test");
|
||||
aliases = client.getAliases(resolved);
|
||||
String resolved = adminClient.resolveAlias("test");
|
||||
aliases = adminClient.getAliases(resolved);
|
||||
assertTrue(aliases.containsKey("a"));
|
||||
assertTrue(aliases.containsKey("b"));
|
||||
assertTrue(aliases.containsKey("c"));
|
||||
assertTrue(aliases.containsKey("test"));
|
||||
|
||||
client.newIndex("test_shift2", settings);
|
||||
bulkClient.newIndex("test_shift2", settings);
|
||||
for (int i = 0; i < 1; i++) {
|
||||
client.index("test_shift2", helper.randomString(1), false,
|
||||
bulkClient.index("test_shift2", helper.randomString(1), false,
|
||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||
}
|
||||
client.flush();
|
||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
|
||||
indexShiftResult = client.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"),
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
indexShiftResult = adminClient.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"),
|
||||
(request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
|
||||
.index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
|
||||
);
|
||||
|
@ -86,32 +88,28 @@ class IndexShiftTest {
|
|||
assertTrue(indexShiftResult.getMovedAliases().contains("a"));
|
||||
assertTrue(indexShiftResult.getMovedAliases().contains("b"));
|
||||
assertTrue(indexShiftResult.getMovedAliases().contains("c"));
|
||||
|
||||
aliases = client.getAliases("test_shift2");
|
||||
aliases = adminClient.getAliases("test_shift2");
|
||||
assertTrue(aliases.containsKey("a"));
|
||||
assertTrue(aliases.containsKey("b"));
|
||||
assertTrue(aliases.containsKey("c"));
|
||||
assertTrue(aliases.containsKey("d"));
|
||||
assertTrue(aliases.containsKey("e"));
|
||||
assertTrue(aliases.containsKey("f"));
|
||||
|
||||
resolved = client.resolveAlias("test");
|
||||
aliases = client.getAliases(resolved);
|
||||
resolved = adminClient.resolveAlias("test");
|
||||
aliases = adminClient.getAliases(resolved);
|
||||
assertTrue(aliases.containsKey("a"));
|
||||
assertTrue(aliases.containsKey("b"));
|
||||
assertTrue(aliases.containsKey("c"));
|
||||
assertTrue(aliases.containsKey("d"));
|
||||
assertTrue(aliases.containsKey("e"));
|
||||
assertTrue(aliases.containsKey("f"));
|
||||
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} finally {
|
||||
client.close();
|
||||
if (client.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", client.getBulkController().getLastBulkError());
|
||||
adminClient.close();
|
||||
bulkClient.close();
|
||||
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(client.getBulkController().getLastBulkError());
|
||||
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,82 @@
|
|||
package org.xbib.elx.node.test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.search.SearchHit;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
import org.xbib.elx.common.Parameters;
|
||||
import org.xbib.elx.node.NodeBulkClient;
|
||||
import org.xbib.elx.node.NodeBulkClientProvider;
|
||||
import org.xbib.elx.node.NodeSearchClient;
|
||||
import org.xbib.elx.node.NodeSearchClientProvider;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@ExtendWith(TestExtension.class)
|
||||
class SearchTest {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(SearchTest.class.getName());
|
||||
|
||||
private static final Long ACTIONS = 100L;
|
||||
|
||||
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
|
||||
|
||||
private final TestExtension.Helper helper;
|
||||
|
||||
SearchTest(TestExtension.Helper helper) {
|
||||
this.helper = helper;
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDocStream() throws Exception {
|
||||
long numactions = ACTIONS;
|
||||
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||
.put(helper.getNodeSettings())
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||
.build();
|
||||
try (bulkClient) {
|
||||
bulkClient.newIndex("test");
|
||||
for (int i = 0; i < ACTIONS; i++) {
|
||||
bulkClient.index("test", null, false,
|
||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||
}
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
bulkClient.refreshIndex("test");
|
||||
assertEquals(numactions, bulkClient.getSearchableDocs("test"));
|
||||
}
|
||||
assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||
try (NodeSearchClient searchClient = ClientBuilder.builder(helper.client("1"))
|
||||
.setSearchClientProvider(NodeSearchClientProvider.class)
|
||||
.put(helper.getNodeSettings())
|
||||
.build()) {
|
||||
Stream<SearchHit> stream = searchClient.search(qb -> qb
|
||||
.setIndices("test")
|
||||
.setQuery(QueryBuilders.matchAllQuery()),
|
||||
TimeValue.timeValueMinutes(1), 10);
|
||||
long count = stream.count();
|
||||
assertEquals(numactions, count);
|
||||
Stream<String> ids = searchClient.getIds(qb -> qb
|
||||
.setIndices("test")
|
||||
.setQuery(QueryBuilders.matchAllQuery()));
|
||||
final AtomicInteger idcount = new AtomicInteger();
|
||||
ids.forEach(id -> {
|
||||
logger.info(id);
|
||||
idcount.incrementAndGet();
|
||||
});
|
||||
assertEquals(numactions, idcount.get());
|
||||
}
|
||||
}
|
||||
}
|
|
@ -2,14 +2,15 @@ package org.xbib.elx.node.test;
|
|||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
import org.xbib.elx.api.IndexDefinition;
|
||||
import org.xbib.elx.node.ExtendedNodeClient;
|
||||
import org.xbib.elx.node.ExtendedNodeClientProvider;
|
||||
import org.xbib.elx.node.NodeAdminClient;
|
||||
import org.xbib.elx.node.NodeAdminClientProvider;
|
||||
import org.xbib.elx.node.NodeBulkClient;
|
||||
import org.xbib.elx.node.NodeBulkClientProvider;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
|
@ -29,46 +30,46 @@ class SmokeTest {
|
|||
|
||||
@Test
|
||||
void smokeTest() throws Exception {
|
||||
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
|
||||
.provider(ExtendedNodeClientProvider.class)
|
||||
try (NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
|
||||
.setAdminClientProvider(NodeAdminClientProvider.class)
|
||||
.put(helper.getNodeSettings())
|
||||
.build();
|
||||
try {
|
||||
assertEquals(helper.getClusterName(), client.getClusterName());
|
||||
client.newIndex("test_smoke");
|
||||
client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
||||
client.flush();
|
||||
client.waitForResponses(30, TimeUnit.SECONDS);
|
||||
client.checkMapping("test_smoke");
|
||||
client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
|
||||
client.delete("test_smoke", "1");
|
||||
client.flush();
|
||||
client.waitForResponses(30, TimeUnit.SECONDS);
|
||||
client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS);
|
||||
client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
|
||||
client.delete("test_smoke", "1");
|
||||
client.flush();
|
||||
client.deleteIndex("test_smoke");
|
||||
IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder()
|
||||
.build());
|
||||
NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||
.put(helper.getNodeSettings())
|
||||
.build()) {
|
||||
IndexDefinition indexDefinition =
|
||||
adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.EMPTY);
|
||||
assertEquals(0, indexDefinition.getReplicaLevel());
|
||||
client.newIndex(indexDefinition);
|
||||
client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
|
||||
client.flush();
|
||||
client.waitForResponses(30, TimeUnit.SECONDS);
|
||||
client.updateReplicaLevel(indexDefinition, 2);
|
||||
int replica = client.getReplicaLevel(indexDefinition);
|
||||
assertEquals(helper.getClusterName(), adminClient.getClusterName());
|
||||
bulkClient.newIndex("test_smoke");
|
||||
bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
|
||||
adminClient.checkMapping("test_smoke");
|
||||
bulkClient.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
|
||||
bulkClient.delete("test_smoke", "1");
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
|
||||
bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
|
||||
bulkClient.delete("test_smoke", "1");
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
|
||||
adminClient.deleteIndex("test_smoke");
|
||||
bulkClient.newIndex(indexDefinition);
|
||||
bulkClient.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
|
||||
adminClient.updateReplicaLevel(indexDefinition, 2);
|
||||
int replica = adminClient.getReplicaLevel(indexDefinition);
|
||||
assertEquals(2, replica);
|
||||
client.deleteIndex(indexDefinition);
|
||||
assertEquals(0, client.getBulkController().getBulkMetric().getFailed().getCount());
|
||||
assertEquals(6, client.getBulkController().getBulkMetric().getSucceeded().getCount());
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} finally {
|
||||
client.close();
|
||||
if (client.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", client.getBulkController().getLastBulkError());
|
||||
assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount());
|
||||
assertEquals(6, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(client.getBulkController().getLastBulkError());
|
||||
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||
adminClient.deleteIndex(indexDefinition);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -193,6 +193,7 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft
|
|||
.put("path.home", getHome())
|
||||
.put("cluster.initial_master_nodes", "1")
|
||||
.put("discovery.seed_hosts", "127.0.0.1:9300")
|
||||
.put("node.max_local_storage_nodes", "2")
|
||||
.build();
|
||||
}
|
||||
|
||||
|
|
|
@ -1,11 +0,0 @@
|
|||
package org.xbib.elx.transport;
|
||||
|
||||
import org.xbib.elx.api.ExtendedClientProvider;
|
||||
|
||||
public class ExtendedTransportClientProvider implements ExtendedClientProvider<ExtendedTransportClient> {
|
||||
|
||||
@Override
|
||||
public ExtendedTransportClient getExtendedClient() {
|
||||
return new ExtendedTransportClient();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,36 @@
|
|||
package org.xbib.elx.transport;
|
||||
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.client.transport.TransportClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.xbib.elx.common.AbstractAdminClient;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Transport admin client.
|
||||
*/
|
||||
public class TransportAdminClient extends AbstractAdminClient {
|
||||
|
||||
private final TransportClientHelper helper;
|
||||
|
||||
public TransportAdminClient() {
|
||||
this.helper = new TransportClientHelper();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||
return helper.createClient(settings);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(Settings settings) throws IOException {
|
||||
super.init(settings);
|
||||
helper.init((TransportClient) getClient(), settings);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void closeClient(Settings settings) {
|
||||
helper.closeClient(settings);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
package org.xbib.elx.transport;
|
||||
|
||||
import org.xbib.elx.api.AdminClientProvider;
|
||||
|
||||
public class TransportAdminClientProvider implements AdminClientProvider<TransportAdminClient> {
|
||||
|
||||
@Override
|
||||
public TransportAdminClient getClient() {
|
||||
return new TransportAdminClient();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,35 @@
|
|||
package org.xbib.elx.transport;
|
||||
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.client.transport.TransportClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.xbib.elx.common.AbstractBulkClient;
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Transport search client with additional methods.
|
||||
*/
|
||||
public class TransportBulkClient extends AbstractBulkClient {
|
||||
|
||||
private final TransportClientHelper helper;
|
||||
|
||||
public TransportBulkClient() {
|
||||
this.helper = new TransportClientHelper();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||
return helper.createClient(settings);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(Settings settings) throws IOException {
|
||||
super.init(settings);
|
||||
helper.init((TransportClient) getClient(), settings);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void closeClient(Settings settings) {
|
||||
helper.closeClient(settings);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
package org.xbib.elx.transport;
|
||||
|
||||
import org.xbib.elx.api.BulkClientProvider;
|
||||
|
||||
public class TransportBulkClientProvider implements BulkClientProvider<TransportBulkClient> {
|
||||
|
||||
@Override
|
||||
public TransportBulkClient getClient() {
|
||||
return new TransportBulkClient();
|
||||
}
|
||||
}
|
|
@ -7,6 +7,7 @@ import org.elasticsearch.Version;
|
|||
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
|
||||
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder;
|
||||
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.elasticsearch.client.transport.TransportClient;
|
||||
|
@ -23,7 +24,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
|
|||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.transport.Netty4Plugin;
|
||||
import org.xbib.elx.common.AbstractExtendedClient;
|
||||
import org.xbib.elx.common.util.NetworkUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
|
@ -31,16 +31,19 @@ import java.net.InetAddress;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Transport client with additional methods using the BulkProcessor.
|
||||
*/
|
||||
public class ExtendedTransportClient extends AbstractExtendedClient {
|
||||
public class TransportClientHelper {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(ExtendedTransportClient.class.getName());
|
||||
private static final Logger logger = LogManager.getLogger(TransportClientHelper.class.getName());
|
||||
|
||||
private static final Map<String, ElasticsearchClient> clientMap = new HashMap<>();
|
||||
|
||||
@Override
|
||||
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||
if (settings != null) {
|
||||
String systemIdentifier = System.getProperty("os.name")
|
||||
|
@ -49,11 +52,9 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
|
|||
+ " " + System.getProperty("java.vm.version")
|
||||
+ " Elasticsearch " + Version.CURRENT.toString();
|
||||
Settings transportClientSettings = getTransportClientSettings(settings);
|
||||
//XContentBuilder settingsBuilder = XContentFactory.jsonBuilder().startObject();
|
||||
XContentBuilder effectiveSettingsBuilder = XContentFactory.jsonBuilder().startObject();
|
||||
logger.log(Level.INFO, "creating transport client on {} with settings {}",
|
||||
systemIdentifier,
|
||||
//Strings.toString(settings.toXContent(settingsBuilder, ToXContent.EMPTY_PARAMS).endObject()),
|
||||
Strings.toString(transportClientSettings.toXContent(effectiveSettingsBuilder,
|
||||
ToXContent.EMPTY_PARAMS).endObject()));
|
||||
return new MyTransportClient(transportClientSettings, Collections.singletonList(Netty4Plugin.class));
|
||||
|
@ -61,29 +62,22 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
|
|||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void closeClient() {
|
||||
if (getClient() != null) {
|
||||
TransportClient client = (TransportClient) getClient();
|
||||
client.close();
|
||||
client.threadPool().shutdown();
|
||||
public void closeClient(Settings settings) {
|
||||
ElasticsearchClient client = clientMap.remove(settings.get("cluster.name"));
|
||||
if (client != null) {
|
||||
if (client instanceof Client) {
|
||||
((Client) client).close();
|
||||
}
|
||||
client.threadPool().shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedTransportClient init(Settings settings) throws IOException {
|
||||
super.init(settings);
|
||||
// additional auto-connect
|
||||
try {
|
||||
Collection<TransportAddress> addrs = findAddresses(settings);
|
||||
if (!connect(addrs, settings.getAsBoolean("autodiscover", false))) {
|
||||
throw new NoNodeAvailableException("no cluster nodes available, check settings "
|
||||
+ settings.toString());
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
public void init(TransportClient transportClient, Settings settings) throws IOException {
|
||||
Collection<TransportAddress> addrs = findAddresses(settings);
|
||||
if (!connect(transportClient, addrs, settings.getAsBoolean("autodiscover", false))) {
|
||||
throw new NoNodeAvailableException("no cluster nodes available, check settings "
|
||||
+ settings.toString());
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
private Collection<TransportAddress> findAddresses(Settings settings) throws IOException {
|
||||
|
@ -112,11 +106,7 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
|
|||
return addresses;
|
||||
}
|
||||
|
||||
private boolean connect(Collection<TransportAddress> addresses, boolean autodiscover) {
|
||||
if (getClient() == null) {
|
||||
throw new IllegalStateException("no client present");
|
||||
}
|
||||
TransportClient transportClient = (TransportClient) getClient();
|
||||
private boolean connect(TransportClient transportClient, Collection<TransportAddress> addresses, boolean autodiscover) {
|
||||
for (TransportAddress address : addresses) {
|
||||
transportClient.addTransportAddresses(address);
|
||||
}
|
||||
|
@ -126,7 +116,7 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
|
|||
if (autodiscover) {
|
||||
logger.debug("trying to discover all nodes...");
|
||||
ClusterStateRequestBuilder clusterStateRequestBuilder =
|
||||
new ClusterStateRequestBuilder(getClient(), ClusterStateAction.INSTANCE);
|
||||
new ClusterStateRequestBuilder(transportClient, ClusterStateAction.INSTANCE);
|
||||
ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet();
|
||||
DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes();
|
||||
for (DiscoveryNode discoveryNode : discoveryNodes) {
|
|
@ -0,0 +1,35 @@
|
|||
package org.xbib.elx.transport;
|
||||
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.client.transport.TransportClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.xbib.elx.common.AbstractSearchClient;
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Transport search client with additional methods.
|
||||
*/
|
||||
public class TransportSearchClient extends AbstractSearchClient {
|
||||
|
||||
private final TransportClientHelper helper;
|
||||
|
||||
public TransportSearchClient() {
|
||||
this.helper = new TransportClientHelper();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||
return helper.createClient(settings);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(Settings settings) throws IOException {
|
||||
super.init(settings);
|
||||
helper.init((TransportClient) getClient(), settings);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void closeClient(Settings settings) {
|
||||
helper.closeClient(settings);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
package org.xbib.elx.transport;
|
||||
|
||||
import org.xbib.elx.api.SearchClientProvider;
|
||||
|
||||
public class TransportSearchClientProvider implements SearchClientProvider<TransportSearchClient> {
|
||||
|
||||
@Override
|
||||
public TransportSearchClient getClient() {
|
||||
return new TransportSearchClient();
|
||||
}
|
||||
}
|
|
@ -0,0 +1 @@
|
|||
org.xbib.elx.transport.TransportAdminClientProvider
|
|
@ -0,0 +1 @@
|
|||
org.xbib.elx.transport.TransportBulkClientProvider
|
|
@ -1 +0,0 @@
|
|||
org.xbib.elx.transport.ExtendedTransportClientProvider
|
|
@ -0,0 +1 @@
|
|||
org.xbib.elx.transport.TransportSearchClientProvider
|
|
@ -0,0 +1,190 @@
|
|||
package org.xbib.elx.transport.test;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
import org.xbib.elx.common.Parameters;
|
||||
import org.xbib.elx.transport.TransportAdminClient;
|
||||
import org.xbib.elx.transport.TransportAdminClientProvider;
|
||||
import org.xbib.elx.transport.TransportBulkClient;
|
||||
import org.xbib.elx.transport.TransportBulkClientProvider;
|
||||
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@ExtendWith(TestExtension.class)
|
||||
class BulkClientTest {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getName());
|
||||
|
||||
private static final Long ACTIONS = 10000L;
|
||||
|
||||
private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
|
||||
|
||||
private final TestExtension.Helper helper;
|
||||
|
||||
BulkClientTest(TestExtension.Helper helper) {
|
||||
this.helper = helper;
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSingleDoc() throws Exception {
|
||||
final TransportBulkClient bulkClient = ClientBuilder.builder()
|
||||
.setBulkClientProvider(TransportBulkClientProvider.class)
|
||||
.put(helper.getTransportSettings())
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
|
||||
.build();
|
||||
try {
|
||||
bulkClient.newIndex("test");
|
||||
bulkClient.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
} finally {
|
||||
assertEquals(1, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||
bulkClient.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testNewIndex() throws Exception {
|
||||
final TransportBulkClient bulkClient = ClientBuilder.builder()
|
||||
.setBulkClientProvider(TransportBulkClientProvider.class)
|
||||
.put(helper.getTransportSettings())
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
||||
.build();
|
||||
bulkClient.newIndex("test");
|
||||
bulkClient.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testMapping() throws Exception {
|
||||
try (TransportAdminClient adminClient = ClientBuilder.builder()
|
||||
.setAdminClientProvider(TransportAdminClientProvider.class)
|
||||
.put(helper.getTransportSettings())
|
||||
.build();
|
||||
TransportBulkClient bulkClient = ClientBuilder.builder()
|
||||
.setBulkClientProvider(TransportBulkClientProvider.class)
|
||||
.put(helper.getTransportSettings())
|
||||
.build()) {
|
||||
XContentBuilder builder = JsonXContent.contentBuilder()
|
||||
.startObject()
|
||||
.startObject("properties")
|
||||
.startObject("location")
|
||||
.field("type", "geo_point")
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject();
|
||||
bulkClient.newIndex("test", Settings.EMPTY, builder);
|
||||
assertTrue(adminClient.getMapping("test", "_doc").containsKey("properties"));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRandomDocs() throws Exception {
|
||||
long numactions = ACTIONS;
|
||||
final TransportBulkClient bulkClient = ClientBuilder.builder()
|
||||
.setBulkClientProvider(TransportBulkClientProvider.class)
|
||||
.put(helper.getTransportSettings())
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
||||
.build();
|
||||
try {
|
||||
bulkClient.newIndex("test");
|
||||
for (int i = 0; i < ACTIONS; i++) {
|
||||
bulkClient.index("test", null, false,
|
||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||
}
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
} finally {
|
||||
assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||
bulkClient.refreshIndex("test");
|
||||
assertEquals(numactions, bulkClient.getSearchableDocs("test"));
|
||||
bulkClient.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testThreadedRandomDocs() throws Exception {
|
||||
int maxthreads = Runtime.getRuntime().availableProcessors();
|
||||
final long actions = ACTIONS;
|
||||
final TransportBulkClient bulkClient = ClientBuilder.builder()
|
||||
.setBulkClientProvider(TransportBulkClientProvider.class)
|
||||
.put(helper.getTransportSettings())
|
||||
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
||||
.put(Parameters.ENABLE_BULK_LOGGING.name(), "true")
|
||||
.build();
|
||||
try {
|
||||
Settings settings = Settings.builder()
|
||||
.put("index.number_of_shards", 1)
|
||||
.put("index.number_of_replicas", 0)
|
||||
.build();
|
||||
bulkClient.newIndex("test", settings);
|
||||
bulkClient.startBulk("test", 0, 1000);
|
||||
logger.info("index created");
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
|
||||
final CountDownLatch latch = new CountDownLatch(maxthreads);
|
||||
for (int i = 0; i < maxthreads; i++) {
|
||||
executorService.execute(() -> {
|
||||
for (int i1 = 0; i1 < actions; i1++) {
|
||||
bulkClient.index("test", null, false,
|
||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||
}
|
||||
latch.countDown();
|
||||
});
|
||||
}
|
||||
logger.info("waiting for latch...");
|
||||
if (latch.await(30L, TimeUnit.SECONDS)) {
|
||||
logger.info("flush...");
|
||||
bulkClient.flush();
|
||||
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
logger.info("got all responses, executor service shutdown...");
|
||||
executorService.shutdown();
|
||||
executorService.awaitTermination(30L, TimeUnit.SECONDS);
|
||||
logger.info("pool is shut down");
|
||||
} else {
|
||||
logger.warn("latch timeout");
|
||||
}
|
||||
bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS);
|
||||
assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
} finally {
|
||||
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||
bulkClient.refreshIndex("test");
|
||||
assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test"));
|
||||
bulkClient.close();
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,247 +0,0 @@
|
|||
package org.xbib.elx.transport.test;
|
||||
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
|
||||
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
|
||||
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
|
||||
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
|
||||
import org.elasticsearch.action.search.SearchAction;
|
||||
import org.elasticsearch.action.search.SearchRequest;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
import org.xbib.elx.common.Parameters;
|
||||
import org.xbib.elx.transport.ExtendedTransportClient;
|
||||
import org.xbib.elx.transport.ExtendedTransportClientProvider;
|
||||
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@ExtendWith(TestExtension.class)
|
||||
class ClientTest {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
|
||||
|
||||
private static final Long ACTIONS = 10000L;
|
||||
|
||||
private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
|
||||
|
||||
private final TestExtension.Helper helper;
|
||||
|
||||
ClientTest(TestExtension.Helper helper) {
|
||||
this.helper = helper;
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSingleDoc() throws Exception {
|
||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
||||
.provider(ExtendedTransportClientProvider.class)
|
||||
.put(helper.getTransportSettings())
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
|
||||
.build();
|
||||
try {
|
||||
client.newIndex("test");
|
||||
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
||||
client.flush();
|
||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} finally {
|
||||
assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount());
|
||||
if (client.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", client.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(client.getBulkController().getLastBulkError());
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testNewIndex() throws Exception {
|
||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
||||
.provider(ExtendedTransportClientProvider.class)
|
||||
.put(helper.getTransportSettings())
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
||||
.build();
|
||||
client.newIndex("test");
|
||||
client.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testNewIndexWithSettings() throws Exception {
|
||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
||||
.provider(ExtendedTransportClientProvider.class)
|
||||
.put(helper.getTransportSettings())
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
||||
.build();
|
||||
Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
|
||||
client.newIndex("test", settings);
|
||||
GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
|
||||
.indices("test");
|
||||
GetSettingsResponse getSettingsResponse =
|
||||
client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet();
|
||||
logger.log(Level.INFO, "settings=" + getSettingsResponse.getSetting("test", "index.number_of_shards"));
|
||||
assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards"));
|
||||
client.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testNewIndexWithSettingsAndMapping() throws Exception {
|
||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
||||
.provider(ExtendedTransportClientProvider.class)
|
||||
.put(helper.getTransportSettings())
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
||||
.build();
|
||||
Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
|
||||
XContentBuilder builder = JsonXContent.contentBuilder()
|
||||
.startObject()
|
||||
.field("date_detection", false)
|
||||
.startObject("properties")
|
||||
.startObject("location")
|
||||
.field("type", "geo_point")
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject();
|
||||
client.newIndex("test", settings, builder);
|
||||
GetMappingsRequest getMappingsRequest = new GetMappingsRequest()
|
||||
.indices("test");
|
||||
GetMappingsResponse getMappingsResponse = client.getClient()
|
||||
.execute(GetMappingsAction.INSTANCE, getMappingsRequest)
|
||||
.actionGet();
|
||||
logger.info("mappings={}", getMappingsResponse.getMappings());
|
||||
assertTrue(getMappingsResponse.getMappings().get("test").containsKey("_doc"));
|
||||
client.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRandomDocs() throws Exception {
|
||||
long numactions = ACTIONS;
|
||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
||||
.provider(ExtendedTransportClientProvider.class)
|
||||
.put(helper.getTransportSettings())
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
||||
.build();
|
||||
try {
|
||||
client.newIndex("test");
|
||||
for (int i = 0; i < ACTIONS; i++) {
|
||||
client.index("test", null, false,
|
||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||
}
|
||||
client.flush();
|
||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
} finally {
|
||||
assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
|
||||
if (client.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", client.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(client.getBulkController().getLastBulkError());
|
||||
client.refreshIndex("test");
|
||||
SearchSourceBuilder builder = new SearchSourceBuilder();
|
||||
builder.query(QueryBuilders.matchAllQuery());
|
||||
builder.size(0);
|
||||
SearchRequest searchRequest = new SearchRequest();
|
||||
searchRequest.indices("test");
|
||||
searchRequest.source(builder);
|
||||
SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
|
||||
logger.log(Level.INFO, searchResponse.toString());
|
||||
assertEquals(numactions, searchResponse.getHits().getTotalHits().value);
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testThreadedRandomDocs() throws Exception {
|
||||
int maxthreads = Runtime.getRuntime().availableProcessors();
|
||||
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
|
||||
final long actions = ACTIONS;
|
||||
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
|
||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
||||
.provider(ExtendedTransportClientProvider.class)
|
||||
.put(helper.getTransportSettings())
|
||||
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
|
||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
||||
.put(Parameters.ENABLE_BULK_LOGGING.name(), "true")
|
||||
.build();
|
||||
try {
|
||||
Settings settings = Settings.builder()
|
||||
.put("index.number_of_shards", 1)
|
||||
.put("index.number_of_replicas", 0)
|
||||
.build();
|
||||
client.newIndex("test", settings, (String)null)
|
||||
.startBulk("test", 0, 1000);
|
||||
logger.info("index created");
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
|
||||
final CountDownLatch latch = new CountDownLatch(maxthreads);
|
||||
for (int i = 0; i < maxthreads; i++) {
|
||||
executorService.execute(() -> {
|
||||
for (int i1 = 0; i1 < actions; i1++) {
|
||||
client.index("test", null, false,
|
||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||
}
|
||||
latch.countDown();
|
||||
});
|
||||
}
|
||||
logger.info("waiting for latch...");
|
||||
if (latch.await(30L, TimeUnit.SECONDS)) {
|
||||
logger.info("flush...");
|
||||
client.flush();
|
||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
||||
logger.info("got all responses, executor service shutdown...");
|
||||
executorService.shutdown();
|
||||
executorService.awaitTermination(30L, TimeUnit.SECONDS);
|
||||
logger.info("pool is shut down");
|
||||
} else {
|
||||
logger.warn("latch timeout");
|
||||
}
|
||||
client.stopBulk("test", 30L, TimeUnit.SECONDS);
|
||||
assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
} finally {
|
||||
if (client.getBulkController().getLastBulkError() != null) {
|
||||
logger.error("error", client.getBulkController().getLastBulkError());
|
||||
}
|
||||
assertNull(client.getBulkController().getLastBulkError());
|
||||
assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
|
||||
logger.log(Level.INFO, "refreshing index test");
|
||||
client.refreshIndex("test");
|
||||
SearchSourceBuilder builder = new SearchSourceBuilder()
|
||||
.query(QueryBuilders.matchAllQuery())
|
||||
.size(0)
|
||||
.trackTotalHits(true);
|
||||
SearchRequest searchRequest = new SearchRequest()
|
||||
.indices("test")
|
||||
.source(builder);
|
||||
SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
|
||||
assertEquals(maxthreads * actions, searchResponse.getHits().getTotalHits().value);
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
}
|
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue