joerg/elx
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

split into admin, bulk, search client, fix HTTP client

Browse source
This commit is contained in:
Jörg Prante 2020-05-21 17:46:23 +02:00
parent 10833f023b
commit 110c7786da
89 changed files with 3003 additions and 2112 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

194
elx-api/src/main/java/org/xbib/elx/api/AdminClient.java Normal file
View file

@ -0,0 +1,194 @@
package org.xbib.elx.api;
import org.elasticsearch.common.settings.Settings;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
/**
* Interface for extended managing and indexing methods of an Elasticsearch client.
*/
public interface AdminClient extends NativeClient {
/**
* Build index definition from settings.
*
* @param index the index name
* @param settings the settings for the index
* @return index definition
* @throws IOException if settings/mapping URL is invalid/malformed
*/
IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException;
/**
* Delete an index.
* @param indexDefinition the index definition
* @return this
*/
AdminClient deleteIndex(IndexDefinition indexDefinition);
/**
* Delete an index.
*
* @param index index
* @return this
*/
AdminClient deleteIndex(String index);
/**
* Update replica level.
* @param indexDefinition the index definition
* @param level the replica level
* @return this
* @throws IOException if replica setting could not be updated
*/
AdminClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException;
/**
* Update replica level.
*
* @param index index
* @param level the replica level
* @param maxWaitTime maximum wait time
* @param timeUnit time unit
* @return this
* @throws IOException if replica setting could not be updated
*/
AdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException;
/**
* Get replica level.
* @param indexDefinition the index name
* @return the replica level of the index
*/
int getReplicaLevel(IndexDefinition indexDefinition);
/**
* Get replica level.
* @param index the index name
* @return the replica level of the index
*/
int getReplicaLevel(String index);
/**
* Force segment merge of an index.
* @param indexDefinition the index definition
* @return this
*/
boolean forceMerge(IndexDefinition indexDefinition);
/**
* Force segment merge of an index.
* @param index the index
* @param maxWaitTime maximum wait time
* @param timeUnit time unit
* @return this
*/
boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit);
/**
* Wait for index recovery (after replica change).
*
* @param index index
* @param maxWaitTime maximum wait time
* @param timeUnit time unit
* @return true if wait succeeded, false if wait timed out
*/
boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit);
/**
* Resolve alias.
*
* @param alias the alias
* @return this index name behind the alias or the alias if there is no index
*/
String resolveAlias(String alias);
/**
* Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index
* name.
*
* @param alias the alias
* @return the most recent index name pointing to the alias
*/
String resolveMostRecentIndex(String alias);
/**
* Get all index aliases.
* @param index the index
* @return map of index aliases
*/
Map<String, String> getAliases(String index);
/**
* Shift from one index to another.
* @param indexDefinition the index definition
* @param additionalAliases new aliases
* @return this
*/
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases);
/**
* Shift from one index to another.
* @param indexDefinition the index definition
* @param additionalAliases new aliases
* @param indexAliasAdder method to add aliases
* @return this
*/
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases,
IndexAliasAdder indexAliasAdder);
/**
* Shift from one index to another.
* @param index the index name
* @param fullIndexName the index name with timestamp
* @param additionalAliases a list of names that should be set as index aliases
* @return this
*/
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases);
/**
* Shift from one index to another.
* @param index the index name
* @param fullIndexName the index name with timestamp
* @param additionalAliases a list of names that should be set as index aliases
* @param adder an adder method to create alias term queries
* @return this
*/
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases,
IndexAliasAdder adder);
/**
* Prune index.
* @param indexDefinition the index definition
* @return the index prune result
*/
IndexPruneResult pruneIndex(IndexDefinition indexDefinition);
/**
* Apply retention policy to prune indices. All indices before delta should be deleted,
* but the number of mintokeep indices must be kept.
*
* @param index index name
* @param fullIndexName index name with timestamp
* @param delta timestamp delta (for index timestamps)
* @param mintokeep minimum number of indices to keep
* @param perform true if pruning should be executed, false if not
* @return the index prune result
*/
IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform);
/**
* Find the timestamp of the most recently indexed document in the index.
*
* @param index the index name
* @param timestampfieldname the timestamp field name
* @return millis UTC millis of the most recent document
* @throws IOException if most rcent document can not be found
*/
Long mostRecentDocument(String index, String timestampfieldname) throws IOException;
}

2
elx-api/src/main/java/org/xbib/elx/api/SeachClientProvider.java → elx-api/src/main/java/org/xbib/elx/api/AdminClientProvider.java
View file

@ -1,7 +1,7 @@
package org.xbib.elx.api;
@FunctionalInterface
public interface SeachClientProvider<C extends SearchClient> {
public interface AdminClientProvider<C extends AdminClient> {
C getClient();
}

227
elx-api/src/main/java/org/xbib/elx/api/BulkClient.java Normal file
View file

@ -0,0 +1,227 @@
package org.xbib.elx.api;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.Flushable;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.TimeUnit;
public interface BulkClient extends NativeClient, Flushable {
/**
* Get bulk metric.
* @return the bulk metric
*/
BulkMetric getBulkMetric();
/**
* Get buulk control.
* @return the bulk control
*/
BulkController getBulkController();
/**
* Create a new index.
*
* @param index index
* @throws IOException if new index creation fails
*/
void newIndex(String index) throws IOException;
/**
* Create a new index.
* @param indexDefinition the index definition
* @throws IOException if settings/mapping is invalid or index creation fails
*/
void newIndex(IndexDefinition indexDefinition) throws IOException;
/**
* Create a new index.
*
* @param index index
* @param settings settings
* @throws IOException if settings is invalid or index creation fails
*/
void newIndex(String index, Settings settings) throws IOException;
/**
* Create a new index.
*
* @param index index
* @param settings settings
* @param mapping mapping
* @throws IOException if settings/mapping is invalid or index creation fails
*/
void newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException;
/**
* Create a new index.
*
* @param index index
* @param settings settings
* @param mapping mapping
* @throws IOException if settings/mapping is invalid or index creation fails
*/
void newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException;
/**
* Add index request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when limits are exceeded.
*
* @param index the index
* @param id the id
* @param create true if document must be created
* @param source the source
* @return this
*/
BulkClient index(String index, String id, boolean create, BytesReference source);
/**
* Index request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when limits are exceeded.
*
* @param index the index
* @param id the id
* @param create true if document is to be created, false otherwise
* @param source the source
* @return this client methods
*/
BulkClient index(String index, String id, boolean create, String source);
/**
* Index request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when bulk limits are exceeded.
*
* @param indexRequest the index request to add
* @return this
*/
BulkClient index(IndexRequest indexRequest);
/**
* Delete request.
*
* @param index the index
* @param id the id
* @return this
*/
BulkClient delete(String index, String id);
/**
* Delete request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when bulk limits are exceeded.
*
* @param deleteRequest the delete request to add
* @return this
*/
BulkClient delete(DeleteRequest deleteRequest);
/**
* Bulked update request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when bulk limits are exceeded.
* Note that updates only work correctly when all operations between nodes are synchronized.
*
* @param index the index
* @param id the id
* @param source the source
* @return this
*/
BulkClient update(String index, String id, BytesReference source);
/**
* Update document. Use with precaution! Does not work in all cases.
*
* @param index the index
* @param id the id
* @param source the source
* @return this
*/
BulkClient update(String index, String id, String source);
/**
* Bulked update request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when bulk limits are exceeded.
* Note that updates only work correctly when all operations between nodes are synchronized.
*
* @param updateRequest the update request to add
* @return this
*/
BulkClient update(UpdateRequest updateRequest);
/**
* Start bulk mode for indexes.
* @param indexDefinition index definition
* @throws IOException if bulk could not be started
*/
void startBulk(IndexDefinition indexDefinition) throws IOException;
/**
* Start bulk mode.
*
* @param index index
* @param startRefreshIntervalSeconds refresh interval before bulk
* @param stopRefreshIntervalSeconds refresh interval after bulk
* @throws IOException if bulk could not be started
*/
void startBulk(String index, long startRefreshIntervalSeconds,
long stopRefreshIntervalSeconds) throws IOException;
/**
* Stop bulk mode.
*
* @param indexDefinition index definition
* @throws IOException if bulk could not be startet
*/
void stopBulk(IndexDefinition indexDefinition) throws IOException;
/**
* Stops bulk mode.
*
* @param index index
* @param timeout maximum wait time
* @param timeUnit time unit for timeout
* @throws IOException if bulk could not be stopped
*/
void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException;
/**
* Wait for all outstanding bulk responses.
*
* @param timeout maximum wait time
* @param timeUnit unit of timeout value
* @return true if wait succeeded, false if wait timed out
*/
boolean waitForResponses(long timeout, TimeUnit timeUnit);
/**
* Update index setting.
* @param index the index
* @param key the key of the value to be updated
* @param value the new value
* @param timeout timeout
* @param timeUnit time unit
* @throws IOException if update index setting failed
*/
void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException;
/**
* Refresh the index.
*
* @param index index
*/
void refreshIndex(String index);
/**
* Flush the index. The cluster clears cache and completes indexing.
*
* @param index index
*/
void flushIndex(String index);
}

7
elx-api/src/main/java/org/xbib/elx/api/BulkClientProvider.java Normal file
View file

@ -0,0 +1,7 @@
package org.xbib.elx.api;
@FunctionalInterface
public interface BulkClientProvider<C extends BulkClient> {
C getClient();
}

480
elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java
View file

@ -1,480 +0,0 @@
package org.xbib.elx.api;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.Settings;
import java.io.Closeable;
import java.io.Flushable;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
/**
* Interface for extended managing and indexing methods of an Elasticsearch client.
*/
public interface ExtendedClient extends Flushable, Closeable {
/**
* Set an Elasticsearch client to extend from it. May be null for TransportClient.
* @param client client
* @return this client
*/
ExtendedClient setClient(ElasticsearchClient client);
/**
* Return Elasticsearch client.
*
* @return Elasticsearch client
*/
ElasticsearchClient getClient();
/**
* Get bulk metric.
* @return the bulk metric
*/
BulkMetric getBulkMetric();
/**
* Get buulk control.
* @return the bulk control
*/
BulkController getBulkController();
/**
* Initiative the extended client, the bulk metric and bulk controller,
* creates instances and connect to cluster, if required.
*
* @param settings settings
* @return this client
* @throws IOException if init fails
*/
ExtendedClient init(Settings settings) throws IOException;
/**
* Build index definition from settings.
*
* @param index the index name
* @param settings the settings for the index
* @return index definition
* @throws IOException if settings/mapping URL is invalid/malformed
*/
IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException;
/**
* Add index request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when limits are exceeded.
*
* @param index the index
* @param id the id
* @param create true if document must be created
* @param source the source
* @return this
*/
ExtendedClient index(String index, String id, boolean create, BytesReference source);
/**
* Index request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when limits are exceeded.
*
* @param index the index
* @param id the id
* @param create true if document is to be created, false otherwise
* @param source the source
* @return this client methods
*/
ExtendedClient index(String index, String id, boolean create, String source);
/**
* Index request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when bulk limits are exceeded.
*
* @param indexRequest the index request to add
* @return this
*/
ExtendedClient index(IndexRequest indexRequest);
/**
* Delete request.
*
* @param index the index
* @param id the id
* @return this
*/
ExtendedClient delete(String index, String id);
/**
* Delete request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when bulk limits are exceeded.
*
* @param deleteRequest the delete request to add
* @return this
*/
ExtendedClient delete(DeleteRequest deleteRequest);
/**
* Bulked update request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when bulk limits are exceeded.
* Note that updates only work correctly when all operations between nodes are synchronized.
*
* @param index the index
* @param id the id
* @param source the source
* @return this
*/
ExtendedClient update(String index, String id, BytesReference source);
/**
* Update document. Use with precaution! Does not work in all cases.
*
* @param index the index
* @param id the id
* @param source the source
* @return this
*/
ExtendedClient update(String index, String id, String source);
/**
* Bulked update request. Each request will be added to a queue for bulking requests.
* Submitting request will be done when bulk limits are exceeded.
* Note that updates only work correctly when all operations between nodes are synchronized.
*
* @param updateRequest the update request to add
* @return this
*/
ExtendedClient update(UpdateRequest updateRequest);
/**
* Create a new index.
*
* @param index index
* @return this
* @throws IOException if new index creation fails
*/
ExtendedClient newIndex(String index) throws IOException;
/**
* Create a new index.
*
* @param index index
* @param settings settings
* @param mapping mapping
* @return this
* @throws IOException if settings/mapping is invalid or index creation fails
*/
ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException;
/**
* Create a new index.
*
* @param index index
* @param settings settings
* @return this
* @throws IOException if settings is invalid or index creation fails
*/
ExtendedClient newIndex(String index, Settings settings) throws IOException;
/**
* Create a new index.
*
* @param index index
* @param settings settings
* @param mapping mapping
* @return this
* @throws IOException if settings/mapping is invalid or index creation fails
*/
ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException;
/**
* Create a new index.
*
* @param index index
* @param settings settings
* @param mapping mapping
* @return this
* @throws IOException if settings/mapping is invalid or index creation fails
*/
ExtendedClient newIndex(String index, Settings settings, Map<String, Object> mapping) throws IOException;
/**
* Create a new index.
* @param indexDefinition the index definition
* @return this
* @throws IOException if settings/mapping is invalid or index creation fails
*/
ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException;
/**
* Delete an index.
* @param indexDefinition the index definition
* @return this
*/
ExtendedClient deleteIndex(IndexDefinition indexDefinition);
/**
* Delete an index.
*
* @param index index
* @return this
*/
ExtendedClient deleteIndex(String index);
/**
* Start bulk mode for indexes.
* @param indexDefinition index definition
* @return this
* @throws IOException if bulk could not be started
*/
ExtendedClient startBulk(IndexDefinition indexDefinition) throws IOException;
/**
* Start bulk mode.
*
* @param index index
* @param startRefreshIntervalSeconds refresh interval before bulk
* @param stopRefreshIntervalSeconds refresh interval after bulk
* @return this
* @throws IOException if bulk could not be started
*/
ExtendedClient startBulk(String index, long startRefreshIntervalSeconds,
long stopRefreshIntervalSeconds) throws IOException;
/**
* Stop bulk mode.
*
* @param indexDefinition index definition
* @return this
* @throws IOException if bulk could not be startet
*/
ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException;
/**
* Stops bulk mode.
*
* @param index index
* @param timeout maximum wait time
* @param timeUnit time unit for timeout
* @return this
* @throws IOException if bulk could not be stopped
*/
ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException;
/**
* Update replica level.
* @param indexDefinition the index definition
* @param level the replica level
* @return this
* @throws IOException if replica setting could not be updated
*/
ExtendedClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException;
/**
* Update replica level.
*
* @param index index
* @param level the replica level
* @param maxWaitTime maximum wait time
* @param timeUnit time unit
* @return this
* @throws IOException if replica setting could not be updated
*/
ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException;
/**
* Get replica level.
* @param indexDefinition the index name
* @return the replica level of the index
*/
int getReplicaLevel(IndexDefinition indexDefinition);
/**
* Get replica level.
* @param index the index name
* @return the replica level of the index
*/
int getReplicaLevel(String index);
/**
* Refresh the index.
*
* @param index index
* @return this
*/
ExtendedClient refreshIndex(String index);
/**
* Flush the index. The cluster clears cache and completes indexing.
*
* @param index index
* @return this
*/
ExtendedClient flushIndex(String index);
/**
* Force segment merge of an index.
* @param indexDefinition the index definition
* @return this
*/
boolean forceMerge(IndexDefinition indexDefinition);
/**
* Force segment merge of an index.
* @param index the index
* @param maxWaitTime maximum wait time
* @param timeUnit time unit
* @return this
*/
boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit);
/**
* Wait for all outstanding bulk responses.
*
* @param timeout maximum wait time
* @param timeUnit unit of timeout value
* @return true if wait succeeded, false if wait timed out
*/
boolean waitForResponses(long timeout, TimeUnit timeUnit);
/**
* Wait for cluster being healthy.
*
* @param healthColor cluster health color to wait for
* @param maxWaitTime time value
* @param timeUnit time unit
* @return true if wait succeeded, false if wait timed out
*/
boolean waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit);
/**
* Get current health color.
*
* @param maxWaitTime maximum wait time
* @param timeUnit time unit
* @return the cluster health color
*/
String getHealthColor(long maxWaitTime, TimeUnit timeUnit);
/**
* Wait for index recovery (after replica change).
*
* @param index index
* @param maxWaitTime maximum wait time
* @param timeUnit time unit
* @return true if wait succeeded, false if wait timed out
*/
boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit);
/**
* Update index setting.
* @param index the index
* @param key the key of the value to be updated
* @param value the new value
* @param timeout timeout
* @param timeUnit time unit
* @throws IOException if update index setting failed
*/
void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException;
/**
* Resolve alias.
*
* @param alias the alias
* @return this index name behind the alias or the alias if there is no index
*/
String resolveAlias(String alias);
/**
* Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index
* name.
*
* @param alias the alias
* @return the most recent index name pointing to the alias
*/
String resolveMostRecentIndex(String alias);
/**
* Get all index aliases.
* @param index the index
* @return map of index aliases
*/
Map<String, String> getAliases(String index);
/**
* Shift from one index to another.
* @param indexDefinition the index definition
* @param additionalAliases new aliases
* @return this
*/
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases);
/**
* Shift from one index to another.
* @param indexDefinition the index definition
* @param additionalAliases new aliases
* @param indexAliasAdder method to add aliases
* @return this
*/
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases,
IndexAliasAdder indexAliasAdder);
/**
* Shift from one index to another.
* @param index the index name
* @param fullIndexName the index name with timestamp
* @param additionalAliases a list of names that should be set as index aliases
* @return this
*/
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases);
/**
* Shift from one index to another.
* @param index the index name
* @param fullIndexName the index name with timestamp
* @param additionalAliases a list of names that should be set as index aliases
* @param adder an adder method to create alias term queries
* @return this
*/
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases,
IndexAliasAdder adder);
/**
* Prune index.
* @param indexDefinition the index definition
* @return the index prune result
*/
IndexPruneResult pruneIndex(IndexDefinition indexDefinition);
/**
* Apply retention policy to prune indices. All indices before delta should be deleted,
* but the number of mintokeep indices must be kept.
*
* @param index index name
* @param fullIndexName index name with timestamp
* @param delta timestamp delta (for index timestamps)
* @param mintokeep minimum number of indices to keep
* @param perform true if pruning should be executed, false if not
* @return the index prune result
*/
IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform);
/**
* Find the timestamp of the most recently indexed document in the index.
*
* @param index the index name
* @param timestampfieldname the timestamp field name
* @return millis UTC millis of the most recent document
* @throws IOException if most rcent document can not be found
*/
Long mostRecentDocument(String index, String timestampfieldname) throws IOException;
/**
* Get cluster name.
* @return the cluster name
*/
String getClusterName();
}

7
elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java
View file

@ -1,7 +0,0 @@
package org.xbib.elx.api;
@FunctionalInterface
public interface ExtendedClientProvider<C extends ExtendedClient> {
C getExtendedClient();
}

14
elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java
View file

@ -1,7 +1,5 @@
package org.xbib.elx.api;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.concurrent.TimeUnit;
public interface IndexDefinition {
@ -14,17 +12,13 @@ public interface IndexDefinition {
String getFullIndexName();
IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException;
IndexDefinition setSettings(String settings);
IndexDefinition setSettingsUrl(URL settingsUrl);
String getSettings();
URL getSettingsUrl();
IndexDefinition setMappings(String mappings);
IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException;
IndexDefinition setMappingsUrl(URL mappingsUrl);
URL getMappingsUrl();
String getMappings();
IndexDefinition setDateTimePattern(String timeWindow);

6
elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java
View file

@ -1,6 +1,6 @@
package org.xbib.elx.api;
import java.util.List;
import java.util.Collection;
public interface IndexPruneResult {
@ -8,9 +8,9 @@ public interface IndexPruneResult {
State getState();
List<String> getCandidateIndices();
Collection<String> getCandidateIndices();
List<String> getDeletedIndices();
Collection<String> getDeletedIndices();
boolean isAcknowledged();
}

6
elx-api/src/main/java/org/xbib/elx/api/IndexShiftResult.java
View file

@ -1,10 +1,10 @@
package org.xbib.elx.api;
import java.util.List;
import java.util.Collection;
public interface IndexShiftResult {
List<String> getMovedAliases();
Collection<String> getMovedAliases();
List<String> getNewAliases();
Collection<String> getNewAliases();
}

63
elx-api/src/main/java/org/xbib/elx/api/NativeClient.java Normal file
View file

@ -0,0 +1,63 @@
package org.xbib.elx.api;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import java.io.Closeable;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.TimeUnit;
public interface NativeClient extends Closeable {
/**
* Set an Elasticsearch client to extend from it. May be null for TransportClient.
* @param client the Elasticsearch client
*/
void setClient(ElasticsearchClient client);
/**
* Return Elasticsearch client.
*
* @return Elasticsearch client
*/
ElasticsearchClient getClient();
/**
* Initiative the extended client, the bulk metric and bulk controller,
* creates instances and connect to cluster, if required.
*
* @param settings settings
* @throws IOException if init fails
*/
void init(Settings settings) throws IOException;
/**
* Get cluster name.
* @return the cluster name
*/
String getClusterName();
/**
* Get current health color.
*
* @param maxWaitTime maximum wait time
* @param timeUnit time unit
* @return the cluster health color
*/
String getHealthColor(long maxWaitTime, TimeUnit timeUnit);
/**
* Wait for cluster being healthy.
*
* @param healthColor cluster health color to wait for
* @param maxWaitTime time value
* @param timeUnit time unit
*/
void waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit);
Map<String, ?> getMapping(String index, String mapping);
long getSearchableDocs(String index);
boolean isIndexExists(String index);
}

28
elx-api/src/main/java/org/xbib/elx/api/SearchClient.java
View file

@ -1,25 +1,27 @@
package org.xbib.elx.api;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetRequestBuilder;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.get.MultiGetRequest;
import org.elasticsearch.action.get.MultiGetRequestBuilder;
import org.elasticsearch.action.get.MultiGetResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.search.SearchHit;
import java.util.Optional;
import java.util.function.Consumer;
import java.util.stream.Stream;
public interface SearchClient {
public interface SearchClient extends NativeClient {
ActionFuture<GetResponse> get(GetRequest getRequest);
Optional<GetResponse> get(Consumer<GetRequestBuilder> getRequestBuilder);
void get(GetRequest request, ActionListener<GetResponse> listener);
Optional<MultiGetResponse> multiGet(Consumer<MultiGetRequestBuilder> multiGetRequestBuilder);
ActionFuture<MultiGetResponse> multiGet(MultiGetRequest request);
Optional<SearchResponse> search(Consumer<SearchRequestBuilder> searchRequestBuilder);
void multiGet(MultiGetRequest request, ActionListener<MultiGetResponse> listener);
Stream<SearchHit> search(Consumer<SearchRequestBuilder> searchRequestBuilder,
TimeValue scrollTime, int scrollSize);
ActionFuture<SearchResponse> search(SearchRequest request);
void search(SearchRequest request, ActionListener<SearchResponse> listener);
Stream<String> getIds(Consumer<SearchRequestBuilder> queryBuilder);
}

7
elx-api/src/main/java/org/xbib/elx/api/SearchClientProvider.java Normal file
View file

@ -0,0 +1,7 @@
package org.xbib.elx.api;
@FunctionalInterface
public interface SearchClientProvider<C extends SearchClient> {
C getClient();
}

511
elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java → elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java
View file

@ -2,10 +2,8 @@ package org.xbib.elx.common;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
@ -18,14 +16,9 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.flush.FlushAction;
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction;
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexAction;
@ -34,27 +27,16 @@ import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.cluster.metadata.AliasOrIndex;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
@ -62,9 +44,8 @@ import org.elasticsearch.common.xcontent.DeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.common.xcontent.yaml.YamlXContent;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
@ -72,9 +53,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.xbib.elx.api.BulkController;
import org.xbib.elx.api.BulkMetric;
import org.xbib.elx.api.ExtendedClient;
import org.xbib.elx.api.AdminClient;
import org.xbib.elx.api.IndexAliasAdder;
import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.api.IndexPruneResult;
@ -83,14 +62,16 @@ import org.xbib.elx.api.IndexShiftResult;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.MalformedInputException;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
@ -102,33 +83,15 @@ import java.util.TreeSet;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public abstract class AbstractExtendedClient implements ExtendedClient {
public abstract class AbstractAdminClient extends AbstractNativeClient implements AdminClient {
private static final Logger logger = LogManager.getLogger(AbstractExtendedClient.class.getName());
/**
* The one and only index type name used in the extended client.
* Notr that all Elasticsearch version < 6.2.0 do not allow a prepending "_".
*/
private static final String TYPE_NAME = "doc";
/**
* The Elasticsearch client.
*/
private ElasticsearchClient client;
private BulkMetric bulkMetric;
private BulkController bulkController;
private AtomicBoolean closed;
private static final Logger logger = LogManager.getLogger(AbstractAdminClient.class.getName());
private static final IndexShiftResult EMPTY_INDEX_SHIFT_RESULT = new IndexShiftResult() {
@Override
@ -164,184 +127,14 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
}
};
protected abstract ElasticsearchClient createClient(Settings settings) throws IOException;
protected abstract void closeClient() throws IOException;
protected AbstractExtendedClient() {
closed = new AtomicBoolean(false);
}
@Override
public AbstractExtendedClient setClient(ElasticsearchClient client) {
this.client = client;
return this;
}
@Override
public ElasticsearchClient getClient() {
return client;
}
@Override
public BulkMetric getBulkMetric() {
return bulkMetric;
}
@Override
public BulkController getBulkController() {
return bulkController;
}
@Override
public AbstractExtendedClient init(Settings settings) throws IOException {
logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(','));
if (client == null) {
client = createClient(settings);
}
if (bulkMetric == null) {
bulkMetric = new DefaultBulkMetric();
bulkMetric.init(settings);
}
if (bulkController == null) {
bulkController = new DefaultBulkController(this, bulkMetric);
bulkController.init(settings);
}
return this;
}
@Override
public void flush() throws IOException {
if (bulkController != null) {
bulkController.flush();
}
}
@Override
public void close() throws IOException {
ensureActive();
if (closed.compareAndSet(false, true)) {
if (bulkMetric != null) {
logger.info("closing bulk metric");
bulkMetric.close();
}
if (bulkController != null) {
logger.info("closing bulk controller");
bulkController.close();
}
closeClient();
}
}
@Override
public String getClusterName() {
ensureActive();
try {
ClusterStateRequest clusterStateRequest = new ClusterStateRequest().clear();
ClusterStateResponse clusterStateResponse =
client.execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
return clusterStateResponse.getClusterName().value();
} catch (ElasticsearchTimeoutException e) {
logger.warn(e.getMessage(), e);
return "TIMEOUT";
} catch (NoNodeAvailableException e) {
logger.warn(e.getMessage(), e);
return "DISCONNECTED";
} catch (Exception e) {
logger.warn(e.getMessage(), e);
return "[" + e.getMessage() + "]";
}
}
@Override
public ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException {
ensureActive();
waitForCluster("YELLOW", 30L, TimeUnit.SECONDS);
URL indexSettings = indexDefinition.getSettingsUrl();
if (indexSettings == null) {
logger.warn("warning while creating index '{}', no settings/mappings",
indexDefinition.getFullIndexName());
newIndex(indexDefinition.getFullIndexName());
return this;
}
URL indexMappings = indexDefinition.getMappingsUrl();
if (indexMappings == null) {
logger.warn("warning while creating index '{}', no mappings",
indexDefinition.getFullIndexName());
newIndex(indexDefinition.getFullIndexName(), indexSettings.openStream(), null);
return this;
}
try (InputStream indexSettingsInput = indexSettings.openStream();
InputStream indexMappingsInput = indexMappings.openStream()) {
newIndex(indexDefinition.getFullIndexName(), indexSettingsInput, indexMappingsInput);
} catch (IOException e) {
if (indexDefinition.ignoreErrors()) {
logger.warn(e.getMessage(), e);
logger.warn("warning while creating index '{}' with settings at {} and mappings at {}",
indexDefinition.getFullIndexName(), indexSettings, indexMappings);
} else {
logger.error("error while creating index '{}' with settings at {} and mappings at {}",
indexDefinition.getFullIndexName(), indexSettings, indexMappings);
throw new IOException(e);
}
}
return this;
}
@Override
public ExtendedClient newIndex(String index) throws IOException {
return newIndex(index, Settings.EMPTY, (Map<String, Object>) null);
}
@Override
public ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException {
return newIndex(index,
Settings.builder().loadFromStream(".json", settings, true).build(),
mapping != null ? JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mapping).mapOrdered() : null);
}
@Override
public ExtendedClient newIndex(String index, Settings settings) throws IOException {
return newIndex(index, settings, (Map<String, Object>) null);
}
@Override
public ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException {
return newIndex(index, settings,
mapping != null ? JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mapping).mapOrdered() : null);
}
@Override
public ExtendedClient newIndex(String index, Settings settings, Map<String, Object> mapping) throws IOException {
ensureActive();
if (index == null) {
logger.warn("no index name given to create index");
return this;
}
CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index);
if (settings != null) {
createIndexRequest.settings(settings);
}
if (mapping != null) {
createIndexRequest.mapping(TYPE_NAME, mapping);
}
CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
XContentBuilder builder = XContentFactory.jsonBuilder();
logger.info("index {} created: {}", index,
Strings.toString(createIndexResponse.toXContent(builder, ToXContent.EMPTY_PARAMS)));
return this;
}
@Override
public ExtendedClient deleteIndex(IndexDefinition indexDefinition) {
public AdminClient deleteIndex(IndexDefinition indexDefinition) {
return deleteIndex(indexDefinition.getFullIndexName());
}
@Override
public ExtendedClient deleteIndex(String index) {
ensureActive();
public AdminClient deleteIndex(String index) {
ensureClientIsPresent();
if (index == null) {
logger.warn("no index name given to delete index");
return this;
@ -351,99 +144,9 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
return this;
}
@Override
public ExtendedClient startBulk(IndexDefinition indexDefinition) throws IOException {
startBulk(indexDefinition.getFullIndexName(), -1, 1);
return this;
}
@Override
public ExtendedClient startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds)
throws IOException {
if (bulkController != null) {
ensureActive();
bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds);
}
return this;
}
@Override
public ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException {
if (bulkController != null) {
ensureActive();
bulkController.stopBulkMode(indexDefinition);
}
return this;
}
@Override
public ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException {
if (bulkController != null) {
ensureActive();
bulkController.stopBulkMode(index, timeout, timeUnit);
}
return this;
}
@Override
public ExtendedClient index(String index, String id, boolean create, String source) {
return index(new IndexRequest(index, TYPE_NAME, id).create(create)
.source(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
}
@Override
public ExtendedClient index(String index, String id, boolean create, BytesReference source) {
return index(new IndexRequest(index, TYPE_NAME, id).create(create)
.source(source, XContentType.JSON));
}
@Override
public ExtendedClient index(IndexRequest indexRequest) {
ensureActive();
bulkController.index(indexRequest);
return this;
}
@Override
public ExtendedClient delete(String index, String id) {
return delete(new DeleteRequest(index, TYPE_NAME, id));
}
@Override
public ExtendedClient delete(DeleteRequest deleteRequest) {
ensureActive();
bulkController.delete(deleteRequest);
return this;
}
@Override
public ExtendedClient update(String index, String id, BytesReference source) {
return update(new UpdateRequest(index, TYPE_NAME, id)
.doc(source, XContentType.JSON));
}
@Override
public ExtendedClient update(String index, String id, String source) {
return update(new UpdateRequest(index, TYPE_NAME, id)
.doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
}
@Override
public ExtendedClient update(UpdateRequest updateRequest) {
ensureActive();
bulkController.update(updateRequest);
return this;
}
@Override
public boolean waitForResponses(long timeout, TimeUnit timeUnit) {
ensureActive();
return bulkController.waitForResponses(timeout, timeUnit);
}
@Override
public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) {
ensureActive();
ensureClientIsPresent();
ensureIndexGiven(index);
GetSettingsRequest settingsRequest = new GetSettingsRequest();
settingsRequest.indices(index);
@ -466,50 +169,13 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
}
@Override
public boolean waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) {
ensureActive();
ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString);
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
if (logger.isErrorEnabled()) {
logger.error("timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name());
}
return false;
}
return true;
}
@Override
public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) {
ensureActive();
try {
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().timeout(timeout)).actionGet();
ClusterHealthStatus status = healthResponse.getStatus();
return status.name();
} catch (ElasticsearchTimeoutException e) {
logger.warn(e.getMessage(), e);
return "TIMEOUT";
} catch (NoNodeAvailableException e) {
logger.warn(e.getMessage(), e);
return "DISCONNECTED";
} catch (Exception e) {
logger.warn(e.getMessage(), e);
return "[" + e.getMessage() + "]";
}
}
@Override
public ExtendedClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException {
public AdminClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException {
return updateReplicaLevel(indexDefinition.getFullIndexName(), level,
indexDefinition.getMaxWaitTime(), indexDefinition.getMaxWaitTimeUnit());
}
@Override
public ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException {
public AdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException {
waitForCluster("YELLOW", maxWaitTime, timeUnit); // let cluster settle down from critical operations
if (level > 0) {
updateIndexSetting(index, "number_of_replicas", level, maxWaitTime, timeUnit);
@ -537,27 +203,10 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
return replica;
}
@Override
public ExtendedClient flushIndex(String index) {
if (index != null) {
ensureActive();
client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet();
}
return this;
}
@Override
public ExtendedClient refreshIndex(String index) {
if (index != null) {
ensureActive();
client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet();
}
return this;
}
@Override
public String resolveMostRecentIndex(String alias) {
ensureActive();
ensureClientIsPresent();
if (alias == null) {
return null;
}
@ -585,7 +234,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
@Override
public String resolveAlias(String alias) {
ensureActive();
ensureClientIsPresent();
ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
clusterStateRequest.blocks(false);
clusterStateRequest.metaData(true);
@ -627,7 +276,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
@Override
public IndexShiftResult shiftIndex(String index, String fullIndexName,
List<String> additionalAliases, IndexAliasAdder adder) {
ensureActive();
ensureClientIsPresent();
if (index == null) {
return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to
}
@ -715,12 +364,13 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
if (index.equals(fullIndexName)) {
return EMPTY_INDEX_PRUNE_RESULT;
}
ensureActive();
GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE);
ensureClientIsPresent();
GetIndexRequestBuilder getIndexRequestBuilder =
new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE);
GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet();
Pattern pattern = Pattern.compile("^(.*?)(\\d+)$");
logger.info("{} indices", getIndexResponse.getIndices().length);
List<String> candidateIndices = new ArrayList<>();
logger.info("found {} indices", getIndexResponse.getIndices().length);
Set<String> candidateIndices = new TreeSet<>();
for (String s : getIndexResponse.getIndices()) {
Matcher m = pattern.matcher(s);
if (m.matches() && index.equals(m.group(1)) && !s.equals(fullIndexName)) {
@ -760,7 +410,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
@Override
public Long mostRecentDocument(String index, String timestampfieldname) {
ensureActive();
ensureClientIsPresent();
SortBuilder<?> sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC);
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.sort(sort);
@ -813,7 +463,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
@Override
public IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings)
throws IOException {
boolean isEnabled = settings.getAsBoolean("enabled", !(client instanceof MockExtendedClient));
boolean isEnabled = settings.getAsBoolean("enabled", !(client instanceof MockAdminClient));
String indexName = settings.get("name", index);
String fullIndexName;
String dateTimePattern = settings.get("dateTimePattern");
@ -833,8 +483,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
.setEnabled(isEnabled)
.setIndex(indexName)
.setFullIndexName(fullIndexName)
.setSettingsUrl(settings.get("settings"))
.setMappingsUrl(settings.get("mapping"))
.setSettings(findSettingsFrom(settings.get("settings")))
.setMappings(findMappingsFrom(settings.get("mapping")))
.setDateTimePattern(dateTimePattern)
.setIgnoreErrors(settings.getAsBoolean("skiperrors", false))
.setShift(settings.getAsBoolean("shift", true))
@ -845,31 +495,48 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
.setStopRefreshInterval(settings.getAsLong("bulk.stoprefreshinterval", -1L));
}
@Override
public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException {
ensureActive();
if (index == null) {
throw new IOException("no index name given");
private static String findSettingsFrom(String string) throws IOException {
if (string == null) {
return null;
}
if (key == null) {
throw new IOException("no key given");
try {
URL url = new URL(string);
try (InputStream inputStream = url.openStream()) {
Settings settings = Settings.builder().loadFromStream(string, inputStream, true).build();
XContentBuilder builder = JsonXContent.contentBuilder();
settings.toXContent(builder, ToXContent.EMPTY_PARAMS);
return Strings.toString(builder);
}
if (value == null) {
throw new IOException("no value given");
} catch (MalformedURLException e) {
return string;
}
Settings.Builder updateSettingsBuilder = Settings.builder();
updateSettingsBuilder.put(key, value.toString());
UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index)
.settings(updateSettingsBuilder).timeout(toTimeValue(timeout, timeUnit));
client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet();
}
private void ensureActive() {
if (this instanceof MockExtendedClient) {
return;
private static String findMappingsFrom(String string) throws IOException {
if (string == null) {
return null;
}
if (client == null) {
throw new IllegalStateException("no client");
try {
URL url = new URL(string);
try (InputStream inputStream = url.openStream()) {
if (string.endsWith(".json")) {
Map<String, ?> mappings = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputStream).mapOrdered();
XContentBuilder builder = JsonXContent.contentBuilder();
builder.startObject().map(mappings).endObject();
return Strings.toString(builder);
}
if (string.endsWith(".yml") || string.endsWith(".yaml")) {
Map<String, ?> mappings = YamlXContent.yamlXContent.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputStream).mapOrdered();
XContentBuilder builder = JsonXContent.contentBuilder();
builder.startObject().map(mappings).endObject();
return Strings.toString(builder);
}
}
return string;
} catch (MalformedInputException e) {
return string;
}
}
@ -896,7 +563,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
}
public void checkMapping(String index) {
ensureActive();
ensureClientIsPresent();
GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(index);
GetMappingsResponse getMappingsResponse = client.execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> map = getMappingsResponse.getMappings();
@ -992,63 +659,43 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
private static <K, V extends Comparable<? super V>> Map<K, V> sortByValue(Map<K, V> map) {
Map<K, V> result = new LinkedHashMap<>();
map.entrySet().stream().sorted(Comparator.comparing(Map.Entry::getValue))
map.entrySet().stream().sorted(Map.Entry.comparingByValue())
.forEachOrdered(e -> result.put(e.getKey(), e.getValue()));
return result;
}
private static TimeValue toTimeValue(long timeValue, TimeUnit timeUnit) {
switch (timeUnit) {
case DAYS:
return TimeValue.timeValueHours(24 * timeValue);
case HOURS:
return TimeValue.timeValueHours(timeValue);
case MINUTES:
return TimeValue.timeValueMinutes(timeValue);
case SECONDS:
return TimeValue.timeValueSeconds(timeValue);
case MILLISECONDS:
return TimeValue.timeValueMillis(timeValue);
case MICROSECONDS:
return TimeValue.timeValueNanos(1000 * timeValue);
case NANOSECONDS:
return TimeValue.timeValueNanos(timeValue);
default:
throw new IllegalArgumentException("unknown time unit: " + timeUnit);
}
}
private static class SuccessIndexShiftResult implements IndexShiftResult {
List<String> movedAliases;
Collection<String> movedAliases;
List<String> newAliases;
Collection<String> newAliases;
SuccessIndexShiftResult(List<String> movedAliases, List<String> newAliases) {
SuccessIndexShiftResult(Collection<String> movedAliases, Collection<String> newAliases) {
this.movedAliases = movedAliases;
this.newAliases = newAliases;
}
@Override
public List<String> getMovedAliases() {
public Collection<String> getMovedAliases() {
return movedAliases;
}
@Override
public List<String> getNewAliases() {
public Collection<String> getNewAliases() {
return newAliases;
}
}
private static class SuccessPruneResult implements IndexPruneResult {
List<String> candidateIndices;
Collection<String> candidateIndices;
List<String> indicesToDelete;
Collection<String> indicesToDelete;
DeleteIndexResponse response;
SuccessPruneResult(List<String> candidateIndices, List<String> indicesToDelete,
SuccessPruneResult(Collection<String> candidateIndices,
Collection<String> indicesToDelete,
DeleteIndexResponse response) {
this.candidateIndices = candidateIndices;
this.indicesToDelete = indicesToDelete;
@ -1061,12 +708,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
}
@Override
public List<String> getCandidateIndices() {
public Collection<String> getCandidateIndices() {
return candidateIndices;
}
@Override
public List<String> getDeletedIndices() {
public Collection<String> getDeletedIndices() {
return indicesToDelete;
}
@ -1078,11 +725,11 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
private static class NothingToDoPruneResult implements IndexPruneResult {
List<String> candidateIndices;
Collection<String> candidateIndices;
List<String> indicesToDelete;
Collection<String> indicesToDelete;
NothingToDoPruneResult(List<String> candidateIndices, List<String> indicesToDelete) {
NothingToDoPruneResult(Collection<String> candidateIndices, List<String> indicesToDelete) {
this.candidateIndices = candidateIndices;
this.indicesToDelete = indicesToDelete;
}
@ -1093,12 +740,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
}
@Override
public List<String> getCandidateIndices() {
public Collection<String> getCandidateIndices() {
return candidateIndices;
}
@Override
public List<String> getDeletedIndices() {
public Collection<String> getDeletedIndices() {
return indicesToDelete;
}

246
elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java Normal file
View file

@ -0,0 +1,246 @@
package org.xbib.elx.common;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.flush.FlushAction;
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.DeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.xbib.elx.api.BulkClient;
import org.xbib.elx.api.BulkController;
import org.xbib.elx.api.BulkMetric;
import org.xbib.elx.api.IndexDefinition;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.concurrent.TimeUnit;
public abstract class AbstractBulkClient extends AbstractNativeClient implements BulkClient {
private static final Logger logger = LogManager.getLogger(AbstractBulkClient.class.getName());
private BulkMetric bulkMetric;
private BulkController bulkController;
@Override
public void init(Settings settings) throws IOException {
logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(','));
super.init(settings);
if (bulkMetric == null) {
bulkMetric = new DefaultBulkMetric();
bulkMetric.init(settings);
}
if (bulkController == null) {
bulkController = new DefaultBulkController(this, bulkMetric);
bulkController.init(settings);
}
}
@Override
public BulkMetric getBulkMetric() {
return bulkMetric;
}
@Override
public BulkController getBulkController() {
return bulkController;
}
@Override
public void flush() throws IOException {
if (bulkController != null) {
bulkController.flush();
}
}
@Override
public void close() throws IOException {
ensureClientIsPresent();
if (closed.compareAndSet(false, true)) {
if (bulkMetric != null) {
logger.info("closing bulk metric");
bulkMetric.close();
}
if (bulkController != null) {
logger.info("closing bulk controller");
bulkController.close();
}
closeClient();
}
}
@Override
public void newIndex(IndexDefinition indexDefinition) throws IOException {
Settings settings = indexDefinition.getSettings() == null ? null :
Settings.builder().loadFromSource(indexDefinition.getSettings(), XContentType.JSON).build();
Map<String, ?> mappings = indexDefinition.getMappings() == null ? null :
JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, indexDefinition.getMappings()).mapOrdered();
newIndex(indexDefinition.getFullIndexName(), settings, mappings);
}
@Override
public void newIndex(String index) throws IOException {
newIndex(index, Settings.EMPTY, (Map<String, ?>) null);
}
@Override
public void newIndex(String index, Settings settings) throws IOException {
newIndex(index, settings, (Map<String, ?>) null);
}
@Override
public void newIndex(String index, Settings settings, XContentBuilder builder) throws IOException {
String mappingString = Strings.toString(builder);
Map<String, ?> mappings = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mappingString).mapOrdered();
newIndex(index, settings, mappings);
}
@Override
public void newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException {
if (index == null) {
logger.warn("no index name given to create index");
return;
}
ensureClientIsPresent();
waitForCluster("YELLOW", 30L, TimeUnit.SECONDS);
CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index);
if (settings != null) {
createIndexRequest.settings(settings);
}
if (mapping != null) {
createIndexRequest.mapping(TYPE_NAME, mapping);
}
CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
XContentBuilder builder = XContentFactory.jsonBuilder();
logger.info("index {} created: {}", index,
Strings.toString(createIndexResponse.toXContent(builder, ToXContent.EMPTY_PARAMS)));
}
@Override
public void startBulk(IndexDefinition indexDefinition) throws IOException {
startBulk(indexDefinition.getFullIndexName(), -1, 1);
}
@Override
public void startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds)
throws IOException {
if (bulkController != null) {
ensureClientIsPresent();
bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds);
}
}
@Override
public void stopBulk(IndexDefinition indexDefinition) throws IOException {
if (bulkController != null) {
ensureClientIsPresent();
bulkController.stopBulkMode(indexDefinition);
}
}
@Override
public void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException {
if (bulkController != null) {
ensureClientIsPresent();
bulkController.stopBulkMode(index, timeout, timeUnit);
}
}
@Override
public BulkClient index(String index, String id, boolean create, String source) {
return index(new IndexRequest(index, TYPE_NAME, id).create(create)
.source(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
}
@Override
public BulkClient index(String index, String id, boolean create, BytesReference source) {
return index(new IndexRequest(index, TYPE_NAME, id).create(create)
.source(source, XContentType.JSON));
}
@Override
public BulkClient index(IndexRequest indexRequest) {
ensureClientIsPresent();
bulkController.index(indexRequest);
return this;
}
@Override
public BulkClient delete(String index, String id) {
return delete(new DeleteRequest(index, TYPE_NAME, id));
}
@Override
public BulkClient delete(DeleteRequest deleteRequest) {
ensureClientIsPresent();
bulkController.delete(deleteRequest);
return this;
}
@Override
public BulkClient update(String index, String id, BytesReference source) {
return update(new UpdateRequest(index, TYPE_NAME, id)
.doc(source, XContentType.JSON));
}
@Override
public BulkClient update(String index, String id, String source) {
return update(new UpdateRequest(index, TYPE_NAME, id)
.doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
}
@Override
public BulkClient update(UpdateRequest updateRequest) {
ensureClientIsPresent();
bulkController.update(updateRequest);
return this;
}
@Override
public boolean waitForResponses(long timeout, TimeUnit timeUnit) {
ensureClientIsPresent();
return bulkController.waitForResponses(timeout, timeUnit);
}
@Override
public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException {
super.updateIndexSetting(index, key, value, timeout, timeUnit);
}
@Override
public void flushIndex(String index) {
if (index != null) {
ensureClientIsPresent();
client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet();
}
}
@Override
public void refreshIndex(String index) {
if (index != null) {
ensureClientIsPresent();
client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet();
}
}
}

218
elx-common/src/main/java/org/xbib/elx/common/AbstractNativeClient.java Normal file
View file

@ -0,0 +1,218 @@
package org.xbib.elx.common;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequestBuilder;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.QueryBuilders;
import org.xbib.elx.api.NativeClient;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
public abstract class AbstractNativeClient implements NativeClient {
private static final Logger logger = LogManager.getLogger(AbstractNativeClient.class.getName());
/**
* The one and only index type name used in the extended client.
* Notr that all Elasticsearch version < 6.2.0 do not allow a prepending "_".
*/
protected static final String TYPE_NAME = "doc";
protected ElasticsearchClient client;
protected final AtomicBoolean closed;
protected AbstractNativeClient() {
closed = new AtomicBoolean(false);
}
@Override
public void setClient(ElasticsearchClient client) {
this.client = client;
}
@Override
public ElasticsearchClient getClient() {
return client;
}
protected abstract ElasticsearchClient createClient(Settings settings) throws IOException;
protected abstract void closeClient() throws IOException;
@Override
public void init(Settings settings) throws IOException {
logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(','));
if (client == null) {
client = createClient(settings);
}
}
@Override
public String getClusterName() {
ensureClientIsPresent();
try {
ClusterStateRequest clusterStateRequest = new ClusterStateRequest().clear();
ClusterStateResponse clusterStateResponse =
getClient().execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
return clusterStateResponse.getClusterName().value();
} catch (ElasticsearchTimeoutException e) {
logger.warn(e.getMessage(), e);
return "TIMEOUT";
} catch (NoNodeAvailableException e) {
logger.warn(e.getMessage(), e);
return "DISCONNECTED";
} catch (Exception e) {
logger.warn(e.getMessage(), e);
return "[" + e.getMessage() + "]";
}
}
@Override
public void waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) {
ensureClientIsPresent();
ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString);
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
String message = "timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name();
if (logger.isErrorEnabled()) {
logger.error(message);
}
throw new IllegalStateException(message);
}
}
@Override
public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) {
ensureClientIsPresent();
try {
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().timeout(timeout)).actionGet();
ClusterHealthStatus status = healthResponse.getStatus();
return status.name();
} catch (ElasticsearchTimeoutException e) {
logger.warn(e.getMessage(), e);
return "TIMEOUT";
} catch (NoNodeAvailableException e) {
logger.warn(e.getMessage(), e);
return "DISCONNECTED";
} catch (Exception e) {
logger.warn(e.getMessage(), e);
return "[" + e.getMessage() + "]";
}
}
@Override
public Map<String, ?> getMapping(String index, String mapping) {
GetMappingsRequestBuilder getMappingsRequestBuilder = new GetMappingsRequestBuilder(client, GetMappingsAction.INSTANCE)
.setIndices(index)
.setTypes(mapping);
GetMappingsResponse getMappingsResponse = getMappingsRequestBuilder.execute().actionGet();
logger.info("get mappings response = {}", getMappingsResponse.getMappings().get(index).get(mapping).getSourceAsMap());
return getMappingsResponse.getMappings().get(index).get(mapping).getSourceAsMap();
}
@Override
public long getSearchableDocs(String index) {
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE)
.setIndices(index)
.setQuery(QueryBuilders.matchAllQuery())
.setSize(0);
return searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
}
@Override
public boolean isIndexExists(String index) {
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
indicesExistsRequest.indices(index);
IndicesExistsResponse indicesExistsResponse =
client.execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
return indicesExistsResponse.isExists();
}
@Override
public void close() throws IOException {
ensureClientIsPresent();
if (closed.compareAndSet(false, true)) {
closeClient();
}
}
protected void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException {
ensureClientIsPresent();
if (index == null) {
throw new IOException("no index name given");
}
if (key == null) {
throw new IOException("no key given");
}
if (value == null) {
throw new IOException("no value given");
}
Settings.Builder updateSettingsBuilder = Settings.builder();
updateSettingsBuilder.put(key, value.toString());
UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index)
.settings(updateSettingsBuilder).timeout(toTimeValue(timeout, timeUnit));
client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet();
}
protected void ensureClientIsPresent() {
if (this instanceof MockAdminClient) {
return;
}
if (client == null) {
throw new IllegalStateException("no client");
}
}
protected static TimeValue toTimeValue(long timeValue, TimeUnit timeUnit) {
switch (timeUnit) {
case DAYS:
return TimeValue.timeValueHours(24 * timeValue);
case HOURS:
return TimeValue.timeValueHours(timeValue);
case MINUTES:
return TimeValue.timeValueMinutes(timeValue);
case SECONDS:
return TimeValue.timeValueSeconds(timeValue);
case MILLISECONDS:
return TimeValue.timeValueMillis(timeValue);
case MICROSECONDS:
return TimeValue.timeValueNanos(1000 * timeValue);
case NANOSECONDS:
return TimeValue.timeValueNanos(timeValue);
default:
throw new IllegalArgumentException("unknown time unit: " + timeUnit);
}
}
}

158
elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java Normal file
View file

@ -0,0 +1,158 @@
package org.xbib.elx.common;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.get.GetAction;
import org.elasticsearch.action.get.GetRequestBuilder;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.get.MultiGetAction;
import org.elasticsearch.action.get.MultiGetRequestBuilder;
import org.elasticsearch.action.get.MultiGetResponse;
import org.elasticsearch.action.search.ClearScrollAction;
import org.elasticsearch.action.search.ClearScrollRequestBuilder;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollAction;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.action.search.ShardSearchFailure;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.search.SearchHit;
import org.xbib.elx.api.SearchClient;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Optional;
import java.util.Spliterator;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
public abstract class AbstractSearchClient extends AbstractNativeClient implements SearchClient {
@Override
public Optional<GetResponse> get(Consumer<GetRequestBuilder> getRequestBuilderConsumer) {
GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE);
getRequestBuilderConsumer.accept(getRequestBuilder);
GetResponse getResponse = getRequestBuilder.execute().actionGet();
return getResponse.isExists() ? Optional.of(getResponse) : Optional.empty();
}
@Override
public Optional<MultiGetResponse> multiGet(Consumer<MultiGetRequestBuilder> multiGetRequestBuilderConsumer) {
MultiGetRequestBuilder multiGetRequestBuilder = new MultiGetRequestBuilder(client, MultiGetAction.INSTANCE);
multiGetRequestBuilderConsumer.accept(multiGetRequestBuilder);
MultiGetResponse multiGetItemResponse = multiGetRequestBuilder.execute().actionGet();
return multiGetItemResponse.getResponses().length == 0 ? Optional.empty() : Optional.of(multiGetItemResponse);
}
@Override
public Optional<SearchResponse> search(Consumer<SearchRequestBuilder> queryBuilder) {
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
queryBuilder.accept(searchRequestBuilder);
SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
if (searchResponse.getFailedShards() > 0) {
StringBuilder sb = new StringBuilder("Search failed:");
for (ShardSearchFailure failure : searchResponse.getShardFailures()) {
sb.append("\n").append(failure.reason());
}
throw new ElasticsearchException(sb.toString());
}
return searchResponse.getHits().getHits().length == 0 ? Optional.empty() : Optional.of(searchResponse);
}
@Override
public Stream<SearchHit> search(Consumer<SearchRequestBuilder> queryBuilder,
TimeValue scrollTime, int scrollSize) {
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
queryBuilder.accept(searchRequestBuilder);
searchRequestBuilder.setScroll(scrollTime).setSize(scrollSize);
SearchResponse originalSearchResponse = searchRequestBuilder.execute().actionGet();
Stream<SearchResponse> infiniteResponses = Stream.iterate(originalSearchResponse,
searchResponse -> new SearchScrollRequestBuilder(client, SearchScrollAction.INSTANCE)
.setScrollId(searchResponse.getScrollId())
.setScroll(scrollTime)
.execute().actionGet());
Predicate<SearchResponse> condition = searchResponse -> searchResponse.getHits().getHits().length > 0;
Consumer<SearchResponse> lastAction = searchResponse -> {
ClearScrollRequestBuilder clearScrollRequestBuilder =
new ClearScrollRequestBuilder(client, ClearScrollAction.INSTANCE)
.addScrollId(searchResponse.getScrollId());
clearScrollRequestBuilder.execute().actionGet();
};
return StreamSupport.stream(TakeWhileSpliterator.over(infiniteResponses.spliterator(),
condition, lastAction), false)
.onClose(infiniteResponses::close)
.flatMap(searchResponse -> Arrays.stream(searchResponse.getHits().getHits()));
}
@Override
public Stream<String> getIds(Consumer<SearchRequestBuilder> queryBuilder) {
return search(queryBuilder, TimeValue.timeValueMinutes(1), 1000).map(SearchHit::getId);
}
static class TakeWhileSpliterator<T> implements Spliterator<T> {
private final Spliterator<T> source;
private final Predicate<T> condition;
private final Consumer<T> lastElement;
private final boolean inclusive;
private final AtomicBoolean checked;
static <T> TakeWhileSpliterator<T> over(Spliterator<T> source,
Predicate<T> condition,
Consumer<T> lastElement) {
return new TakeWhileSpliterator<>(source, condition, lastElement, true);
}
private TakeWhileSpliterator(Spliterator<T> source,
Predicate<T> condition,
Consumer<T> lastElement,
boolean inclusive) {
this.source = source;
this.condition = condition;
this.lastElement = lastElement;
this.inclusive = inclusive;
this.checked = new AtomicBoolean(true);
}
@Override
public boolean tryAdvance(Consumer<? super T> action) {
return checked.get() && source.tryAdvance(e -> {
if (condition.test(e)) {
action.accept(e);
} else {
if (inclusive && checked.get()) {
action.accept(e);
}
lastElement.accept(e);
checked.set(false);
}
});
}
@Override
public Spliterator<T> trySplit() {
return null;
}
@Override
public long estimateSize() {
return checked.get() ? source.estimateSize() : 0;
}
@Override
public int characteristics() {
return source.characteristics() &~ Spliterator.SIZED;
}
@Override
public Comparator<? super T> getComparator() {
return source.getComparator();
}
}
}

76
elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java
View file

@ -8,12 +8,12 @@ import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.xbib.elx.api.ExtendedClient;
import org.xbib.elx.api.ExtendedClientProvider;
import org.xbib.elx.api.AdminClientProvider;
import org.xbib.elx.api.BulkClientProvider;
import org.xbib.elx.api.NativeClient;
import org.xbib.elx.api.SearchClientProvider;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.ServiceLoader;
@SuppressWarnings("rawtypes")
@ -23,11 +23,15 @@ public class ClientBuilder {
private final ElasticsearchClient client;
private final ClassLoader classLoader;
private final Settings.Builder settingsBuilder;
private Map<Class<? extends ExtendedClientProvider>, ExtendedClientProvider> providerMap;
private Class<? extends AdminClientProvider> adminClientProvider;
private Class<? extends ExtendedClientProvider> provider;
private Class<? extends BulkClientProvider> bulkClientProvider;
private Class<? extends SearchClientProvider> searchClientProvider;
public ClientBuilder() {
this(null);
@ -39,14 +43,9 @@ public class ClientBuilder {
public ClientBuilder(ElasticsearchClient client, ClassLoader classLoader) {
this.client = client;
this.classLoader = classLoader;
this.settingsBuilder = Settings.builder();
settingsBuilder.put("node.name", "elx-client-" + Version.CURRENT);
this.providerMap = new HashMap<>();
ServiceLoader<ExtendedClientProvider> serviceLoader = ServiceLoader.load(ExtendedClientProvider.class,
classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader());
for (ExtendedClientProvider provider : serviceLoader) {
providerMap.put(provider.getClass(), provider);
}
}
public static ClientBuilder builder() {
@ -57,8 +56,18 @@ public class ClientBuilder {
return new ClientBuilder(client);
}
public ClientBuilder provider(Class<? extends ExtendedClientProvider> provider) {
this.provider = provider;
public ClientBuilder setAdminClientProvider(Class<? extends AdminClientProvider> adminClientProvider) {
this.adminClientProvider = adminClientProvider;
return this;
}
public ClientBuilder setBulkClientProvider(Class<? extends BulkClientProvider> bulkClientProvider) {
this.bulkClientProvider = bulkClientProvider;
return this;
}
public ClientBuilder setSearchClientProvider(Class<? extends SearchClientProvider> searchClientProvider) {
this.searchClientProvider = searchClientProvider;
return this;
}
@ -98,14 +107,39 @@ public class ClientBuilder {
}
@SuppressWarnings("unchecked")
public <C extends ExtendedClient> C build() throws IOException {
if (provider == null) {
throw new IllegalArgumentException("no provider");
}
public <C extends NativeClient> C build() throws IOException {
Settings settings = settingsBuilder.build();
logger.log(Level.INFO, "settings = " + settings.toDelimitedString(','));
return (C) providerMap.get(provider).getExtendedClient()
.setClient(client)
.init(settings);
if (adminClientProvider != null) {
for (AdminClientProvider provider : ServiceLoader.load(AdminClientProvider.class, classLoader)) {
if (provider.getClass().isAssignableFrom(adminClientProvider)) {
C c = (C) provider.getClient();
c.setClient(client);
c.init(settings);
return c;
}
}
}
if (bulkClientProvider != null) {
for (BulkClientProvider provider : ServiceLoader.load(BulkClientProvider.class, classLoader)) {
if (provider.getClass().isAssignableFrom(bulkClientProvider)) {
C c = (C) provider.getClient();
c.setClient(client);
c.init(settings);
return c;
}
}
}
if (searchClientProvider != null) {
for (SearchClientProvider provider : ServiceLoader.load(SearchClientProvider.class, classLoader)) {
if (provider.getClass().isAssignableFrom(searchClientProvider)) {
C c = (C) provider.getClient();
c.setClient(client);
c.init(settings);
return c;
}
}
}
throw new IllegalArgumentException("no provider");
}
}

10
elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java
View file

@ -11,10 +11,10 @@ import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.xbib.elx.api.BulkClient;
import org.xbib.elx.api.BulkController;
import org.xbib.elx.api.BulkMetric;
import org.xbib.elx.api.BulkProcessor;
import org.xbib.elx.api.ExtendedClient;
import org.xbib.elx.api.IndexDefinition;
import java.io.IOException;
@ -29,7 +29,7 @@ public class DefaultBulkController implements BulkController {
private static final Logger logger = LogManager.getLogger(DefaultBulkController.class);
private final ExtendedClient client;
private final BulkClient client;
private final BulkMetric bulkMetric;
@ -39,9 +39,9 @@ public class DefaultBulkController implements BulkController {
private final Map<String, Long> stopBulkRefreshIntervals;
private long maxWaitTime;
private final long maxWaitTime;
private TimeUnit maxWaitTimeUnit;
private final TimeUnit maxWaitTimeUnit;
private BulkProcessor bulkProcessor;
@ -51,7 +51,7 @@ public class DefaultBulkController implements BulkController {
private boolean enableBulkLogging;
public DefaultBulkController(ExtendedClient client, BulkMetric bulkMetric) {
public DefaultBulkController(BulkClient client, BulkMetric bulkMetric) {
this.client = client;
this.bulkMetric = bulkMetric;
this.indexNames = new ArrayList<>();

40
elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java
View file

@ -3,8 +3,6 @@ package org.xbib.elx.common;
import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.api.IndexRetention;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.concurrent.TimeUnit;
public class DefaultIndexDefinition implements IndexDefinition {
@ -15,9 +13,9 @@ public class DefaultIndexDefinition implements IndexDefinition {
private String dateTimePattern;
private URL settingsUrl;
private String settings;
private URL mappingsUrl;
private String mappings;
private boolean enabled;
@ -62,37 +60,25 @@ public class DefaultIndexDefinition implements IndexDefinition {
}
@Override
public IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException {
this.settingsUrl = settingsUrlString != null ? new URL(settingsUrlString) : null;
public IndexDefinition setSettings(String settings) {
this.settings = settings;
return this;
}
@Override
public IndexDefinition setSettingsUrl(URL settingsUrl) {
this.settingsUrl = settingsUrl;
public String getSettings() {
return settings;
}
@Override
public IndexDefinition setMappings(String mappings) {
this.mappings = mappings;
return this;
}
@Override
public URL getSettingsUrl() {
return settingsUrl;
}
@Override
public IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException {
this.mappingsUrl = mappingsUrlString != null ? new URL(mappingsUrlString) : null;
return this;
}
@Override
public IndexDefinition setMappingsUrl(URL mappingsUrl) {
this.mappingsUrl = mappingsUrl;
return this;
}
@Override
public URL getMappingsUrl() {
return mappingsUrl;
public String getMappings() {
return mappings;
}
@Override

59
elx-common/src/main/java/org/xbib/elx/common/MockAdminClient.java Normal file
View file

@ -0,0 +1,59 @@
package org.xbib.elx.common;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import java.util.concurrent.TimeUnit;
/**
* A mocked client, it does not perform any actions on a cluster. Useful for testing.
*/
public class MockAdminClient extends AbstractAdminClient {
@Override
public ElasticsearchClient getClient() {
return null;
}
@Override
public void init(Settings settings) {
}
@Override
protected ElasticsearchClient createClient(Settings settings) {
return null;
}
@Override
protected void closeClient() {
}
@Override
public MockAdminClient deleteIndex(String index) {
return this;
}
@Override
public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) {
return true;
}
@Override
public void waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) {
}
@Override
public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) {
return true;
}
@Override
public MockAdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) {
return this;
}
@Override
public void close() {
// nothing to do
}
}

10
elx-common/src/main/java/org/xbib/elx/common/MockAdminClientProvider.java Normal file
View file

@ -0,0 +1,10 @@
package org.xbib.elx.common;
import org.xbib.elx.api.AdminClientProvider;
public class MockAdminClientProvider implements AdminClientProvider<MockAdminClient> {
@Override
public MockAdminClient getClient() {
return new MockAdminClient();
}
}

99
elx-common/src/main/java/org/xbib/elx/common/MockBulkClient.java Normal file
View file

@ -0,0 +1,99 @@
package org.xbib.elx.common;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import java.util.concurrent.TimeUnit;
/**
* A mocked client, it does not perform any actions on a cluster. Useful for testing.
*/
public class MockBulkClient extends AbstractBulkClient {
@Override
public ElasticsearchClient getClient() {
return null;
}
@Override
public void init(Settings settings) {
}
@Override
public String getClusterName() {
return null;
}
@Override
protected ElasticsearchClient createClient(Settings settings) {
return null;
}
@Override
protected void closeClient() {
}
@Override
public MockBulkClient index(String index, String id, boolean create, String source) {
return this;
}
@Override
public MockBulkClient delete(String index, String id) {
return this;
}
@Override
public MockBulkClient update(String index, String id, String source) {
return this;
}
@Override
public MockBulkClient index(IndexRequest indexRequest) {
return this;
}
@Override
public MockBulkClient delete(DeleteRequest deleteRequest) {
return this;
}
@Override
public MockBulkClient update(UpdateRequest updateRequest) {
return this;
}
@Override
public void startBulk(String index, long startRefreshInterval, long stopRefreshIterval) {
}
@Override
public void stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) {
}
@Override
public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) {
return true;
}
@Override
public void refreshIndex(String index) {
}
@Override
public void flushIndex(String index) {
}
@Override
public void flush() {
// nothing to do
}
@Override
public void close() {
// nothing to do
}
}

11
elx-common/src/main/java/org/xbib/elx/common/MockBulkClientProvider.java Normal file
View file

@ -0,0 +1,11 @@
package org.xbib.elx.common;
import org.xbib.elx.api.BulkClientProvider;
public class MockBulkClientProvider implements BulkClientProvider<MockBulkClient> {
@Override
public MockBulkClient getClient() {
return new MockBulkClient();
}
}

129
elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java
View file

@ -1,129 +0,0 @@
package org.xbib.elx.common;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import java.util.concurrent.TimeUnit;
/**
* A mocked client, it does not perform any actions on a cluster. Useful for testing.
*/
public class MockExtendedClient extends AbstractExtendedClient {
@Override
public ElasticsearchClient getClient() {
return null;
}
@Override
public MockExtendedClient init(Settings settings) {
return this;
}
@Override
protected ElasticsearchClient createClient(Settings settings) {
return null;
}
@Override
protected void closeClient() {
}
@Override
public MockExtendedClient index(String index, String id, boolean create, String source) {
return this;
}
@Override
public MockExtendedClient delete(String index, String id) {
return this;
}
@Override
public MockExtendedClient update(String index, String id, String source) {
return this;
}
@Override
public MockExtendedClient index(IndexRequest indexRequest) {
return this;
}
@Override
public MockExtendedClient delete(DeleteRequest deleteRequest) {
return this;
}
@Override
public MockExtendedClient update(UpdateRequest updateRequest) {
return this;
}
@Override
public MockExtendedClient startBulk(String index, long startRefreshInterval, long stopRefreshIterval) {
return this;
}
@Override
public MockExtendedClient stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) {
return this;
}
@Override
public MockExtendedClient newIndex(String index) {
return this;
}
@Override
public MockExtendedClient deleteIndex(String index) {
return this;
}
@Override
public MockExtendedClient refreshIndex(String index) {
return this;
}
@Override
public MockExtendedClient flushIndex(String index) {
return this;
}
@Override
public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) {
return true;
}
@Override
public boolean waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) {
return true;
}
@Override
public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) {
return true;
}
@Override
public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) {
return true;
}
@Override
public MockExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) {
return this;
}
@Override
public void flush() {
// nothing to do
}
@Override
public void close() {
// nothing to do
}
}

10
elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java
View file

@ -1,10 +0,0 @@
package org.xbib.elx.common;
import org.xbib.elx.api.ExtendedClientProvider;
public class MockExtendedClientProvider implements ExtendedClientProvider<MockExtendedClient> {
@Override
public MockExtendedClient getExtendedClient() {
return new MockExtendedClient();
}
}

38
elx-common/src/main/java/org/xbib/elx/common/MockSearchClient.java Normal file
View file

@ -0,0 +1,38 @@
package org.xbib.elx.common;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
/**
* A mocked client, it does not perform any actions on a cluster. Useful for testing.
*/
public class MockSearchClient extends AbstractSearchClient {
@Override
public ElasticsearchClient getClient() {
return null;
}
@Override
public void init(Settings settings) {
}
@Override
public String getClusterName() {
return null;
}
@Override
protected ElasticsearchClient createClient(Settings settings) {
return null;
}
@Override
protected void closeClient() {
}
@Override
public void close() {
// nothing to do
}
}

11
elx-common/src/main/java/org/xbib/elx/common/MockSearchClientProvider.java Normal file
View file

@ -0,0 +1,11 @@
package org.xbib.elx.common;
import org.xbib.elx.api.SearchClientProvider;
public class MockSearchClientProvider implements SearchClientProvider<MockSearchClient> {
@Override
public MockSearchClient getClient() {
return new MockSearchClient();
}
}

1
elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider Normal file
View file

@ -0,0 +1 @@
org.xbib.elx.common.MockAdminClientProvider

1
elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider
View file

@ -1 +0,0 @@
org.xbib.elx.common.MockExtendedClientProvider

21
elx-common/src/test/java/org/xbib/elx/common/test/MockAdminClientProviderTest.java Normal file
View file

@ -0,0 +1,21 @@
package org.xbib.elx.common.test;
import org.junit.jupiter.api.Test;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.MockAdminClient;
import org.xbib.elx.common.MockAdminClientProvider;
import java.io.IOException;
import static org.junit.jupiter.api.Assertions.assertNotNull;
class MockAdminClientProviderTest {
@Test
void testMockAdminProvider() throws IOException {
MockAdminClient client = ClientBuilder.builder()
.setAdminClientProvider(MockAdminClientProvider.class)
.build();
assertNotNull(client);
}
}

19
elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java
View file

@ -1,19 +0,0 @@
package org.xbib.elx.common.test;
import org.junit.jupiter.api.Test;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.MockExtendedClient;
import org.xbib.elx.common.MockExtendedClientProvider;
import java.io.IOException;
import static org.junit.jupiter.api.Assertions.assertNotNull;
class MockExtendedClientProviderTest {
@Test
void testMockExtendedProvider() throws IOException {
MockExtendedClient client = ClientBuilder.builder().provider(MockExtendedClientProvider.class).build();
assertNotNull(client);
}
}

155
elx-http/src/main/java/org/elasticsearch/action/admin/indices/get/HttpGetIndexAction.java Normal file
View file

@ -0,0 +1,155 @@
package org.elasticsearch.action.admin.indices.get;
import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
import org.apache.lucene.util.CollectionUtil;
import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentParser;
import org.xbib.elx.http.HttpAction;
import org.xbib.netty.http.client.api.Request;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
public class HttpGetIndexAction extends HttpAction<GetIndexRequest, GetIndexResponse> {
@Override
public GetIndexAction getActionInstance() {
return GetIndexAction.INSTANCE;
}
@Override
protected Request.Builder createHttpRequest(String url, GetIndexRequest getIndexRequest) throws IOException {
List<String> list = getIndexRequest.indices().length == 0 ?
List.of("*") : Arrays.asList(getIndexRequest.indices());
String command = "/" + String.join(",", list);
logger.info("command = " + command);
return newGetRequest(url, command);
}
@Override
protected CheckedFunction<XContentParser, GetIndexResponse, IOException> entityParser() {
return this::fromXContent;
}
@Override
protected GetIndexResponse emptyResponse() {
return new GetIndexResponse();
}
private GetIndexResponse fromXContent(XContentParser parser) throws IOException {
ImmutableOpenMap.Builder<String, List<AliasMetaData>> aliases = ImmutableOpenMap.builder();
ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> mappings = ImmutableOpenMap.builder();
ImmutableOpenMap.Builder<String, Settings> settings = ImmutableOpenMap.builder();
List<String> indices = new ArrayList<>();
if (parser.currentToken() == null) {
parser.nextToken();
}
ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
parser.nextToken();
while (!parser.isClosed()) {
if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
String indexName = parser.currentName();
indices.add(indexName);
IndexEntry indexEntry = parseIndexEntry(parser);
CollectionUtil.timSort(indexEntry.indexAliases, Comparator.comparing(AliasMetaData::alias));
aliases.put(indexName, Collections.unmodifiableList(indexEntry.indexAliases));
mappings.put(indexName, indexEntry.indexMappings);
settings.put(indexName, indexEntry.indexSettings);
} else if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
parser.skipChildren();
} else {
parser.nextToken();
}
}
return new GetIndexResponse(indices.toArray(new String[0]),
mappings.build(),
aliases.build(),
settings.build());
}
private static IndexEntry parseIndexEntry(XContentParser parser) throws IOException {
List<AliasMetaData> indexAliases = null;
ImmutableOpenMap<String, MappingMetaData> indexMappings = null;
Settings indexSettings = null;
Settings indexDefaultSettings = null;
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
parser.nextToken();
if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
switch (parser.currentName()) {
case "aliases":
indexAliases = parseAliases(parser);
break;
case "mappings":
indexMappings = parseMappings(parser);
break;
case "settings":
indexSettings = Settings.fromXContent(parser);
break;
case "defaults":
indexDefaultSettings = Settings.fromXContent(parser);
break;
default:
parser.skipChildren();
}
} else if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
parser.skipChildren();
}
}
return new IndexEntry(indexAliases, indexMappings, indexSettings, indexDefaultSettings);
}
private static List<AliasMetaData> parseAliases(XContentParser parser) throws IOException {
List<AliasMetaData> indexAliases = new ArrayList<>();
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
indexAliases.add(AliasMetaData.Builder.fromXContent(parser));
}
return indexAliases;
}
private static ImmutableOpenMap<String, MappingMetaData> parseMappings(XContentParser parser) throws IOException {
ImmutableOpenMap.Builder<String, MappingMetaData> indexMappings = ImmutableOpenMap.builder();
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
parser.nextToken();
if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
String mappingType = parser.currentName();
indexMappings.put(mappingType, new MappingMetaData(mappingType, parser.map()));
} else if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
parser.skipChildren();
}
}
return indexMappings.build();
}
private static class IndexEntry {
List<AliasMetaData> indexAliases = new ArrayList<>();
ImmutableOpenMap<String, MappingMetaData> indexMappings = ImmutableOpenMap.of();
Settings indexSettings = Settings.EMPTY;
Settings indexDefaultSettings = Settings.EMPTY;
IndexEntry(List<AliasMetaData> indexAliases, ImmutableOpenMap<String, MappingMetaData> indexMappings,
Settings indexSettings, Settings indexDefaultSettings) {
if (indexAliases != null) {
this.indexAliases = indexAliases;
}
if (indexMappings != null) {
this.indexMappings = indexMappings;
}
if (indexSettings != null) {
this.indexSettings = indexSettings;
}
if (indexDefaultSettings != null) {
this.indexDefaultSettings = indexDefaultSettings;
}
}
}
}

12
elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClientProvider.java
View file

@ -1,12 +0,0 @@
package org.xbib.elx.http;
import org.xbib.elx.api.ExtendedClientProvider;
import java.util.Collections;
public class ExtendedHttpClientProvider implements ExtendedClientProvider<ExtendedHttpClient> {
@Override
public ExtendedHttpClient getExtendedClient() {
return new ExtendedHttpClient(Collections.emptyList(), Thread.currentThread().getContextClassLoader());
}
}

6
elx-http/src/main/java/org/xbib/elx/http/HttpAction.java
View file

@ -77,7 +77,7 @@ public abstract class HttpAction<R extends ActionRequest, T extends ActionRespon
}
}
});
Transport transport = httpActionContext.getExtendedHttpClient().internalClient().execute(httpRequest);
Transport transport = httpActionContext.getHelper().internalClient().execute(httpRequest);
httpActionContext.setHttpClientTransport(transport);
if (transport.isFailed()) {
listener.onFailure(new Exception(transport.getFailure()));
@ -155,7 +155,7 @@ public abstract class HttpAction<R extends ActionRequest, T extends ActionRespon
throw new IllegalStateException("unsupported content-type: " + mediaType);
}
try (XContentParser parser = xContentType.xContent()
.createParser(httpActionContext.getExtendedHttpClient().getRegistry(),
.createParser(httpActionContext.getHelper().getRegistry(),
DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
httpActionContext.getHttpResponse().getBody().toString(StandardCharsets.UTF_8))) {
return entityParser().apply(parser);
@ -167,7 +167,7 @@ public abstract class HttpAction<R extends ActionRequest, T extends ActionRespon
protected ElasticsearchStatusException parseToError(HttpActionContext<R, T> httpActionContext) {
try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
.createParser(httpActionContext.getExtendedHttpClient().getRegistry(),
.createParser(httpActionContext.getHelper().getRegistry(),
DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
httpActionContext.getHttpResponse().getBody().toString(StandardCharsets.UTF_8))) {
return errorParser().apply(parser);

10
elx-http/src/main/java/org/xbib/elx/http/HttpActionContext.java
View file

@ -13,7 +13,7 @@ import org.xbib.netty.http.common.HttpResponse;
*/
public class HttpActionContext<R extends ActionRequest, T extends ActionResponse> {
private final ExtendedHttpClient extendedHttpClient;
private final HttpClientHelper helper;
private final R request;
@ -23,14 +23,14 @@ public class HttpActionContext<R extends ActionRequest, T extends ActionResponse
private HttpResponse httpResponse;
HttpActionContext(ExtendedHttpClient extendedHttpClient, R request, String url) {
this.extendedHttpClient = extendedHttpClient;
HttpActionContext(HttpClientHelper helper, R request, String url) {
this.helper = helper;
this.request = request;
this.url = url;
}
public ExtendedHttpClient getExtendedHttpClient() {
return extendedHttpClient;
public HttpClientHelper getHelper() {
return helper;
}
public R getRequest() {

61
elx-http/src/main/java/org/xbib/elx/http/HttpAdminClient.java Normal file
View file

@ -0,0 +1,61 @@
package org.xbib.elx.http;
import org.elasticsearch.action.Action;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.threadpool.ThreadPool;
import org.xbib.elx.common.AbstractAdminClient;
import java.io.IOException;
/**
* Elasticsearch HTTP admin client.
*/
public class HttpAdminClient extends AbstractAdminClient implements ElasticsearchClient {
private final HttpClientHelper helper;
public HttpAdminClient() {
this.helper = new HttpClientHelper();
}
@Override
public void init(Settings settings) throws IOException {
super.init(settings);
helper.init(settings);
}
@Override
protected ElasticsearchClient createClient(Settings settings) throws IOException {
return helper.createClient(settings);
}
@Override
protected void closeClient() throws IOException {
helper.closeClient();
}
@Override
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(Action<Request, Response, RequestBuilder> action, Request request) {
return helper.execute(action, request);
}
@Override
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
helper.execute(action, request, listener);
}
@Override
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(Action<Request, Response, RequestBuilder> action) {
return helper.prepareExecute(action);
}
@Override
public ThreadPool threadPool() {
return helper.threadPool();
}
}

10
elx-http/src/main/java/org/xbib/elx/http/HttpAdminClientProvider.java Normal file
View file

@ -0,0 +1,10 @@
package org.xbib.elx.http;
import org.xbib.elx.api.AdminClientProvider;
public class HttpAdminClientProvider implements AdminClientProvider<HttpAdminClient> {
@Override
public HttpAdminClient getClient() {
return new HttpAdminClient();
}
}

61
elx-http/src/main/java/org/xbib/elx/http/HttpBulkClient.java Normal file
View file

@ -0,0 +1,61 @@
package org.xbib.elx.http;
import org.elasticsearch.action.Action;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.threadpool.ThreadPool;
import org.xbib.elx.common.AbstractBulkClient;
import java.io.IOException;
/**
* Elasticsearch HTTP bulk client.
*/
public class HttpBulkClient extends AbstractBulkClient implements ElasticsearchClient {
private final HttpClientHelper helper;
public HttpBulkClient() {
this.helper = new HttpClientHelper();
}
@Override
public void init(Settings settings) throws IOException {
super.init(settings);
helper.init(settings);
}
@Override
protected ElasticsearchClient createClient(Settings settings) throws IOException {
return helper.createClient(settings);
}
@Override
protected void closeClient() throws IOException {
helper.closeClient();
}
@Override
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(Action<Request, Response, RequestBuilder> action, Request request) {
return helper.execute(action, request);
}
@Override
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
helper.execute(action, request, listener);
}
@Override
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(Action<Request, Response, RequestBuilder> action) {
return helper.prepareExecute(action);
}
@Override
public ThreadPool threadPool() {
return helper.threadPool();
}
}

11
elx-http/src/main/java/org/xbib/elx/http/HttpBulkClientProvider.java Normal file
View file

@ -0,0 +1,11 @@
package org.xbib.elx.http;
import org.xbib.elx.api.BulkClientProvider;
public class HttpBulkClientProvider implements BulkClientProvider<HttpBulkClient> {
@Override
public HttpBulkClient getClient() {
return new HttpBulkClient();
}
}

17
elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java → elx-http/src/main/java/org/xbib/elx/http/HttpClientHelper.java
View file

@ -15,12 +15,12 @@ import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.threadpool.ThreadPool;
import org.xbib.elx.common.AbstractExtendedClient;
import org.xbib.elx.common.AbstractAdminClient;
import org.xbib.net.URL;
import org.xbib.netty.http.client.Client;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -32,9 +32,9 @@ import java.util.stream.Stream;
/**
* Elasticsearch HTTP client.
*/
public class ExtendedHttpClient extends AbstractExtendedClient implements ElasticsearchClient {
public class HttpClientHelper extends AbstractAdminClient implements ElasticsearchClient {
private static final Logger logger = LogManager.getLogger(ExtendedHttpClient.class);
private static final Logger logger = LogManager.getLogger(HttpClientHelper.class);
private Client nettyHttpClient;
@ -47,7 +47,11 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
private String url;
public ExtendedHttpClient(List<NamedXContentRegistry.Entry> namedXContentEntries, ClassLoader classLoader) {
public HttpClientHelper() {
this(Collections.emptyList(), Thread.currentThread().getContextClassLoader());
}
public HttpClientHelper(List<NamedXContentRegistry.Entry> namedXContentEntries, ClassLoader classLoader) {
this.registry = new NamedXContentRegistry(Stream.of(getNamedXContents().stream(),
namedXContentEntries.stream()).flatMap(Function.identity()).collect(Collectors.toList()));
this.classLoader = classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader();
@ -56,7 +60,7 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
@Override
@SuppressWarnings({"unchecked", "rawtypes"})
public ExtendedHttpClient init(Settings settings) throws IOException {
public void init(Settings settings) throws IOException {
super.init(settings);
if (settings.hasValue("url")) {
this.url = settings.get("url");
@ -78,7 +82,6 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
this.nettyHttpClient = clientBuilder.build();
logger.log(Level.DEBUG, "extended HTTP client initialized, settings = {}, url = {}, {} actions",
settings, url, actionMap.size());
return this;
}
private static List<NamedXContentRegistry.Entry> getNamedXContents() {

61
elx-http/src/main/java/org/xbib/elx/http/HttpSearchClient.java Normal file
View file

@ -0,0 +1,61 @@
package org.xbib.elx.http;
import org.elasticsearch.action.Action;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.threadpool.ThreadPool;
import org.xbib.elx.common.AbstractSearchClient;
import java.io.IOException;
/**
* Elasticsearch HTTP search client.
*/
public class HttpSearchClient extends AbstractSearchClient implements ElasticsearchClient {
private final HttpClientHelper helper;
public HttpSearchClient() {
this.helper = new HttpClientHelper();
}
@Override
public void init(Settings settings) throws IOException {
super.init(settings);
helper.init(settings);
}
@Override
protected ElasticsearchClient createClient(Settings settings) throws IOException {
return helper.createClient(settings);
}
@Override
protected void closeClient() throws IOException {
helper.closeClient();
}
@Override
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(Action<Request, Response, RequestBuilder> action, Request request) {
return helper.execute(action, request);
}
@Override
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
helper.execute(action, request, listener);
}
@Override
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(Action<Request, Response, RequestBuilder> action) {
return helper.prepareExecute(action);
}
@Override
public ThreadPool threadPool() {
return helper.threadPool();
}
}

11
elx-http/src/main/java/org/xbib/elx/http/HttpSearchClientProvider.java Normal file
View file

@ -0,0 +1,11 @@
package org.xbib.elx.http;
import org.xbib.elx.api.SearchClientProvider;
public class HttpSearchClientProvider implements SearchClientProvider<HttpSearchClient> {
@Override
public HttpSearchClient getClient() {
return new HttpSearchClient();
}
}

1
elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider Normal file
View file

@ -0,0 +1 @@
org.xbib.elx.http.HttpAdminClientProvider

1
elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider Normal file
View file

@ -0,0 +1 @@
org.xbib.elx.http.HttpBulkClientProvider

1
elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider
View file

@ -1 +0,0 @@
org.xbib.elx.http.ExtendedHttpClientProvider

1
elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider Normal file
View file

@ -0,0 +1 @@
org.xbib.elx.http.HttpSearchClientProvider

1
elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction
View file

@ -1,4 +1,5 @@
org.elasticsearch.action.admin.indices.mapping.get.HttpGetMappingsAction
org.elasticsearch.action.admin.indices.get.HttpGetIndexAction
org.xbib.elx.http.action.admin.cluster.health.HttpClusterHealthAction
org.xbib.elx.http.action.admin.cluster.node.info.HttpNodesInfoAction
org.xbib.elx.http.action.admin.cluster.settings.HttpClusterUpdateSettingsAction

92
elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java → elx-http/src/test/java/org/xbib/elx/http/test/BulkClientTest.java
View file

@ -2,24 +2,16 @@ package org.xbib.elx.http.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.http.ExtendedHttpClient;
import org.xbib.elx.http.ExtendedHttpClientProvider;
import org.xbib.elx.http.HttpBulkClient;
import org.xbib.elx.http.HttpBulkClientProvider;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
@ -31,9 +23,9 @@ import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ExtendWith(TestExtension.class)
class ClientTest {
class BulkClientTest {
private static final Logger logger = LogManager.getLogger(ClientTest.class.getSimpleName());
private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getSimpleName());
private static final Long ACTIONS = 100L;
@ -41,14 +33,14 @@ class ClientTest {
private final TestExtension.Helper helper;
ClientTest(TestExtension.Helper helper) {
BulkClientTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
void testSingleDoc() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
final HttpBulkClient client = ClientBuilder.builder()
.setBulkClientProvider(HttpBulkClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
@ -58,8 +50,6 @@ class ClientTest {
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
assertEquals(1, client.getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) {
@ -72,8 +62,8 @@ class ClientTest {
@Test
void testNewIndex() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
final HttpBulkClient client = ClientBuilder.builder()
.setBulkClientProvider(HttpBulkClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
@ -83,8 +73,8 @@ class ClientTest {
@Test
void testMapping() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
final HttpBulkClient client = ClientBuilder.builder()
.setBulkClientProvider(HttpBulkClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
@ -98,20 +88,16 @@ class ClientTest {
.endObject()
.endObject()
.endObject();
client.newIndex("test", Settings.EMPTY, Strings.toString(builder));
GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test");
GetMappingsResponse getMappingsResponse =
client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
logger.info("mappings={}", getMappingsResponse.getMappings());
assertTrue(getMappingsResponse.getMappings().get("test").containsKey("doc"));
client.newIndex("test", Settings.EMPTY, builder);
assertTrue(client.getMapping("test", "doc").containsKey("properties"));
client.close();
}
@Test
void testRandomDocs() throws Exception {
long numactions = ACTIONS;
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
final HttpBulkClient client = ClientBuilder.builder()
.setBulkClientProvider(HttpBulkClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
@ -123,10 +109,6 @@ class ClientTest {
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) {
@ -134,10 +116,7 @@ class ClientTest {
}
assertNull(client.getBulkController().getLastBulkError());
client.refreshIndex("test");
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
.setQuery(QueryBuilders.matchAllQuery()).setSize(0);
assertEquals(numactions,
searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
assertEquals(numactions, client.getSearchableDocs("test"));
client.close();
}
}
@ -146,10 +125,10 @@ class ClientTest {
void testThreadedRandomDocs() throws Exception {
int maxthreads = Runtime.getRuntime().availableProcessors();
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
final Long actions = ACTIONS;
final long actions = ACTIONS;
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
final HttpBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(HttpBulkClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
@ -160,15 +139,15 @@ class ClientTest {
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
client.newIndex("test", settings, (String)null)
.startBulk("test", 0, 1000);
bulkClient.newIndex("test", settings);
bulkClient.startBulk("test", 0, 1000);
logger.info("index created");
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
final CountDownLatch latch = new CountDownLatch(maxthreads);
for (int i = 0; i < maxthreads; i++) {
executorService.execute(() -> {
for (int i1 = 0; i1 < actions; i1++) {
client.index("test", null, false,"{ \"name\" : \"" + helper.randomString(32) + "\"}");
bulkClient.index("test", null, false,"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
latch.countDown();
});
@ -176,8 +155,8 @@ class ClientTest {
logger.info("waiting for latch...");
if (latch.await(60L, TimeUnit.SECONDS)) {
logger.info("flush...");
client.flush();
client.waitForResponses(60L, TimeUnit.SECONDS);
bulkClient.flush();
bulkClient.waitForResponses(60L, TimeUnit.SECONDS);
logger.info("got all responses, executor service shutdown...");
executorService.shutdown();
executorService.awaitTermination(60L, TimeUnit.SECONDS);
@ -185,23 +164,16 @@ class ClientTest {
} else {
logger.warn("latch timeout");
}
client.stopBulk("test", 30L, TimeUnit.SECONDS);
assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount());
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} catch (Exception e) {
logger.error(e.getMessage(), e);
bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS);
assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount());
} finally {
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
client.refreshIndex("test");
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
.setQuery(QueryBuilders.matchAllQuery()).setSize(0);
assertEquals(maxthreads * actions,
searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
client.close();
assertNull(bulkClient.getBulkController().getLastBulkError());
bulkClient.refreshIndex("test");
assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test"));
bulkClient.close();
}
}
}

25
elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java
View file

@ -2,19 +2,15 @@ package org.xbib.elx.http.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.http.ExtendedHttpClient;
import org.xbib.elx.http.ExtendedHttpClientProvider;
import org.xbib.elx.http.HttpBulkClient;
import org.xbib.elx.http.HttpBulkClientProvider;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ -24,9 +20,9 @@ class DuplicateIDTest {
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getSimpleName());
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
private static final Long ACTIONS = 100L;
private static final Long ACTIONS = 50L;
private static final Long MAX_ACTIONS_PER_REQUEST = 5L;
private final TestExtension.Helper helper;
@ -37,8 +33,8 @@ class DuplicateIDTest {
@Test
void testDuplicateDocIDs() throws Exception {
long numactions = ACTIONS;
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
final HttpBulkClient client = ClientBuilder.builder()
.setBulkClientProvider(HttpBulkClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.build();
@ -51,15 +47,8 @@ class DuplicateIDTest {
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
client.refreshIndex("test");
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
.setIndices("test")
.setTypes("test")
.setQuery(matchAllQuery());
long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
logger.info("hits = {}", hits);
long hits = client.getSearchableDocs("test");
assertTrue(hits < ACTIONS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.close();
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());

53
elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java
View file

@ -2,17 +2,15 @@ package org.xbib.elx.http.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexPruneResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.http.ExtendedHttpClient;
import org.xbib.elx.http.ExtendedHttpClientProvider;
import org.xbib.elx.http.HttpAdminClient;
import org.xbib.elx.http.HttpAdminClientProvider;
import org.xbib.elx.http.HttpBulkClient;
import org.xbib.elx.http.HttpBulkClientProvider;
import java.io.IOException;
import java.util.ArrayList;
@ -37,50 +35,49 @@ class IndexPruneTest {
@Test
void testPrune() throws IOException {
final ExtendedHttpClient client = ClientBuilder.builder(helper.client("1"))
final HttpAdminClient adminClient = ClientBuilder.builder()
.setAdminClientProvider(HttpAdminClientProvider.class)
.put(helper.getHttpSettings())
.build();
final HttpBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(HttpBulkClientProvider.class)
.put(helper.getHttpSettings())
.provider(ExtendedHttpClientProvider.class)
.build();
try {
Settings settings = Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
client.newIndex("test1", settings);
client.shiftIndex("test", "test1", Collections.emptyList());
client.newIndex("test2", settings);
client.shiftIndex("test", "test2", Collections.emptyList());
client.newIndex("test3", settings);
client.shiftIndex("test", "test3", Collections.emptyList());
client.newIndex("test4", settings);
client.shiftIndex("test", "test4", Collections.emptyList());
bulkClient.newIndex("test1", settings);
adminClient.shiftIndex("test", "test1", Collections.emptyList());
bulkClient.newIndex("test2", settings);
adminClient.shiftIndex("test", "test2", Collections.emptyList());
bulkClient.newIndex("test3", settings);
adminClient.shiftIndex("test", "test3", Collections.emptyList());
bulkClient.newIndex("test4", settings);
adminClient.shiftIndex("test", "test4", Collections.emptyList());
IndexPruneResult indexPruneResult =
client.pruneIndex("test", "test4", 2, 2, true);
adminClient.pruneIndex("test", "test4", 2, 2, true);
assertTrue(indexPruneResult.getDeletedIndices().contains("test1"));
assertTrue(indexPruneResult.getDeletedIndices().contains("test2"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test3"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test4"));
List<Boolean> list = new ArrayList<>();
for (String index : Arrays.asList("test1", "test2", "test3", "test4")) {
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
indicesExistsRequest.indices(index);
IndicesExistsResponse indicesExistsResponse =
client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
list.add(indicesExistsResponse.isExists());
list.add(bulkClient.isIndexExists(index));
}
logger.info(list);
assertFalse(list.get(0));
assertFalse(list.get(1));
assertTrue(list.get(2));
assertTrue(list.get(3));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
bulkClient.close();
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
assertNull(bulkClient.getBulkController().getLastBulkError());
adminClient.close();
}
}
}

67
elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java
View file

@ -4,16 +4,16 @@ import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexShiftResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.http.ExtendedHttpClient;
import org.xbib.elx.http.ExtendedHttpClientProvider;
import org.xbib.elx.http.HttpAdminClient;
import org.xbib.elx.http.HttpAdminClientProvider;
import org.xbib.elx.http.HttpBulkClient;
import org.xbib.elx.http.HttpBulkClientProvider;
import java.util.Arrays;
import java.util.Map;
@ -35,56 +35,54 @@ class IndexShiftTest {
@Test
void testIndexShift() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder()
final HttpAdminClient adminClient = ClientBuilder.builder()
.setAdminClientProvider(HttpAdminClientProvider.class)
.put(helper.getHttpSettings())
.build();
final HttpBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(HttpBulkClientProvider.class)
.put(helper.getHttpSettings())
.provider(ExtendedHttpClientProvider.class)
.build();
try {
Settings settings = Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
client.newIndex("test1234", settings);
bulkClient.newIndex("test1234", settings);
for (int i = 0; i < 1; i++) {
client.index("test1234", helper.randomString(1), false,
bulkClient.index("test1234", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
IndexShiftResult indexShiftResult =
client.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c"));
adminClient.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c"));
assertTrue(indexShiftResult.getNewAliases().contains("a"));
assertTrue(indexShiftResult.getNewAliases().contains("b"));
assertTrue(indexShiftResult.getNewAliases().contains("c"));
assertTrue(indexShiftResult.getMovedAliases().isEmpty());
Map<String, String> aliases = client.getAliases("test1234");
Map<String, String> aliases = adminClient.getAliases("test1234");
logger.log(Level.DEBUG, "aliases = " + aliases);
assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("test"));
String resolved = client.resolveAlias("test");
String resolved = adminClient.resolveAlias("test");
logger.log(Level.DEBUG, "resolved = " + resolved);
aliases = client.getAliases(resolved);
aliases = adminClient.getAliases(resolved);
logger.log(Level.DEBUG, "aliases = " + aliases);
assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("test"));
client.newIndex("test5678", settings);
bulkClient.newIndex("test5678", settings);
for (int i = 0; i < 1; i++) {
client.index("test5678", helper.randomString(1), false,
bulkClient.index("test5678", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
indexShiftResult = client.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"),
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
indexShiftResult = adminClient.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"),
(request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
.index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
);
@ -94,31 +92,28 @@ class IndexShiftTest {
assertTrue(indexShiftResult.getMovedAliases().contains("a"));
assertTrue(indexShiftResult.getMovedAliases().contains("b"));
assertTrue(indexShiftResult.getMovedAliases().contains("c"));
aliases = client.getAliases("test5678");
aliases = adminClient.getAliases("test5678");
assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("d"));
assertTrue(aliases.containsKey("e"));
assertTrue(aliases.containsKey("f"));
resolved = client.resolveAlias("test");
aliases = client.getAliases(resolved);
resolved = adminClient.resolveAlias("test");
aliases = adminClient.getAliases(resolved);
assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("d"));
assertTrue(aliases.containsKey("e"));
assertTrue(aliases.containsKey("f"));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
bulkClient.close();
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
assertNull(bulkClient.getBulkController().getLastBulkError());
adminClient.close();
}
}
}

79
elx-http/src/test/java/org/xbib/elx/http/test/SmokeTest.java
View file

@ -2,14 +2,15 @@ package org.xbib.elx.http.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.http.ExtendedHttpClient;
import org.xbib.elx.http.ExtendedHttpClientProvider;
import org.xbib.elx.http.HttpAdminClient;
import org.xbib.elx.http.HttpAdminClientProvider;
import org.xbib.elx.http.HttpBulkClient;
import org.xbib.elx.http.HttpBulkClientProvider;
import java.util.concurrent.TimeUnit;
@ -29,47 +30,51 @@ class SmokeTest {
@Test
void smokeTest() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
final HttpAdminClient adminClient = ClientBuilder.builder()
.setAdminClientProvider(HttpAdminClientProvider.class)
.put(helper.getHttpSettings())
.build();
try {
assertEquals(helper.getClusterName(), client.getClusterName());
client.newIndex("test_smoke");
client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
client.flush();
client.waitForResponses(30, TimeUnit.SECONDS);
client.checkMapping("test_smoke");
client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
client.delete("test_smoke", "1");
client.flush();
client.waitForResponses(30, TimeUnit.SECONDS);
client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS);
client.delete("test_smoke", "1");
client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
client.flush();
client.deleteIndex("test_smoke");
IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder()
final HttpBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(HttpBulkClientProvider.class)
.put(helper.getHttpSettings())
.build();
IndexDefinition indexDefinition = adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.builder()
.build());
try {
assertEquals(helper.getClusterName(), adminClient.getClusterName());
bulkClient.newIndex("test_smoke");
bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
bulkClient.flush();
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
adminClient.checkMapping("test_smoke");
bulkClient.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
bulkClient.delete("test_smoke", "1");
bulkClient.flush();
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
adminClient.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS);
bulkClient.delete("test_smoke", "1");
bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
bulkClient.flush();
adminClient.deleteIndex("test_smoke");
assertEquals(0, indexDefinition.getReplicaLevel());
client.newIndex(indexDefinition);
client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
client.flush();
client.waitForResponses(30, TimeUnit.SECONDS);
client.updateReplicaLevel(indexDefinition, 2);
int replica = client.getReplicaLevel(indexDefinition);
bulkClient.newIndex(indexDefinition);
bulkClient.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
bulkClient.flush();
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
adminClient.updateReplicaLevel(indexDefinition, 2);
int replica = adminClient.getReplicaLevel(indexDefinition);
assertEquals(2, replica);
client.deleteIndex(indexDefinition);
assertEquals(0, client.getBulkMetric().getFailed().getCount());
assertEquals(6, client.getBulkMetric().getSucceeded().getCount());
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
bulkClient.close();
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount());
assertEquals(6, bulkClient.getBulkMetric().getSucceeded().getCount());
assertNull(bulkClient.getBulkController().getLastBulkError());
// close admin after bulk
adminClient.deleteIndex(indexDefinition);
adminClient.close();
}
}
}

0
elx-http/src/test/resources/log4j2.xml → elx-http/src/test/resources/log4j2-test.xml
View file

10
elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java
View file

@ -1,10 +0,0 @@
package org.xbib.elx.node;
import org.xbib.elx.api.ExtendedClientProvider;
public class ExtendedNodeClientProvider implements ExtendedClientProvider<ExtendedNodeClient> {
@Override
public ExtendedNodeClient getExtendedClient() {
return new ExtendedNodeClient();
}
}

6
elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java → elx-node/src/main/java/org/xbib/elx/node/NodeAdminClient.java
View file

@ -11,15 +11,15 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.env.Environment;
import org.elasticsearch.node.Node;
import org.elasticsearch.plugins.Plugin;
import org.xbib.elx.common.AbstractExtendedClient;
import org.xbib.elx.common.AbstractAdminClient;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
public class ExtendedNodeClient extends AbstractExtendedClient {
public class NodeAdminClient extends AbstractAdminClient {
private static final Logger logger = LogManager.getLogger(ExtendedNodeClient.class.getName());
private static final Logger logger = LogManager.getLogger(NodeAdminClient.class.getName());
private Node node;

11
elx-node/src/main/java/org/xbib/elx/node/NodeAdminClientProvider.java Normal file
View file

@ -0,0 +1,11 @@
package org.xbib.elx.node;
import org.xbib.elx.api.AdminClientProvider;
public class NodeAdminClientProvider implements AdminClientProvider<NodeAdminClient> {
@Override
public NodeAdminClient getClient() {
return new NodeAdminClient();
}
}

68
elx-node/src/main/java/org/xbib/elx/node/NodeBulkClient.java Normal file
View file

@ -0,0 +1,68 @@
package org.xbib.elx.node;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.env.Environment;
import org.elasticsearch.node.Node;
import org.elasticsearch.plugins.Plugin;
import org.xbib.elx.common.AbstractBulkClient;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
public class NodeBulkClient extends AbstractBulkClient {
private static final Logger logger = LogManager.getLogger(NodeBulkClient.class.getName());
private Node node;
@Override
protected ElasticsearchClient createClient(Settings settings) throws IOException {
if (settings == null) {
return null;
}
String version = System.getProperty("os.name")
+ " " + System.getProperty("java.vm.name")
+ " " + System.getProperty("java.vm.vendor")
+ " " + System.getProperty("java.runtime.version")
+ " " + System.getProperty("java.vm.version");
Settings effectiveSettings = Settings.builder().put(settings)
.put("node.client", true)
.put("node.master", false)
.put("node.data", false)
.build();
XContentBuilder builder = XContentFactory.jsonBuilder();
effectiveSettings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true")));
logger.info("creating node client on {} with effective settings {}",
version, Strings.toString(builder));
Collection<Class<? extends Plugin>> plugins = Collections.emptyList();
this.node = new BulkNode(new Environment(effectiveSettings, null), plugins);
try {
node.start();
} catch (Exception e) {
throw new IOException(e);
}
return node.client();
}
@Override
protected void closeClient() throws IOException {
if (node != null) {
logger.debug("closing node client");
node.close();
}
}
private static class BulkNode extends Node {
BulkNode(Environment env, Collection<Class<? extends Plugin>> classpathPlugins) {
super(env, classpathPlugins);
}
}
}

11
elx-node/src/main/java/org/xbib/elx/node/NodeBulkClientProvider.java Normal file
View file

@ -0,0 +1,11 @@
package org.xbib.elx.node;
import org.xbib.elx.api.BulkClientProvider;
public class NodeBulkClientProvider implements BulkClientProvider<NodeBulkClient> {
@Override
public NodeBulkClient getClient() {
return new NodeBulkClient();
}
}

68
elx-node/src/main/java/org/xbib/elx/node/NodeSearchClient.java Normal file
View file

@ -0,0 +1,68 @@
package org.xbib.elx.node;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.env.Environment;
import org.elasticsearch.node.Node;
import org.elasticsearch.plugins.Plugin;
import org.xbib.elx.common.AbstractSearchClient;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
public class NodeSearchClient extends AbstractSearchClient {
private static final Logger logger = LogManager.getLogger(NodeSearchClient.class.getName());
private Node node;
@Override
protected ElasticsearchClient createClient(Settings settings) throws IOException {
if (settings == null) {
return null;
}
String version = System.getProperty("os.name")
+ " " + System.getProperty("java.vm.name")
+ " " + System.getProperty("java.vm.vendor")
+ " " + System.getProperty("java.runtime.version")
+ " " + System.getProperty("java.vm.version");
Settings effectiveSettings = Settings.builder().put(settings)
.put("node.client", true)
.put("node.master", false)
.put("node.data", false)
.build();
XContentBuilder builder = XContentFactory.jsonBuilder();
effectiveSettings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true")));
logger.info("creating node client on {} with effective settings {}",
version, Strings.toString(builder));
Collection<Class<? extends Plugin>> plugins = Collections.emptyList();
this.node = new BulkNode(new Environment(effectiveSettings, null), plugins);
try {
node.start();
} catch (Exception e) {
throw new IOException(e);
}
return node.client();
}
@Override
protected void closeClient() throws IOException {
if (node != null) {
logger.debug("closing node client");
node.close();
}
}
private static class BulkNode extends Node {
BulkNode(Environment env, Collection<Class<? extends Plugin>> classpathPlugins) {
super(env, classpathPlugins);
}
}
}

11
elx-node/src/main/java/org/xbib/elx/node/NodeSearchClientProvider.java Normal file
View file

@ -0,0 +1,11 @@
package org.xbib.elx.node;
import org.xbib.elx.api.SearchClientProvider;
public class NodeSearchClientProvider implements SearchClientProvider<NodeSearchClient> {
@Override
public NodeSearchClient getClient() {
return new NodeSearchClient();
}
}

1
elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider Normal file
View file

@ -0,0 +1 @@
org.xbib.elx.node.NodeAdminClientProvider

1
elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider Normal file
View file

@ -0,0 +1 @@
org.xbib.elx.node.NodeBulkClientProvider

1
elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider
View file

@ -1 +0,0 @@
org.xbib.elx.node.ExtendedNodeClientProvider

1
elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider Normal file
View file

@ -0,0 +1 @@
org.xbib.elx.node.NodeSearchClientProvider

174
elx-node/src/test/java/org/xbib/elx/node/test/BulkClientTest.java Normal file
View file

@ -0,0 +1,174 @@
package org.xbib.elx.node.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.node.NodeBulkClient;
import org.xbib.elx.node.NodeBulkClientProvider;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ExtendWith(TestExtension.class)
class BulkClientTest {
private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getName());
private static final Long ACTIONS = 10000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
private final TestExtension.Helper helper;
BulkClientTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
void testSingleDoc() throws Exception {
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
.setBulkClientProvider(NodeBulkClientProvider.class)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
.build();
try {
bulkClient.newIndex("test");
bulkClient.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
} finally {
assertEquals(1, bulkClient.getBulkMetric().getSucceeded().getCount());
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(bulkClient.getBulkController().getLastBulkError());
bulkClient.close();
}
}
@Test
void testNewIndex() throws Exception {
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
.setBulkClientProvider(NodeBulkClientProvider.class)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
bulkClient.newIndex("test");
bulkClient.close();
}
@Test
void testMapping() throws Exception {
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
.setBulkClientProvider(NodeBulkClientProvider.class)
.build();
XContentBuilder builder = JsonXContent.contentBuilder()
.startObject()
.startObject("doc")
.startObject("properties")
.startObject("location")
.field("type", "geo_point")
.endObject()
.endObject()
.endObject()
.endObject();
bulkClient.newIndex("test", Settings.EMPTY, builder);
assertTrue(bulkClient.getMapping("test", "doc").containsKey("properties"));
bulkClient.close();
}
@Test
void testRandomDocs() throws Exception {
long numactions = ACTIONS;
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
.setBulkClientProvider(NodeBulkClientProvider.class)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.build();
try {
bulkClient.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
bulkClient.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
} finally {
assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(bulkClient.getBulkController().getLastBulkError());
bulkClient.refreshIndex("test");
assertEquals(numactions, bulkClient.getSearchableDocs("test"));
bulkClient.close();
}
}
@Test
void testThreadedRandomDocs() throws Exception {
int maxthreads = Runtime.getRuntime().availableProcessors();
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
final long actions = ACTIONS;
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
.setBulkClientProvider(NodeBulkClientProvider.class)
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.build();
try {
Settings settings = Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
bulkClient.newIndex("test", settings);
bulkClient.startBulk("test", 0, 1000);
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
final CountDownLatch latch = new CountDownLatch(maxthreads);
for (int i = 0; i < maxthreads; i++) {
executorService.execute(() -> {
for (int i1 = 0; i1 < actions; i1++) {
bulkClient.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
latch.countDown();
});
}
logger.info("waiting for latch...");
if (latch.await(30L, TimeUnit.SECONDS)) {
logger.info("flush...");
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
logger.info("got all responses, executor service shutdown...");
executorService.shutdown();
executorService.awaitTermination(30L, TimeUnit.SECONDS);
logger.info("pool is shut down");
} else {
logger.warn("latch timeout");
}
bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS);
assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount());
} finally {
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(bulkClient.getBulkController().getLastBulkError());
bulkClient.refreshIndex("test");
assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test"));
bulkClient.close();
}
}
}

200
elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java
View file

@ -1,200 +0,0 @@
package org.xbib.elx.node.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.node.ExtendedNodeClient;
import org.xbib.elx.node.ExtendedNodeClientProvider;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ExtendWith(TestExtension.class)
class ClientTest {
private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
private static final Long ACTIONS = 10000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
private final TestExtension.Helper helper;
ClientTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
void testSingleDoc() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
.build();
try {
client.newIndex("test");
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
assertEquals(1, client.getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
client.close();
}
}
@Test
void testNewIndex() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
client.newIndex("test");
client.close();
}
@Test
void testMapping() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
XContentBuilder builder = JsonXContent.contentBuilder()
.startObject()
.startObject("doc")
.startObject("properties")
.startObject("location")
.field("type", "geo_point")
.endObject()
.endObject()
.endObject()
.endObject();
client.newIndex("test", Settings.EMPTY, Strings.toString(builder));
GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test");
GetMappingsResponse getMappingsResponse =
client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
logger.info("mappings={}", getMappingsResponse.getMappings());
assertTrue(getMappingsResponse.getMappings().get("test").containsKey("doc"));
client.close();
}
@Test
void testRandomDocs() throws Exception {
long numactions = ACTIONS;
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.build();
try {
client.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
client.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
client.refreshIndex("test");
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
.setQuery(QueryBuilders.matchAllQuery()).setSize(0);
assertEquals(numactions,
searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
client.close();
}
}
@Test
void testThreadedRandomDocs() throws Exception {
int maxthreads = Runtime.getRuntime().availableProcessors();
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
final long actions = ACTIONS;
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.build();
try {
Settings settings = Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
client.newIndex("test", settings, (String)null)
.startBulk("test", 0, 1000);
logger.info("index created");
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
final CountDownLatch latch = new CountDownLatch(maxthreads);
for (int i = 0; i < maxthreads; i++) {
executorService.execute(() -> {
for (int i1 = 0; i1 < actions; i1++) {
client.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
latch.countDown();
});
}
logger.info("waiting for latch...");
if (latch.await(30L, TimeUnit.SECONDS)) {
logger.info("flush...");
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
logger.info("got all responses, executor service shutdown...");
executorService.shutdown();
executorService.awaitTermination(30L, TimeUnit.SECONDS);
logger.info("pool is shut down");
} else {
logger.warn("latch timeout");
}
client.stopBulk("test", 30L, TimeUnit.SECONDS);
assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount());
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
client.refreshIndex("test");
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
.setQuery(QueryBuilders.matchAllQuery()).setSize(0);
assertEquals(maxthreads * actions,
searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
client.close();
}
}
}

49
elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java
View file

@ -2,17 +2,12 @@ package org.xbib.elx.node.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.node.ExtendedNodeClient;
import org.xbib.elx.node.ExtendedNodeClientProvider;
import org.xbib.elx.node.NodeBulkClient;
import org.xbib.elx.node.NodeBulkClientProvider;
import java.util.concurrent.TimeUnit;
@ -25,9 +20,9 @@ class DuplicateIDTest {
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName());
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
private static final Long ACTIONS = 100L;
private static final Long ACTIONS = 50L;
private static final Long MAX_ACTIONS_PER_REQUEST = 5L;
private final TestExtension.Helper helper;
@ -38,37 +33,27 @@ class DuplicateIDTest {
@Test
void testDuplicateDocIDs() throws Exception {
long numactions = ACTIONS;
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
.setBulkClientProvider(NodeBulkClientProvider.class)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.build();
try {
client.newIndex("test");
bulkClient.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
client.index("test", helper.randomString(1), false,
bulkClient.index("test", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
client.refreshIndex("test");
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.query(QueryBuilders.matchAllQuery());
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices("test");
searchRequest.types("test");
searchRequest.source(builder);
long hits = helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits();
logger.info("hits = {}", hits);
assertTrue(hits < ACTIONS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
bulkClient.refreshIndex("test");
assertTrue(bulkClient.getSearchableDocs("test") < ACTIONS);
} finally {
client.close();
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
bulkClient.close();
assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
assertNull(bulkClient.getBulkController().getLastBulkError());
}
}
}

52
elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java
View file

@ -2,17 +2,15 @@ package org.xbib.elx.node.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexPruneResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.node.ExtendedNodeClient;
import org.xbib.elx.node.ExtendedNodeClientProvider;
import org.xbib.elx.node.NodeAdminClient;
import org.xbib.elx.node.NodeAdminClientProvider;
import org.xbib.elx.node.NodeBulkClient;
import org.xbib.elx.node.NodeBulkClientProvider;
import java.io.IOException;
import java.util.ArrayList;
@ -37,49 +35,47 @@ class IndexPruneTest {
@Test
void testPrune() throws IOException {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
.setAdminClientProvider(NodeAdminClientProvider.class)
.build();
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
.setBulkClientProvider(NodeBulkClientProvider.class)
.build();
try {
Settings settings = Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
client.newIndex("test_prune1", settings);
client.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
client.newIndex("test_prune2", settings);
client.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
client.newIndex("test_prune3", settings);
client.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
client.newIndex("test_prune4", settings);
client.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
bulkClient.newIndex("test_prune1", settings);
adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
bulkClient.newIndex("test_prune2", settings);
adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
bulkClient.newIndex("test_prune3", settings);
adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
bulkClient.newIndex("test_prune4", settings);
adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
IndexPruneResult indexPruneResult =
client.pruneIndex("test_prune", "test_prune4", 2, 2, true);
adminClient.pruneIndex("test_prune", "test_prune4", 2, 2, true);
assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1"));
assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune4"));
List<Boolean> list = new ArrayList<>();
for (String index : Arrays.asList("test_prune1", "test_prune2", "test_prune3", "test_prune4")) {
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
indicesExistsRequest.indices(index);
IndicesExistsResponse indicesExistsResponse =
client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
list.add(indicesExistsResponse.isExists());
list.add(bulkClient.isIndexExists(index));
}
logger.info(list);
assertFalse(list.get(0));
assertFalse(list.get(1));
assertTrue(list.get(2));
assertTrue(list.get(3));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
adminClient.close();
bulkClient.close();
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
assertNull(bulkClient.getBulkController().getLastBulkError());
}
}
}

64
elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java
View file

@ -3,15 +3,16 @@ package org.xbib.elx.node.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexShiftResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.node.ExtendedNodeClient;
import org.xbib.elx.node.ExtendedNodeClientProvider;
import org.xbib.elx.node.NodeAdminClient;
import org.xbib.elx.node.NodeAdminClientProvider;
import org.xbib.elx.node.NodeBulkClient;
import org.xbib.elx.node.NodeBulkClientProvider;
import java.util.Arrays;
import java.util.Map;
@ -33,50 +34,49 @@ class IndexShiftTest {
@Test
void testIndexShift() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
.setAdminClientProvider(NodeAdminClientProvider.class)
.build();
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
.setBulkClientProvider(NodeBulkClientProvider.class)
.build();
try {
Settings settings = Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
client.newIndex("test_shift", settings);
bulkClient.newIndex("test_shift", settings);
for (int i = 0; i < 1; i++) {
client.index("test_shift", helper.randomString(1), false,
bulkClient.index("test_shift", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
IndexShiftResult indexShiftResult =
client.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c"));
adminClient.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c"));
assertTrue(indexShiftResult.getNewAliases().contains("a"));
assertTrue(indexShiftResult.getNewAliases().contains("b"));
assertTrue(indexShiftResult.getNewAliases().contains("c"));
assertTrue(indexShiftResult.getMovedAliases().isEmpty());
Map<String, String> aliases = client.getAliases("test_shift");
Map<String, String> aliases = adminClient.getAliases("test_shift");
assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("test"));
String resolved = client.resolveAlias("test");
aliases = client.getAliases(resolved);
String resolved = adminClient.resolveAlias("test");
aliases = adminClient.getAliases(resolved);
assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("test"));
client.newIndex("test_shift2", settings);
bulkClient.newIndex("test_shift2", settings);
for (int i = 0; i < 1; i++) {
client.index("test_shift2", helper.randomString(1), false,
bulkClient.index("test_shift2", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
indexShiftResult = client.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"),
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
indexShiftResult = adminClient.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"),
(request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
.index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
);
@ -86,32 +86,28 @@ class IndexShiftTest {
assertTrue(indexShiftResult.getMovedAliases().contains("a"));
assertTrue(indexShiftResult.getMovedAliases().contains("b"));
assertTrue(indexShiftResult.getMovedAliases().contains("c"));
aliases = client.getAliases("test_shift2");
aliases = adminClient.getAliases("test_shift2");
assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("d"));
assertTrue(aliases.containsKey("e"));
assertTrue(aliases.containsKey("f"));
resolved = client.resolveAlias("test");
aliases = client.getAliases(resolved);
resolved = adminClient.resolveAlias("test");
aliases = adminClient.getAliases(resolved);
assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("d"));
assertTrue(aliases.containsKey("e"));
assertTrue(aliases.containsKey("f"));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
adminClient.close();
bulkClient.close();
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
assertNull(bulkClient.getBulkController().getLastBulkError());
}
}
}

78
elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java
View file

@ -2,14 +2,15 @@ package org.xbib.elx.node.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.node.ExtendedNodeClient;
import org.xbib.elx.node.ExtendedNodeClientProvider;
import org.xbib.elx.node.NodeAdminClient;
import org.xbib.elx.node.NodeAdminClientProvider;
import org.xbib.elx.node.NodeBulkClient;
import org.xbib.elx.node.NodeBulkClientProvider;
import java.util.concurrent.TimeUnit;
@ -29,46 +30,49 @@ class SmokeTest {
@Test
void smokeTest() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
.setAdminClientProvider(NodeAdminClientProvider.class)
.build();
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
.setBulkClientProvider(NodeBulkClientProvider.class)
.build();
IndexDefinition indexDefinition =
adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.EMPTY);
try {
assertEquals(helper.getClusterName(), client.getClusterName());
client.newIndex("test_smoke");
client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
client.flush();
client.waitForResponses(30, TimeUnit.SECONDS);
client.checkMapping("test_smoke");
client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
client.delete("test_smoke", "1");
client.flush();
client.waitForResponses(30, TimeUnit.SECONDS);
client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS);
client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
client.delete("test_smoke", "1");
client.flush();
client.deleteIndex("test_smoke");
IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder()
.build());
assertEquals(helper.getClusterName(), adminClient.getClusterName());
bulkClient.newIndex("test_smoke");
bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
bulkClient.flush();
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
adminClient.checkMapping("test_smoke");
bulkClient.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
bulkClient.delete("test_smoke", "1");
bulkClient.flush();
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
adminClient.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS);
bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
bulkClient.delete("test_smoke", "1");
bulkClient.flush();
adminClient.deleteIndex("test_smoke");
assertEquals(0, indexDefinition.getReplicaLevel());
client.newIndex(indexDefinition);
client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
client.flush();
client.waitForResponses(30, TimeUnit.SECONDS);
client.updateReplicaLevel(indexDefinition, 2);
int replica = client.getReplicaLevel(indexDefinition);
bulkClient.newIndex(indexDefinition);
bulkClient.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
bulkClient.flush();
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
adminClient.updateReplicaLevel(indexDefinition, 2);
int replica = adminClient.getReplicaLevel(indexDefinition);
assertEquals(2, replica);
client.deleteIndex(indexDefinition);
//assertEquals(0, client.getBulkMetric().getFailed().getCount());
//assertEquals(6, client.getBulkMetric().getSucceeded().getCount());
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
bulkClient.close();
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount());
assertEquals(6, bulkClient.getBulkMetric().getSucceeded().getCount());
assertNull(bulkClient.getBulkController().getLastBulkError());
// close admin after bulk
adminClient.deleteIndex(indexDefinition);
adminClient.close();
}
}
}

0
elx-node/src/test/resources/log4j2.xml → elx-node/src/test/resources/log4j2-test.xml
View file

11
elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java
View file

@ -1,11 +0,0 @@
package org.xbib.elx.transport;
import org.xbib.elx.api.ExtendedClientProvider;
public class ExtendedTransportClientProvider implements ExtendedClientProvider<ExtendedTransportClient> {
@Override
public ExtendedTransportClient getExtendedClient() {
return new ExtendedTransportClient();
}
}

40
elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClient.java Normal file
View file

@ -0,0 +1,40 @@
package org.xbib.elx.transport;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.xbib.elx.common.AbstractAdminClient;
import java.io.IOException;
/**
* Transport admin client.
*/
public class TransportAdminClient extends AbstractAdminClient {
private final TransportClientHelper helper;
public TransportAdminClient() {
this.helper = new TransportClientHelper();
}
@Override
protected ElasticsearchClient createClient(Settings settings) throws IOException {
return helper.createClient(settings);
}
@Override
protected void closeClient() {
if (getClient() != null) {
TransportClient client = (TransportClient) getClient();
client.close();
client.threadPool().shutdown();
}
}
@Override
public void init(Settings settings) throws IOException {
super.init(settings);
helper.init((TransportClient) getClient(), settings);
}
}

11
elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClientProvider.java Normal file
View file

@ -0,0 +1,11 @@
package org.xbib.elx.transport;
import org.xbib.elx.api.AdminClientProvider;
public class TransportAdminClientProvider implements AdminClientProvider<TransportAdminClient> {
@Override
public TransportAdminClient getClient() {
return new TransportAdminClient();
}
}

39
elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClient.java Normal file
View file

@ -0,0 +1,39 @@
package org.xbib.elx.transport;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.xbib.elx.common.AbstractBulkClient;
import java.io.IOException;
/**
* Transport search client with additional methods.
*/
public class TransportBulkClient extends AbstractBulkClient {
private final TransportClientHelper helper;
public TransportBulkClient() {
this.helper = new TransportClientHelper();
}
@Override
protected ElasticsearchClient createClient(Settings settings) throws IOException {
return helper.createClient(settings);
}
@Override
protected void closeClient() {
if (getClient() != null) {
TransportClient client = (TransportClient) getClient();
client.close();
client.threadPool().shutdown();
}
}
@Override
public void init(Settings settings) throws IOException {
super.init(settings);
helper.init((TransportClient) getClient(), settings);
}
}

11
elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClientProvider.java Normal file
View file

@ -0,0 +1,11 @@
package org.xbib.elx.transport;
import org.xbib.elx.api.BulkClientProvider;
public class TransportBulkClientProvider implements BulkClientProvider<TransportBulkClient> {
@Override
public TransportBulkClient getClient() {
return new TransportBulkClient();
}
}

42
elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java → elx-transport/src/main/java/org/xbib/elx/transport/TransportClientHelper.java
View file

@ -23,9 +23,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.netty4.Netty4Plugin;
import org.xbib.elx.common.AbstractExtendedClient;
import org.xbib.elx.common.util.NetworkUtils;
import java.io.IOException;
import java.net.InetAddress;
import java.util.ArrayList;
@ -33,14 +31,11 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
/**
* Transport client with additional methods using the BulkProcessor.
*/
public class ExtendedTransportClient extends AbstractExtendedClient {
public class TransportClientHelper {
private static final Logger logger = LogManager.getLogger(ExtendedTransportClient.class.getName());
@Override
private static final Logger logger = LogManager.getLogger(TransportAdminClient.class.getName());
protected ElasticsearchClient createClient(Settings settings) throws IOException {
if (settings != null) {
String systemIdentifier = System.getProperty("os.name")
@ -49,11 +44,9 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
+ " " + System.getProperty("java.vm.version")
+ " Elasticsearch " + Version.CURRENT.toString();
Settings transportClientSettings = getTransportClientSettings(settings);
//XContentBuilder settingsBuilder = XContentFactory.jsonBuilder().startObject();
XContentBuilder effectiveSettingsBuilder = XContentFactory.jsonBuilder().startObject();
logger.log(Level.INFO, "creating transport client on {} with settings {}",
systemIdentifier,
//Strings.toString(settings.toXContent(settingsBuilder, ToXContent.EMPTY_PARAMS).endObject()),
Strings.toString(transportClientSettings.toXContent(effectiveSettingsBuilder,
ToXContent.EMPTY_PARAMS).endObject()));
return new MyTransportClient(transportClientSettings, Collections.singletonList(Netty4Plugin.class));
@ -61,29 +54,12 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
return null;
}
@Override
protected void closeClient() {
if (getClient() != null) {
TransportClient client = (TransportClient) getClient();
client.close();
client.threadPool().shutdown();
}
}
@Override
public ExtendedTransportClient init(Settings settings) throws IOException {
super.init(settings);
// additional auto-connect
try {
public void init(TransportClient transportClient, Settings settings) throws IOException {
Collection<TransportAddress> addrs = findAddresses(settings);
if (!connect(addrs, settings.getAsBoolean("autodiscover", false))) {
if (!connect(transportClient, addrs, settings.getAsBoolean("autodiscover", false))) {
throw new NoNodeAvailableException("no cluster nodes available, check settings "
+ settings.toString());
}
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
return this;
}
private Collection<TransportAddress> findAddresses(Settings settings) throws IOException {
@ -112,11 +88,7 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
return addresses;
}
private boolean connect(Collection<TransportAddress> addresses, boolean autodiscover) {
if (getClient() == null) {
throw new IllegalStateException("no client present");
}
TransportClient transportClient = (TransportClient) getClient();
private boolean connect(TransportClient transportClient, Collection<TransportAddress> addresses, boolean autodiscover) {
for (TransportAddress address : addresses) {
transportClient.addTransportAddresses(address);
}
@ -126,7 +98,7 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
if (autodiscover) {
logger.debug("trying to auto-discover all nodes...");
ClusterStateRequestBuilder clusterStateRequestBuilder =
new ClusterStateRequestBuilder(getClient(), ClusterStateAction.INSTANCE);
new ClusterStateRequestBuilder(transportClient, ClusterStateAction.INSTANCE);
ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet();
DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes();
addDiscoveryNodes(transportClient, discoveryNodes);

39
elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClient.java Normal file
View file

@ -0,0 +1,39 @@
package org.xbib.elx.transport;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.xbib.elx.common.AbstractSearchClient;
import java.io.IOException;
/**
* Transport search client with additional methods.
*/
public class TransportSearchClient extends AbstractSearchClient {
private final TransportClientHelper helper;
public TransportSearchClient() {
this.helper = new TransportClientHelper();
}
@Override
protected ElasticsearchClient createClient(Settings settings) throws IOException {
return helper.createClient(settings);
}
@Override
protected void closeClient() {
if (getClient() != null) {
TransportClient client = (TransportClient) getClient();
client.close();
client.threadPool().shutdown();
}
}
@Override
public void init(Settings settings) throws IOException {
super.init(settings);
helper.init((TransportClient) getClient(), settings);
}
}

11
elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClientProvider.java Normal file
View file

@ -0,0 +1,11 @@
package org.xbib.elx.transport;
import org.xbib.elx.api.SearchClientProvider;
public class TransportSearchClientProvider implements SearchClientProvider<TransportSearchClient> {
@Override
public TransportSearchClient getClient() {
return new TransportSearchClient();
}
}

1
elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider Normal file
View file

@ -0,0 +1 @@
org.xbib.elx.transport.TransportAdminClientProvider

1
elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider Normal file
View file

@ -0,0 +1 @@
org.xbib.elx.transport.TransportBulkClientProvider

1
elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider
View file

@ -1 +0,0 @@
org.xbib.elx.transport.ExtendedTransportClientProvider

1
elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider Normal file
View file

@ -0,0 +1 @@
org.xbib.elx.transport.TransportSearchClientProvider

188
elx-transport/src/test/java/org/xbib/elx/transport/test/BulkClientTest.java Normal file
View file

@ -0,0 +1,188 @@
package org.xbib.elx.transport.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.transport.TransportBulkClient;
import org.xbib.elx.transport.TransportBulkClientProvider;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ExtendWith(TestExtension.class)
class BulkClientTest {
private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getName());
private static final Long ACTIONS = 10000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
private final TestExtension.Helper helper;
BulkClientTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
void testSingleDoc() throws Exception {
final TransportBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(TransportBulkClientProvider.class)
.put(helper.getTransportSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
.build();
try {
bulkClient.newIndex("test");
bulkClient.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
} finally {
assertEquals(1, bulkClient.getBulkMetric().getSucceeded().getCount());
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(bulkClient.getBulkController().getLastBulkError());
bulkClient.close();
}
}
@Test
void testNewIndex() throws Exception {
final TransportBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(TransportBulkClientProvider.class)
.put(helper.getTransportSettings())
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
bulkClient.newIndex("test");
bulkClient.close();
}
@Test
void testMapping() throws Exception {
final TransportBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(TransportBulkClientProvider.class)
.put(helper.getTransportSettings())
.build();
XContentBuilder builder = JsonXContent.contentBuilder()
.startObject()
.startObject("doc")
.startObject("properties")
.startObject("location")
.field("type", "geo_point")
.endObject()
.endObject()
.endObject()
.endObject();
bulkClient.newIndex("test", Settings.EMPTY, builder);
assertTrue(bulkClient.getMapping("test", "doc").containsKey("properties"));
bulkClient.close();
}
@Test
void testRandomDocs() throws Exception {
long numactions = ACTIONS;
final TransportBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(TransportBulkClientProvider.class)
.put(helper.getTransportSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.build();
try {
bulkClient.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
bulkClient.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(bulkClient.getBulkController().getLastBulkError());
bulkClient.refreshIndex("test");
assertEquals(numactions, bulkClient.getSearchableDocs("test"));
bulkClient.close();
}
}
@Test
void testThreadedRandomDocs() throws Exception {
int maxthreads = Runtime.getRuntime().availableProcessors();
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
final long actions = ACTIONS;
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
final TransportBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(TransportBulkClientProvider.class)
.put(helper.getTransportSettings())
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.put(Parameters.ENABLE_BULK_LOGGING.name(), "true")
.build();
try {
Settings settings = Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
bulkClient.newIndex("test", settings);
bulkClient.startBulk("test", 0, 1000);
logger.info("index created");
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
final CountDownLatch latch = new CountDownLatch(maxthreads);
for (int i = 0; i < maxthreads; i++) {
executorService.execute(() -> {
for (int i1 = 0; i1 < actions; i1++) {
bulkClient.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
latch.countDown();
});
}
logger.info("waiting for latch...");
if (latch.await(30L, TimeUnit.SECONDS)) {
logger.info("flush...");
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
logger.info("got all responses, executor service shutdown...");
executorService.shutdown();
executorService.awaitTermination(30L, TimeUnit.SECONDS);
logger.info("pool is shut down");
} else {
logger.warn("latch timeout");
}
bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS);
assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount());
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(bulkClient.getBulkController().getLastBulkError());
bulkClient.refreshIndex("test");
assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test"));
bulkClient.close();
}
}
}

223
elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java
View file

@ -1,223 +0,0 @@
package org.xbib.elx.transport.test;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.transport.ExtendedTransportClient;
import org.xbib.elx.transport.ExtendedTransportClientProvider;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ExtendWith(TestExtension.class)
class ClientTest {
private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
private static final Long ACTIONS = 10000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
private final TestExtension.Helper helper;
ClientTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
void testSingleDoc() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(helper.getTransportSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
.build();
try {
client.newIndex("test");
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
assertEquals(1, client.getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
client.close();
}
}
@Test
void testNewIndex() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(helper.getTransportSettings())
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
client.newIndex("test");
client.close();
}
@Test
void testMapping() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(helper.getTransportSettings())
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
XContentBuilder builder = JsonXContent.contentBuilder()
.startObject()
.startObject("doc")
.startObject("properties")
.startObject("location")
.field("type", "geo_point")
.endObject()
.endObject()
.endObject()
.endObject();
client.newIndex("test", Settings.EMPTY, Strings.toString(builder));
GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test");
GetMappingsResponse getMappingsResponse =
client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
logger.info("mappings={}", getMappingsResponse.getMappings());
assertTrue(getMappingsResponse.getMappings().get("test").containsKey("doc"));
client.close();
}
@Test
void testRandomDocs() throws Exception {
long numactions = ACTIONS;
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(helper.getTransportSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.build();
try {
client.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
client.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
client.refreshIndex("test");
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.query(QueryBuilders.matchAllQuery());
builder.size(0);
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices("test");
searchRequest.source(builder);
SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
logger.log(Level.INFO, searchResponse.toString());
assertEquals(numactions, searchResponse.getHits().getTotalHits());
client.close();
}
}
@Test
void testThreadedRandomDocs() throws Exception {
int maxthreads = Runtime.getRuntime().availableProcessors();
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
final long actions = ACTIONS;
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(helper.getTransportSettings())
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.put(Parameters.ENABLE_BULK_LOGGING.name(), "true")
.build();
try {
Settings settings = Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
client.newIndex("test", settings, (String)null)
.startBulk("test", 0, 1000);
logger.info("index created");
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
final CountDownLatch latch = new CountDownLatch(maxthreads);
for (int i = 0; i < maxthreads; i++) {
executorService.execute(() -> {
for (int i1 = 0; i1 < actions; i1++) {
client.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
latch.countDown();
});
}
logger.info("waiting for latch...");
if (latch.await(30L, TimeUnit.SECONDS)) {
logger.info("flush...");
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
logger.info("got all responses, executor service shutdown...");
executorService.shutdown();
executorService.awaitTermination(30L, TimeUnit.SECONDS);
logger.info("pool is shut down");
} else {
logger.warn("latch timeout");
}
client.stopBulk("test", 30L, TimeUnit.SECONDS);
assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount());
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
client.refreshIndex("test");
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.query(QueryBuilders.matchAllQuery());
builder.size(0);
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices("test");
searchRequest.source(builder);
SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
logger.log(Level.INFO, searchResponse.toString());
assertEquals(maxthreads * actions, searchResponse.getHits().getTotalHits());
client.close();
}
}
}

53
elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java
View file

@ -2,18 +2,12 @@ package org.xbib.elx.transport.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.transport.ExtendedTransportClient;
import org.xbib.elx.transport.ExtendedTransportClientProvider;
import org.xbib.elx.transport.TransportBulkClient;
import org.xbib.elx.transport.TransportBulkClientProvider;
import java.util.concurrent.TimeUnit;
@ -26,9 +20,9 @@ class DuplicateIDTest {
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName());
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
private static final Long ACTIONS = 100L;
private static final Long ACTIONS = 5L;
private static final Long MAX_ACTIONS_PER_REQUEST = 5L;
private final TestExtension.Helper helper;
@ -39,41 +33,28 @@ class DuplicateIDTest {
@Test
void testDuplicateDocIDs() throws Exception {
long numactions = ACTIONS;
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
final TransportBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(TransportBulkClientProvider.class)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(helper.getTransportSettings())
.build();
try {
client.newIndex("test_dup");
bulkClient.newIndex("test_dup");
for (int i = 0; i < ACTIONS; i++) {
client.index("test_dup", helper.randomString(1), false,
bulkClient.index("test_dup", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
client.refreshIndex("test_dup");
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.query(QueryBuilders.matchAllQuery());
builder.size(0);
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices("test_dup");
searchRequest.types("test_dup");
searchRequest.source(builder);
SearchResponse searchResponse =
helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet();
long hits = searchResponse.getHits().getTotalHits();
logger.info("hits = {}", hits);
assertTrue(hits < ACTIONS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
bulkClient.refreshIndex("test_dup");
assertTrue(bulkClient.getSearchableDocs("test_dup") < ACTIONS);
} finally {
client.close();
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
bulkClient.close();
assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
assertNull(bulkClient.getBulkController().getLastBulkError());
}
}
}

53
elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java
View file

@ -2,17 +2,15 @@ package org.xbib.elx.transport.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexPruneResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.transport.ExtendedTransportClient;
import org.xbib.elx.transport.ExtendedTransportClientProvider;
import org.xbib.elx.transport.TransportAdminClient;
import org.xbib.elx.transport.TransportAdminClientProvider;
import org.xbib.elx.transport.TransportBulkClient;
import org.xbib.elx.transport.TransportBulkClientProvider;
import java.io.IOException;
import java.util.ArrayList;
@ -37,8 +35,12 @@ class IndexPruneTest {
@Test
void testPrune() throws IOException {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
final TransportAdminClient adminClient = ClientBuilder.builder()
.setAdminClientProvider(TransportAdminClientProvider.class)
.put(helper.getTransportSettings())
.build();
final TransportBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(TransportBulkClientProvider.class)
.put(helper.getTransportSettings())
.build();
try {
@ -46,41 +48,36 @@ class IndexPruneTest {
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
client.newIndex("test_prune1", settings);
client.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
client.newIndex("test_prune2", settings);
client.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
client.newIndex("test_prune3", settings);
client.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
client.newIndex("test_prune4", settings);
client.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
bulkClient.newIndex("test_prune1", settings);
adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
bulkClient.newIndex("test_prune2", settings);
adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
bulkClient.newIndex("test_prune3", settings);
adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
bulkClient.newIndex("test_prune4", settings);
adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
IndexPruneResult indexPruneResult =
client.pruneIndex("test_prune", "test_prune4", 2, 2, true);
adminClient.pruneIndex("test_prune", "test_prune4", 2, 2, true);
assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1"));
assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune4"));
List<Boolean> list = new ArrayList<>();
for (String index : Arrays.asList("test_prune1", "test_prune2", "test_prune3", "test_prune4")) {
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
indicesExistsRequest.indices(index);
IndicesExistsResponse indicesExistsResponse =
client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
list.add(indicesExistsResponse.isExists());
list.add(bulkClient.isIndexExists(index));
}
logger.info(list);
assertFalse(list.get(0));
assertFalse(list.get(1));
assertTrue(list.get(2));
assertTrue(list.get(3));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
adminClient.close();
bulkClient.close();
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
assertNull(bulkClient.getBulkController().getLastBulkError());
}
}
}

58
elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java
View file

@ -3,15 +3,16 @@ package org.xbib.elx.transport.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexShiftResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.transport.ExtendedTransportClient;
import org.xbib.elx.transport.ExtendedTransportClientProvider;
import org.xbib.elx.transport.TransportAdminClient;
import org.xbib.elx.transport.TransportAdminClientProvider;
import org.xbib.elx.transport.TransportBulkClient;
import org.xbib.elx.transport.TransportBulkClientProvider;
import java.util.Arrays;
import java.util.Map;
@ -33,8 +34,12 @@ class IndexShiftTest {
@Test
void testIndexShift() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
final TransportAdminClient adminClient = ClientBuilder.builder()
.setAdminClientProvider(TransportAdminClientProvider.class)
.put(helper.getTransportSettings())
.build();
final TransportBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(TransportBulkClientProvider.class)
.put(helper.getTransportSettings())
.build();
try {
@ -42,38 +47,38 @@ class IndexShiftTest {
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
client.newIndex("test_shift1234", settings);
bulkClient.newIndex("test_shift1234", settings);
for (int i = 0; i < 1; i++) {
client.index("test_shift1234", helper.randomString(1), false,
bulkClient.index("test_shift1234", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
IndexShiftResult indexShiftResult =
client.shiftIndex("test_shift", "test_shift1234", Arrays.asList("a", "b", "c"));
adminClient.shiftIndex("test_shift", "test_shift1234", Arrays.asList("a", "b", "c"));
assertTrue(indexShiftResult.getNewAliases().contains("a"));
assertTrue(indexShiftResult.getNewAliases().contains("b"));
assertTrue(indexShiftResult.getNewAliases().contains("c"));
assertTrue(indexShiftResult.getMovedAliases().isEmpty());
Map<String, String> aliases = client.getAliases("test_shift1234");
Map<String, String> aliases = adminClient.getAliases("test_shift1234");
assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("test_shift"));
String resolved = client.resolveAlias("test_shift");
aliases = client.getAliases(resolved);
String resolved = adminClient.resolveAlias("test_shift");
aliases = adminClient.getAliases(resolved);
assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("test_shift"));
client.newIndex("test_shift5678", settings);
bulkClient.newIndex("test_shift5678", settings);
for (int i = 0; i < 1; i++) {
client.index("test_shift5678", helper.randomString(1), false,
bulkClient.index("test_shift5678", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
indexShiftResult = client.shiftIndex("test_shift", "test_shift5678", Arrays.asList("d", "e", "f"),
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
indexShiftResult = adminClient.shiftIndex("test_shift", "test_shift5678", Arrays.asList("d", "e", "f"),
(request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
.index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
);
@ -83,29 +88,28 @@ class IndexShiftTest {
assertTrue(indexShiftResult.getMovedAliases().contains("a"));
assertTrue(indexShiftResult.getMovedAliases().contains("b"));
assertTrue(indexShiftResult.getMovedAliases().contains("c"));
aliases = client.getAliases("test_shift5678");
aliases = adminClient.getAliases("test_shift5678");
assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("d"));
assertTrue(aliases.containsKey("e"));
assertTrue(aliases.containsKey("f"));
resolved = client.resolveAlias("test_shift");
aliases = client.getAliases(resolved);
resolved = adminClient.resolveAlias("test_shift");
aliases = adminClient.getAliases(resolved);
assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("d"));
assertTrue(aliases.containsKey("e"));
assertTrue(aliases.containsKey("f"));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
adminClient.close();
bulkClient.close();
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
assertNull(bulkClient.getBulkController().getLastBulkError());
}
}
}

73
elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java
View file

@ -2,14 +2,15 @@ package org.xbib.elx.transport.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.transport.ExtendedTransportClient;
import org.xbib.elx.transport.ExtendedTransportClientProvider;
import org.xbib.elx.transport.TransportAdminClient;
import org.xbib.elx.transport.TransportAdminClientProvider;
import org.xbib.elx.transport.TransportBulkClient;
import org.xbib.elx.transport.TransportBulkClientProvider;
import java.util.concurrent.TimeUnit;
@ -29,44 +30,48 @@ class SmokeTest {
@Test
void smokeTest() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
final TransportAdminClient adminClient = ClientBuilder.builder()
.setAdminClientProvider(TransportAdminClientProvider.class)
.put(helper.getTransportSettings())
.build();
final TransportBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(TransportBulkClientProvider.class)
.put(helper.getTransportSettings())
.build();
IndexDefinition indexDefinition =
adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.EMPTY);
try {
assertEquals(helper.getClusterName(), client.getClusterName());
client.newIndex("test_smoke");
client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
client.flush();
client.waitForResponses(30, TimeUnit.SECONDS);
client.checkMapping("test_smoke");
client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
client.delete("test_smoke", "1");
client.flush();
client.waitForResponses(30, TimeUnit.SECONDS);
client.checkMapping("test_smoke");
client.deleteIndex("test_smoke");
IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder()
.build());
assertEquals(helper.getClusterName(), adminClient.getClusterName());
bulkClient.newIndex("test_smoke");
bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
bulkClient.flush();
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
adminClient.checkMapping("test_smoke");
bulkClient.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
bulkClient.delete("test_smoke", "1");
bulkClient.flush();
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
adminClient.checkMapping("test_smoke");
adminClient.deleteIndex("test_smoke");
assertEquals(0, indexDefinition.getReplicaLevel());
client.newIndex(indexDefinition);
client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
client.flush();
client.waitForResponses(30, TimeUnit.SECONDS);
client.updateReplicaLevel(indexDefinition, 2);
int replica = client.getReplicaLevel(indexDefinition);
bulkClient.newIndex(indexDefinition);
bulkClient.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
bulkClient.flush();
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
adminClient.updateReplicaLevel(indexDefinition, 2);
int replica = adminClient.getReplicaLevel(indexDefinition);
assertEquals(2, replica);
client.deleteIndex(indexDefinition);
assertEquals(0, client.getBulkMetric().getFailed().getCount());
assertEquals(4, client.getBulkMetric().getSucceeded().getCount());
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
bulkClient.close();
assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount());
assertEquals(4, bulkClient.getBulkMetric().getSucceeded().getCount());
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
assertNull(bulkClient.getBulkController().getLastBulkError());
// close admin after bulk
adminClient.deleteIndex(indexDefinition);
adminClient.close();
}
}
}