split into admin, bulk, search client, fix HTTP client
This commit is contained in:
parent
10833f023b
commit
110c7786da
89 changed files with 3003 additions and 2112 deletions
194
elx-api/src/main/java/org/xbib/elx/api/AdminClient.java
Normal file
194
elx-api/src/main/java/org/xbib/elx/api/AdminClient.java
Normal file
|
@ -0,0 +1,194 @@
|
||||||
|
package org.xbib.elx.api;
|
||||||
|
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Interface for extended managing and indexing methods of an Elasticsearch client.
|
||||||
|
*/
|
||||||
|
public interface AdminClient extends NativeClient {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build index definition from settings.
|
||||||
|
*
|
||||||
|
* @param index the index name
|
||||||
|
* @param settings the settings for the index
|
||||||
|
* @return index definition
|
||||||
|
* @throws IOException if settings/mapping URL is invalid/malformed
|
||||||
|
*/
|
||||||
|
IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete an index.
|
||||||
|
* @param indexDefinition the index definition
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
AdminClient deleteIndex(IndexDefinition indexDefinition);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete an index.
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
AdminClient deleteIndex(String index);
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update replica level.
|
||||||
|
* @param indexDefinition the index definition
|
||||||
|
* @param level the replica level
|
||||||
|
* @return this
|
||||||
|
* @throws IOException if replica setting could not be updated
|
||||||
|
*/
|
||||||
|
AdminClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update replica level.
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
* @param level the replica level
|
||||||
|
* @param maxWaitTime maximum wait time
|
||||||
|
* @param timeUnit time unit
|
||||||
|
* @return this
|
||||||
|
* @throws IOException if replica setting could not be updated
|
||||||
|
*/
|
||||||
|
AdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get replica level.
|
||||||
|
* @param indexDefinition the index name
|
||||||
|
* @return the replica level of the index
|
||||||
|
*/
|
||||||
|
int getReplicaLevel(IndexDefinition indexDefinition);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get replica level.
|
||||||
|
* @param index the index name
|
||||||
|
* @return the replica level of the index
|
||||||
|
*/
|
||||||
|
int getReplicaLevel(String index);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Force segment merge of an index.
|
||||||
|
* @param indexDefinition the index definition
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
boolean forceMerge(IndexDefinition indexDefinition);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Force segment merge of an index.
|
||||||
|
* @param index the index
|
||||||
|
* @param maxWaitTime maximum wait time
|
||||||
|
* @param timeUnit time unit
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit);
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wait for index recovery (after replica change).
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
* @param maxWaitTime maximum wait time
|
||||||
|
* @param timeUnit time unit
|
||||||
|
* @return true if wait succeeded, false if wait timed out
|
||||||
|
*/
|
||||||
|
boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve alias.
|
||||||
|
*
|
||||||
|
* @param alias the alias
|
||||||
|
* @return this index name behind the alias or the alias if there is no index
|
||||||
|
*/
|
||||||
|
String resolveAlias(String alias);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index
|
||||||
|
* name.
|
||||||
|
*
|
||||||
|
* @param alias the alias
|
||||||
|
* @return the most recent index name pointing to the alias
|
||||||
|
*/
|
||||||
|
String resolveMostRecentIndex(String alias);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get all index aliases.
|
||||||
|
* @param index the index
|
||||||
|
* @return map of index aliases
|
||||||
|
*/
|
||||||
|
Map<String, String> getAliases(String index);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Shift from one index to another.
|
||||||
|
* @param indexDefinition the index definition
|
||||||
|
* @param additionalAliases new aliases
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Shift from one index to another.
|
||||||
|
* @param indexDefinition the index definition
|
||||||
|
* @param additionalAliases new aliases
|
||||||
|
* @param indexAliasAdder method to add aliases
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases,
|
||||||
|
IndexAliasAdder indexAliasAdder);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Shift from one index to another.
|
||||||
|
* @param index the index name
|
||||||
|
* @param fullIndexName the index name with timestamp
|
||||||
|
* @param additionalAliases a list of names that should be set as index aliases
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Shift from one index to another.
|
||||||
|
* @param index the index name
|
||||||
|
* @param fullIndexName the index name with timestamp
|
||||||
|
* @param additionalAliases a list of names that should be set as index aliases
|
||||||
|
* @param adder an adder method to create alias term queries
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases,
|
||||||
|
IndexAliasAdder adder);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Prune index.
|
||||||
|
* @param indexDefinition the index definition
|
||||||
|
* @return the index prune result
|
||||||
|
*/
|
||||||
|
IndexPruneResult pruneIndex(IndexDefinition indexDefinition);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Apply retention policy to prune indices. All indices before delta should be deleted,
|
||||||
|
* but the number of mintokeep indices must be kept.
|
||||||
|
*
|
||||||
|
* @param index index name
|
||||||
|
* @param fullIndexName index name with timestamp
|
||||||
|
* @param delta timestamp delta (for index timestamps)
|
||||||
|
* @param mintokeep minimum number of indices to keep
|
||||||
|
* @param perform true if pruning should be executed, false if not
|
||||||
|
* @return the index prune result
|
||||||
|
*/
|
||||||
|
IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find the timestamp of the most recently indexed document in the index.
|
||||||
|
*
|
||||||
|
* @param index the index name
|
||||||
|
* @param timestampfieldname the timestamp field name
|
||||||
|
* @return millis UTC millis of the most recent document
|
||||||
|
* @throws IOException if most rcent document can not be found
|
||||||
|
*/
|
||||||
|
Long mostRecentDocument(String index, String timestampfieldname) throws IOException;
|
||||||
|
}
|
|
@ -1,7 +1,7 @@
|
||||||
package org.xbib.elx.api;
|
package org.xbib.elx.api;
|
||||||
|
|
||||||
@FunctionalInterface
|
@FunctionalInterface
|
||||||
public interface SeachClientProvider<C extends SearchClient> {
|
public interface AdminClientProvider<C extends AdminClient> {
|
||||||
|
|
||||||
C getClient();
|
C getClient();
|
||||||
}
|
}
|
227
elx-api/src/main/java/org/xbib/elx/api/BulkClient.java
Normal file
227
elx-api/src/main/java/org/xbib/elx/api/BulkClient.java
Normal file
|
@ -0,0 +1,227 @@
|
||||||
|
package org.xbib.elx.api;
|
||||||
|
|
||||||
|
import org.elasticsearch.action.delete.DeleteRequest;
|
||||||
|
import org.elasticsearch.action.index.IndexRequest;
|
||||||
|
import org.elasticsearch.action.update.UpdateRequest;
|
||||||
|
import org.elasticsearch.common.bytes.BytesReference;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
|
import java.io.Flushable;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
public interface BulkClient extends NativeClient, Flushable {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get bulk metric.
|
||||||
|
* @return the bulk metric
|
||||||
|
*/
|
||||||
|
BulkMetric getBulkMetric();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get buulk control.
|
||||||
|
* @return the bulk control
|
||||||
|
*/
|
||||||
|
BulkController getBulkController();
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new index.
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
* @throws IOException if new index creation fails
|
||||||
|
*/
|
||||||
|
void newIndex(String index) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new index.
|
||||||
|
* @param indexDefinition the index definition
|
||||||
|
* @throws IOException if settings/mapping is invalid or index creation fails
|
||||||
|
*/
|
||||||
|
void newIndex(IndexDefinition indexDefinition) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new index.
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
* @param settings settings
|
||||||
|
* @throws IOException if settings is invalid or index creation fails
|
||||||
|
*/
|
||||||
|
void newIndex(String index, Settings settings) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new index.
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
* @param settings settings
|
||||||
|
* @param mapping mapping
|
||||||
|
* @throws IOException if settings/mapping is invalid or index creation fails
|
||||||
|
*/
|
||||||
|
void newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new index.
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
* @param settings settings
|
||||||
|
* @param mapping mapping
|
||||||
|
* @throws IOException if settings/mapping is invalid or index creation fails
|
||||||
|
*/
|
||||||
|
void newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add index request. Each request will be added to a queue for bulking requests.
|
||||||
|
* Submitting request will be done when limits are exceeded.
|
||||||
|
*
|
||||||
|
* @param index the index
|
||||||
|
* @param id the id
|
||||||
|
* @param create true if document must be created
|
||||||
|
* @param source the source
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
BulkClient index(String index, String id, boolean create, BytesReference source);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Index request. Each request will be added to a queue for bulking requests.
|
||||||
|
* Submitting request will be done when limits are exceeded.
|
||||||
|
*
|
||||||
|
* @param index the index
|
||||||
|
* @param id the id
|
||||||
|
* @param create true if document is to be created, false otherwise
|
||||||
|
* @param source the source
|
||||||
|
* @return this client methods
|
||||||
|
*/
|
||||||
|
BulkClient index(String index, String id, boolean create, String source);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Index request. Each request will be added to a queue for bulking requests.
|
||||||
|
* Submitting request will be done when bulk limits are exceeded.
|
||||||
|
*
|
||||||
|
* @param indexRequest the index request to add
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
BulkClient index(IndexRequest indexRequest);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete request.
|
||||||
|
*
|
||||||
|
* @param index the index
|
||||||
|
* @param id the id
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
BulkClient delete(String index, String id);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete request. Each request will be added to a queue for bulking requests.
|
||||||
|
* Submitting request will be done when bulk limits are exceeded.
|
||||||
|
*
|
||||||
|
* @param deleteRequest the delete request to add
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
BulkClient delete(DeleteRequest deleteRequest);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Bulked update request. Each request will be added to a queue for bulking requests.
|
||||||
|
* Submitting request will be done when bulk limits are exceeded.
|
||||||
|
* Note that updates only work correctly when all operations between nodes are synchronized.
|
||||||
|
*
|
||||||
|
* @param index the index
|
||||||
|
* @param id the id
|
||||||
|
* @param source the source
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
BulkClient update(String index, String id, BytesReference source);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update document. Use with precaution! Does not work in all cases.
|
||||||
|
*
|
||||||
|
* @param index the index
|
||||||
|
* @param id the id
|
||||||
|
* @param source the source
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
BulkClient update(String index, String id, String source);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Bulked update request. Each request will be added to a queue for bulking requests.
|
||||||
|
* Submitting request will be done when bulk limits are exceeded.
|
||||||
|
* Note that updates only work correctly when all operations between nodes are synchronized.
|
||||||
|
*
|
||||||
|
* @param updateRequest the update request to add
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
BulkClient update(UpdateRequest updateRequest);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start bulk mode for indexes.
|
||||||
|
* @param indexDefinition index definition
|
||||||
|
* @throws IOException if bulk could not be started
|
||||||
|
*/
|
||||||
|
void startBulk(IndexDefinition indexDefinition) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start bulk mode.
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
* @param startRefreshIntervalSeconds refresh interval before bulk
|
||||||
|
* @param stopRefreshIntervalSeconds refresh interval after bulk
|
||||||
|
* @throws IOException if bulk could not be started
|
||||||
|
*/
|
||||||
|
void startBulk(String index, long startRefreshIntervalSeconds,
|
||||||
|
long stopRefreshIntervalSeconds) throws IOException;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stop bulk mode.
|
||||||
|
*
|
||||||
|
* @param indexDefinition index definition
|
||||||
|
* @throws IOException if bulk could not be startet
|
||||||
|
*/
|
||||||
|
void stopBulk(IndexDefinition indexDefinition) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stops bulk mode.
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
* @param timeout maximum wait time
|
||||||
|
* @param timeUnit time unit for timeout
|
||||||
|
* @throws IOException if bulk could not be stopped
|
||||||
|
*/
|
||||||
|
void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wait for all outstanding bulk responses.
|
||||||
|
*
|
||||||
|
* @param timeout maximum wait time
|
||||||
|
* @param timeUnit unit of timeout value
|
||||||
|
* @return true if wait succeeded, false if wait timed out
|
||||||
|
*/
|
||||||
|
boolean waitForResponses(long timeout, TimeUnit timeUnit);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update index setting.
|
||||||
|
* @param index the index
|
||||||
|
* @param key the key of the value to be updated
|
||||||
|
* @param value the new value
|
||||||
|
* @param timeout timeout
|
||||||
|
* @param timeUnit time unit
|
||||||
|
* @throws IOException if update index setting failed
|
||||||
|
*/
|
||||||
|
void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Refresh the index.
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
*/
|
||||||
|
void refreshIndex(String index);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Flush the index. The cluster clears cache and completes indexing.
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
*/
|
||||||
|
void flushIndex(String index);
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,7 @@
|
||||||
|
package org.xbib.elx.api;
|
||||||
|
|
||||||
|
@FunctionalInterface
|
||||||
|
public interface BulkClientProvider<C extends BulkClient> {
|
||||||
|
|
||||||
|
C getClient();
|
||||||
|
}
|
|
@ -1,480 +0,0 @@
|
||||||
package org.xbib.elx.api;
|
|
||||||
|
|
||||||
import org.elasticsearch.action.delete.DeleteRequest;
|
|
||||||
import org.elasticsearch.action.index.IndexRequest;
|
|
||||||
import org.elasticsearch.action.update.UpdateRequest;
|
|
||||||
import org.elasticsearch.client.ElasticsearchClient;
|
|
||||||
import org.elasticsearch.common.bytes.BytesReference;
|
|
||||||
import org.elasticsearch.common.settings.Settings;
|
|
||||||
|
|
||||||
import java.io.Closeable;
|
|
||||||
import java.io.Flushable;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.InputStream;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Interface for extended managing and indexing methods of an Elasticsearch client.
|
|
||||||
*/
|
|
||||||
public interface ExtendedClient extends Flushable, Closeable {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Set an Elasticsearch client to extend from it. May be null for TransportClient.
|
|
||||||
* @param client client
|
|
||||||
* @return this client
|
|
||||||
*/
|
|
||||||
ExtendedClient setClient(ElasticsearchClient client);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Return Elasticsearch client.
|
|
||||||
*
|
|
||||||
* @return Elasticsearch client
|
|
||||||
*/
|
|
||||||
ElasticsearchClient getClient();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get bulk metric.
|
|
||||||
* @return the bulk metric
|
|
||||||
*/
|
|
||||||
BulkMetric getBulkMetric();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get buulk control.
|
|
||||||
* @return the bulk control
|
|
||||||
*/
|
|
||||||
BulkController getBulkController();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initiative the extended client, the bulk metric and bulk controller,
|
|
||||||
* creates instances and connect to cluster, if required.
|
|
||||||
*
|
|
||||||
* @param settings settings
|
|
||||||
* @return this client
|
|
||||||
* @throws IOException if init fails
|
|
||||||
*/
|
|
||||||
ExtendedClient init(Settings settings) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Build index definition from settings.
|
|
||||||
*
|
|
||||||
* @param index the index name
|
|
||||||
* @param settings the settings for the index
|
|
||||||
* @return index definition
|
|
||||||
* @throws IOException if settings/mapping URL is invalid/malformed
|
|
||||||
*/
|
|
||||||
IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Add index request. Each request will be added to a queue for bulking requests.
|
|
||||||
* Submitting request will be done when limits are exceeded.
|
|
||||||
*
|
|
||||||
* @param index the index
|
|
||||||
* @param id the id
|
|
||||||
* @param create true if document must be created
|
|
||||||
* @param source the source
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient index(String index, String id, boolean create, BytesReference source);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Index request. Each request will be added to a queue for bulking requests.
|
|
||||||
* Submitting request will be done when limits are exceeded.
|
|
||||||
*
|
|
||||||
* @param index the index
|
|
||||||
* @param id the id
|
|
||||||
* @param create true if document is to be created, false otherwise
|
|
||||||
* @param source the source
|
|
||||||
* @return this client methods
|
|
||||||
*/
|
|
||||||
ExtendedClient index(String index, String id, boolean create, String source);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Index request. Each request will be added to a queue for bulking requests.
|
|
||||||
* Submitting request will be done when bulk limits are exceeded.
|
|
||||||
*
|
|
||||||
* @param indexRequest the index request to add
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient index(IndexRequest indexRequest);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Delete request.
|
|
||||||
*
|
|
||||||
* @param index the index
|
|
||||||
* @param id the id
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient delete(String index, String id);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Delete request. Each request will be added to a queue for bulking requests.
|
|
||||||
* Submitting request will be done when bulk limits are exceeded.
|
|
||||||
*
|
|
||||||
* @param deleteRequest the delete request to add
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient delete(DeleteRequest deleteRequest);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Bulked update request. Each request will be added to a queue for bulking requests.
|
|
||||||
* Submitting request will be done when bulk limits are exceeded.
|
|
||||||
* Note that updates only work correctly when all operations between nodes are synchronized.
|
|
||||||
*
|
|
||||||
* @param index the index
|
|
||||||
* @param id the id
|
|
||||||
* @param source the source
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient update(String index, String id, BytesReference source);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Update document. Use with precaution! Does not work in all cases.
|
|
||||||
*
|
|
||||||
* @param index the index
|
|
||||||
* @param id the id
|
|
||||||
* @param source the source
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient update(String index, String id, String source);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Bulked update request. Each request will be added to a queue for bulking requests.
|
|
||||||
* Submitting request will be done when bulk limits are exceeded.
|
|
||||||
* Note that updates only work correctly when all operations between nodes are synchronized.
|
|
||||||
*
|
|
||||||
* @param updateRequest the update request to add
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient update(UpdateRequest updateRequest);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a new index.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if new index creation fails
|
|
||||||
*/
|
|
||||||
ExtendedClient newIndex(String index) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a new index.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @param settings settings
|
|
||||||
* @param mapping mapping
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if settings/mapping is invalid or index creation fails
|
|
||||||
*/
|
|
||||||
ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a new index.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @param settings settings
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if settings is invalid or index creation fails
|
|
||||||
*/
|
|
||||||
ExtendedClient newIndex(String index, Settings settings) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a new index.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @param settings settings
|
|
||||||
* @param mapping mapping
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if settings/mapping is invalid or index creation fails
|
|
||||||
*/
|
|
||||||
ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a new index.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @param settings settings
|
|
||||||
* @param mapping mapping
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if settings/mapping is invalid or index creation fails
|
|
||||||
*/
|
|
||||||
ExtendedClient newIndex(String index, Settings settings, Map<String, Object> mapping) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a new index.
|
|
||||||
* @param indexDefinition the index definition
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if settings/mapping is invalid or index creation fails
|
|
||||||
*/
|
|
||||||
ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Delete an index.
|
|
||||||
* @param indexDefinition the index definition
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient deleteIndex(IndexDefinition indexDefinition);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Delete an index.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient deleteIndex(String index);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Start bulk mode for indexes.
|
|
||||||
* @param indexDefinition index definition
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if bulk could not be started
|
|
||||||
*/
|
|
||||||
ExtendedClient startBulk(IndexDefinition indexDefinition) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Start bulk mode.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @param startRefreshIntervalSeconds refresh interval before bulk
|
|
||||||
* @param stopRefreshIntervalSeconds refresh interval after bulk
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if bulk could not be started
|
|
||||||
*/
|
|
||||||
ExtendedClient startBulk(String index, long startRefreshIntervalSeconds,
|
|
||||||
long stopRefreshIntervalSeconds) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Stop bulk mode.
|
|
||||||
*
|
|
||||||
* @param indexDefinition index definition
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if bulk could not be startet
|
|
||||||
*/
|
|
||||||
ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Stops bulk mode.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @param timeout maximum wait time
|
|
||||||
* @param timeUnit time unit for timeout
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if bulk could not be stopped
|
|
||||||
*/
|
|
||||||
ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Update replica level.
|
|
||||||
* @param indexDefinition the index definition
|
|
||||||
* @param level the replica level
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if replica setting could not be updated
|
|
||||||
*/
|
|
||||||
ExtendedClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Update replica level.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @param level the replica level
|
|
||||||
* @param maxWaitTime maximum wait time
|
|
||||||
* @param timeUnit time unit
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if replica setting could not be updated
|
|
||||||
*/
|
|
||||||
ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get replica level.
|
|
||||||
* @param indexDefinition the index name
|
|
||||||
* @return the replica level of the index
|
|
||||||
*/
|
|
||||||
int getReplicaLevel(IndexDefinition indexDefinition);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get replica level.
|
|
||||||
* @param index the index name
|
|
||||||
* @return the replica level of the index
|
|
||||||
*/
|
|
||||||
int getReplicaLevel(String index);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Refresh the index.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient refreshIndex(String index);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Flush the index. The cluster clears cache and completes indexing.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient flushIndex(String index);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Force segment merge of an index.
|
|
||||||
* @param indexDefinition the index definition
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
boolean forceMerge(IndexDefinition indexDefinition);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Force segment merge of an index.
|
|
||||||
* @param index the index
|
|
||||||
* @param maxWaitTime maximum wait time
|
|
||||||
* @param timeUnit time unit
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Wait for all outstanding bulk responses.
|
|
||||||
*
|
|
||||||
* @param timeout maximum wait time
|
|
||||||
* @param timeUnit unit of timeout value
|
|
||||||
* @return true if wait succeeded, false if wait timed out
|
|
||||||
*/
|
|
||||||
boolean waitForResponses(long timeout, TimeUnit timeUnit);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Wait for cluster being healthy.
|
|
||||||
*
|
|
||||||
* @param healthColor cluster health color to wait for
|
|
||||||
* @param maxWaitTime time value
|
|
||||||
* @param timeUnit time unit
|
|
||||||
* @return true if wait succeeded, false if wait timed out
|
|
||||||
*/
|
|
||||||
boolean waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get current health color.
|
|
||||||
*
|
|
||||||
* @param maxWaitTime maximum wait time
|
|
||||||
* @param timeUnit time unit
|
|
||||||
* @return the cluster health color
|
|
||||||
*/
|
|
||||||
String getHealthColor(long maxWaitTime, TimeUnit timeUnit);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Wait for index recovery (after replica change).
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @param maxWaitTime maximum wait time
|
|
||||||
* @param timeUnit time unit
|
|
||||||
* @return true if wait succeeded, false if wait timed out
|
|
||||||
*/
|
|
||||||
boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Update index setting.
|
|
||||||
* @param index the index
|
|
||||||
* @param key the key of the value to be updated
|
|
||||||
* @param value the new value
|
|
||||||
* @param timeout timeout
|
|
||||||
* @param timeUnit time unit
|
|
||||||
* @throws IOException if update index setting failed
|
|
||||||
*/
|
|
||||||
void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Resolve alias.
|
|
||||||
*
|
|
||||||
* @param alias the alias
|
|
||||||
* @return this index name behind the alias or the alias if there is no index
|
|
||||||
*/
|
|
||||||
String resolveAlias(String alias);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index
|
|
||||||
* name.
|
|
||||||
*
|
|
||||||
* @param alias the alias
|
|
||||||
* @return the most recent index name pointing to the alias
|
|
||||||
*/
|
|
||||||
String resolveMostRecentIndex(String alias);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all index aliases.
|
|
||||||
* @param index the index
|
|
||||||
* @return map of index aliases
|
|
||||||
*/
|
|
||||||
Map<String, String> getAliases(String index);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Shift from one index to another.
|
|
||||||
* @param indexDefinition the index definition
|
|
||||||
* @param additionalAliases new aliases
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Shift from one index to another.
|
|
||||||
* @param indexDefinition the index definition
|
|
||||||
* @param additionalAliases new aliases
|
|
||||||
* @param indexAliasAdder method to add aliases
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases,
|
|
||||||
IndexAliasAdder indexAliasAdder);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Shift from one index to another.
|
|
||||||
* @param index the index name
|
|
||||||
* @param fullIndexName the index name with timestamp
|
|
||||||
* @param additionalAliases a list of names that should be set as index aliases
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Shift from one index to another.
|
|
||||||
* @param index the index name
|
|
||||||
* @param fullIndexName the index name with timestamp
|
|
||||||
* @param additionalAliases a list of names that should be set as index aliases
|
|
||||||
* @param adder an adder method to create alias term queries
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases,
|
|
||||||
IndexAliasAdder adder);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Prune index.
|
|
||||||
* @param indexDefinition the index definition
|
|
||||||
* @return the index prune result
|
|
||||||
*/
|
|
||||||
IndexPruneResult pruneIndex(IndexDefinition indexDefinition);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Apply retention policy to prune indices. All indices before delta should be deleted,
|
|
||||||
* but the number of mintokeep indices must be kept.
|
|
||||||
*
|
|
||||||
* @param index index name
|
|
||||||
* @param fullIndexName index name with timestamp
|
|
||||||
* @param delta timestamp delta (for index timestamps)
|
|
||||||
* @param mintokeep minimum number of indices to keep
|
|
||||||
* @param perform true if pruning should be executed, false if not
|
|
||||||
* @return the index prune result
|
|
||||||
*/
|
|
||||||
IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Find the timestamp of the most recently indexed document in the index.
|
|
||||||
*
|
|
||||||
* @param index the index name
|
|
||||||
* @param timestampfieldname the timestamp field name
|
|
||||||
* @return millis UTC millis of the most recent document
|
|
||||||
* @throws IOException if most rcent document can not be found
|
|
||||||
*/
|
|
||||||
Long mostRecentDocument(String index, String timestampfieldname) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get cluster name.
|
|
||||||
* @return the cluster name
|
|
||||||
*/
|
|
||||||
String getClusterName();
|
|
||||||
}
|
|
|
@ -1,7 +0,0 @@
|
||||||
package org.xbib.elx.api;
|
|
||||||
|
|
||||||
@FunctionalInterface
|
|
||||||
public interface ExtendedClientProvider<C extends ExtendedClient> {
|
|
||||||
|
|
||||||
C getExtendedClient();
|
|
||||||
}
|
|
|
@ -1,7 +1,5 @@
|
||||||
package org.xbib.elx.api;
|
package org.xbib.elx.api;
|
||||||
|
|
||||||
import java.net.MalformedURLException;
|
|
||||||
import java.net.URL;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
public interface IndexDefinition {
|
public interface IndexDefinition {
|
||||||
|
@ -14,17 +12,13 @@ public interface IndexDefinition {
|
||||||
|
|
||||||
String getFullIndexName();
|
String getFullIndexName();
|
||||||
|
|
||||||
IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException;
|
IndexDefinition setSettings(String settings);
|
||||||
|
|
||||||
IndexDefinition setSettingsUrl(URL settingsUrl);
|
String getSettings();
|
||||||
|
|
||||||
URL getSettingsUrl();
|
IndexDefinition setMappings(String mappings);
|
||||||
|
|
||||||
IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException;
|
String getMappings();
|
||||||
|
|
||||||
IndexDefinition setMappingsUrl(URL mappingsUrl);
|
|
||||||
|
|
||||||
URL getMappingsUrl();
|
|
||||||
|
|
||||||
IndexDefinition setDateTimePattern(String timeWindow);
|
IndexDefinition setDateTimePattern(String timeWindow);
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
package org.xbib.elx.api;
|
package org.xbib.elx.api;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.Collection;
|
||||||
|
|
||||||
public interface IndexPruneResult {
|
public interface IndexPruneResult {
|
||||||
|
|
||||||
|
@ -8,9 +8,9 @@ public interface IndexPruneResult {
|
||||||
|
|
||||||
State getState();
|
State getState();
|
||||||
|
|
||||||
List<String> getCandidateIndices();
|
Collection<String> getCandidateIndices();
|
||||||
|
|
||||||
List<String> getDeletedIndices();
|
Collection<String> getDeletedIndices();
|
||||||
|
|
||||||
boolean isAcknowledged();
|
boolean isAcknowledged();
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,10 +1,10 @@
|
||||||
package org.xbib.elx.api;
|
package org.xbib.elx.api;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.Collection;
|
||||||
|
|
||||||
public interface IndexShiftResult {
|
public interface IndexShiftResult {
|
||||||
|
|
||||||
List<String> getMovedAliases();
|
Collection<String> getMovedAliases();
|
||||||
|
|
||||||
List<String> getNewAliases();
|
Collection<String> getNewAliases();
|
||||||
}
|
}
|
||||||
|
|
63
elx-api/src/main/java/org/xbib/elx/api/NativeClient.java
Normal file
63
elx-api/src/main/java/org/xbib/elx/api/NativeClient.java
Normal file
|
@ -0,0 +1,63 @@
|
||||||
|
package org.xbib.elx.api;
|
||||||
|
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import java.io.Closeable;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
public interface NativeClient extends Closeable {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set an Elasticsearch client to extend from it. May be null for TransportClient.
|
||||||
|
* @param client the Elasticsearch client
|
||||||
|
*/
|
||||||
|
void setClient(ElasticsearchClient client);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return Elasticsearch client.
|
||||||
|
*
|
||||||
|
* @return Elasticsearch client
|
||||||
|
*/
|
||||||
|
ElasticsearchClient getClient();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initiative the extended client, the bulk metric and bulk controller,
|
||||||
|
* creates instances and connect to cluster, if required.
|
||||||
|
*
|
||||||
|
* @param settings settings
|
||||||
|
* @throws IOException if init fails
|
||||||
|
*/
|
||||||
|
void init(Settings settings) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get cluster name.
|
||||||
|
* @return the cluster name
|
||||||
|
*/
|
||||||
|
String getClusterName();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get current health color.
|
||||||
|
*
|
||||||
|
* @param maxWaitTime maximum wait time
|
||||||
|
* @param timeUnit time unit
|
||||||
|
* @return the cluster health color
|
||||||
|
*/
|
||||||
|
String getHealthColor(long maxWaitTime, TimeUnit timeUnit);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wait for cluster being healthy.
|
||||||
|
*
|
||||||
|
* @param healthColor cluster health color to wait for
|
||||||
|
* @param maxWaitTime time value
|
||||||
|
* @param timeUnit time unit
|
||||||
|
*/
|
||||||
|
void waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit);
|
||||||
|
|
||||||
|
Map<String, ?> getMapping(String index, String mapping);
|
||||||
|
|
||||||
|
long getSearchableDocs(String index);
|
||||||
|
|
||||||
|
boolean isIndexExists(String index);
|
||||||
|
}
|
|
@ -1,25 +1,27 @@
|
||||||
package org.xbib.elx.api;
|
package org.xbib.elx.api;
|
||||||
|
|
||||||
import org.elasticsearch.action.ActionFuture;
|
import org.elasticsearch.action.get.GetRequestBuilder;
|
||||||
import org.elasticsearch.action.ActionListener;
|
|
||||||
import org.elasticsearch.action.get.GetRequest;
|
|
||||||
import org.elasticsearch.action.get.GetResponse;
|
import org.elasticsearch.action.get.GetResponse;
|
||||||
import org.elasticsearch.action.get.MultiGetRequest;
|
import org.elasticsearch.action.get.MultiGetRequestBuilder;
|
||||||
import org.elasticsearch.action.get.MultiGetResponse;
|
import org.elasticsearch.action.get.MultiGetResponse;
|
||||||
import org.elasticsearch.action.search.SearchRequest;
|
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||||
import org.elasticsearch.action.search.SearchResponse;
|
import org.elasticsearch.action.search.SearchResponse;
|
||||||
|
import org.elasticsearch.common.unit.TimeValue;
|
||||||
|
import org.elasticsearch.search.SearchHit;
|
||||||
|
import java.util.Optional;
|
||||||
|
import java.util.function.Consumer;
|
||||||
|
import java.util.stream.Stream;
|
||||||
|
|
||||||
public interface SearchClient {
|
public interface SearchClient extends NativeClient {
|
||||||
|
|
||||||
ActionFuture<GetResponse> get(GetRequest getRequest);
|
Optional<GetResponse> get(Consumer<GetRequestBuilder> getRequestBuilder);
|
||||||
|
|
||||||
void get(GetRequest request, ActionListener<GetResponse> listener);
|
Optional<MultiGetResponse> multiGet(Consumer<MultiGetRequestBuilder> multiGetRequestBuilder);
|
||||||
|
|
||||||
ActionFuture<MultiGetResponse> multiGet(MultiGetRequest request);
|
Optional<SearchResponse> search(Consumer<SearchRequestBuilder> searchRequestBuilder);
|
||||||
|
|
||||||
void multiGet(MultiGetRequest request, ActionListener<MultiGetResponse> listener);
|
Stream<SearchHit> search(Consumer<SearchRequestBuilder> searchRequestBuilder,
|
||||||
|
TimeValue scrollTime, int scrollSize);
|
||||||
|
|
||||||
ActionFuture<SearchResponse> search(SearchRequest request);
|
Stream<String> getIds(Consumer<SearchRequestBuilder> queryBuilder);
|
||||||
|
|
||||||
void search(SearchRequest request, ActionListener<SearchResponse> listener);
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,7 @@
|
||||||
|
package org.xbib.elx.api;
|
||||||
|
|
||||||
|
@FunctionalInterface
|
||||||
|
public interface SearchClientProvider<C extends SearchClient> {
|
||||||
|
|
||||||
|
C getClient();
|
||||||
|
}
|
|
@ -2,10 +2,8 @@ package org.xbib.elx.common;
|
||||||
|
|
||||||
import com.carrotsearch.hppc.cursors.ObjectCursor;
|
import com.carrotsearch.hppc.cursors.ObjectCursor;
|
||||||
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
|
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
|
||||||
import org.apache.logging.log4j.Level;
|
|
||||||
import org.apache.logging.log4j.LogManager;
|
import org.apache.logging.log4j.LogManager;
|
||||||
import org.apache.logging.log4j.Logger;
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.elasticsearch.ElasticsearchTimeoutException;
|
|
||||||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
|
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
|
||||||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
|
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
|
||||||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
|
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
|
||||||
|
@ -18,14 +16,9 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
|
||||||
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction;
|
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction;
|
||||||
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
|
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
|
||||||
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
|
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
|
||||||
import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
|
|
||||||
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
|
|
||||||
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
|
|
||||||
import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
|
import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
|
||||||
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
|
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
|
||||||
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
|
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
|
||||||
import org.elasticsearch.action.admin.indices.flush.FlushAction;
|
|
||||||
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
|
|
||||||
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction;
|
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction;
|
||||||
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
|
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
|
||||||
import org.elasticsearch.action.admin.indices.get.GetIndexAction;
|
import org.elasticsearch.action.admin.indices.get.GetIndexAction;
|
||||||
|
@ -34,27 +27,16 @@ import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
|
||||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
|
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
|
||||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
|
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
|
||||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
|
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
|
||||||
import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
|
|
||||||
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
|
|
||||||
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
|
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
|
||||||
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
|
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
|
||||||
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
|
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
|
||||||
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
|
|
||||||
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
|
|
||||||
import org.elasticsearch.action.delete.DeleteRequest;
|
|
||||||
import org.elasticsearch.action.index.IndexRequest;
|
|
||||||
import org.elasticsearch.action.search.SearchAction;
|
import org.elasticsearch.action.search.SearchAction;
|
||||||
import org.elasticsearch.action.search.SearchRequest;
|
import org.elasticsearch.action.search.SearchRequest;
|
||||||
import org.elasticsearch.action.search.SearchResponse;
|
import org.elasticsearch.action.search.SearchResponse;
|
||||||
import org.elasticsearch.action.update.UpdateRequest;
|
|
||||||
import org.elasticsearch.client.ElasticsearchClient;
|
|
||||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
|
||||||
import org.elasticsearch.cluster.health.ClusterHealthStatus;
|
|
||||||
import org.elasticsearch.cluster.metadata.AliasMetaData;
|
import org.elasticsearch.cluster.metadata.AliasMetaData;
|
||||||
import org.elasticsearch.cluster.metadata.AliasOrIndex;
|
import org.elasticsearch.cluster.metadata.AliasOrIndex;
|
||||||
import org.elasticsearch.cluster.metadata.MappingMetaData;
|
import org.elasticsearch.cluster.metadata.MappingMetaData;
|
||||||
import org.elasticsearch.common.Strings;
|
import org.elasticsearch.common.Strings;
|
||||||
import org.elasticsearch.common.bytes.BytesReference;
|
|
||||||
import org.elasticsearch.common.collect.ImmutableOpenMap;
|
import org.elasticsearch.common.collect.ImmutableOpenMap;
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.common.unit.TimeValue;
|
import org.elasticsearch.common.unit.TimeValue;
|
||||||
|
@ -62,9 +44,8 @@ import org.elasticsearch.common.xcontent.DeprecationHandler;
|
||||||
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
|
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
|
||||||
import org.elasticsearch.common.xcontent.ToXContent;
|
import org.elasticsearch.common.xcontent.ToXContent;
|
||||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
|
||||||
import org.elasticsearch.common.xcontent.XContentType;
|
|
||||||
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||||
|
import org.elasticsearch.common.xcontent.yaml.YamlXContent;
|
||||||
import org.elasticsearch.index.query.QueryBuilder;
|
import org.elasticsearch.index.query.QueryBuilder;
|
||||||
import org.elasticsearch.index.query.QueryBuilders;
|
import org.elasticsearch.index.query.QueryBuilders;
|
||||||
import org.elasticsearch.search.SearchHit;
|
import org.elasticsearch.search.SearchHit;
|
||||||
|
@ -72,9 +53,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
|
||||||
import org.elasticsearch.search.sort.SortBuilder;
|
import org.elasticsearch.search.sort.SortBuilder;
|
||||||
import org.elasticsearch.search.sort.SortBuilders;
|
import org.elasticsearch.search.sort.SortBuilders;
|
||||||
import org.elasticsearch.search.sort.SortOrder;
|
import org.elasticsearch.search.sort.SortOrder;
|
||||||
import org.xbib.elx.api.BulkController;
|
import org.xbib.elx.api.AdminClient;
|
||||||
import org.xbib.elx.api.BulkMetric;
|
|
||||||
import org.xbib.elx.api.ExtendedClient;
|
|
||||||
import org.xbib.elx.api.IndexAliasAdder;
|
import org.xbib.elx.api.IndexAliasAdder;
|
||||||
import org.xbib.elx.api.IndexDefinition;
|
import org.xbib.elx.api.IndexDefinition;
|
||||||
import org.xbib.elx.api.IndexPruneResult;
|
import org.xbib.elx.api.IndexPruneResult;
|
||||||
|
@ -83,14 +62,16 @@ import org.xbib.elx.api.IndexShiftResult;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
|
import java.net.MalformedURLException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
|
import java.nio.charset.MalformedInputException;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.time.LocalDate;
|
import java.time.LocalDate;
|
||||||
import java.time.ZoneId;
|
import java.time.ZoneId;
|
||||||
import java.time.format.DateTimeFormatter;
|
import java.time.format.DateTimeFormatter;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collection;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.Comparator;
|
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
@ -102,33 +83,15 @@ import java.util.TreeSet;
|
||||||
import java.util.concurrent.ExecutionException;
|
import java.util.concurrent.ExecutionException;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import java.util.concurrent.TimeoutException;
|
import java.util.concurrent.TimeoutException;
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
import java.util.function.Consumer;
|
import java.util.function.Consumer;
|
||||||
import java.util.regex.Matcher;
|
import java.util.regex.Matcher;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
public abstract class AbstractExtendedClient implements ExtendedClient {
|
public abstract class AbstractAdminClient extends AbstractNativeClient implements AdminClient {
|
||||||
|
|
||||||
private static final Logger logger = LogManager.getLogger(AbstractExtendedClient.class.getName());
|
private static final Logger logger = LogManager.getLogger(AbstractAdminClient.class.getName());
|
||||||
|
|
||||||
/**
|
|
||||||
* The one and only index type name used in the extended client.
|
|
||||||
* Notr that all Elasticsearch version < 6.2.0 do not allow a prepending "_".
|
|
||||||
*/
|
|
||||||
private static final String TYPE_NAME = "doc";
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The Elasticsearch client.
|
|
||||||
*/
|
|
||||||
private ElasticsearchClient client;
|
|
||||||
|
|
||||||
private BulkMetric bulkMetric;
|
|
||||||
|
|
||||||
private BulkController bulkController;
|
|
||||||
|
|
||||||
private AtomicBoolean closed;
|
|
||||||
|
|
||||||
private static final IndexShiftResult EMPTY_INDEX_SHIFT_RESULT = new IndexShiftResult() {
|
private static final IndexShiftResult EMPTY_INDEX_SHIFT_RESULT = new IndexShiftResult() {
|
||||||
@Override
|
@Override
|
||||||
|
@ -164,184 +127,14 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
protected abstract ElasticsearchClient createClient(Settings settings) throws IOException;
|
|
||||||
|
|
||||||
protected abstract void closeClient() throws IOException;
|
|
||||||
|
|
||||||
protected AbstractExtendedClient() {
|
|
||||||
closed = new AtomicBoolean(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public AbstractExtendedClient setClient(ElasticsearchClient client) {
|
public AdminClient deleteIndex(IndexDefinition indexDefinition) {
|
||||||
this.client = client;
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ElasticsearchClient getClient() {
|
|
||||||
return client;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public BulkMetric getBulkMetric() {
|
|
||||||
return bulkMetric;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public BulkController getBulkController() {
|
|
||||||
return bulkController;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public AbstractExtendedClient init(Settings settings) throws IOException {
|
|
||||||
logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(','));
|
|
||||||
if (client == null) {
|
|
||||||
client = createClient(settings);
|
|
||||||
}
|
|
||||||
if (bulkMetric == null) {
|
|
||||||
bulkMetric = new DefaultBulkMetric();
|
|
||||||
bulkMetric.init(settings);
|
|
||||||
}
|
|
||||||
if (bulkController == null) {
|
|
||||||
bulkController = new DefaultBulkController(this, bulkMetric);
|
|
||||||
bulkController.init(settings);
|
|
||||||
}
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void flush() throws IOException {
|
|
||||||
if (bulkController != null) {
|
|
||||||
bulkController.flush();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void close() throws IOException {
|
|
||||||
ensureActive();
|
|
||||||
if (closed.compareAndSet(false, true)) {
|
|
||||||
if (bulkMetric != null) {
|
|
||||||
logger.info("closing bulk metric");
|
|
||||||
bulkMetric.close();
|
|
||||||
}
|
|
||||||
if (bulkController != null) {
|
|
||||||
logger.info("closing bulk controller");
|
|
||||||
bulkController.close();
|
|
||||||
}
|
|
||||||
closeClient();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getClusterName() {
|
|
||||||
ensureActive();
|
|
||||||
try {
|
|
||||||
ClusterStateRequest clusterStateRequest = new ClusterStateRequest().clear();
|
|
||||||
ClusterStateResponse clusterStateResponse =
|
|
||||||
client.execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
|
|
||||||
return clusterStateResponse.getClusterName().value();
|
|
||||||
} catch (ElasticsearchTimeoutException e) {
|
|
||||||
logger.warn(e.getMessage(), e);
|
|
||||||
return "TIMEOUT";
|
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn(e.getMessage(), e);
|
|
||||||
return "DISCONNECTED";
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.warn(e.getMessage(), e);
|
|
||||||
return "[" + e.getMessage() + "]";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException {
|
|
||||||
ensureActive();
|
|
||||||
waitForCluster("YELLOW", 30L, TimeUnit.SECONDS);
|
|
||||||
URL indexSettings = indexDefinition.getSettingsUrl();
|
|
||||||
if (indexSettings == null) {
|
|
||||||
logger.warn("warning while creating index '{}', no settings/mappings",
|
|
||||||
indexDefinition.getFullIndexName());
|
|
||||||
newIndex(indexDefinition.getFullIndexName());
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
URL indexMappings = indexDefinition.getMappingsUrl();
|
|
||||||
if (indexMappings == null) {
|
|
||||||
logger.warn("warning while creating index '{}', no mappings",
|
|
||||||
indexDefinition.getFullIndexName());
|
|
||||||
newIndex(indexDefinition.getFullIndexName(), indexSettings.openStream(), null);
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
try (InputStream indexSettingsInput = indexSettings.openStream();
|
|
||||||
InputStream indexMappingsInput = indexMappings.openStream()) {
|
|
||||||
newIndex(indexDefinition.getFullIndexName(), indexSettingsInput, indexMappingsInput);
|
|
||||||
} catch (IOException e) {
|
|
||||||
if (indexDefinition.ignoreErrors()) {
|
|
||||||
logger.warn(e.getMessage(), e);
|
|
||||||
logger.warn("warning while creating index '{}' with settings at {} and mappings at {}",
|
|
||||||
indexDefinition.getFullIndexName(), indexSettings, indexMappings);
|
|
||||||
} else {
|
|
||||||
logger.error("error while creating index '{}' with settings at {} and mappings at {}",
|
|
||||||
indexDefinition.getFullIndexName(), indexSettings, indexMappings);
|
|
||||||
throw new IOException(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient newIndex(String index) throws IOException {
|
|
||||||
return newIndex(index, Settings.EMPTY, (Map<String, Object>) null);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException {
|
|
||||||
return newIndex(index,
|
|
||||||
Settings.builder().loadFromStream(".json", settings, true).build(),
|
|
||||||
mapping != null ? JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
|
|
||||||
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mapping).mapOrdered() : null);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient newIndex(String index, Settings settings) throws IOException {
|
|
||||||
return newIndex(index, settings, (Map<String, Object>) null);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException {
|
|
||||||
return newIndex(index, settings,
|
|
||||||
mapping != null ? JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
|
|
||||||
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mapping).mapOrdered() : null);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient newIndex(String index, Settings settings, Map<String, Object> mapping) throws IOException {
|
|
||||||
ensureActive();
|
|
||||||
if (index == null) {
|
|
||||||
logger.warn("no index name given to create index");
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index);
|
|
||||||
if (settings != null) {
|
|
||||||
createIndexRequest.settings(settings);
|
|
||||||
}
|
|
||||||
if (mapping != null) {
|
|
||||||
createIndexRequest.mapping(TYPE_NAME, mapping);
|
|
||||||
}
|
|
||||||
CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
|
|
||||||
XContentBuilder builder = XContentFactory.jsonBuilder();
|
|
||||||
logger.info("index {} created: {}", index,
|
|
||||||
Strings.toString(createIndexResponse.toXContent(builder, ToXContent.EMPTY_PARAMS)));
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient deleteIndex(IndexDefinition indexDefinition) {
|
|
||||||
return deleteIndex(indexDefinition.getFullIndexName());
|
return deleteIndex(indexDefinition.getFullIndexName());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ExtendedClient deleteIndex(String index) {
|
public AdminClient deleteIndex(String index) {
|
||||||
ensureActive();
|
ensureClientIsPresent();
|
||||||
if (index == null) {
|
if (index == null) {
|
||||||
logger.warn("no index name given to delete index");
|
logger.warn("no index name given to delete index");
|
||||||
return this;
|
return this;
|
||||||
|
@ -351,99 +144,9 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient startBulk(IndexDefinition indexDefinition) throws IOException {
|
|
||||||
startBulk(indexDefinition.getFullIndexName(), -1, 1);
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds)
|
|
||||||
throws IOException {
|
|
||||||
if (bulkController != null) {
|
|
||||||
ensureActive();
|
|
||||||
bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds);
|
|
||||||
}
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException {
|
|
||||||
if (bulkController != null) {
|
|
||||||
ensureActive();
|
|
||||||
bulkController.stopBulkMode(indexDefinition);
|
|
||||||
}
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException {
|
|
||||||
if (bulkController != null) {
|
|
||||||
ensureActive();
|
|
||||||
bulkController.stopBulkMode(index, timeout, timeUnit);
|
|
||||||
}
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient index(String index, String id, boolean create, String source) {
|
|
||||||
return index(new IndexRequest(index, TYPE_NAME, id).create(create)
|
|
||||||
.source(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient index(String index, String id, boolean create, BytesReference source) {
|
|
||||||
return index(new IndexRequest(index, TYPE_NAME, id).create(create)
|
|
||||||
.source(source, XContentType.JSON));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient index(IndexRequest indexRequest) {
|
|
||||||
ensureActive();
|
|
||||||
bulkController.index(indexRequest);
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient delete(String index, String id) {
|
|
||||||
return delete(new DeleteRequest(index, TYPE_NAME, id));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient delete(DeleteRequest deleteRequest) {
|
|
||||||
ensureActive();
|
|
||||||
bulkController.delete(deleteRequest);
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient update(String index, String id, BytesReference source) {
|
|
||||||
return update(new UpdateRequest(index, TYPE_NAME, id)
|
|
||||||
.doc(source, XContentType.JSON));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient update(String index, String id, String source) {
|
|
||||||
return update(new UpdateRequest(index, TYPE_NAME, id)
|
|
||||||
.doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient update(UpdateRequest updateRequest) {
|
|
||||||
ensureActive();
|
|
||||||
bulkController.update(updateRequest);
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean waitForResponses(long timeout, TimeUnit timeUnit) {
|
|
||||||
ensureActive();
|
|
||||||
return bulkController.waitForResponses(timeout, timeUnit);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) {
|
public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) {
|
||||||
ensureActive();
|
ensureClientIsPresent();
|
||||||
ensureIndexGiven(index);
|
ensureIndexGiven(index);
|
||||||
GetSettingsRequest settingsRequest = new GetSettingsRequest();
|
GetSettingsRequest settingsRequest = new GetSettingsRequest();
|
||||||
settingsRequest.indices(index);
|
settingsRequest.indices(index);
|
||||||
|
@ -466,50 +169,13 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) {
|
public AdminClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException {
|
||||||
ensureActive();
|
|
||||||
ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString);
|
|
||||||
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
|
|
||||||
ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
|
|
||||||
new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet();
|
|
||||||
if (healthResponse != null && healthResponse.isTimedOut()) {
|
|
||||||
if (logger.isErrorEnabled()) {
|
|
||||||
logger.error("timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name());
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) {
|
|
||||||
ensureActive();
|
|
||||||
try {
|
|
||||||
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
|
|
||||||
ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
|
|
||||||
new ClusterHealthRequest().timeout(timeout)).actionGet();
|
|
||||||
ClusterHealthStatus status = healthResponse.getStatus();
|
|
||||||
return status.name();
|
|
||||||
} catch (ElasticsearchTimeoutException e) {
|
|
||||||
logger.warn(e.getMessage(), e);
|
|
||||||
return "TIMEOUT";
|
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn(e.getMessage(), e);
|
|
||||||
return "DISCONNECTED";
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.warn(e.getMessage(), e);
|
|
||||||
return "[" + e.getMessage() + "]";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException {
|
|
||||||
return updateReplicaLevel(indexDefinition.getFullIndexName(), level,
|
return updateReplicaLevel(indexDefinition.getFullIndexName(), level,
|
||||||
indexDefinition.getMaxWaitTime(), indexDefinition.getMaxWaitTimeUnit());
|
indexDefinition.getMaxWaitTime(), indexDefinition.getMaxWaitTimeUnit());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException {
|
public AdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException {
|
||||||
waitForCluster("YELLOW", maxWaitTime, timeUnit); // let cluster settle down from critical operations
|
waitForCluster("YELLOW", maxWaitTime, timeUnit); // let cluster settle down from critical operations
|
||||||
if (level > 0) {
|
if (level > 0) {
|
||||||
updateIndexSetting(index, "number_of_replicas", level, maxWaitTime, timeUnit);
|
updateIndexSetting(index, "number_of_replicas", level, maxWaitTime, timeUnit);
|
||||||
|
@ -537,27 +203,10 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
||||||
return replica;
|
return replica;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient flushIndex(String index) {
|
|
||||||
if (index != null) {
|
|
||||||
ensureActive();
|
|
||||||
client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet();
|
|
||||||
}
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedClient refreshIndex(String index) {
|
|
||||||
if (index != null) {
|
|
||||||
ensureActive();
|
|
||||||
client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet();
|
|
||||||
}
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String resolveMostRecentIndex(String alias) {
|
public String resolveMostRecentIndex(String alias) {
|
||||||
ensureActive();
|
ensureClientIsPresent();
|
||||||
if (alias == null) {
|
if (alias == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -585,7 +234,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String resolveAlias(String alias) {
|
public String resolveAlias(String alias) {
|
||||||
ensureActive();
|
ensureClientIsPresent();
|
||||||
ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
|
ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
|
||||||
clusterStateRequest.blocks(false);
|
clusterStateRequest.blocks(false);
|
||||||
clusterStateRequest.metaData(true);
|
clusterStateRequest.metaData(true);
|
||||||
|
@ -627,7 +276,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
||||||
@Override
|
@Override
|
||||||
public IndexShiftResult shiftIndex(String index, String fullIndexName,
|
public IndexShiftResult shiftIndex(String index, String fullIndexName,
|
||||||
List<String> additionalAliases, IndexAliasAdder adder) {
|
List<String> additionalAliases, IndexAliasAdder adder) {
|
||||||
ensureActive();
|
ensureClientIsPresent();
|
||||||
if (index == null) {
|
if (index == null) {
|
||||||
return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to
|
return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to
|
||||||
}
|
}
|
||||||
|
@ -715,12 +364,13 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
||||||
if (index.equals(fullIndexName)) {
|
if (index.equals(fullIndexName)) {
|
||||||
return EMPTY_INDEX_PRUNE_RESULT;
|
return EMPTY_INDEX_PRUNE_RESULT;
|
||||||
}
|
}
|
||||||
ensureActive();
|
ensureClientIsPresent();
|
||||||
GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE);
|
GetIndexRequestBuilder getIndexRequestBuilder =
|
||||||
|
new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE);
|
||||||
GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet();
|
GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet();
|
||||||
Pattern pattern = Pattern.compile("^(.*?)(\\d+)$");
|
Pattern pattern = Pattern.compile("^(.*?)(\\d+)$");
|
||||||
logger.info("{} indices", getIndexResponse.getIndices().length);
|
logger.info("found {} indices", getIndexResponse.getIndices().length);
|
||||||
List<String> candidateIndices = new ArrayList<>();
|
Set<String> candidateIndices = new TreeSet<>();
|
||||||
for (String s : getIndexResponse.getIndices()) {
|
for (String s : getIndexResponse.getIndices()) {
|
||||||
Matcher m = pattern.matcher(s);
|
Matcher m = pattern.matcher(s);
|
||||||
if (m.matches() && index.equals(m.group(1)) && !s.equals(fullIndexName)) {
|
if (m.matches() && index.equals(m.group(1)) && !s.equals(fullIndexName)) {
|
||||||
|
@ -760,7 +410,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Long mostRecentDocument(String index, String timestampfieldname) {
|
public Long mostRecentDocument(String index, String timestampfieldname) {
|
||||||
ensureActive();
|
ensureClientIsPresent();
|
||||||
SortBuilder<?> sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC);
|
SortBuilder<?> sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC);
|
||||||
SearchSourceBuilder builder = new SearchSourceBuilder();
|
SearchSourceBuilder builder = new SearchSourceBuilder();
|
||||||
builder.sort(sort);
|
builder.sort(sort);
|
||||||
|
@ -813,7 +463,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
||||||
@Override
|
@Override
|
||||||
public IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings)
|
public IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
boolean isEnabled = settings.getAsBoolean("enabled", !(client instanceof MockExtendedClient));
|
boolean isEnabled = settings.getAsBoolean("enabled", !(client instanceof MockAdminClient));
|
||||||
String indexName = settings.get("name", index);
|
String indexName = settings.get("name", index);
|
||||||
String fullIndexName;
|
String fullIndexName;
|
||||||
String dateTimePattern = settings.get("dateTimePattern");
|
String dateTimePattern = settings.get("dateTimePattern");
|
||||||
|
@ -833,8 +483,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
||||||
.setEnabled(isEnabled)
|
.setEnabled(isEnabled)
|
||||||
.setIndex(indexName)
|
.setIndex(indexName)
|
||||||
.setFullIndexName(fullIndexName)
|
.setFullIndexName(fullIndexName)
|
||||||
.setSettingsUrl(settings.get("settings"))
|
.setSettings(findSettingsFrom(settings.get("settings")))
|
||||||
.setMappingsUrl(settings.get("mapping"))
|
.setMappings(findMappingsFrom(settings.get("mapping")))
|
||||||
.setDateTimePattern(dateTimePattern)
|
.setDateTimePattern(dateTimePattern)
|
||||||
.setIgnoreErrors(settings.getAsBoolean("skiperrors", false))
|
.setIgnoreErrors(settings.getAsBoolean("skiperrors", false))
|
||||||
.setShift(settings.getAsBoolean("shift", true))
|
.setShift(settings.getAsBoolean("shift", true))
|
||||||
|
@ -845,31 +495,48 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
||||||
.setStopRefreshInterval(settings.getAsLong("bulk.stoprefreshinterval", -1L));
|
.setStopRefreshInterval(settings.getAsLong("bulk.stoprefreshinterval", -1L));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
private static String findSettingsFrom(String string) throws IOException {
|
||||||
public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException {
|
if (string == null) {
|
||||||
ensureActive();
|
return null;
|
||||||
if (index == null) {
|
|
||||||
throw new IOException("no index name given");
|
|
||||||
}
|
}
|
||||||
if (key == null) {
|
try {
|
||||||
throw new IOException("no key given");
|
URL url = new URL(string);
|
||||||
|
try (InputStream inputStream = url.openStream()) {
|
||||||
|
Settings settings = Settings.builder().loadFromStream(string, inputStream, true).build();
|
||||||
|
XContentBuilder builder = JsonXContent.contentBuilder();
|
||||||
|
settings.toXContent(builder, ToXContent.EMPTY_PARAMS);
|
||||||
|
return Strings.toString(builder);
|
||||||
}
|
}
|
||||||
if (value == null) {
|
} catch (MalformedURLException e) {
|
||||||
throw new IOException("no value given");
|
return string;
|
||||||
}
|
}
|
||||||
Settings.Builder updateSettingsBuilder = Settings.builder();
|
|
||||||
updateSettingsBuilder.put(key, value.toString());
|
|
||||||
UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index)
|
|
||||||
.settings(updateSettingsBuilder).timeout(toTimeValue(timeout, timeUnit));
|
|
||||||
client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void ensureActive() {
|
private static String findMappingsFrom(String string) throws IOException {
|
||||||
if (this instanceof MockExtendedClient) {
|
if (string == null) {
|
||||||
return;
|
return null;
|
||||||
}
|
}
|
||||||
if (client == null) {
|
try {
|
||||||
throw new IllegalStateException("no client");
|
URL url = new URL(string);
|
||||||
|
try (InputStream inputStream = url.openStream()) {
|
||||||
|
if (string.endsWith(".json")) {
|
||||||
|
Map<String, ?> mappings = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
|
||||||
|
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputStream).mapOrdered();
|
||||||
|
XContentBuilder builder = JsonXContent.contentBuilder();
|
||||||
|
builder.startObject().map(mappings).endObject();
|
||||||
|
return Strings.toString(builder);
|
||||||
|
}
|
||||||
|
if (string.endsWith(".yml") || string.endsWith(".yaml")) {
|
||||||
|
Map<String, ?> mappings = YamlXContent.yamlXContent.createParser(NamedXContentRegistry.EMPTY,
|
||||||
|
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputStream).mapOrdered();
|
||||||
|
XContentBuilder builder = JsonXContent.contentBuilder();
|
||||||
|
builder.startObject().map(mappings).endObject();
|
||||||
|
return Strings.toString(builder);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return string;
|
||||||
|
} catch (MalformedInputException e) {
|
||||||
|
return string;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -896,7 +563,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
public void checkMapping(String index) {
|
public void checkMapping(String index) {
|
||||||
ensureActive();
|
ensureClientIsPresent();
|
||||||
GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(index);
|
GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(index);
|
||||||
GetMappingsResponse getMappingsResponse = client.execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
|
GetMappingsResponse getMappingsResponse = client.execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
|
||||||
ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> map = getMappingsResponse.getMappings();
|
ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> map = getMappingsResponse.getMappings();
|
||||||
|
@ -992,63 +659,43 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
||||||
|
|
||||||
private static <K, V extends Comparable<? super V>> Map<K, V> sortByValue(Map<K, V> map) {
|
private static <K, V extends Comparable<? super V>> Map<K, V> sortByValue(Map<K, V> map) {
|
||||||
Map<K, V> result = new LinkedHashMap<>();
|
Map<K, V> result = new LinkedHashMap<>();
|
||||||
map.entrySet().stream().sorted(Comparator.comparing(Map.Entry::getValue))
|
map.entrySet().stream().sorted(Map.Entry.comparingByValue())
|
||||||
.forEachOrdered(e -> result.put(e.getKey(), e.getValue()));
|
.forEachOrdered(e -> result.put(e.getKey(), e.getValue()));
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static TimeValue toTimeValue(long timeValue, TimeUnit timeUnit) {
|
|
||||||
switch (timeUnit) {
|
|
||||||
case DAYS:
|
|
||||||
return TimeValue.timeValueHours(24 * timeValue);
|
|
||||||
case HOURS:
|
|
||||||
return TimeValue.timeValueHours(timeValue);
|
|
||||||
case MINUTES:
|
|
||||||
return TimeValue.timeValueMinutes(timeValue);
|
|
||||||
case SECONDS:
|
|
||||||
return TimeValue.timeValueSeconds(timeValue);
|
|
||||||
case MILLISECONDS:
|
|
||||||
return TimeValue.timeValueMillis(timeValue);
|
|
||||||
case MICROSECONDS:
|
|
||||||
return TimeValue.timeValueNanos(1000 * timeValue);
|
|
||||||
case NANOSECONDS:
|
|
||||||
return TimeValue.timeValueNanos(timeValue);
|
|
||||||
default:
|
|
||||||
throw new IllegalArgumentException("unknown time unit: " + timeUnit);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static class SuccessIndexShiftResult implements IndexShiftResult {
|
private static class SuccessIndexShiftResult implements IndexShiftResult {
|
||||||
|
|
||||||
List<String> movedAliases;
|
Collection<String> movedAliases;
|
||||||
|
|
||||||
List<String> newAliases;
|
Collection<String> newAliases;
|
||||||
|
|
||||||
SuccessIndexShiftResult(List<String> movedAliases, List<String> newAliases) {
|
SuccessIndexShiftResult(Collection<String> movedAliases, Collection<String> newAliases) {
|
||||||
this.movedAliases = movedAliases;
|
this.movedAliases = movedAliases;
|
||||||
this.newAliases = newAliases;
|
this.newAliases = newAliases;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<String> getMovedAliases() {
|
public Collection<String> getMovedAliases() {
|
||||||
return movedAliases;
|
return movedAliases;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<String> getNewAliases() {
|
public Collection<String> getNewAliases() {
|
||||||
return newAliases;
|
return newAliases;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static class SuccessPruneResult implements IndexPruneResult {
|
private static class SuccessPruneResult implements IndexPruneResult {
|
||||||
|
|
||||||
List<String> candidateIndices;
|
Collection<String> candidateIndices;
|
||||||
|
|
||||||
List<String> indicesToDelete;
|
Collection<String> indicesToDelete;
|
||||||
|
|
||||||
DeleteIndexResponse response;
|
DeleteIndexResponse response;
|
||||||
|
|
||||||
SuccessPruneResult(List<String> candidateIndices, List<String> indicesToDelete,
|
SuccessPruneResult(Collection<String> candidateIndices,
|
||||||
|
Collection<String> indicesToDelete,
|
||||||
DeleteIndexResponse response) {
|
DeleteIndexResponse response) {
|
||||||
this.candidateIndices = candidateIndices;
|
this.candidateIndices = candidateIndices;
|
||||||
this.indicesToDelete = indicesToDelete;
|
this.indicesToDelete = indicesToDelete;
|
||||||
|
@ -1061,12 +708,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<String> getCandidateIndices() {
|
public Collection<String> getCandidateIndices() {
|
||||||
return candidateIndices;
|
return candidateIndices;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<String> getDeletedIndices() {
|
public Collection<String> getDeletedIndices() {
|
||||||
return indicesToDelete;
|
return indicesToDelete;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1078,11 +725,11 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
||||||
|
|
||||||
private static class NothingToDoPruneResult implements IndexPruneResult {
|
private static class NothingToDoPruneResult implements IndexPruneResult {
|
||||||
|
|
||||||
List<String> candidateIndices;
|
Collection<String> candidateIndices;
|
||||||
|
|
||||||
List<String> indicesToDelete;
|
Collection<String> indicesToDelete;
|
||||||
|
|
||||||
NothingToDoPruneResult(List<String> candidateIndices, List<String> indicesToDelete) {
|
NothingToDoPruneResult(Collection<String> candidateIndices, List<String> indicesToDelete) {
|
||||||
this.candidateIndices = candidateIndices;
|
this.candidateIndices = candidateIndices;
|
||||||
this.indicesToDelete = indicesToDelete;
|
this.indicesToDelete = indicesToDelete;
|
||||||
}
|
}
|
||||||
|
@ -1093,12 +740,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<String> getCandidateIndices() {
|
public Collection<String> getCandidateIndices() {
|
||||||
return candidateIndices;
|
return candidateIndices;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<String> getDeletedIndices() {
|
public Collection<String> getDeletedIndices() {
|
||||||
return indicesToDelete;
|
return indicesToDelete;
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,246 @@
|
||||||
|
package org.xbib.elx.common;
|
||||||
|
|
||||||
|
import org.apache.logging.log4j.Level;
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
|
||||||
|
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
|
||||||
|
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
|
||||||
|
import org.elasticsearch.action.admin.indices.flush.FlushAction;
|
||||||
|
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
|
||||||
|
import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
|
||||||
|
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
|
||||||
|
import org.elasticsearch.action.delete.DeleteRequest;
|
||||||
|
import org.elasticsearch.action.index.IndexRequest;
|
||||||
|
import org.elasticsearch.action.update.UpdateRequest;
|
||||||
|
import org.elasticsearch.common.Strings;
|
||||||
|
import org.elasticsearch.common.bytes.BytesReference;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.common.xcontent.DeprecationHandler;
|
||||||
|
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
|
||||||
|
import org.elasticsearch.common.xcontent.ToXContent;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentType;
|
||||||
|
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||||
|
import org.xbib.elx.api.BulkClient;
|
||||||
|
import org.xbib.elx.api.BulkController;
|
||||||
|
import org.xbib.elx.api.BulkMetric;
|
||||||
|
import org.xbib.elx.api.IndexDefinition;
|
||||||
|
import java.io.ByteArrayOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
public abstract class AbstractBulkClient extends AbstractNativeClient implements BulkClient {
|
||||||
|
|
||||||
|
private static final Logger logger = LogManager.getLogger(AbstractBulkClient.class.getName());
|
||||||
|
|
||||||
|
private BulkMetric bulkMetric;
|
||||||
|
|
||||||
|
private BulkController bulkController;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) throws IOException {
|
||||||
|
logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(','));
|
||||||
|
super.init(settings);
|
||||||
|
if (bulkMetric == null) {
|
||||||
|
bulkMetric = new DefaultBulkMetric();
|
||||||
|
bulkMetric.init(settings);
|
||||||
|
}
|
||||||
|
if (bulkController == null) {
|
||||||
|
bulkController = new DefaultBulkController(this, bulkMetric);
|
||||||
|
bulkController.init(settings);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BulkMetric getBulkMetric() {
|
||||||
|
return bulkMetric;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BulkController getBulkController() {
|
||||||
|
return bulkController;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void flush() throws IOException {
|
||||||
|
if (bulkController != null) {
|
||||||
|
bulkController.flush();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() throws IOException {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
if (closed.compareAndSet(false, true)) {
|
||||||
|
if (bulkMetric != null) {
|
||||||
|
logger.info("closing bulk metric");
|
||||||
|
bulkMetric.close();
|
||||||
|
}
|
||||||
|
if (bulkController != null) {
|
||||||
|
logger.info("closing bulk controller");
|
||||||
|
bulkController.close();
|
||||||
|
}
|
||||||
|
closeClient();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void newIndex(IndexDefinition indexDefinition) throws IOException {
|
||||||
|
Settings settings = indexDefinition.getSettings() == null ? null :
|
||||||
|
Settings.builder().loadFromSource(indexDefinition.getSettings(), XContentType.JSON).build();
|
||||||
|
Map<String, ?> mappings = indexDefinition.getMappings() == null ? null :
|
||||||
|
JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
|
||||||
|
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, indexDefinition.getMappings()).mapOrdered();
|
||||||
|
newIndex(indexDefinition.getFullIndexName(), settings, mappings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void newIndex(String index) throws IOException {
|
||||||
|
newIndex(index, Settings.EMPTY, (Map<String, ?>) null);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void newIndex(String index, Settings settings) throws IOException {
|
||||||
|
newIndex(index, settings, (Map<String, ?>) null);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void newIndex(String index, Settings settings, XContentBuilder builder) throws IOException {
|
||||||
|
String mappingString = Strings.toString(builder);
|
||||||
|
Map<String, ?> mappings = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
|
||||||
|
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mappingString).mapOrdered();
|
||||||
|
newIndex(index, settings, mappings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException {
|
||||||
|
if (index == null) {
|
||||||
|
logger.warn("no index name given to create index");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
ensureClientIsPresent();
|
||||||
|
waitForCluster("YELLOW", 30L, TimeUnit.SECONDS);
|
||||||
|
CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index);
|
||||||
|
if (settings != null) {
|
||||||
|
createIndexRequest.settings(settings);
|
||||||
|
}
|
||||||
|
if (mapping != null) {
|
||||||
|
createIndexRequest.mapping(TYPE_NAME, mapping);
|
||||||
|
}
|
||||||
|
CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
|
||||||
|
XContentBuilder builder = XContentFactory.jsonBuilder();
|
||||||
|
logger.info("index {} created: {}", index,
|
||||||
|
Strings.toString(createIndexResponse.toXContent(builder, ToXContent.EMPTY_PARAMS)));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void startBulk(IndexDefinition indexDefinition) throws IOException {
|
||||||
|
startBulk(indexDefinition.getFullIndexName(), -1, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds)
|
||||||
|
throws IOException {
|
||||||
|
if (bulkController != null) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void stopBulk(IndexDefinition indexDefinition) throws IOException {
|
||||||
|
if (bulkController != null) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
bulkController.stopBulkMode(indexDefinition);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException {
|
||||||
|
if (bulkController != null) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
bulkController.stopBulkMode(index, timeout, timeUnit);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BulkClient index(String index, String id, boolean create, String source) {
|
||||||
|
return index(new IndexRequest(index, TYPE_NAME, id).create(create)
|
||||||
|
.source(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BulkClient index(String index, String id, boolean create, BytesReference source) {
|
||||||
|
return index(new IndexRequest(index, TYPE_NAME, id).create(create)
|
||||||
|
.source(source, XContentType.JSON));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BulkClient index(IndexRequest indexRequest) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
bulkController.index(indexRequest);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BulkClient delete(String index, String id) {
|
||||||
|
return delete(new DeleteRequest(index, TYPE_NAME, id));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BulkClient delete(DeleteRequest deleteRequest) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
bulkController.delete(deleteRequest);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BulkClient update(String index, String id, BytesReference source) {
|
||||||
|
return update(new UpdateRequest(index, TYPE_NAME, id)
|
||||||
|
.doc(source, XContentType.JSON));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BulkClient update(String index, String id, String source) {
|
||||||
|
return update(new UpdateRequest(index, TYPE_NAME, id)
|
||||||
|
.doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BulkClient update(UpdateRequest updateRequest) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
bulkController.update(updateRequest);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean waitForResponses(long timeout, TimeUnit timeUnit) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
return bulkController.waitForResponses(timeout, timeUnit);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException {
|
||||||
|
super.updateIndexSetting(index, key, value, timeout, timeUnit);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void flushIndex(String index) {
|
||||||
|
if (index != null) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void refreshIndex(String index) {
|
||||||
|
if (index != null) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,218 @@
|
||||||
|
package org.xbib.elx.common;
|
||||||
|
|
||||||
|
import org.apache.logging.log4j.Level;
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
import org.elasticsearch.ElasticsearchTimeoutException;
|
||||||
|
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
|
||||||
|
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
|
||||||
|
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
|
||||||
|
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
|
||||||
|
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
|
||||||
|
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
|
||||||
|
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
|
||||||
|
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
|
||||||
|
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
|
||||||
|
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
|
||||||
|
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequestBuilder;
|
||||||
|
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
|
||||||
|
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
|
||||||
|
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
|
||||||
|
import org.elasticsearch.action.search.SearchAction;
|
||||||
|
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||||
|
import org.elasticsearch.cluster.health.ClusterHealthStatus;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.common.unit.TimeValue;
|
||||||
|
import org.elasticsearch.index.query.QueryBuilders;
|
||||||
|
import org.xbib.elx.api.NativeClient;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
|
|
||||||
|
public abstract class AbstractNativeClient implements NativeClient {
|
||||||
|
|
||||||
|
private static final Logger logger = LogManager.getLogger(AbstractNativeClient.class.getName());
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The one and only index type name used in the extended client.
|
||||||
|
* Notr that all Elasticsearch version < 6.2.0 do not allow a prepending "_".
|
||||||
|
*/
|
||||||
|
protected static final String TYPE_NAME = "doc";
|
||||||
|
|
||||||
|
protected ElasticsearchClient client;
|
||||||
|
|
||||||
|
protected final AtomicBoolean closed;
|
||||||
|
|
||||||
|
protected AbstractNativeClient() {
|
||||||
|
closed = new AtomicBoolean(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setClient(ElasticsearchClient client) {
|
||||||
|
this.client = client;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ElasticsearchClient getClient() {
|
||||||
|
return client;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected abstract ElasticsearchClient createClient(Settings settings) throws IOException;
|
||||||
|
|
||||||
|
protected abstract void closeClient() throws IOException;
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) throws IOException {
|
||||||
|
logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(','));
|
||||||
|
if (client == null) {
|
||||||
|
client = createClient(settings);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getClusterName() {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
try {
|
||||||
|
ClusterStateRequest clusterStateRequest = new ClusterStateRequest().clear();
|
||||||
|
ClusterStateResponse clusterStateResponse =
|
||||||
|
getClient().execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
|
||||||
|
return clusterStateResponse.getClusterName().value();
|
||||||
|
} catch (ElasticsearchTimeoutException e) {
|
||||||
|
logger.warn(e.getMessage(), e);
|
||||||
|
return "TIMEOUT";
|
||||||
|
} catch (NoNodeAvailableException e) {
|
||||||
|
logger.warn(e.getMessage(), e);
|
||||||
|
return "DISCONNECTED";
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.warn(e.getMessage(), e);
|
||||||
|
return "[" + e.getMessage() + "]";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString);
|
||||||
|
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
|
||||||
|
ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
|
||||||
|
new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet();
|
||||||
|
if (healthResponse != null && healthResponse.isTimedOut()) {
|
||||||
|
String message = "timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name();
|
||||||
|
if (logger.isErrorEnabled()) {
|
||||||
|
logger.error(message);
|
||||||
|
}
|
||||||
|
throw new IllegalStateException(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
try {
|
||||||
|
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
|
||||||
|
ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
|
||||||
|
new ClusterHealthRequest().timeout(timeout)).actionGet();
|
||||||
|
ClusterHealthStatus status = healthResponse.getStatus();
|
||||||
|
return status.name();
|
||||||
|
} catch (ElasticsearchTimeoutException e) {
|
||||||
|
logger.warn(e.getMessage(), e);
|
||||||
|
return "TIMEOUT";
|
||||||
|
} catch (NoNodeAvailableException e) {
|
||||||
|
logger.warn(e.getMessage(), e);
|
||||||
|
return "DISCONNECTED";
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.warn(e.getMessage(), e);
|
||||||
|
return "[" + e.getMessage() + "]";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<String, ?> getMapping(String index, String mapping) {
|
||||||
|
GetMappingsRequestBuilder getMappingsRequestBuilder = new GetMappingsRequestBuilder(client, GetMappingsAction.INSTANCE)
|
||||||
|
.setIndices(index)
|
||||||
|
.setTypes(mapping);
|
||||||
|
GetMappingsResponse getMappingsResponse = getMappingsRequestBuilder.execute().actionGet();
|
||||||
|
logger.info("get mappings response = {}", getMappingsResponse.getMappings().get(index).get(mapping).getSourceAsMap());
|
||||||
|
return getMappingsResponse.getMappings().get(index).get(mapping).getSourceAsMap();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long getSearchableDocs(String index) {
|
||||||
|
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE)
|
||||||
|
.setIndices(index)
|
||||||
|
.setQuery(QueryBuilders.matchAllQuery())
|
||||||
|
.setSize(0);
|
||||||
|
return searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isIndexExists(String index) {
|
||||||
|
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
|
||||||
|
indicesExistsRequest.indices(index);
|
||||||
|
IndicesExistsResponse indicesExistsResponse =
|
||||||
|
client.execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
|
||||||
|
return indicesExistsResponse.isExists();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() throws IOException {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
if (closed.compareAndSet(false, true)) {
|
||||||
|
closeClient();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
if (index == null) {
|
||||||
|
throw new IOException("no index name given");
|
||||||
|
}
|
||||||
|
if (key == null) {
|
||||||
|
throw new IOException("no key given");
|
||||||
|
}
|
||||||
|
if (value == null) {
|
||||||
|
throw new IOException("no value given");
|
||||||
|
}
|
||||||
|
Settings.Builder updateSettingsBuilder = Settings.builder();
|
||||||
|
updateSettingsBuilder.put(key, value.toString());
|
||||||
|
UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index)
|
||||||
|
.settings(updateSettingsBuilder).timeout(toTimeValue(timeout, timeUnit));
|
||||||
|
client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void ensureClientIsPresent() {
|
||||||
|
if (this instanceof MockAdminClient) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (client == null) {
|
||||||
|
throw new IllegalStateException("no client");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected static TimeValue toTimeValue(long timeValue, TimeUnit timeUnit) {
|
||||||
|
switch (timeUnit) {
|
||||||
|
case DAYS:
|
||||||
|
return TimeValue.timeValueHours(24 * timeValue);
|
||||||
|
case HOURS:
|
||||||
|
return TimeValue.timeValueHours(timeValue);
|
||||||
|
case MINUTES:
|
||||||
|
return TimeValue.timeValueMinutes(timeValue);
|
||||||
|
case SECONDS:
|
||||||
|
return TimeValue.timeValueSeconds(timeValue);
|
||||||
|
case MILLISECONDS:
|
||||||
|
return TimeValue.timeValueMillis(timeValue);
|
||||||
|
case MICROSECONDS:
|
||||||
|
return TimeValue.timeValueNanos(1000 * timeValue);
|
||||||
|
case NANOSECONDS:
|
||||||
|
return TimeValue.timeValueNanos(timeValue);
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException("unknown time unit: " + timeUnit);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,158 @@
|
||||||
|
package org.xbib.elx.common;
|
||||||
|
|
||||||
|
import org.elasticsearch.ElasticsearchException;
|
||||||
|
import org.elasticsearch.action.get.GetAction;
|
||||||
|
import org.elasticsearch.action.get.GetRequestBuilder;
|
||||||
|
import org.elasticsearch.action.get.GetResponse;
|
||||||
|
import org.elasticsearch.action.get.MultiGetAction;
|
||||||
|
import org.elasticsearch.action.get.MultiGetRequestBuilder;
|
||||||
|
import org.elasticsearch.action.get.MultiGetResponse;
|
||||||
|
import org.elasticsearch.action.search.ClearScrollAction;
|
||||||
|
import org.elasticsearch.action.search.ClearScrollRequestBuilder;
|
||||||
|
import org.elasticsearch.action.search.SearchAction;
|
||||||
|
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||||
|
import org.elasticsearch.action.search.SearchResponse;
|
||||||
|
import org.elasticsearch.action.search.SearchScrollAction;
|
||||||
|
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
|
||||||
|
import org.elasticsearch.action.search.ShardSearchFailure;
|
||||||
|
import org.elasticsearch.common.unit.TimeValue;
|
||||||
|
import org.elasticsearch.search.SearchHit;
|
||||||
|
import org.xbib.elx.api.SearchClient;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Comparator;
|
||||||
|
import java.util.Optional;
|
||||||
|
import java.util.Spliterator;
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
|
import java.util.function.Consumer;
|
||||||
|
import java.util.function.Predicate;
|
||||||
|
import java.util.stream.Stream;
|
||||||
|
import java.util.stream.StreamSupport;
|
||||||
|
|
||||||
|
public abstract class AbstractSearchClient extends AbstractNativeClient implements SearchClient {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Optional<GetResponse> get(Consumer<GetRequestBuilder> getRequestBuilderConsumer) {
|
||||||
|
GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE);
|
||||||
|
getRequestBuilderConsumer.accept(getRequestBuilder);
|
||||||
|
GetResponse getResponse = getRequestBuilder.execute().actionGet();
|
||||||
|
return getResponse.isExists() ? Optional.of(getResponse) : Optional.empty();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Optional<MultiGetResponse> multiGet(Consumer<MultiGetRequestBuilder> multiGetRequestBuilderConsumer) {
|
||||||
|
MultiGetRequestBuilder multiGetRequestBuilder = new MultiGetRequestBuilder(client, MultiGetAction.INSTANCE);
|
||||||
|
multiGetRequestBuilderConsumer.accept(multiGetRequestBuilder);
|
||||||
|
MultiGetResponse multiGetItemResponse = multiGetRequestBuilder.execute().actionGet();
|
||||||
|
return multiGetItemResponse.getResponses().length == 0 ? Optional.empty() : Optional.of(multiGetItemResponse);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Optional<SearchResponse> search(Consumer<SearchRequestBuilder> queryBuilder) {
|
||||||
|
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
|
||||||
|
queryBuilder.accept(searchRequestBuilder);
|
||||||
|
SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
|
||||||
|
if (searchResponse.getFailedShards() > 0) {
|
||||||
|
StringBuilder sb = new StringBuilder("Search failed:");
|
||||||
|
for (ShardSearchFailure failure : searchResponse.getShardFailures()) {
|
||||||
|
sb.append("\n").append(failure.reason());
|
||||||
|
}
|
||||||
|
throw new ElasticsearchException(sb.toString());
|
||||||
|
}
|
||||||
|
return searchResponse.getHits().getHits().length == 0 ? Optional.empty() : Optional.of(searchResponse);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Stream<SearchHit> search(Consumer<SearchRequestBuilder> queryBuilder,
|
||||||
|
TimeValue scrollTime, int scrollSize) {
|
||||||
|
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
|
||||||
|
queryBuilder.accept(searchRequestBuilder);
|
||||||
|
searchRequestBuilder.setScroll(scrollTime).setSize(scrollSize);
|
||||||
|
SearchResponse originalSearchResponse = searchRequestBuilder.execute().actionGet();
|
||||||
|
Stream<SearchResponse> infiniteResponses = Stream.iterate(originalSearchResponse,
|
||||||
|
searchResponse -> new SearchScrollRequestBuilder(client, SearchScrollAction.INSTANCE)
|
||||||
|
.setScrollId(searchResponse.getScrollId())
|
||||||
|
.setScroll(scrollTime)
|
||||||
|
.execute().actionGet());
|
||||||
|
Predicate<SearchResponse> condition = searchResponse -> searchResponse.getHits().getHits().length > 0;
|
||||||
|
Consumer<SearchResponse> lastAction = searchResponse -> {
|
||||||
|
ClearScrollRequestBuilder clearScrollRequestBuilder =
|
||||||
|
new ClearScrollRequestBuilder(client, ClearScrollAction.INSTANCE)
|
||||||
|
.addScrollId(searchResponse.getScrollId());
|
||||||
|
clearScrollRequestBuilder.execute().actionGet();
|
||||||
|
};
|
||||||
|
return StreamSupport.stream(TakeWhileSpliterator.over(infiniteResponses.spliterator(),
|
||||||
|
condition, lastAction), false)
|
||||||
|
.onClose(infiniteResponses::close)
|
||||||
|
.flatMap(searchResponse -> Arrays.stream(searchResponse.getHits().getHits()));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Stream<String> getIds(Consumer<SearchRequestBuilder> queryBuilder) {
|
||||||
|
return search(queryBuilder, TimeValue.timeValueMinutes(1), 1000).map(SearchHit::getId);
|
||||||
|
}
|
||||||
|
|
||||||
|
static class TakeWhileSpliterator<T> implements Spliterator<T> {
|
||||||
|
|
||||||
|
private final Spliterator<T> source;
|
||||||
|
|
||||||
|
private final Predicate<T> condition;
|
||||||
|
|
||||||
|
private final Consumer<T> lastElement;
|
||||||
|
|
||||||
|
private final boolean inclusive;
|
||||||
|
|
||||||
|
private final AtomicBoolean checked;
|
||||||
|
|
||||||
|
static <T> TakeWhileSpliterator<T> over(Spliterator<T> source,
|
||||||
|
Predicate<T> condition,
|
||||||
|
Consumer<T> lastElement) {
|
||||||
|
return new TakeWhileSpliterator<>(source, condition, lastElement, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
private TakeWhileSpliterator(Spliterator<T> source,
|
||||||
|
Predicate<T> condition,
|
||||||
|
Consumer<T> lastElement,
|
||||||
|
boolean inclusive) {
|
||||||
|
this.source = source;
|
||||||
|
this.condition = condition;
|
||||||
|
this.lastElement = lastElement;
|
||||||
|
this.inclusive = inclusive;
|
||||||
|
this.checked = new AtomicBoolean(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean tryAdvance(Consumer<? super T> action) {
|
||||||
|
return checked.get() && source.tryAdvance(e -> {
|
||||||
|
if (condition.test(e)) {
|
||||||
|
action.accept(e);
|
||||||
|
} else {
|
||||||
|
if (inclusive && checked.get()) {
|
||||||
|
action.accept(e);
|
||||||
|
}
|
||||||
|
lastElement.accept(e);
|
||||||
|
checked.set(false);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Spliterator<T> trySplit() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long estimateSize() {
|
||||||
|
return checked.get() ? source.estimateSize() : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int characteristics() {
|
||||||
|
return source.characteristics() &~ Spliterator.SIZED;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Comparator<? super T> getComparator() {
|
||||||
|
return source.getComparator();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -8,12 +8,12 @@ import org.elasticsearch.client.ElasticsearchClient;
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.common.unit.ByteSizeValue;
|
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||||
import org.elasticsearch.common.unit.TimeValue;
|
import org.elasticsearch.common.unit.TimeValue;
|
||||||
import org.xbib.elx.api.ExtendedClient;
|
import org.xbib.elx.api.AdminClientProvider;
|
||||||
import org.xbib.elx.api.ExtendedClientProvider;
|
import org.xbib.elx.api.BulkClientProvider;
|
||||||
|
import org.xbib.elx.api.NativeClient;
|
||||||
|
import org.xbib.elx.api.SearchClientProvider;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.ServiceLoader;
|
import java.util.ServiceLoader;
|
||||||
|
|
||||||
@SuppressWarnings("rawtypes")
|
@SuppressWarnings("rawtypes")
|
||||||
|
@ -23,11 +23,15 @@ public class ClientBuilder {
|
||||||
|
|
||||||
private final ElasticsearchClient client;
|
private final ElasticsearchClient client;
|
||||||
|
|
||||||
|
private final ClassLoader classLoader;
|
||||||
|
|
||||||
private final Settings.Builder settingsBuilder;
|
private final Settings.Builder settingsBuilder;
|
||||||
|
|
||||||
private Map<Class<? extends ExtendedClientProvider>, ExtendedClientProvider> providerMap;
|
private Class<? extends AdminClientProvider> adminClientProvider;
|
||||||
|
|
||||||
private Class<? extends ExtendedClientProvider> provider;
|
private Class<? extends BulkClientProvider> bulkClientProvider;
|
||||||
|
|
||||||
|
private Class<? extends SearchClientProvider> searchClientProvider;
|
||||||
|
|
||||||
public ClientBuilder() {
|
public ClientBuilder() {
|
||||||
this(null);
|
this(null);
|
||||||
|
@ -39,14 +43,9 @@ public class ClientBuilder {
|
||||||
|
|
||||||
public ClientBuilder(ElasticsearchClient client, ClassLoader classLoader) {
|
public ClientBuilder(ElasticsearchClient client, ClassLoader classLoader) {
|
||||||
this.client = client;
|
this.client = client;
|
||||||
|
this.classLoader = classLoader;
|
||||||
this.settingsBuilder = Settings.builder();
|
this.settingsBuilder = Settings.builder();
|
||||||
settingsBuilder.put("node.name", "elx-client-" + Version.CURRENT);
|
settingsBuilder.put("node.name", "elx-client-" + Version.CURRENT);
|
||||||
this.providerMap = new HashMap<>();
|
|
||||||
ServiceLoader<ExtendedClientProvider> serviceLoader = ServiceLoader.load(ExtendedClientProvider.class,
|
|
||||||
classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader());
|
|
||||||
for (ExtendedClientProvider provider : serviceLoader) {
|
|
||||||
providerMap.put(provider.getClass(), provider);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static ClientBuilder builder() {
|
public static ClientBuilder builder() {
|
||||||
|
@ -57,8 +56,18 @@ public class ClientBuilder {
|
||||||
return new ClientBuilder(client);
|
return new ClientBuilder(client);
|
||||||
}
|
}
|
||||||
|
|
||||||
public ClientBuilder provider(Class<? extends ExtendedClientProvider> provider) {
|
public ClientBuilder setAdminClientProvider(Class<? extends AdminClientProvider> adminClientProvider) {
|
||||||
this.provider = provider;
|
this.adminClientProvider = adminClientProvider;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ClientBuilder setBulkClientProvider(Class<? extends BulkClientProvider> bulkClientProvider) {
|
||||||
|
this.bulkClientProvider = bulkClientProvider;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ClientBuilder setSearchClientProvider(Class<? extends SearchClientProvider> searchClientProvider) {
|
||||||
|
this.searchClientProvider = searchClientProvider;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -98,14 +107,39 @@ public class ClientBuilder {
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
public <C extends ExtendedClient> C build() throws IOException {
|
public <C extends NativeClient> C build() throws IOException {
|
||||||
if (provider == null) {
|
|
||||||
throw new IllegalArgumentException("no provider");
|
|
||||||
}
|
|
||||||
Settings settings = settingsBuilder.build();
|
Settings settings = settingsBuilder.build();
|
||||||
logger.log(Level.INFO, "settings = " + settings.toDelimitedString(','));
|
logger.log(Level.INFO, "settings = " + settings.toDelimitedString(','));
|
||||||
return (C) providerMap.get(provider).getExtendedClient()
|
if (adminClientProvider != null) {
|
||||||
.setClient(client)
|
for (AdminClientProvider provider : ServiceLoader.load(AdminClientProvider.class, classLoader)) {
|
||||||
.init(settings);
|
if (provider.getClass().isAssignableFrom(adminClientProvider)) {
|
||||||
|
C c = (C) provider.getClient();
|
||||||
|
c.setClient(client);
|
||||||
|
c.init(settings);
|
||||||
|
return c;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (bulkClientProvider != null) {
|
||||||
|
for (BulkClientProvider provider : ServiceLoader.load(BulkClientProvider.class, classLoader)) {
|
||||||
|
if (provider.getClass().isAssignableFrom(bulkClientProvider)) {
|
||||||
|
C c = (C) provider.getClient();
|
||||||
|
c.setClient(client);
|
||||||
|
c.init(settings);
|
||||||
|
return c;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (searchClientProvider != null) {
|
||||||
|
for (SearchClientProvider provider : ServiceLoader.load(SearchClientProvider.class, classLoader)) {
|
||||||
|
if (provider.getClass().isAssignableFrom(searchClientProvider)) {
|
||||||
|
C c = (C) provider.getClient();
|
||||||
|
c.setClient(client);
|
||||||
|
c.init(settings);
|
||||||
|
return c;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new IllegalArgumentException("no provider");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,10 +11,10 @@ import org.elasticsearch.action.update.UpdateRequest;
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.common.unit.ByteSizeValue;
|
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||||
import org.elasticsearch.common.unit.TimeValue;
|
import org.elasticsearch.common.unit.TimeValue;
|
||||||
|
import org.xbib.elx.api.BulkClient;
|
||||||
import org.xbib.elx.api.BulkController;
|
import org.xbib.elx.api.BulkController;
|
||||||
import org.xbib.elx.api.BulkMetric;
|
import org.xbib.elx.api.BulkMetric;
|
||||||
import org.xbib.elx.api.BulkProcessor;
|
import org.xbib.elx.api.BulkProcessor;
|
||||||
import org.xbib.elx.api.ExtendedClient;
|
|
||||||
import org.xbib.elx.api.IndexDefinition;
|
import org.xbib.elx.api.IndexDefinition;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
@ -29,7 +29,7 @@ public class DefaultBulkController implements BulkController {
|
||||||
|
|
||||||
private static final Logger logger = LogManager.getLogger(DefaultBulkController.class);
|
private static final Logger logger = LogManager.getLogger(DefaultBulkController.class);
|
||||||
|
|
||||||
private final ExtendedClient client;
|
private final BulkClient client;
|
||||||
|
|
||||||
private final BulkMetric bulkMetric;
|
private final BulkMetric bulkMetric;
|
||||||
|
|
||||||
|
@ -39,9 +39,9 @@ public class DefaultBulkController implements BulkController {
|
||||||
|
|
||||||
private final Map<String, Long> stopBulkRefreshIntervals;
|
private final Map<String, Long> stopBulkRefreshIntervals;
|
||||||
|
|
||||||
private long maxWaitTime;
|
private final long maxWaitTime;
|
||||||
|
|
||||||
private TimeUnit maxWaitTimeUnit;
|
private final TimeUnit maxWaitTimeUnit;
|
||||||
|
|
||||||
private BulkProcessor bulkProcessor;
|
private BulkProcessor bulkProcessor;
|
||||||
|
|
||||||
|
@ -51,7 +51,7 @@ public class DefaultBulkController implements BulkController {
|
||||||
|
|
||||||
private boolean enableBulkLogging;
|
private boolean enableBulkLogging;
|
||||||
|
|
||||||
public DefaultBulkController(ExtendedClient client, BulkMetric bulkMetric) {
|
public DefaultBulkController(BulkClient client, BulkMetric bulkMetric) {
|
||||||
this.client = client;
|
this.client = client;
|
||||||
this.bulkMetric = bulkMetric;
|
this.bulkMetric = bulkMetric;
|
||||||
this.indexNames = new ArrayList<>();
|
this.indexNames = new ArrayList<>();
|
||||||
|
|
|
@ -3,8 +3,6 @@ package org.xbib.elx.common;
|
||||||
import org.xbib.elx.api.IndexDefinition;
|
import org.xbib.elx.api.IndexDefinition;
|
||||||
import org.xbib.elx.api.IndexRetention;
|
import org.xbib.elx.api.IndexRetention;
|
||||||
|
|
||||||
import java.net.MalformedURLException;
|
|
||||||
import java.net.URL;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
public class DefaultIndexDefinition implements IndexDefinition {
|
public class DefaultIndexDefinition implements IndexDefinition {
|
||||||
|
@ -15,9 +13,9 @@ public class DefaultIndexDefinition implements IndexDefinition {
|
||||||
|
|
||||||
private String dateTimePattern;
|
private String dateTimePattern;
|
||||||
|
|
||||||
private URL settingsUrl;
|
private String settings;
|
||||||
|
|
||||||
private URL mappingsUrl;
|
private String mappings;
|
||||||
|
|
||||||
private boolean enabled;
|
private boolean enabled;
|
||||||
|
|
||||||
|
@ -62,37 +60,25 @@ public class DefaultIndexDefinition implements IndexDefinition {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException {
|
public IndexDefinition setSettings(String settings) {
|
||||||
this.settingsUrl = settingsUrlString != null ? new URL(settingsUrlString) : null;
|
this.settings = settings;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public IndexDefinition setSettingsUrl(URL settingsUrl) {
|
public String getSettings() {
|
||||||
this.settingsUrl = settingsUrl;
|
return settings;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public IndexDefinition setMappings(String mappings) {
|
||||||
|
this.mappings = mappings;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public URL getSettingsUrl() {
|
public String getMappings() {
|
||||||
return settingsUrl;
|
return mappings;
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException {
|
|
||||||
this.mappingsUrl = mappingsUrlString != null ? new URL(mappingsUrlString) : null;
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public IndexDefinition setMappingsUrl(URL mappingsUrl) {
|
|
||||||
this.mappingsUrl = mappingsUrl;
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public URL getMappingsUrl() {
|
|
||||||
return mappingsUrl;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -0,0 +1,59 @@
|
||||||
|
package org.xbib.elx.common;
|
||||||
|
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A mocked client, it does not perform any actions on a cluster. Useful for testing.
|
||||||
|
*/
|
||||||
|
public class MockAdminClient extends AbstractAdminClient {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ElasticsearchClient getClient() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MockAdminClient deleteIndex(String index) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MockAdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() {
|
||||||
|
// nothing to do
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,10 @@
|
||||||
|
package org.xbib.elx.common;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.AdminClientProvider;
|
||||||
|
|
||||||
|
public class MockAdminClientProvider implements AdminClientProvider<MockAdminClient> {
|
||||||
|
@Override
|
||||||
|
public MockAdminClient getClient() {
|
||||||
|
return new MockAdminClient();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,99 @@
|
||||||
|
package org.xbib.elx.common;
|
||||||
|
|
||||||
|
import org.elasticsearch.action.delete.DeleteRequest;
|
||||||
|
import org.elasticsearch.action.index.IndexRequest;
|
||||||
|
import org.elasticsearch.action.update.UpdateRequest;
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A mocked client, it does not perform any actions on a cluster. Useful for testing.
|
||||||
|
*/
|
||||||
|
public class MockBulkClient extends AbstractBulkClient {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ElasticsearchClient getClient() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getClusterName() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MockBulkClient index(String index, String id, boolean create, String source) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MockBulkClient delete(String index, String id) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MockBulkClient update(String index, String id, String source) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MockBulkClient index(IndexRequest indexRequest) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MockBulkClient delete(DeleteRequest deleteRequest) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MockBulkClient update(UpdateRequest updateRequest) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void startBulk(String index, long startRefreshInterval, long stopRefreshIterval) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void refreshIndex(String index) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void flushIndex(String index) {
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void flush() {
|
||||||
|
// nothing to do
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() {
|
||||||
|
// nothing to do
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,11 @@
|
||||||
|
package org.xbib.elx.common;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.BulkClientProvider;
|
||||||
|
|
||||||
|
public class MockBulkClientProvider implements BulkClientProvider<MockBulkClient> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MockBulkClient getClient() {
|
||||||
|
return new MockBulkClient();
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,129 +0,0 @@
|
||||||
package org.xbib.elx.common;
|
|
||||||
|
|
||||||
import org.elasticsearch.action.delete.DeleteRequest;
|
|
||||||
import org.elasticsearch.action.index.IndexRequest;
|
|
||||||
import org.elasticsearch.action.update.UpdateRequest;
|
|
||||||
import org.elasticsearch.client.ElasticsearchClient;
|
|
||||||
import org.elasticsearch.common.settings.Settings;
|
|
||||||
|
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A mocked client, it does not perform any actions on a cluster. Useful for testing.
|
|
||||||
*/
|
|
||||||
public class MockExtendedClient extends AbstractExtendedClient {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ElasticsearchClient getClient() {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient init(Settings settings) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected ElasticsearchClient createClient(Settings settings) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void closeClient() {
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient index(String index, String id, boolean create, String source) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient delete(String index, String id) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient update(String index, String id, String source) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient index(IndexRequest indexRequest) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient delete(DeleteRequest deleteRequest) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient update(UpdateRequest updateRequest) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient startBulk(String index, long startRefreshInterval, long stopRefreshIterval) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient newIndex(String index) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient deleteIndex(String index) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient refreshIndex(String index) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient flushIndex(String index) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void flush() {
|
|
||||||
// nothing to do
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void close() {
|
|
||||||
// nothing to do
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,10 +0,0 @@
|
||||||
package org.xbib.elx.common;
|
|
||||||
|
|
||||||
import org.xbib.elx.api.ExtendedClientProvider;
|
|
||||||
|
|
||||||
public class MockExtendedClientProvider implements ExtendedClientProvider<MockExtendedClient> {
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient getExtendedClient() {
|
|
||||||
return new MockExtendedClient();
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -0,0 +1,38 @@
|
||||||
|
package org.xbib.elx.common;
|
||||||
|
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A mocked client, it does not perform any actions on a cluster. Useful for testing.
|
||||||
|
*/
|
||||||
|
public class MockSearchClient extends AbstractSearchClient {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ElasticsearchClient getClient() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getClusterName() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() {
|
||||||
|
// nothing to do
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,11 @@
|
||||||
|
package org.xbib.elx.common;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.SearchClientProvider;
|
||||||
|
|
||||||
|
public class MockSearchClientProvider implements SearchClientProvider<MockSearchClient> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MockSearchClient getClient() {
|
||||||
|
return new MockSearchClient();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1 @@
|
||||||
|
org.xbib.elx.common.MockAdminClientProvider
|
|
@ -1 +0,0 @@
|
||||||
org.xbib.elx.common.MockExtendedClientProvider
|
|
|
@ -0,0 +1,21 @@
|
||||||
|
package org.xbib.elx.common.test;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.xbib.elx.common.ClientBuilder;
|
||||||
|
import org.xbib.elx.common.MockAdminClient;
|
||||||
|
import org.xbib.elx.common.MockAdminClientProvider;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
|
|
||||||
|
class MockAdminClientProviderTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testMockAdminProvider() throws IOException {
|
||||||
|
MockAdminClient client = ClientBuilder.builder()
|
||||||
|
.setAdminClientProvider(MockAdminClientProvider.class)
|
||||||
|
.build();
|
||||||
|
assertNotNull(client);
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,19 +0,0 @@
|
||||||
package org.xbib.elx.common.test;
|
|
||||||
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.xbib.elx.common.ClientBuilder;
|
|
||||||
import org.xbib.elx.common.MockExtendedClient;
|
|
||||||
import org.xbib.elx.common.MockExtendedClientProvider;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
|
||||||
|
|
||||||
class MockExtendedClientProviderTest {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testMockExtendedProvider() throws IOException {
|
|
||||||
MockExtendedClient client = ClientBuilder.builder().provider(MockExtendedClientProvider.class).build();
|
|
||||||
assertNotNull(client);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -0,0 +1,155 @@
|
||||||
|
package org.elasticsearch.action.admin.indices.get;
|
||||||
|
|
||||||
|
import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
|
||||||
|
import org.apache.lucene.util.CollectionUtil;
|
||||||
|
import org.elasticsearch.cluster.metadata.AliasMetaData;
|
||||||
|
import org.elasticsearch.cluster.metadata.MappingMetaData;
|
||||||
|
import org.elasticsearch.common.CheckedFunction;
|
||||||
|
import org.elasticsearch.common.collect.ImmutableOpenMap;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentParser;
|
||||||
|
import org.xbib.elx.http.HttpAction;
|
||||||
|
import org.xbib.netty.http.client.api.Request;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.Comparator;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class HttpGetIndexAction extends HttpAction<GetIndexRequest, GetIndexResponse> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetIndexAction getActionInstance() {
|
||||||
|
return GetIndexAction.INSTANCE;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Request.Builder createHttpRequest(String url, GetIndexRequest getIndexRequest) throws IOException {
|
||||||
|
List<String> list = getIndexRequest.indices().length == 0 ?
|
||||||
|
List.of("*") : Arrays.asList(getIndexRequest.indices());
|
||||||
|
String command = "/" + String.join(",", list);
|
||||||
|
logger.info("command = " + command);
|
||||||
|
return newGetRequest(url, command);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected CheckedFunction<XContentParser, GetIndexResponse, IOException> entityParser() {
|
||||||
|
return this::fromXContent;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected GetIndexResponse emptyResponse() {
|
||||||
|
return new GetIndexResponse();
|
||||||
|
}
|
||||||
|
|
||||||
|
private GetIndexResponse fromXContent(XContentParser parser) throws IOException {
|
||||||
|
ImmutableOpenMap.Builder<String, List<AliasMetaData>> aliases = ImmutableOpenMap.builder();
|
||||||
|
ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> mappings = ImmutableOpenMap.builder();
|
||||||
|
ImmutableOpenMap.Builder<String, Settings> settings = ImmutableOpenMap.builder();
|
||||||
|
List<String> indices = new ArrayList<>();
|
||||||
|
if (parser.currentToken() == null) {
|
||||||
|
parser.nextToken();
|
||||||
|
}
|
||||||
|
ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
|
||||||
|
parser.nextToken();
|
||||||
|
while (!parser.isClosed()) {
|
||||||
|
if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
|
||||||
|
String indexName = parser.currentName();
|
||||||
|
indices.add(indexName);
|
||||||
|
IndexEntry indexEntry = parseIndexEntry(parser);
|
||||||
|
CollectionUtil.timSort(indexEntry.indexAliases, Comparator.comparing(AliasMetaData::alias));
|
||||||
|
aliases.put(indexName, Collections.unmodifiableList(indexEntry.indexAliases));
|
||||||
|
mappings.put(indexName, indexEntry.indexMappings);
|
||||||
|
settings.put(indexName, indexEntry.indexSettings);
|
||||||
|
} else if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
|
||||||
|
parser.skipChildren();
|
||||||
|
} else {
|
||||||
|
parser.nextToken();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return new GetIndexResponse(indices.toArray(new String[0]),
|
||||||
|
mappings.build(),
|
||||||
|
aliases.build(),
|
||||||
|
settings.build());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static IndexEntry parseIndexEntry(XContentParser parser) throws IOException {
|
||||||
|
List<AliasMetaData> indexAliases = null;
|
||||||
|
ImmutableOpenMap<String, MappingMetaData> indexMappings = null;
|
||||||
|
Settings indexSettings = null;
|
||||||
|
Settings indexDefaultSettings = null;
|
||||||
|
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
|
||||||
|
ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
|
||||||
|
parser.nextToken();
|
||||||
|
if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
|
||||||
|
switch (parser.currentName()) {
|
||||||
|
case "aliases":
|
||||||
|
indexAliases = parseAliases(parser);
|
||||||
|
break;
|
||||||
|
case "mappings":
|
||||||
|
indexMappings = parseMappings(parser);
|
||||||
|
break;
|
||||||
|
case "settings":
|
||||||
|
indexSettings = Settings.fromXContent(parser);
|
||||||
|
break;
|
||||||
|
case "defaults":
|
||||||
|
indexDefaultSettings = Settings.fromXContent(parser);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
parser.skipChildren();
|
||||||
|
}
|
||||||
|
} else if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
|
||||||
|
parser.skipChildren();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return new IndexEntry(indexAliases, indexMappings, indexSettings, indexDefaultSettings);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static List<AliasMetaData> parseAliases(XContentParser parser) throws IOException {
|
||||||
|
List<AliasMetaData> indexAliases = new ArrayList<>();
|
||||||
|
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
|
||||||
|
ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
|
||||||
|
indexAliases.add(AliasMetaData.Builder.fromXContent(parser));
|
||||||
|
}
|
||||||
|
return indexAliases;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ImmutableOpenMap<String, MappingMetaData> parseMappings(XContentParser parser) throws IOException {
|
||||||
|
ImmutableOpenMap.Builder<String, MappingMetaData> indexMappings = ImmutableOpenMap.builder();
|
||||||
|
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
|
||||||
|
ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
|
||||||
|
parser.nextToken();
|
||||||
|
if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
|
||||||
|
String mappingType = parser.currentName();
|
||||||
|
indexMappings.put(mappingType, new MappingMetaData(mappingType, parser.map()));
|
||||||
|
} else if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
|
||||||
|
parser.skipChildren();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return indexMappings.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static class IndexEntry {
|
||||||
|
List<AliasMetaData> indexAliases = new ArrayList<>();
|
||||||
|
ImmutableOpenMap<String, MappingMetaData> indexMappings = ImmutableOpenMap.of();
|
||||||
|
Settings indexSettings = Settings.EMPTY;
|
||||||
|
Settings indexDefaultSettings = Settings.EMPTY;
|
||||||
|
|
||||||
|
IndexEntry(List<AliasMetaData> indexAliases, ImmutableOpenMap<String, MappingMetaData> indexMappings,
|
||||||
|
Settings indexSettings, Settings indexDefaultSettings) {
|
||||||
|
if (indexAliases != null) {
|
||||||
|
this.indexAliases = indexAliases;
|
||||||
|
}
|
||||||
|
if (indexMappings != null) {
|
||||||
|
this.indexMappings = indexMappings;
|
||||||
|
}
|
||||||
|
if (indexSettings != null) {
|
||||||
|
this.indexSettings = indexSettings;
|
||||||
|
}
|
||||||
|
if (indexDefaultSettings != null) {
|
||||||
|
this.indexDefaultSettings = indexDefaultSettings;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,12 +0,0 @@
|
||||||
package org.xbib.elx.http;
|
|
||||||
|
|
||||||
import org.xbib.elx.api.ExtendedClientProvider;
|
|
||||||
|
|
||||||
import java.util.Collections;
|
|
||||||
|
|
||||||
public class ExtendedHttpClientProvider implements ExtendedClientProvider<ExtendedHttpClient> {
|
|
||||||
@Override
|
|
||||||
public ExtendedHttpClient getExtendedClient() {
|
|
||||||
return new ExtendedHttpClient(Collections.emptyList(), Thread.currentThread().getContextClassLoader());
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -77,7 +77,7 @@ public abstract class HttpAction<R extends ActionRequest, T extends ActionRespon
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
Transport transport = httpActionContext.getExtendedHttpClient().internalClient().execute(httpRequest);
|
Transport transport = httpActionContext.getHelper().internalClient().execute(httpRequest);
|
||||||
httpActionContext.setHttpClientTransport(transport);
|
httpActionContext.setHttpClientTransport(transport);
|
||||||
if (transport.isFailed()) {
|
if (transport.isFailed()) {
|
||||||
listener.onFailure(new Exception(transport.getFailure()));
|
listener.onFailure(new Exception(transport.getFailure()));
|
||||||
|
@ -155,7 +155,7 @@ public abstract class HttpAction<R extends ActionRequest, T extends ActionRespon
|
||||||
throw new IllegalStateException("unsupported content-type: " + mediaType);
|
throw new IllegalStateException("unsupported content-type: " + mediaType);
|
||||||
}
|
}
|
||||||
try (XContentParser parser = xContentType.xContent()
|
try (XContentParser parser = xContentType.xContent()
|
||||||
.createParser(httpActionContext.getExtendedHttpClient().getRegistry(),
|
.createParser(httpActionContext.getHelper().getRegistry(),
|
||||||
DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
|
DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
|
||||||
httpActionContext.getHttpResponse().getBody().toString(StandardCharsets.UTF_8))) {
|
httpActionContext.getHttpResponse().getBody().toString(StandardCharsets.UTF_8))) {
|
||||||
return entityParser().apply(parser);
|
return entityParser().apply(parser);
|
||||||
|
@ -167,7 +167,7 @@ public abstract class HttpAction<R extends ActionRequest, T extends ActionRespon
|
||||||
|
|
||||||
protected ElasticsearchStatusException parseToError(HttpActionContext<R, T> httpActionContext) {
|
protected ElasticsearchStatusException parseToError(HttpActionContext<R, T> httpActionContext) {
|
||||||
try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
|
try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
|
||||||
.createParser(httpActionContext.getExtendedHttpClient().getRegistry(),
|
.createParser(httpActionContext.getHelper().getRegistry(),
|
||||||
DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
|
DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
|
||||||
httpActionContext.getHttpResponse().getBody().toString(StandardCharsets.UTF_8))) {
|
httpActionContext.getHttpResponse().getBody().toString(StandardCharsets.UTF_8))) {
|
||||||
return errorParser().apply(parser);
|
return errorParser().apply(parser);
|
||||||
|
|
|
@ -13,7 +13,7 @@ import org.xbib.netty.http.common.HttpResponse;
|
||||||
*/
|
*/
|
||||||
public class HttpActionContext<R extends ActionRequest, T extends ActionResponse> {
|
public class HttpActionContext<R extends ActionRequest, T extends ActionResponse> {
|
||||||
|
|
||||||
private final ExtendedHttpClient extendedHttpClient;
|
private final HttpClientHelper helper;
|
||||||
|
|
||||||
private final R request;
|
private final R request;
|
||||||
|
|
||||||
|
@ -23,14 +23,14 @@ public class HttpActionContext<R extends ActionRequest, T extends ActionResponse
|
||||||
|
|
||||||
private HttpResponse httpResponse;
|
private HttpResponse httpResponse;
|
||||||
|
|
||||||
HttpActionContext(ExtendedHttpClient extendedHttpClient, R request, String url) {
|
HttpActionContext(HttpClientHelper helper, R request, String url) {
|
||||||
this.extendedHttpClient = extendedHttpClient;
|
this.helper = helper;
|
||||||
this.request = request;
|
this.request = request;
|
||||||
this.url = url;
|
this.url = url;
|
||||||
}
|
}
|
||||||
|
|
||||||
public ExtendedHttpClient getExtendedHttpClient() {
|
public HttpClientHelper getHelper() {
|
||||||
return extendedHttpClient;
|
return helper;
|
||||||
}
|
}
|
||||||
|
|
||||||
public R getRequest() {
|
public R getRequest() {
|
||||||
|
|
|
@ -0,0 +1,61 @@
|
||||||
|
package org.xbib.elx.http;
|
||||||
|
|
||||||
|
import org.elasticsearch.action.Action;
|
||||||
|
import org.elasticsearch.action.ActionFuture;
|
||||||
|
import org.elasticsearch.action.ActionListener;
|
||||||
|
import org.elasticsearch.action.ActionRequest;
|
||||||
|
import org.elasticsearch.action.ActionRequestBuilder;
|
||||||
|
import org.elasticsearch.action.ActionResponse;
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.threadpool.ThreadPool;
|
||||||
|
import org.xbib.elx.common.AbstractAdminClient;
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Elasticsearch HTTP admin client.
|
||||||
|
*/
|
||||||
|
public class HttpAdminClient extends AbstractAdminClient implements ElasticsearchClient {
|
||||||
|
|
||||||
|
private final HttpClientHelper helper;
|
||||||
|
|
||||||
|
public HttpAdminClient() {
|
||||||
|
this.helper = new HttpClientHelper();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) throws IOException {
|
||||||
|
super.init(settings);
|
||||||
|
helper.init(settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||||
|
return helper.createClient(settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() throws IOException {
|
||||||
|
helper.closeClient();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(Action<Request, Response, RequestBuilder> action, Request request) {
|
||||||
|
return helper.execute(action, request);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
|
||||||
|
helper.execute(action, request, listener);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(Action<Request, Response, RequestBuilder> action) {
|
||||||
|
return helper.prepareExecute(action);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ThreadPool threadPool() {
|
||||||
|
return helper.threadPool();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,10 @@
|
||||||
|
package org.xbib.elx.http;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.AdminClientProvider;
|
||||||
|
|
||||||
|
public class HttpAdminClientProvider implements AdminClientProvider<HttpAdminClient> {
|
||||||
|
@Override
|
||||||
|
public HttpAdminClient getClient() {
|
||||||
|
return new HttpAdminClient();
|
||||||
|
}
|
||||||
|
}
|
61
elx-http/src/main/java/org/xbib/elx/http/HttpBulkClient.java
Normal file
61
elx-http/src/main/java/org/xbib/elx/http/HttpBulkClient.java
Normal file
|
@ -0,0 +1,61 @@
|
||||||
|
package org.xbib.elx.http;
|
||||||
|
|
||||||
|
import org.elasticsearch.action.Action;
|
||||||
|
import org.elasticsearch.action.ActionFuture;
|
||||||
|
import org.elasticsearch.action.ActionListener;
|
||||||
|
import org.elasticsearch.action.ActionRequest;
|
||||||
|
import org.elasticsearch.action.ActionRequestBuilder;
|
||||||
|
import org.elasticsearch.action.ActionResponse;
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.threadpool.ThreadPool;
|
||||||
|
import org.xbib.elx.common.AbstractBulkClient;
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Elasticsearch HTTP bulk client.
|
||||||
|
*/
|
||||||
|
public class HttpBulkClient extends AbstractBulkClient implements ElasticsearchClient {
|
||||||
|
|
||||||
|
private final HttpClientHelper helper;
|
||||||
|
|
||||||
|
public HttpBulkClient() {
|
||||||
|
this.helper = new HttpClientHelper();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) throws IOException {
|
||||||
|
super.init(settings);
|
||||||
|
helper.init(settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||||
|
return helper.createClient(settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() throws IOException {
|
||||||
|
helper.closeClient();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(Action<Request, Response, RequestBuilder> action, Request request) {
|
||||||
|
return helper.execute(action, request);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
|
||||||
|
helper.execute(action, request, listener);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(Action<Request, Response, RequestBuilder> action) {
|
||||||
|
return helper.prepareExecute(action);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ThreadPool threadPool() {
|
||||||
|
return helper.threadPool();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,11 @@
|
||||||
|
package org.xbib.elx.http;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.BulkClientProvider;
|
||||||
|
|
||||||
|
public class HttpBulkClientProvider implements BulkClientProvider<HttpBulkClient> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public HttpBulkClient getClient() {
|
||||||
|
return new HttpBulkClient();
|
||||||
|
}
|
||||||
|
}
|
|
@ -15,12 +15,12 @@ import org.elasticsearch.client.ElasticsearchClient;
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
|
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
|
||||||
import org.elasticsearch.threadpool.ThreadPool;
|
import org.elasticsearch.threadpool.ThreadPool;
|
||||||
import org.xbib.elx.common.AbstractExtendedClient;
|
import org.xbib.elx.common.AbstractAdminClient;
|
||||||
import org.xbib.net.URL;
|
import org.xbib.net.URL;
|
||||||
import org.xbib.netty.http.client.Client;
|
import org.xbib.netty.http.client.Client;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collections;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
@ -32,9 +32,9 @@ import java.util.stream.Stream;
|
||||||
/**
|
/**
|
||||||
* Elasticsearch HTTP client.
|
* Elasticsearch HTTP client.
|
||||||
*/
|
*/
|
||||||
public class ExtendedHttpClient extends AbstractExtendedClient implements ElasticsearchClient {
|
public class HttpClientHelper extends AbstractAdminClient implements ElasticsearchClient {
|
||||||
|
|
||||||
private static final Logger logger = LogManager.getLogger(ExtendedHttpClient.class);
|
private static final Logger logger = LogManager.getLogger(HttpClientHelper.class);
|
||||||
|
|
||||||
private Client nettyHttpClient;
|
private Client nettyHttpClient;
|
||||||
|
|
||||||
|
@ -47,7 +47,11 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
|
||||||
|
|
||||||
private String url;
|
private String url;
|
||||||
|
|
||||||
public ExtendedHttpClient(List<NamedXContentRegistry.Entry> namedXContentEntries, ClassLoader classLoader) {
|
public HttpClientHelper() {
|
||||||
|
this(Collections.emptyList(), Thread.currentThread().getContextClassLoader());
|
||||||
|
}
|
||||||
|
|
||||||
|
public HttpClientHelper(List<NamedXContentRegistry.Entry> namedXContentEntries, ClassLoader classLoader) {
|
||||||
this.registry = new NamedXContentRegistry(Stream.of(getNamedXContents().stream(),
|
this.registry = new NamedXContentRegistry(Stream.of(getNamedXContents().stream(),
|
||||||
namedXContentEntries.stream()).flatMap(Function.identity()).collect(Collectors.toList()));
|
namedXContentEntries.stream()).flatMap(Function.identity()).collect(Collectors.toList()));
|
||||||
this.classLoader = classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader();
|
this.classLoader = classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader();
|
||||||
|
@ -56,7 +60,7 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@SuppressWarnings({"unchecked", "rawtypes"})
|
@SuppressWarnings({"unchecked", "rawtypes"})
|
||||||
public ExtendedHttpClient init(Settings settings) throws IOException {
|
public void init(Settings settings) throws IOException {
|
||||||
super.init(settings);
|
super.init(settings);
|
||||||
if (settings.hasValue("url")) {
|
if (settings.hasValue("url")) {
|
||||||
this.url = settings.get("url");
|
this.url = settings.get("url");
|
||||||
|
@ -78,7 +82,6 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
|
||||||
this.nettyHttpClient = clientBuilder.build();
|
this.nettyHttpClient = clientBuilder.build();
|
||||||
logger.log(Level.DEBUG, "extended HTTP client initialized, settings = {}, url = {}, {} actions",
|
logger.log(Level.DEBUG, "extended HTTP client initialized, settings = {}, url = {}, {} actions",
|
||||||
settings, url, actionMap.size());
|
settings, url, actionMap.size());
|
||||||
return this;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static List<NamedXContentRegistry.Entry> getNamedXContents() {
|
private static List<NamedXContentRegistry.Entry> getNamedXContents() {
|
|
@ -0,0 +1,61 @@
|
||||||
|
package org.xbib.elx.http;
|
||||||
|
|
||||||
|
import org.elasticsearch.action.Action;
|
||||||
|
import org.elasticsearch.action.ActionFuture;
|
||||||
|
import org.elasticsearch.action.ActionListener;
|
||||||
|
import org.elasticsearch.action.ActionRequest;
|
||||||
|
import org.elasticsearch.action.ActionRequestBuilder;
|
||||||
|
import org.elasticsearch.action.ActionResponse;
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.threadpool.ThreadPool;
|
||||||
|
import org.xbib.elx.common.AbstractSearchClient;
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Elasticsearch HTTP search client.
|
||||||
|
*/
|
||||||
|
public class HttpSearchClient extends AbstractSearchClient implements ElasticsearchClient {
|
||||||
|
|
||||||
|
private final HttpClientHelper helper;
|
||||||
|
|
||||||
|
public HttpSearchClient() {
|
||||||
|
this.helper = new HttpClientHelper();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) throws IOException {
|
||||||
|
super.init(settings);
|
||||||
|
helper.init(settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||||
|
return helper.createClient(settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() throws IOException {
|
||||||
|
helper.closeClient();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(Action<Request, Response, RequestBuilder> action, Request request) {
|
||||||
|
return helper.execute(action, request);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
|
||||||
|
helper.execute(action, request, listener);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(Action<Request, Response, RequestBuilder> action) {
|
||||||
|
return helper.prepareExecute(action);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ThreadPool threadPool() {
|
||||||
|
return helper.threadPool();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,11 @@
|
||||||
|
package org.xbib.elx.http;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.SearchClientProvider;
|
||||||
|
|
||||||
|
public class HttpSearchClientProvider implements SearchClientProvider<HttpSearchClient> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public HttpSearchClient getClient() {
|
||||||
|
return new HttpSearchClient();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1 @@
|
||||||
|
org.xbib.elx.http.HttpAdminClientProvider
|
|
@ -0,0 +1 @@
|
||||||
|
org.xbib.elx.http.HttpBulkClientProvider
|
|
@ -1 +0,0 @@
|
||||||
org.xbib.elx.http.ExtendedHttpClientProvider
|
|
|
@ -0,0 +1 @@
|
||||||
|
org.xbib.elx.http.HttpSearchClientProvider
|
|
@ -1,4 +1,5 @@
|
||||||
org.elasticsearch.action.admin.indices.mapping.get.HttpGetMappingsAction
|
org.elasticsearch.action.admin.indices.mapping.get.HttpGetMappingsAction
|
||||||
|
org.elasticsearch.action.admin.indices.get.HttpGetIndexAction
|
||||||
org.xbib.elx.http.action.admin.cluster.health.HttpClusterHealthAction
|
org.xbib.elx.http.action.admin.cluster.health.HttpClusterHealthAction
|
||||||
org.xbib.elx.http.action.admin.cluster.node.info.HttpNodesInfoAction
|
org.xbib.elx.http.action.admin.cluster.node.info.HttpNodesInfoAction
|
||||||
org.xbib.elx.http.action.admin.cluster.settings.HttpClusterUpdateSettingsAction
|
org.xbib.elx.http.action.admin.cluster.settings.HttpClusterUpdateSettingsAction
|
||||||
|
|
|
@ -2,24 +2,16 @@ package org.xbib.elx.http.test;
|
||||||
|
|
||||||
import org.apache.logging.log4j.LogManager;
|
import org.apache.logging.log4j.LogManager;
|
||||||
import org.apache.logging.log4j.Logger;
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
|
|
||||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
|
|
||||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
|
|
||||||
import org.elasticsearch.action.search.SearchAction;
|
|
||||||
import org.elasticsearch.action.search.SearchRequestBuilder;
|
|
||||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
|
||||||
import org.elasticsearch.common.Strings;
|
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.common.unit.TimeValue;
|
import org.elasticsearch.common.unit.TimeValue;
|
||||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||||
import org.elasticsearch.index.query.QueryBuilders;
|
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
import org.xbib.elx.common.ClientBuilder;
|
import org.xbib.elx.common.ClientBuilder;
|
||||||
import org.xbib.elx.common.Parameters;
|
import org.xbib.elx.common.Parameters;
|
||||||
import org.xbib.elx.http.ExtendedHttpClient;
|
import org.xbib.elx.http.HttpBulkClient;
|
||||||
import org.xbib.elx.http.ExtendedHttpClientProvider;
|
import org.xbib.elx.http.HttpBulkClientProvider;
|
||||||
|
|
||||||
import java.util.concurrent.CountDownLatch;
|
import java.util.concurrent.CountDownLatch;
|
||||||
import java.util.concurrent.ExecutorService;
|
import java.util.concurrent.ExecutorService;
|
||||||
|
@ -31,9 +23,9 @@ import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
@ExtendWith(TestExtension.class)
|
@ExtendWith(TestExtension.class)
|
||||||
class ClientTest {
|
class BulkClientTest {
|
||||||
|
|
||||||
private static final Logger logger = LogManager.getLogger(ClientTest.class.getSimpleName());
|
private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getSimpleName());
|
||||||
|
|
||||||
private static final Long ACTIONS = 100L;
|
private static final Long ACTIONS = 100L;
|
||||||
|
|
||||||
|
@ -41,14 +33,14 @@ class ClientTest {
|
||||||
|
|
||||||
private final TestExtension.Helper helper;
|
private final TestExtension.Helper helper;
|
||||||
|
|
||||||
ClientTest(TestExtension.Helper helper) {
|
BulkClientTest(TestExtension.Helper helper) {
|
||||||
this.helper = helper;
|
this.helper = helper;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testSingleDoc() throws Exception {
|
void testSingleDoc() throws Exception {
|
||||||
final ExtendedHttpClient client = ClientBuilder.builder()
|
final HttpBulkClient client = ClientBuilder.builder()
|
||||||
.provider(ExtendedHttpClientProvider.class)
|
.setBulkClientProvider(HttpBulkClientProvider.class)
|
||||||
.put(helper.getHttpSettings())
|
.put(helper.getHttpSettings())
|
||||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
|
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
|
||||||
|
@ -58,8 +50,6 @@ class ClientTest {
|
||||||
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
||||||
client.flush();
|
client.flush();
|
||||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
client.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} finally {
|
} finally {
|
||||||
assertEquals(1, client.getBulkMetric().getSucceeded().getCount());
|
assertEquals(1, client.getBulkMetric().getSucceeded().getCount());
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
if (client.getBulkController().getLastBulkError() != null) {
|
||||||
|
@ -72,8 +62,8 @@ class ClientTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testNewIndex() throws Exception {
|
void testNewIndex() throws Exception {
|
||||||
final ExtendedHttpClient client = ClientBuilder.builder()
|
final HttpBulkClient client = ClientBuilder.builder()
|
||||||
.provider(ExtendedHttpClientProvider.class)
|
.setBulkClientProvider(HttpBulkClientProvider.class)
|
||||||
.put(helper.getHttpSettings())
|
.put(helper.getHttpSettings())
|
||||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
||||||
.build();
|
.build();
|
||||||
|
@ -83,8 +73,8 @@ class ClientTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testMapping() throws Exception {
|
void testMapping() throws Exception {
|
||||||
final ExtendedHttpClient client = ClientBuilder.builder()
|
final HttpBulkClient client = ClientBuilder.builder()
|
||||||
.provider(ExtendedHttpClientProvider.class)
|
.setBulkClientProvider(HttpBulkClientProvider.class)
|
||||||
.put(helper.getHttpSettings())
|
.put(helper.getHttpSettings())
|
||||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
||||||
.build();
|
.build();
|
||||||
|
@ -98,20 +88,16 @@ class ClientTest {
|
||||||
.endObject()
|
.endObject()
|
||||||
.endObject()
|
.endObject()
|
||||||
.endObject();
|
.endObject();
|
||||||
client.newIndex("test", Settings.EMPTY, Strings.toString(builder));
|
client.newIndex("test", Settings.EMPTY, builder);
|
||||||
GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test");
|
assertTrue(client.getMapping("test", "doc").containsKey("properties"));
|
||||||
GetMappingsResponse getMappingsResponse =
|
|
||||||
client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
|
|
||||||
logger.info("mappings={}", getMappingsResponse.getMappings());
|
|
||||||
assertTrue(getMappingsResponse.getMappings().get("test").containsKey("doc"));
|
|
||||||
client.close();
|
client.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testRandomDocs() throws Exception {
|
void testRandomDocs() throws Exception {
|
||||||
long numactions = ACTIONS;
|
long numactions = ACTIONS;
|
||||||
final ExtendedHttpClient client = ClientBuilder.builder()
|
final HttpBulkClient client = ClientBuilder.builder()
|
||||||
.provider(ExtendedHttpClientProvider.class)
|
.setBulkClientProvider(HttpBulkClientProvider.class)
|
||||||
.put(helper.getHttpSettings())
|
.put(helper.getHttpSettings())
|
||||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
||||||
|
@ -123,10 +109,6 @@ class ClientTest {
|
||||||
}
|
}
|
||||||
client.flush();
|
client.flush();
|
||||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
client.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error(e.getMessage(), e);
|
|
||||||
} finally {
|
} finally {
|
||||||
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
|
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
if (client.getBulkController().getLastBulkError() != null) {
|
||||||
|
@ -134,10 +116,7 @@ class ClientTest {
|
||||||
}
|
}
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
assertNull(client.getBulkController().getLastBulkError());
|
||||||
client.refreshIndex("test");
|
client.refreshIndex("test");
|
||||||
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
|
assertEquals(numactions, client.getSearchableDocs("test"));
|
||||||
.setQuery(QueryBuilders.matchAllQuery()).setSize(0);
|
|
||||||
assertEquals(numactions,
|
|
||||||
searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
|
|
||||||
client.close();
|
client.close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -146,10 +125,10 @@ class ClientTest {
|
||||||
void testThreadedRandomDocs() throws Exception {
|
void testThreadedRandomDocs() throws Exception {
|
||||||
int maxthreads = Runtime.getRuntime().availableProcessors();
|
int maxthreads = Runtime.getRuntime().availableProcessors();
|
||||||
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
|
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
|
||||||
final Long actions = ACTIONS;
|
final long actions = ACTIONS;
|
||||||
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
|
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
|
||||||
final ExtendedHttpClient client = ClientBuilder.builder()
|
final HttpBulkClient bulkClient = ClientBuilder.builder()
|
||||||
.provider(ExtendedHttpClientProvider.class)
|
.setBulkClientProvider(HttpBulkClientProvider.class)
|
||||||
.put(helper.getHttpSettings())
|
.put(helper.getHttpSettings())
|
||||||
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
|
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
|
||||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
|
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
|
||||||
|
@ -160,15 +139,15 @@ class ClientTest {
|
||||||
.put("index.number_of_shards", 1)
|
.put("index.number_of_shards", 1)
|
||||||
.put("index.number_of_replicas", 0)
|
.put("index.number_of_replicas", 0)
|
||||||
.build();
|
.build();
|
||||||
client.newIndex("test", settings, (String)null)
|
bulkClient.newIndex("test", settings);
|
||||||
.startBulk("test", 0, 1000);
|
bulkClient.startBulk("test", 0, 1000);
|
||||||
logger.info("index created");
|
logger.info("index created");
|
||||||
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
|
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
|
||||||
final CountDownLatch latch = new CountDownLatch(maxthreads);
|
final CountDownLatch latch = new CountDownLatch(maxthreads);
|
||||||
for (int i = 0; i < maxthreads; i++) {
|
for (int i = 0; i < maxthreads; i++) {
|
||||||
executorService.execute(() -> {
|
executorService.execute(() -> {
|
||||||
for (int i1 = 0; i1 < actions; i1++) {
|
for (int i1 = 0; i1 < actions; i1++) {
|
||||||
client.index("test", null, false,"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
bulkClient.index("test", null, false,"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||||
}
|
}
|
||||||
latch.countDown();
|
latch.countDown();
|
||||||
});
|
});
|
||||||
|
@ -176,8 +155,8 @@ class ClientTest {
|
||||||
logger.info("waiting for latch...");
|
logger.info("waiting for latch...");
|
||||||
if (latch.await(60L, TimeUnit.SECONDS)) {
|
if (latch.await(60L, TimeUnit.SECONDS)) {
|
||||||
logger.info("flush...");
|
logger.info("flush...");
|
||||||
client.flush();
|
bulkClient.flush();
|
||||||
client.waitForResponses(60L, TimeUnit.SECONDS);
|
bulkClient.waitForResponses(60L, TimeUnit.SECONDS);
|
||||||
logger.info("got all responses, executor service shutdown...");
|
logger.info("got all responses, executor service shutdown...");
|
||||||
executorService.shutdown();
|
executorService.shutdown();
|
||||||
executorService.awaitTermination(60L, TimeUnit.SECONDS);
|
executorService.awaitTermination(60L, TimeUnit.SECONDS);
|
||||||
|
@ -185,23 +164,16 @@ class ClientTest {
|
||||||
} else {
|
} else {
|
||||||
logger.warn("latch timeout");
|
logger.warn("latch timeout");
|
||||||
}
|
}
|
||||||
client.stopBulk("test", 30L, TimeUnit.SECONDS);
|
bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS);
|
||||||
assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount());
|
assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error(e.getMessage(), e);
|
|
||||||
} finally {
|
} finally {
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
}
|
}
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
client.refreshIndex("test");
|
bulkClient.refreshIndex("test");
|
||||||
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
|
assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test"));
|
||||||
.setQuery(QueryBuilders.matchAllQuery()).setSize(0);
|
bulkClient.close();
|
||||||
assertEquals(maxthreads * actions,
|
|
||||||
searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
|
|
||||||
client.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -2,19 +2,15 @@ package org.xbib.elx.http.test;
|
||||||
|
|
||||||
import org.apache.logging.log4j.LogManager;
|
import org.apache.logging.log4j.LogManager;
|
||||||
import org.apache.logging.log4j.Logger;
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.elasticsearch.action.search.SearchAction;
|
|
||||||
import org.elasticsearch.action.search.SearchRequestBuilder;
|
|
||||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
import org.xbib.elx.common.ClientBuilder;
|
import org.xbib.elx.common.ClientBuilder;
|
||||||
import org.xbib.elx.common.Parameters;
|
import org.xbib.elx.common.Parameters;
|
||||||
import org.xbib.elx.http.ExtendedHttpClient;
|
import org.xbib.elx.http.HttpBulkClient;
|
||||||
import org.xbib.elx.http.ExtendedHttpClientProvider;
|
import org.xbib.elx.http.HttpBulkClientProvider;
|
||||||
|
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
@ -24,9 +20,9 @@ class DuplicateIDTest {
|
||||||
|
|
||||||
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getSimpleName());
|
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getSimpleName());
|
||||||
|
|
||||||
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
|
private static final Long ACTIONS = 100L;
|
||||||
|
|
||||||
private static final Long ACTIONS = 50L;
|
private static final Long MAX_ACTIONS_PER_REQUEST = 5L;
|
||||||
|
|
||||||
private final TestExtension.Helper helper;
|
private final TestExtension.Helper helper;
|
||||||
|
|
||||||
|
@ -37,8 +33,8 @@ class DuplicateIDTest {
|
||||||
@Test
|
@Test
|
||||||
void testDuplicateDocIDs() throws Exception {
|
void testDuplicateDocIDs() throws Exception {
|
||||||
long numactions = ACTIONS;
|
long numactions = ACTIONS;
|
||||||
final ExtendedHttpClient client = ClientBuilder.builder()
|
final HttpBulkClient client = ClientBuilder.builder()
|
||||||
.provider(ExtendedHttpClientProvider.class)
|
.setBulkClientProvider(HttpBulkClientProvider.class)
|
||||||
.put(helper.getHttpSettings())
|
.put(helper.getHttpSettings())
|
||||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||||
.build();
|
.build();
|
||||||
|
@ -51,15 +47,8 @@ class DuplicateIDTest {
|
||||||
client.flush();
|
client.flush();
|
||||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
client.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
client.refreshIndex("test");
|
client.refreshIndex("test");
|
||||||
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
|
long hits = client.getSearchableDocs("test");
|
||||||
.setIndices("test")
|
|
||||||
.setTypes("test")
|
|
||||||
.setQuery(matchAllQuery());
|
|
||||||
long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
|
|
||||||
logger.info("hits = {}", hits);
|
|
||||||
assertTrue(hits < ACTIONS);
|
assertTrue(hits < ACTIONS);
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} finally {
|
} finally {
|
||||||
client.close();
|
client.close();
|
||||||
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
|
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
|
||||||
|
|
|
@ -2,17 +2,15 @@ package org.xbib.elx.http.test;
|
||||||
|
|
||||||
import org.apache.logging.log4j.LogManager;
|
import org.apache.logging.log4j.LogManager;
|
||||||
import org.apache.logging.log4j.Logger;
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
|
|
||||||
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
|
|
||||||
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
|
|
||||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
import org.xbib.elx.api.IndexPruneResult;
|
import org.xbib.elx.api.IndexPruneResult;
|
||||||
import org.xbib.elx.common.ClientBuilder;
|
import org.xbib.elx.common.ClientBuilder;
|
||||||
import org.xbib.elx.http.ExtendedHttpClient;
|
import org.xbib.elx.http.HttpAdminClient;
|
||||||
import org.xbib.elx.http.ExtendedHttpClientProvider;
|
import org.xbib.elx.http.HttpAdminClientProvider;
|
||||||
|
import org.xbib.elx.http.HttpBulkClient;
|
||||||
|
import org.xbib.elx.http.HttpBulkClientProvider;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
@ -37,50 +35,49 @@ class IndexPruneTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testPrune() throws IOException {
|
void testPrune() throws IOException {
|
||||||
final ExtendedHttpClient client = ClientBuilder.builder(helper.client("1"))
|
final HttpAdminClient adminClient = ClientBuilder.builder()
|
||||||
|
.setAdminClientProvider(HttpAdminClientProvider.class)
|
||||||
|
.put(helper.getHttpSettings())
|
||||||
|
.build();
|
||||||
|
final HttpBulkClient bulkClient = ClientBuilder.builder()
|
||||||
|
.setBulkClientProvider(HttpBulkClientProvider.class)
|
||||||
.put(helper.getHttpSettings())
|
.put(helper.getHttpSettings())
|
||||||
.provider(ExtendedHttpClientProvider.class)
|
|
||||||
.build();
|
.build();
|
||||||
try {
|
try {
|
||||||
Settings settings = Settings.builder()
|
Settings settings = Settings.builder()
|
||||||
.put("index.number_of_shards", 1)
|
.put("index.number_of_shards", 1)
|
||||||
.put("index.number_of_replicas", 0)
|
.put("index.number_of_replicas", 0)
|
||||||
.build();
|
.build();
|
||||||
client.newIndex("test1", settings);
|
bulkClient.newIndex("test1", settings);
|
||||||
client.shiftIndex("test", "test1", Collections.emptyList());
|
adminClient.shiftIndex("test", "test1", Collections.emptyList());
|
||||||
client.newIndex("test2", settings);
|
bulkClient.newIndex("test2", settings);
|
||||||
client.shiftIndex("test", "test2", Collections.emptyList());
|
adminClient.shiftIndex("test", "test2", Collections.emptyList());
|
||||||
client.newIndex("test3", settings);
|
bulkClient.newIndex("test3", settings);
|
||||||
client.shiftIndex("test", "test3", Collections.emptyList());
|
adminClient.shiftIndex("test", "test3", Collections.emptyList());
|
||||||
client.newIndex("test4", settings);
|
bulkClient.newIndex("test4", settings);
|
||||||
client.shiftIndex("test", "test4", Collections.emptyList());
|
adminClient.shiftIndex("test", "test4", Collections.emptyList());
|
||||||
IndexPruneResult indexPruneResult =
|
IndexPruneResult indexPruneResult =
|
||||||
client.pruneIndex("test", "test4", 2, 2, true);
|
adminClient.pruneIndex("test", "test4", 2, 2, true);
|
||||||
assertTrue(indexPruneResult.getDeletedIndices().contains("test1"));
|
assertTrue(indexPruneResult.getDeletedIndices().contains("test1"));
|
||||||
assertTrue(indexPruneResult.getDeletedIndices().contains("test2"));
|
assertTrue(indexPruneResult.getDeletedIndices().contains("test2"));
|
||||||
assertFalse(indexPruneResult.getDeletedIndices().contains("test3"));
|
assertFalse(indexPruneResult.getDeletedIndices().contains("test3"));
|
||||||
assertFalse(indexPruneResult.getDeletedIndices().contains("test4"));
|
assertFalse(indexPruneResult.getDeletedIndices().contains("test4"));
|
||||||
List<Boolean> list = new ArrayList<>();
|
List<Boolean> list = new ArrayList<>();
|
||||||
for (String index : Arrays.asList("test1", "test2", "test3", "test4")) {
|
for (String index : Arrays.asList("test1", "test2", "test3", "test4")) {
|
||||||
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
|
list.add(bulkClient.isIndexExists(index));
|
||||||
indicesExistsRequest.indices(index);
|
|
||||||
IndicesExistsResponse indicesExistsResponse =
|
|
||||||
client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
|
|
||||||
list.add(indicesExistsResponse.isExists());
|
|
||||||
}
|
}
|
||||||
logger.info(list);
|
logger.info(list);
|
||||||
assertFalse(list.get(0));
|
assertFalse(list.get(0));
|
||||||
assertFalse(list.get(1));
|
assertFalse(list.get(1));
|
||||||
assertTrue(list.get(2));
|
assertTrue(list.get(2));
|
||||||
assertTrue(list.get(3));
|
assertTrue(list.get(3));
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} finally {
|
} finally {
|
||||||
client.close();
|
bulkClient.close();
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
}
|
}
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
|
adminClient.close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,16 +4,16 @@ import org.apache.logging.log4j.Level;
|
||||||
import org.apache.logging.log4j.LogManager;
|
import org.apache.logging.log4j.LogManager;
|
||||||
import org.apache.logging.log4j.Logger;
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
|
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
|
||||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.index.query.QueryBuilders;
|
import org.elasticsearch.index.query.QueryBuilders;
|
||||||
import org.junit.jupiter.api.Disabled;
|
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
import org.xbib.elx.api.IndexShiftResult;
|
import org.xbib.elx.api.IndexShiftResult;
|
||||||
import org.xbib.elx.common.ClientBuilder;
|
import org.xbib.elx.common.ClientBuilder;
|
||||||
import org.xbib.elx.http.ExtendedHttpClient;
|
import org.xbib.elx.http.HttpAdminClient;
|
||||||
import org.xbib.elx.http.ExtendedHttpClientProvider;
|
import org.xbib.elx.http.HttpAdminClientProvider;
|
||||||
|
import org.xbib.elx.http.HttpBulkClient;
|
||||||
|
import org.xbib.elx.http.HttpBulkClientProvider;
|
||||||
|
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
@ -35,56 +35,54 @@ class IndexShiftTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testIndexShift() throws Exception {
|
void testIndexShift() throws Exception {
|
||||||
final ExtendedHttpClient client = ClientBuilder.builder()
|
final HttpAdminClient adminClient = ClientBuilder.builder()
|
||||||
|
.setAdminClientProvider(HttpAdminClientProvider.class)
|
||||||
|
.put(helper.getHttpSettings())
|
||||||
|
.build();
|
||||||
|
final HttpBulkClient bulkClient = ClientBuilder.builder()
|
||||||
|
.setBulkClientProvider(HttpBulkClientProvider.class)
|
||||||
.put(helper.getHttpSettings())
|
.put(helper.getHttpSettings())
|
||||||
.provider(ExtendedHttpClientProvider.class)
|
|
||||||
.build();
|
.build();
|
||||||
try {
|
try {
|
||||||
Settings settings = Settings.builder()
|
Settings settings = Settings.builder()
|
||||||
.put("index.number_of_shards", 1)
|
.put("index.number_of_shards", 1)
|
||||||
.put("index.number_of_replicas", 0)
|
.put("index.number_of_replicas", 0)
|
||||||
.build();
|
.build();
|
||||||
client.newIndex("test1234", settings);
|
bulkClient.newIndex("test1234", settings);
|
||||||
for (int i = 0; i < 1; i++) {
|
for (int i = 0; i < 1; i++) {
|
||||||
client.index("test1234", helper.randomString(1), false,
|
bulkClient.index("test1234", helper.randomString(1), false,
|
||||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||||
}
|
}
|
||||||
client.flush();
|
bulkClient.flush();
|
||||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
|
|
||||||
IndexShiftResult indexShiftResult =
|
IndexShiftResult indexShiftResult =
|
||||||
client.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c"));
|
adminClient.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c"));
|
||||||
|
|
||||||
assertTrue(indexShiftResult.getNewAliases().contains("a"));
|
assertTrue(indexShiftResult.getNewAliases().contains("a"));
|
||||||
assertTrue(indexShiftResult.getNewAliases().contains("b"));
|
assertTrue(indexShiftResult.getNewAliases().contains("b"));
|
||||||
assertTrue(indexShiftResult.getNewAliases().contains("c"));
|
assertTrue(indexShiftResult.getNewAliases().contains("c"));
|
||||||
assertTrue(indexShiftResult.getMovedAliases().isEmpty());
|
assertTrue(indexShiftResult.getMovedAliases().isEmpty());
|
||||||
|
Map<String, String> aliases = adminClient.getAliases("test1234");
|
||||||
Map<String, String> aliases = client.getAliases("test1234");
|
|
||||||
logger.log(Level.DEBUG, "aliases = " + aliases);
|
logger.log(Level.DEBUG, "aliases = " + aliases);
|
||||||
assertTrue(aliases.containsKey("a"));
|
assertTrue(aliases.containsKey("a"));
|
||||||
assertTrue(aliases.containsKey("b"));
|
assertTrue(aliases.containsKey("b"));
|
||||||
assertTrue(aliases.containsKey("c"));
|
assertTrue(aliases.containsKey("c"));
|
||||||
assertTrue(aliases.containsKey("test"));
|
assertTrue(aliases.containsKey("test"));
|
||||||
|
String resolved = adminClient.resolveAlias("test");
|
||||||
String resolved = client.resolveAlias("test");
|
|
||||||
logger.log(Level.DEBUG, "resolved = " + resolved);
|
logger.log(Level.DEBUG, "resolved = " + resolved);
|
||||||
aliases = client.getAliases(resolved);
|
aliases = adminClient.getAliases(resolved);
|
||||||
logger.log(Level.DEBUG, "aliases = " + aliases);
|
logger.log(Level.DEBUG, "aliases = " + aliases);
|
||||||
assertTrue(aliases.containsKey("a"));
|
assertTrue(aliases.containsKey("a"));
|
||||||
assertTrue(aliases.containsKey("b"));
|
assertTrue(aliases.containsKey("b"));
|
||||||
assertTrue(aliases.containsKey("c"));
|
assertTrue(aliases.containsKey("c"));
|
||||||
assertTrue(aliases.containsKey("test"));
|
assertTrue(aliases.containsKey("test"));
|
||||||
|
bulkClient.newIndex("test5678", settings);
|
||||||
client.newIndex("test5678", settings);
|
|
||||||
for (int i = 0; i < 1; i++) {
|
for (int i = 0; i < 1; i++) {
|
||||||
client.index("test5678", helper.randomString(1), false,
|
bulkClient.index("test5678", helper.randomString(1), false,
|
||||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||||
}
|
}
|
||||||
client.flush();
|
bulkClient.flush();
|
||||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
|
indexShiftResult = adminClient.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"),
|
||||||
indexShiftResult = client.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"),
|
|
||||||
(request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
|
(request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
|
||||||
.index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
|
.index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
|
||||||
);
|
);
|
||||||
|
@ -94,31 +92,28 @@ class IndexShiftTest {
|
||||||
assertTrue(indexShiftResult.getMovedAliases().contains("a"));
|
assertTrue(indexShiftResult.getMovedAliases().contains("a"));
|
||||||
assertTrue(indexShiftResult.getMovedAliases().contains("b"));
|
assertTrue(indexShiftResult.getMovedAliases().contains("b"));
|
||||||
assertTrue(indexShiftResult.getMovedAliases().contains("c"));
|
assertTrue(indexShiftResult.getMovedAliases().contains("c"));
|
||||||
|
aliases = adminClient.getAliases("test5678");
|
||||||
aliases = client.getAliases("test5678");
|
|
||||||
assertTrue(aliases.containsKey("a"));
|
assertTrue(aliases.containsKey("a"));
|
||||||
assertTrue(aliases.containsKey("b"));
|
assertTrue(aliases.containsKey("b"));
|
||||||
assertTrue(aliases.containsKey("c"));
|
assertTrue(aliases.containsKey("c"));
|
||||||
assertTrue(aliases.containsKey("d"));
|
assertTrue(aliases.containsKey("d"));
|
||||||
assertTrue(aliases.containsKey("e"));
|
assertTrue(aliases.containsKey("e"));
|
||||||
assertTrue(aliases.containsKey("f"));
|
assertTrue(aliases.containsKey("f"));
|
||||||
|
resolved = adminClient.resolveAlias("test");
|
||||||
resolved = client.resolveAlias("test");
|
aliases = adminClient.getAliases(resolved);
|
||||||
aliases = client.getAliases(resolved);
|
|
||||||
assertTrue(aliases.containsKey("a"));
|
assertTrue(aliases.containsKey("a"));
|
||||||
assertTrue(aliases.containsKey("b"));
|
assertTrue(aliases.containsKey("b"));
|
||||||
assertTrue(aliases.containsKey("c"));
|
assertTrue(aliases.containsKey("c"));
|
||||||
assertTrue(aliases.containsKey("d"));
|
assertTrue(aliases.containsKey("d"));
|
||||||
assertTrue(aliases.containsKey("e"));
|
assertTrue(aliases.containsKey("e"));
|
||||||
assertTrue(aliases.containsKey("f"));
|
assertTrue(aliases.containsKey("f"));
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} finally {
|
} finally {
|
||||||
client.close();
|
bulkClient.close();
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
}
|
}
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
|
adminClient.close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,14 +2,15 @@ package org.xbib.elx.http.test;
|
||||||
|
|
||||||
import org.apache.logging.log4j.LogManager;
|
import org.apache.logging.log4j.LogManager;
|
||||||
import org.apache.logging.log4j.Logger;
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
import org.xbib.elx.api.IndexDefinition;
|
import org.xbib.elx.api.IndexDefinition;
|
||||||
import org.xbib.elx.common.ClientBuilder;
|
import org.xbib.elx.common.ClientBuilder;
|
||||||
import org.xbib.elx.http.ExtendedHttpClient;
|
import org.xbib.elx.http.HttpAdminClient;
|
||||||
import org.xbib.elx.http.ExtendedHttpClientProvider;
|
import org.xbib.elx.http.HttpAdminClientProvider;
|
||||||
|
import org.xbib.elx.http.HttpBulkClient;
|
||||||
|
import org.xbib.elx.http.HttpBulkClientProvider;
|
||||||
|
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
@ -29,47 +30,51 @@ class SmokeTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void smokeTest() throws Exception {
|
void smokeTest() throws Exception {
|
||||||
final ExtendedHttpClient client = ClientBuilder.builder()
|
final HttpAdminClient adminClient = ClientBuilder.builder()
|
||||||
.provider(ExtendedHttpClientProvider.class)
|
.setAdminClientProvider(HttpAdminClientProvider.class)
|
||||||
.put(helper.getHttpSettings())
|
.put(helper.getHttpSettings())
|
||||||
.build();
|
.build();
|
||||||
try {
|
final HttpBulkClient bulkClient = ClientBuilder.builder()
|
||||||
assertEquals(helper.getClusterName(), client.getClusterName());
|
.setBulkClientProvider(HttpBulkClientProvider.class)
|
||||||
client.newIndex("test_smoke");
|
.put(helper.getHttpSettings())
|
||||||
client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
.build();
|
||||||
client.flush();
|
IndexDefinition indexDefinition = adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.builder()
|
||||||
client.waitForResponses(30, TimeUnit.SECONDS);
|
|
||||||
client.checkMapping("test_smoke");
|
|
||||||
client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
|
|
||||||
client.delete("test_smoke", "1");
|
|
||||||
client.flush();
|
|
||||||
client.waitForResponses(30, TimeUnit.SECONDS);
|
|
||||||
client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS);
|
|
||||||
client.delete("test_smoke", "1");
|
|
||||||
client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
|
|
||||||
client.flush();
|
|
||||||
client.deleteIndex("test_smoke");
|
|
||||||
IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder()
|
|
||||||
.build());
|
.build());
|
||||||
|
try {
|
||||||
|
assertEquals(helper.getClusterName(), adminClient.getClusterName());
|
||||||
|
bulkClient.newIndex("test_smoke");
|
||||||
|
bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
||||||
|
bulkClient.flush();
|
||||||
|
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
|
||||||
|
adminClient.checkMapping("test_smoke");
|
||||||
|
bulkClient.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
|
||||||
|
bulkClient.delete("test_smoke", "1");
|
||||||
|
bulkClient.flush();
|
||||||
|
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
|
||||||
|
adminClient.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS);
|
||||||
|
bulkClient.delete("test_smoke", "1");
|
||||||
|
bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
|
||||||
|
bulkClient.flush();
|
||||||
|
adminClient.deleteIndex("test_smoke");
|
||||||
assertEquals(0, indexDefinition.getReplicaLevel());
|
assertEquals(0, indexDefinition.getReplicaLevel());
|
||||||
client.newIndex(indexDefinition);
|
bulkClient.newIndex(indexDefinition);
|
||||||
client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
|
bulkClient.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
|
||||||
client.flush();
|
bulkClient.flush();
|
||||||
client.waitForResponses(30, TimeUnit.SECONDS);
|
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
|
||||||
client.updateReplicaLevel(indexDefinition, 2);
|
adminClient.updateReplicaLevel(indexDefinition, 2);
|
||||||
int replica = client.getReplicaLevel(indexDefinition);
|
int replica = adminClient.getReplicaLevel(indexDefinition);
|
||||||
assertEquals(2, replica);
|
assertEquals(2, replica);
|
||||||
client.deleteIndex(indexDefinition);
|
|
||||||
assertEquals(0, client.getBulkMetric().getFailed().getCount());
|
|
||||||
assertEquals(6, client.getBulkMetric().getSucceeded().getCount());
|
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} finally {
|
} finally {
|
||||||
client.close();
|
bulkClient.close();
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
}
|
}
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount());
|
||||||
|
assertEquals(6, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||||
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
|
// close admin after bulk
|
||||||
|
adminClient.deleteIndex(indexDefinition);
|
||||||
|
adminClient.close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,10 +0,0 @@
|
||||||
package org.xbib.elx.node;
|
|
||||||
|
|
||||||
import org.xbib.elx.api.ExtendedClientProvider;
|
|
||||||
|
|
||||||
public class ExtendedNodeClientProvider implements ExtendedClientProvider<ExtendedNodeClient> {
|
|
||||||
@Override
|
|
||||||
public ExtendedNodeClient getExtendedClient() {
|
|
||||||
return new ExtendedNodeClient();
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -11,15 +11,15 @@ import org.elasticsearch.common.xcontent.XContentFactory;
|
||||||
import org.elasticsearch.env.Environment;
|
import org.elasticsearch.env.Environment;
|
||||||
import org.elasticsearch.node.Node;
|
import org.elasticsearch.node.Node;
|
||||||
import org.elasticsearch.plugins.Plugin;
|
import org.elasticsearch.plugins.Plugin;
|
||||||
import org.xbib.elx.common.AbstractExtendedClient;
|
import org.xbib.elx.common.AbstractAdminClient;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
|
|
||||||
public class ExtendedNodeClient extends AbstractExtendedClient {
|
public class NodeAdminClient extends AbstractAdminClient {
|
||||||
|
|
||||||
private static final Logger logger = LogManager.getLogger(ExtendedNodeClient.class.getName());
|
private static final Logger logger = LogManager.getLogger(NodeAdminClient.class.getName());
|
||||||
|
|
||||||
private Node node;
|
private Node node;
|
||||||
|
|
|
@ -0,0 +1,11 @@
|
||||||
|
package org.xbib.elx.node;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.AdminClientProvider;
|
||||||
|
|
||||||
|
public class NodeAdminClientProvider implements AdminClientProvider<NodeAdminClient> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public NodeAdminClient getClient() {
|
||||||
|
return new NodeAdminClient();
|
||||||
|
}
|
||||||
|
}
|
68
elx-node/src/main/java/org/xbib/elx/node/NodeBulkClient.java
Normal file
68
elx-node/src/main/java/org/xbib/elx/node/NodeBulkClient.java
Normal file
|
@ -0,0 +1,68 @@
|
||||||
|
package org.xbib.elx.node;
|
||||||
|
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.common.Strings;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.common.xcontent.ToXContent;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||||
|
import org.elasticsearch.env.Environment;
|
||||||
|
import org.elasticsearch.node.Node;
|
||||||
|
import org.elasticsearch.plugins.Plugin;
|
||||||
|
import org.xbib.elx.common.AbstractBulkClient;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.Collections;
|
||||||
|
|
||||||
|
public class NodeBulkClient extends AbstractBulkClient {
|
||||||
|
|
||||||
|
private static final Logger logger = LogManager.getLogger(NodeBulkClient.class.getName());
|
||||||
|
|
||||||
|
private Node node;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||||
|
if (settings == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String version = System.getProperty("os.name")
|
||||||
|
+ " " + System.getProperty("java.vm.name")
|
||||||
|
+ " " + System.getProperty("java.vm.vendor")
|
||||||
|
+ " " + System.getProperty("java.runtime.version")
|
||||||
|
+ " " + System.getProperty("java.vm.version");
|
||||||
|
Settings effectiveSettings = Settings.builder().put(settings)
|
||||||
|
.put("node.client", true)
|
||||||
|
.put("node.master", false)
|
||||||
|
.put("node.data", false)
|
||||||
|
.build();
|
||||||
|
XContentBuilder builder = XContentFactory.jsonBuilder();
|
||||||
|
effectiveSettings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true")));
|
||||||
|
logger.info("creating node client on {} with effective settings {}",
|
||||||
|
version, Strings.toString(builder));
|
||||||
|
Collection<Class<? extends Plugin>> plugins = Collections.emptyList();
|
||||||
|
this.node = new BulkNode(new Environment(effectiveSettings, null), plugins);
|
||||||
|
try {
|
||||||
|
node.start();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
return node.client();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() throws IOException {
|
||||||
|
if (node != null) {
|
||||||
|
logger.debug("closing node client");
|
||||||
|
node.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static class BulkNode extends Node {
|
||||||
|
|
||||||
|
BulkNode(Environment env, Collection<Class<? extends Plugin>> classpathPlugins) {
|
||||||
|
super(env, classpathPlugins);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,11 @@
|
||||||
|
package org.xbib.elx.node;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.BulkClientProvider;
|
||||||
|
|
||||||
|
public class NodeBulkClientProvider implements BulkClientProvider<NodeBulkClient> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public NodeBulkClient getClient() {
|
||||||
|
return new NodeBulkClient();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,68 @@
|
||||||
|
package org.xbib.elx.node;
|
||||||
|
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.common.Strings;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.common.xcontent.ToXContent;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||||
|
import org.elasticsearch.env.Environment;
|
||||||
|
import org.elasticsearch.node.Node;
|
||||||
|
import org.elasticsearch.plugins.Plugin;
|
||||||
|
import org.xbib.elx.common.AbstractSearchClient;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.Collections;
|
||||||
|
|
||||||
|
public class NodeSearchClient extends AbstractSearchClient {
|
||||||
|
|
||||||
|
private static final Logger logger = LogManager.getLogger(NodeSearchClient.class.getName());
|
||||||
|
|
||||||
|
private Node node;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||||
|
if (settings == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String version = System.getProperty("os.name")
|
||||||
|
+ " " + System.getProperty("java.vm.name")
|
||||||
|
+ " " + System.getProperty("java.vm.vendor")
|
||||||
|
+ " " + System.getProperty("java.runtime.version")
|
||||||
|
+ " " + System.getProperty("java.vm.version");
|
||||||
|
Settings effectiveSettings = Settings.builder().put(settings)
|
||||||
|
.put("node.client", true)
|
||||||
|
.put("node.master", false)
|
||||||
|
.put("node.data", false)
|
||||||
|
.build();
|
||||||
|
XContentBuilder builder = XContentFactory.jsonBuilder();
|
||||||
|
effectiveSettings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true")));
|
||||||
|
logger.info("creating node client on {} with effective settings {}",
|
||||||
|
version, Strings.toString(builder));
|
||||||
|
Collection<Class<? extends Plugin>> plugins = Collections.emptyList();
|
||||||
|
this.node = new BulkNode(new Environment(effectiveSettings, null), plugins);
|
||||||
|
try {
|
||||||
|
node.start();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
return node.client();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() throws IOException {
|
||||||
|
if (node != null) {
|
||||||
|
logger.debug("closing node client");
|
||||||
|
node.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static class BulkNode extends Node {
|
||||||
|
|
||||||
|
BulkNode(Environment env, Collection<Class<? extends Plugin>> classpathPlugins) {
|
||||||
|
super(env, classpathPlugins);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,11 @@
|
||||||
|
package org.xbib.elx.node;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.SearchClientProvider;
|
||||||
|
|
||||||
|
public class NodeSearchClientProvider implements SearchClientProvider<NodeSearchClient> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public NodeSearchClient getClient() {
|
||||||
|
return new NodeSearchClient();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1 @@
|
||||||
|
org.xbib.elx.node.NodeAdminClientProvider
|
|
@ -0,0 +1 @@
|
||||||
|
org.xbib.elx.node.NodeBulkClientProvider
|
|
@ -1 +0,0 @@
|
||||||
org.xbib.elx.node.ExtendedNodeClientProvider
|
|
|
@ -0,0 +1 @@
|
||||||
|
org.xbib.elx.node.NodeSearchClientProvider
|
|
@ -0,0 +1,174 @@
|
||||||
|
package org.xbib.elx.node.test;
|
||||||
|
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.common.unit.TimeValue;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
|
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.xbib.elx.common.ClientBuilder;
|
||||||
|
import org.xbib.elx.common.Parameters;
|
||||||
|
import org.xbib.elx.node.NodeBulkClient;
|
||||||
|
import org.xbib.elx.node.NodeBulkClientProvider;
|
||||||
|
|
||||||
|
import java.util.concurrent.CountDownLatch;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
@ExtendWith(TestExtension.class)
|
||||||
|
class BulkClientTest {
|
||||||
|
|
||||||
|
private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getName());
|
||||||
|
|
||||||
|
private static final Long ACTIONS = 10000L;
|
||||||
|
|
||||||
|
private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
|
||||||
|
|
||||||
|
private final TestExtension.Helper helper;
|
||||||
|
|
||||||
|
BulkClientTest(TestExtension.Helper helper) {
|
||||||
|
this.helper = helper;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testSingleDoc() throws Exception {
|
||||||
|
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||||
|
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||||
|
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||||
|
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
|
||||||
|
.build();
|
||||||
|
try {
|
||||||
|
bulkClient.newIndex("test");
|
||||||
|
bulkClient.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
||||||
|
bulkClient.flush();
|
||||||
|
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
|
} finally {
|
||||||
|
assertEquals(1, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||||
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
|
}
|
||||||
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
|
bulkClient.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testNewIndex() throws Exception {
|
||||||
|
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||||
|
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||||
|
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
||||||
|
.build();
|
||||||
|
bulkClient.newIndex("test");
|
||||||
|
bulkClient.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testMapping() throws Exception {
|
||||||
|
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||||
|
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||||
|
.build();
|
||||||
|
XContentBuilder builder = JsonXContent.contentBuilder()
|
||||||
|
.startObject()
|
||||||
|
.startObject("doc")
|
||||||
|
.startObject("properties")
|
||||||
|
.startObject("location")
|
||||||
|
.field("type", "geo_point")
|
||||||
|
.endObject()
|
||||||
|
.endObject()
|
||||||
|
.endObject()
|
||||||
|
.endObject();
|
||||||
|
bulkClient.newIndex("test", Settings.EMPTY, builder);
|
||||||
|
assertTrue(bulkClient.getMapping("test", "doc").containsKey("properties"));
|
||||||
|
bulkClient.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testRandomDocs() throws Exception {
|
||||||
|
long numactions = ACTIONS;
|
||||||
|
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||||
|
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||||
|
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||||
|
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
||||||
|
.build();
|
||||||
|
try {
|
||||||
|
bulkClient.newIndex("test");
|
||||||
|
for (int i = 0; i < ACTIONS; i++) {
|
||||||
|
bulkClient.index("test", null, false,
|
||||||
|
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||||
|
}
|
||||||
|
bulkClient.flush();
|
||||||
|
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
|
} finally {
|
||||||
|
assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||||
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
|
}
|
||||||
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
|
bulkClient.refreshIndex("test");
|
||||||
|
assertEquals(numactions, bulkClient.getSearchableDocs("test"));
|
||||||
|
bulkClient.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testThreadedRandomDocs() throws Exception {
|
||||||
|
int maxthreads = Runtime.getRuntime().availableProcessors();
|
||||||
|
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
|
||||||
|
final long actions = ACTIONS;
|
||||||
|
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
|
||||||
|
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||||
|
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||||
|
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads)
|
||||||
|
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
|
||||||
|
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
||||||
|
.build();
|
||||||
|
try {
|
||||||
|
Settings settings = Settings.builder()
|
||||||
|
.put("index.number_of_shards", 1)
|
||||||
|
.put("index.number_of_replicas", 0)
|
||||||
|
.build();
|
||||||
|
bulkClient.newIndex("test", settings);
|
||||||
|
bulkClient.startBulk("test", 0, 1000);
|
||||||
|
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
|
||||||
|
final CountDownLatch latch = new CountDownLatch(maxthreads);
|
||||||
|
for (int i = 0; i < maxthreads; i++) {
|
||||||
|
executorService.execute(() -> {
|
||||||
|
for (int i1 = 0; i1 < actions; i1++) {
|
||||||
|
bulkClient.index("test", null, false,
|
||||||
|
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||||
|
}
|
||||||
|
latch.countDown();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
logger.info("waiting for latch...");
|
||||||
|
if (latch.await(30L, TimeUnit.SECONDS)) {
|
||||||
|
logger.info("flush...");
|
||||||
|
bulkClient.flush();
|
||||||
|
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
|
logger.info("got all responses, executor service shutdown...");
|
||||||
|
executorService.shutdown();
|
||||||
|
executorService.awaitTermination(30L, TimeUnit.SECONDS);
|
||||||
|
logger.info("pool is shut down");
|
||||||
|
} else {
|
||||||
|
logger.warn("latch timeout");
|
||||||
|
}
|
||||||
|
bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS);
|
||||||
|
assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||||
|
} finally {
|
||||||
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
|
}
|
||||||
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
|
bulkClient.refreshIndex("test");
|
||||||
|
assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test"));
|
||||||
|
bulkClient.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,200 +0,0 @@
|
||||||
package org.xbib.elx.node.test;
|
|
||||||
|
|
||||||
import org.apache.logging.log4j.LogManager;
|
|
||||||
import org.apache.logging.log4j.Logger;
|
|
||||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
|
|
||||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
|
|
||||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
|
|
||||||
import org.elasticsearch.action.search.SearchAction;
|
|
||||||
import org.elasticsearch.action.search.SearchRequestBuilder;
|
|
||||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
|
||||||
import org.elasticsearch.common.Strings;
|
|
||||||
import org.elasticsearch.common.settings.Settings;
|
|
||||||
import org.elasticsearch.common.unit.TimeValue;
|
|
||||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
|
||||||
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
|
||||||
import org.elasticsearch.index.query.QueryBuilders;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
|
||||||
import org.xbib.elx.common.ClientBuilder;
|
|
||||||
import org.xbib.elx.common.Parameters;
|
|
||||||
import org.xbib.elx.node.ExtendedNodeClient;
|
|
||||||
import org.xbib.elx.node.ExtendedNodeClientProvider;
|
|
||||||
|
|
||||||
import java.util.concurrent.CountDownLatch;
|
|
||||||
import java.util.concurrent.ExecutorService;
|
|
||||||
import java.util.concurrent.Executors;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
||||||
|
|
||||||
@ExtendWith(TestExtension.class)
|
|
||||||
class ClientTest {
|
|
||||||
|
|
||||||
private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
|
|
||||||
|
|
||||||
private static final Long ACTIONS = 10000L;
|
|
||||||
|
|
||||||
private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
|
|
||||||
|
|
||||||
private final TestExtension.Helper helper;
|
|
||||||
|
|
||||||
ClientTest(TestExtension.Helper helper) {
|
|
||||||
this.helper = helper;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testSingleDoc() throws Exception {
|
|
||||||
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
|
|
||||||
.provider(ExtendedNodeClientProvider.class)
|
|
||||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
|
||||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
|
|
||||||
.build();
|
|
||||||
try {
|
|
||||||
client.newIndex("test");
|
|
||||||
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
|
||||||
client.flush();
|
|
||||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} finally {
|
|
||||||
assertEquals(1, client.getBulkMetric().getSucceeded().getCount());
|
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
|
||||||
}
|
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
|
||||||
client.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testNewIndex() throws Exception {
|
|
||||||
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
|
|
||||||
.provider(ExtendedNodeClientProvider.class)
|
|
||||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
|
||||||
.build();
|
|
||||||
client.newIndex("test");
|
|
||||||
client.close();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testMapping() throws Exception {
|
|
||||||
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
|
|
||||||
.provider(ExtendedNodeClientProvider.class)
|
|
||||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
|
||||||
.build();
|
|
||||||
XContentBuilder builder = JsonXContent.contentBuilder()
|
|
||||||
.startObject()
|
|
||||||
.startObject("doc")
|
|
||||||
.startObject("properties")
|
|
||||||
.startObject("location")
|
|
||||||
.field("type", "geo_point")
|
|
||||||
.endObject()
|
|
||||||
.endObject()
|
|
||||||
.endObject()
|
|
||||||
.endObject();
|
|
||||||
client.newIndex("test", Settings.EMPTY, Strings.toString(builder));
|
|
||||||
GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test");
|
|
||||||
GetMappingsResponse getMappingsResponse =
|
|
||||||
client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
|
|
||||||
logger.info("mappings={}", getMappingsResponse.getMappings());
|
|
||||||
assertTrue(getMappingsResponse.getMappings().get("test").containsKey("doc"));
|
|
||||||
client.close();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testRandomDocs() throws Exception {
|
|
||||||
long numactions = ACTIONS;
|
|
||||||
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
|
|
||||||
.provider(ExtendedNodeClientProvider.class)
|
|
||||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
|
||||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
|
||||||
.build();
|
|
||||||
try {
|
|
||||||
client.newIndex("test");
|
|
||||||
for (int i = 0; i < ACTIONS; i++) {
|
|
||||||
client.index("test", null, false,
|
|
||||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
|
||||||
}
|
|
||||||
client.flush();
|
|
||||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} finally {
|
|
||||||
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
|
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
|
||||||
}
|
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
|
||||||
client.refreshIndex("test");
|
|
||||||
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
|
|
||||||
.setQuery(QueryBuilders.matchAllQuery()).setSize(0);
|
|
||||||
assertEquals(numactions,
|
|
||||||
searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
|
|
||||||
client.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testThreadedRandomDocs() throws Exception {
|
|
||||||
int maxthreads = Runtime.getRuntime().availableProcessors();
|
|
||||||
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
|
|
||||||
final long actions = ACTIONS;
|
|
||||||
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
|
|
||||||
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
|
|
||||||
.provider(ExtendedNodeClientProvider.class)
|
|
||||||
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads)
|
|
||||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
|
|
||||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
|
||||||
.build();
|
|
||||||
try {
|
|
||||||
Settings settings = Settings.builder()
|
|
||||||
.put("index.number_of_shards", 1)
|
|
||||||
.put("index.number_of_replicas", 0)
|
|
||||||
.build();
|
|
||||||
client.newIndex("test", settings, (String)null)
|
|
||||||
.startBulk("test", 0, 1000);
|
|
||||||
logger.info("index created");
|
|
||||||
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
|
|
||||||
final CountDownLatch latch = new CountDownLatch(maxthreads);
|
|
||||||
for (int i = 0; i < maxthreads; i++) {
|
|
||||||
executorService.execute(() -> {
|
|
||||||
for (int i1 = 0; i1 < actions; i1++) {
|
|
||||||
client.index("test", null, false,
|
|
||||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
|
||||||
}
|
|
||||||
latch.countDown();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
logger.info("waiting for latch...");
|
|
||||||
if (latch.await(30L, TimeUnit.SECONDS)) {
|
|
||||||
logger.info("flush...");
|
|
||||||
client.flush();
|
|
||||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
|
||||||
logger.info("got all responses, executor service shutdown...");
|
|
||||||
executorService.shutdown();
|
|
||||||
executorService.awaitTermination(30L, TimeUnit.SECONDS);
|
|
||||||
logger.info("pool is shut down");
|
|
||||||
} else {
|
|
||||||
logger.warn("latch timeout");
|
|
||||||
}
|
|
||||||
client.stopBulk("test", 30L, TimeUnit.SECONDS);
|
|
||||||
assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount());
|
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} finally {
|
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
|
||||||
}
|
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
|
||||||
client.refreshIndex("test");
|
|
||||||
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
|
|
||||||
.setQuery(QueryBuilders.matchAllQuery()).setSize(0);
|
|
||||||
assertEquals(maxthreads * actions,
|
|
||||||
searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
|
|
||||||
client.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -2,17 +2,12 @@ package org.xbib.elx.node.test;
|
||||||
|
|
||||||
import org.apache.logging.log4j.LogManager;
|
import org.apache.logging.log4j.LogManager;
|
||||||
import org.apache.logging.log4j.Logger;
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.elasticsearch.action.search.SearchAction;
|
|
||||||
import org.elasticsearch.action.search.SearchRequest;
|
|
||||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
|
||||||
import org.elasticsearch.index.query.QueryBuilders;
|
|
||||||
import org.elasticsearch.search.builder.SearchSourceBuilder;
|
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
import org.xbib.elx.common.ClientBuilder;
|
import org.xbib.elx.common.ClientBuilder;
|
||||||
import org.xbib.elx.common.Parameters;
|
import org.xbib.elx.common.Parameters;
|
||||||
import org.xbib.elx.node.ExtendedNodeClient;
|
import org.xbib.elx.node.NodeBulkClient;
|
||||||
import org.xbib.elx.node.ExtendedNodeClientProvider;
|
import org.xbib.elx.node.NodeBulkClientProvider;
|
||||||
|
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
@ -25,9 +20,9 @@ class DuplicateIDTest {
|
||||||
|
|
||||||
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName());
|
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName());
|
||||||
|
|
||||||
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
|
private static final Long ACTIONS = 100L;
|
||||||
|
|
||||||
private static final Long ACTIONS = 50L;
|
private static final Long MAX_ACTIONS_PER_REQUEST = 5L;
|
||||||
|
|
||||||
private final TestExtension.Helper helper;
|
private final TestExtension.Helper helper;
|
||||||
|
|
||||||
|
@ -38,37 +33,27 @@ class DuplicateIDTest {
|
||||||
@Test
|
@Test
|
||||||
void testDuplicateDocIDs() throws Exception {
|
void testDuplicateDocIDs() throws Exception {
|
||||||
long numactions = ACTIONS;
|
long numactions = ACTIONS;
|
||||||
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
|
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||||
.provider(ExtendedNodeClientProvider.class)
|
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||||
.build();
|
.build();
|
||||||
try {
|
try {
|
||||||
client.newIndex("test");
|
bulkClient.newIndex("test");
|
||||||
for (int i = 0; i < ACTIONS; i++) {
|
for (int i = 0; i < ACTIONS; i++) {
|
||||||
client.index("test", helper.randomString(1), false,
|
bulkClient.index("test", helper.randomString(1), false,
|
||||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||||
}
|
}
|
||||||
client.flush();
|
bulkClient.flush();
|
||||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
client.refreshIndex("test");
|
bulkClient.refreshIndex("test");
|
||||||
SearchSourceBuilder builder = new SearchSourceBuilder();
|
assertTrue(bulkClient.getSearchableDocs("test") < ACTIONS);
|
||||||
builder.query(QueryBuilders.matchAllQuery());
|
|
||||||
SearchRequest searchRequest = new SearchRequest();
|
|
||||||
searchRequest.indices("test");
|
|
||||||
searchRequest.types("test");
|
|
||||||
searchRequest.source(builder);
|
|
||||||
long hits = helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits();
|
|
||||||
logger.info("hits = {}", hits);
|
|
||||||
assertTrue(hits < ACTIONS);
|
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} finally {
|
} finally {
|
||||||
client.close();
|
bulkClient.close();
|
||||||
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
|
assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
}
|
}
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,17 +2,15 @@ package org.xbib.elx.node.test;
|
||||||
|
|
||||||
import org.apache.logging.log4j.LogManager;
|
import org.apache.logging.log4j.LogManager;
|
||||||
import org.apache.logging.log4j.Logger;
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
|
|
||||||
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
|
|
||||||
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
|
|
||||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
import org.xbib.elx.api.IndexPruneResult;
|
import org.xbib.elx.api.IndexPruneResult;
|
||||||
import org.xbib.elx.common.ClientBuilder;
|
import org.xbib.elx.common.ClientBuilder;
|
||||||
import org.xbib.elx.node.ExtendedNodeClient;
|
import org.xbib.elx.node.NodeAdminClient;
|
||||||
import org.xbib.elx.node.ExtendedNodeClientProvider;
|
import org.xbib.elx.node.NodeAdminClientProvider;
|
||||||
|
import org.xbib.elx.node.NodeBulkClient;
|
||||||
|
import org.xbib.elx.node.NodeBulkClientProvider;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
@ -37,49 +35,47 @@ class IndexPruneTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testPrune() throws IOException {
|
void testPrune() throws IOException {
|
||||||
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
|
final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
|
||||||
.provider(ExtendedNodeClientProvider.class)
|
.setAdminClientProvider(NodeAdminClientProvider.class)
|
||||||
|
.build();
|
||||||
|
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||||
|
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||||
.build();
|
.build();
|
||||||
try {
|
try {
|
||||||
Settings settings = Settings.builder()
|
Settings settings = Settings.builder()
|
||||||
.put("index.number_of_shards", 1)
|
.put("index.number_of_shards", 1)
|
||||||
.put("index.number_of_replicas", 0)
|
.put("index.number_of_replicas", 0)
|
||||||
.build();
|
.build();
|
||||||
client.newIndex("test_prune1", settings);
|
bulkClient.newIndex("test_prune1", settings);
|
||||||
client.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
|
adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
|
||||||
client.newIndex("test_prune2", settings);
|
bulkClient.newIndex("test_prune2", settings);
|
||||||
client.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
|
adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
|
||||||
client.newIndex("test_prune3", settings);
|
bulkClient.newIndex("test_prune3", settings);
|
||||||
client.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
|
adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
|
||||||
client.newIndex("test_prune4", settings);
|
bulkClient.newIndex("test_prune4", settings);
|
||||||
client.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
|
adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
|
||||||
IndexPruneResult indexPruneResult =
|
IndexPruneResult indexPruneResult =
|
||||||
client.pruneIndex("test_prune", "test_prune4", 2, 2, true);
|
adminClient.pruneIndex("test_prune", "test_prune4", 2, 2, true);
|
||||||
assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1"));
|
assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1"));
|
||||||
assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2"));
|
assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2"));
|
||||||
assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3"));
|
assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3"));
|
||||||
assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune4"));
|
assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune4"));
|
||||||
List<Boolean> list = new ArrayList<>();
|
List<Boolean> list = new ArrayList<>();
|
||||||
for (String index : Arrays.asList("test_prune1", "test_prune2", "test_prune3", "test_prune4")) {
|
for (String index : Arrays.asList("test_prune1", "test_prune2", "test_prune3", "test_prune4")) {
|
||||||
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
|
list.add(bulkClient.isIndexExists(index));
|
||||||
indicesExistsRequest.indices(index);
|
|
||||||
IndicesExistsResponse indicesExistsResponse =
|
|
||||||
client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
|
|
||||||
list.add(indicesExistsResponse.isExists());
|
|
||||||
}
|
}
|
||||||
logger.info(list);
|
logger.info(list);
|
||||||
assertFalse(list.get(0));
|
assertFalse(list.get(0));
|
||||||
assertFalse(list.get(1));
|
assertFalse(list.get(1));
|
||||||
assertTrue(list.get(2));
|
assertTrue(list.get(2));
|
||||||
assertTrue(list.get(3));
|
assertTrue(list.get(3));
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} finally {
|
} finally {
|
||||||
client.close();
|
adminClient.close();
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
bulkClient.close();
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
}
|
}
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,15 +3,16 @@ package org.xbib.elx.node.test;
|
||||||
import org.apache.logging.log4j.LogManager;
|
import org.apache.logging.log4j.LogManager;
|
||||||
import org.apache.logging.log4j.Logger;
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
|
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
|
||||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.index.query.QueryBuilders;
|
import org.elasticsearch.index.query.QueryBuilders;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
import org.xbib.elx.api.IndexShiftResult;
|
import org.xbib.elx.api.IndexShiftResult;
|
||||||
import org.xbib.elx.common.ClientBuilder;
|
import org.xbib.elx.common.ClientBuilder;
|
||||||
import org.xbib.elx.node.ExtendedNodeClient;
|
import org.xbib.elx.node.NodeAdminClient;
|
||||||
import org.xbib.elx.node.ExtendedNodeClientProvider;
|
import org.xbib.elx.node.NodeAdminClientProvider;
|
||||||
|
import org.xbib.elx.node.NodeBulkClient;
|
||||||
|
import org.xbib.elx.node.NodeBulkClientProvider;
|
||||||
|
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
@ -33,50 +34,49 @@ class IndexShiftTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testIndexShift() throws Exception {
|
void testIndexShift() throws Exception {
|
||||||
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
|
final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
|
||||||
.provider(ExtendedNodeClientProvider.class)
|
.setAdminClientProvider(NodeAdminClientProvider.class)
|
||||||
|
.build();
|
||||||
|
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||||
|
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||||
.build();
|
.build();
|
||||||
try {
|
try {
|
||||||
Settings settings = Settings.builder()
|
Settings settings = Settings.builder()
|
||||||
.put("index.number_of_shards", 1)
|
.put("index.number_of_shards", 1)
|
||||||
.put("index.number_of_replicas", 0)
|
.put("index.number_of_replicas", 0)
|
||||||
.build();
|
.build();
|
||||||
client.newIndex("test_shift", settings);
|
bulkClient.newIndex("test_shift", settings);
|
||||||
for (int i = 0; i < 1; i++) {
|
for (int i = 0; i < 1; i++) {
|
||||||
client.index("test_shift", helper.randomString(1), false,
|
bulkClient.index("test_shift", helper.randomString(1), false,
|
||||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||||
}
|
}
|
||||||
client.flush();
|
bulkClient.flush();
|
||||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
IndexShiftResult indexShiftResult =
|
IndexShiftResult indexShiftResult =
|
||||||
client.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c"));
|
adminClient.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c"));
|
||||||
assertTrue(indexShiftResult.getNewAliases().contains("a"));
|
assertTrue(indexShiftResult.getNewAliases().contains("a"));
|
||||||
assertTrue(indexShiftResult.getNewAliases().contains("b"));
|
assertTrue(indexShiftResult.getNewAliases().contains("b"));
|
||||||
assertTrue(indexShiftResult.getNewAliases().contains("c"));
|
assertTrue(indexShiftResult.getNewAliases().contains("c"));
|
||||||
assertTrue(indexShiftResult.getMovedAliases().isEmpty());
|
assertTrue(indexShiftResult.getMovedAliases().isEmpty());
|
||||||
|
Map<String, String> aliases = adminClient.getAliases("test_shift");
|
||||||
Map<String, String> aliases = client.getAliases("test_shift");
|
|
||||||
assertTrue(aliases.containsKey("a"));
|
assertTrue(aliases.containsKey("a"));
|
||||||
assertTrue(aliases.containsKey("b"));
|
assertTrue(aliases.containsKey("b"));
|
||||||
assertTrue(aliases.containsKey("c"));
|
assertTrue(aliases.containsKey("c"));
|
||||||
assertTrue(aliases.containsKey("test"));
|
assertTrue(aliases.containsKey("test"));
|
||||||
|
String resolved = adminClient.resolveAlias("test");
|
||||||
String resolved = client.resolveAlias("test");
|
aliases = adminClient.getAliases(resolved);
|
||||||
aliases = client.getAliases(resolved);
|
|
||||||
assertTrue(aliases.containsKey("a"));
|
assertTrue(aliases.containsKey("a"));
|
||||||
assertTrue(aliases.containsKey("b"));
|
assertTrue(aliases.containsKey("b"));
|
||||||
assertTrue(aliases.containsKey("c"));
|
assertTrue(aliases.containsKey("c"));
|
||||||
assertTrue(aliases.containsKey("test"));
|
assertTrue(aliases.containsKey("test"));
|
||||||
|
bulkClient.newIndex("test_shift2", settings);
|
||||||
client.newIndex("test_shift2", settings);
|
|
||||||
for (int i = 0; i < 1; i++) {
|
for (int i = 0; i < 1; i++) {
|
||||||
client.index("test_shift2", helper.randomString(1), false,
|
bulkClient.index("test_shift2", helper.randomString(1), false,
|
||||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||||
}
|
}
|
||||||
client.flush();
|
bulkClient.flush();
|
||||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
|
indexShiftResult = adminClient.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"),
|
||||||
indexShiftResult = client.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"),
|
|
||||||
(request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
|
(request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
|
||||||
.index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
|
.index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
|
||||||
);
|
);
|
||||||
|
@ -86,32 +86,28 @@ class IndexShiftTest {
|
||||||
assertTrue(indexShiftResult.getMovedAliases().contains("a"));
|
assertTrue(indexShiftResult.getMovedAliases().contains("a"));
|
||||||
assertTrue(indexShiftResult.getMovedAliases().contains("b"));
|
assertTrue(indexShiftResult.getMovedAliases().contains("b"));
|
||||||
assertTrue(indexShiftResult.getMovedAliases().contains("c"));
|
assertTrue(indexShiftResult.getMovedAliases().contains("c"));
|
||||||
|
aliases = adminClient.getAliases("test_shift2");
|
||||||
aliases = client.getAliases("test_shift2");
|
|
||||||
assertTrue(aliases.containsKey("a"));
|
assertTrue(aliases.containsKey("a"));
|
||||||
assertTrue(aliases.containsKey("b"));
|
assertTrue(aliases.containsKey("b"));
|
||||||
assertTrue(aliases.containsKey("c"));
|
assertTrue(aliases.containsKey("c"));
|
||||||
assertTrue(aliases.containsKey("d"));
|
assertTrue(aliases.containsKey("d"));
|
||||||
assertTrue(aliases.containsKey("e"));
|
assertTrue(aliases.containsKey("e"));
|
||||||
assertTrue(aliases.containsKey("f"));
|
assertTrue(aliases.containsKey("f"));
|
||||||
|
resolved = adminClient.resolveAlias("test");
|
||||||
resolved = client.resolveAlias("test");
|
aliases = adminClient.getAliases(resolved);
|
||||||
aliases = client.getAliases(resolved);
|
|
||||||
assertTrue(aliases.containsKey("a"));
|
assertTrue(aliases.containsKey("a"));
|
||||||
assertTrue(aliases.containsKey("b"));
|
assertTrue(aliases.containsKey("b"));
|
||||||
assertTrue(aliases.containsKey("c"));
|
assertTrue(aliases.containsKey("c"));
|
||||||
assertTrue(aliases.containsKey("d"));
|
assertTrue(aliases.containsKey("d"));
|
||||||
assertTrue(aliases.containsKey("e"));
|
assertTrue(aliases.containsKey("e"));
|
||||||
assertTrue(aliases.containsKey("f"));
|
assertTrue(aliases.containsKey("f"));
|
||||||
|
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} finally {
|
} finally {
|
||||||
client.close();
|
adminClient.close();
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
bulkClient.close();
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
}
|
}
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,14 +2,15 @@ package org.xbib.elx.node.test;
|
||||||
|
|
||||||
import org.apache.logging.log4j.LogManager;
|
import org.apache.logging.log4j.LogManager;
|
||||||
import org.apache.logging.log4j.Logger;
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
import org.xbib.elx.common.ClientBuilder;
|
import org.xbib.elx.common.ClientBuilder;
|
||||||
import org.xbib.elx.api.IndexDefinition;
|
import org.xbib.elx.api.IndexDefinition;
|
||||||
import org.xbib.elx.node.ExtendedNodeClient;
|
import org.xbib.elx.node.NodeAdminClient;
|
||||||
import org.xbib.elx.node.ExtendedNodeClientProvider;
|
import org.xbib.elx.node.NodeAdminClientProvider;
|
||||||
|
import org.xbib.elx.node.NodeBulkClient;
|
||||||
|
import org.xbib.elx.node.NodeBulkClientProvider;
|
||||||
|
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
@ -29,46 +30,49 @@ class SmokeTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void smokeTest() throws Exception {
|
void smokeTest() throws Exception {
|
||||||
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
|
final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
|
||||||
.provider(ExtendedNodeClientProvider.class)
|
.setAdminClientProvider(NodeAdminClientProvider.class)
|
||||||
.build();
|
.build();
|
||||||
|
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||||
|
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||||
|
.build();
|
||||||
|
IndexDefinition indexDefinition =
|
||||||
|
adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.EMPTY);
|
||||||
try {
|
try {
|
||||||
assertEquals(helper.getClusterName(), client.getClusterName());
|
assertEquals(helper.getClusterName(), adminClient.getClusterName());
|
||||||
client.newIndex("test_smoke");
|
bulkClient.newIndex("test_smoke");
|
||||||
client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
||||||
client.flush();
|
bulkClient.flush();
|
||||||
client.waitForResponses(30, TimeUnit.SECONDS);
|
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
|
||||||
client.checkMapping("test_smoke");
|
adminClient.checkMapping("test_smoke");
|
||||||
client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
|
bulkClient.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
|
||||||
client.delete("test_smoke", "1");
|
bulkClient.delete("test_smoke", "1");
|
||||||
client.flush();
|
bulkClient.flush();
|
||||||
client.waitForResponses(30, TimeUnit.SECONDS);
|
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
|
||||||
client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS);
|
adminClient.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS);
|
||||||
client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
|
bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
|
||||||
client.delete("test_smoke", "1");
|
bulkClient.delete("test_smoke", "1");
|
||||||
client.flush();
|
bulkClient.flush();
|
||||||
client.deleteIndex("test_smoke");
|
adminClient.deleteIndex("test_smoke");
|
||||||
IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder()
|
|
||||||
.build());
|
|
||||||
assertEquals(0, indexDefinition.getReplicaLevel());
|
assertEquals(0, indexDefinition.getReplicaLevel());
|
||||||
client.newIndex(indexDefinition);
|
bulkClient.newIndex(indexDefinition);
|
||||||
client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
|
bulkClient.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
|
||||||
client.flush();
|
bulkClient.flush();
|
||||||
client.waitForResponses(30, TimeUnit.SECONDS);
|
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
|
||||||
client.updateReplicaLevel(indexDefinition, 2);
|
adminClient.updateReplicaLevel(indexDefinition, 2);
|
||||||
int replica = client.getReplicaLevel(indexDefinition);
|
int replica = adminClient.getReplicaLevel(indexDefinition);
|
||||||
assertEquals(2, replica);
|
assertEquals(2, replica);
|
||||||
client.deleteIndex(indexDefinition);
|
|
||||||
//assertEquals(0, client.getBulkMetric().getFailed().getCount());
|
|
||||||
//assertEquals(6, client.getBulkMetric().getSucceeded().getCount());
|
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} finally {
|
} finally {
|
||||||
client.close();
|
bulkClient.close();
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
}
|
}
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount());
|
||||||
|
assertEquals(6, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||||
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
|
// close admin after bulk
|
||||||
|
adminClient.deleteIndex(indexDefinition);
|
||||||
|
adminClient.close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,11 +0,0 @@
|
||||||
package org.xbib.elx.transport;
|
|
||||||
|
|
||||||
import org.xbib.elx.api.ExtendedClientProvider;
|
|
||||||
|
|
||||||
public class ExtendedTransportClientProvider implements ExtendedClientProvider<ExtendedTransportClient> {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedTransportClient getExtendedClient() {
|
|
||||||
return new ExtendedTransportClient();
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -0,0 +1,40 @@
|
||||||
|
package org.xbib.elx.transport;
|
||||||
|
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.client.transport.TransportClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.xbib.elx.common.AbstractAdminClient;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Transport admin client.
|
||||||
|
*/
|
||||||
|
public class TransportAdminClient extends AbstractAdminClient {
|
||||||
|
|
||||||
|
private final TransportClientHelper helper;
|
||||||
|
|
||||||
|
public TransportAdminClient() {
|
||||||
|
this.helper = new TransportClientHelper();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||||
|
return helper.createClient(settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() {
|
||||||
|
if (getClient() != null) {
|
||||||
|
TransportClient client = (TransportClient) getClient();
|
||||||
|
client.close();
|
||||||
|
client.threadPool().shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) throws IOException {
|
||||||
|
super.init(settings);
|
||||||
|
helper.init((TransportClient) getClient(), settings);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,11 @@
|
||||||
|
package org.xbib.elx.transport;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.AdminClientProvider;
|
||||||
|
|
||||||
|
public class TransportAdminClientProvider implements AdminClientProvider<TransportAdminClient> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public TransportAdminClient getClient() {
|
||||||
|
return new TransportAdminClient();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,39 @@
|
||||||
|
package org.xbib.elx.transport;
|
||||||
|
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.client.transport.TransportClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.xbib.elx.common.AbstractBulkClient;
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Transport search client with additional methods.
|
||||||
|
*/
|
||||||
|
public class TransportBulkClient extends AbstractBulkClient {
|
||||||
|
|
||||||
|
private final TransportClientHelper helper;
|
||||||
|
|
||||||
|
public TransportBulkClient() {
|
||||||
|
this.helper = new TransportClientHelper();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||||
|
return helper.createClient(settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() {
|
||||||
|
if (getClient() != null) {
|
||||||
|
TransportClient client = (TransportClient) getClient();
|
||||||
|
client.close();
|
||||||
|
client.threadPool().shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) throws IOException {
|
||||||
|
super.init(settings);
|
||||||
|
helper.init((TransportClient) getClient(), settings);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,11 @@
|
||||||
|
package org.xbib.elx.transport;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.BulkClientProvider;
|
||||||
|
|
||||||
|
public class TransportBulkClientProvider implements BulkClientProvider<TransportBulkClient> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public TransportBulkClient getClient() {
|
||||||
|
return new TransportBulkClient();
|
||||||
|
}
|
||||||
|
}
|
|
@ -23,9 +23,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||||
import org.elasticsearch.plugins.Plugin;
|
import org.elasticsearch.plugins.Plugin;
|
||||||
import org.elasticsearch.transport.netty4.Netty4Plugin;
|
import org.elasticsearch.transport.netty4.Netty4Plugin;
|
||||||
import org.xbib.elx.common.AbstractExtendedClient;
|
|
||||||
import org.xbib.elx.common.util.NetworkUtils;
|
import org.xbib.elx.common.util.NetworkUtils;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.InetAddress;
|
import java.net.InetAddress;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
@ -33,14 +31,11 @@ import java.util.Collection;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
/**
|
public class TransportClientHelper {
|
||||||
* Transport client with additional methods using the BulkProcessor.
|
|
||||||
*/
|
|
||||||
public class ExtendedTransportClient extends AbstractExtendedClient {
|
|
||||||
|
|
||||||
private static final Logger logger = LogManager.getLogger(ExtendedTransportClient.class.getName());
|
|
||||||
|
|
||||||
@Override
|
private static final Logger logger = LogManager.getLogger(TransportAdminClient.class.getName());
|
||||||
|
|
||||||
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||||
if (settings != null) {
|
if (settings != null) {
|
||||||
String systemIdentifier = System.getProperty("os.name")
|
String systemIdentifier = System.getProperty("os.name")
|
||||||
|
@ -49,11 +44,9 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
|
||||||
+ " " + System.getProperty("java.vm.version")
|
+ " " + System.getProperty("java.vm.version")
|
||||||
+ " Elasticsearch " + Version.CURRENT.toString();
|
+ " Elasticsearch " + Version.CURRENT.toString();
|
||||||
Settings transportClientSettings = getTransportClientSettings(settings);
|
Settings transportClientSettings = getTransportClientSettings(settings);
|
||||||
//XContentBuilder settingsBuilder = XContentFactory.jsonBuilder().startObject();
|
|
||||||
XContentBuilder effectiveSettingsBuilder = XContentFactory.jsonBuilder().startObject();
|
XContentBuilder effectiveSettingsBuilder = XContentFactory.jsonBuilder().startObject();
|
||||||
logger.log(Level.INFO, "creating transport client on {} with settings {}",
|
logger.log(Level.INFO, "creating transport client on {} with settings {}",
|
||||||
systemIdentifier,
|
systemIdentifier,
|
||||||
//Strings.toString(settings.toXContent(settingsBuilder, ToXContent.EMPTY_PARAMS).endObject()),
|
|
||||||
Strings.toString(transportClientSettings.toXContent(effectiveSettingsBuilder,
|
Strings.toString(transportClientSettings.toXContent(effectiveSettingsBuilder,
|
||||||
ToXContent.EMPTY_PARAMS).endObject()));
|
ToXContent.EMPTY_PARAMS).endObject()));
|
||||||
return new MyTransportClient(transportClientSettings, Collections.singletonList(Netty4Plugin.class));
|
return new MyTransportClient(transportClientSettings, Collections.singletonList(Netty4Plugin.class));
|
||||||
|
@ -61,29 +54,12 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
public void init(TransportClient transportClient, Settings settings) throws IOException {
|
||||||
protected void closeClient() {
|
|
||||||
if (getClient() != null) {
|
|
||||||
TransportClient client = (TransportClient) getClient();
|
|
||||||
client.close();
|
|
||||||
client.threadPool().shutdown();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedTransportClient init(Settings settings) throws IOException {
|
|
||||||
super.init(settings);
|
|
||||||
// additional auto-connect
|
|
||||||
try {
|
|
||||||
Collection<TransportAddress> addrs = findAddresses(settings);
|
Collection<TransportAddress> addrs = findAddresses(settings);
|
||||||
if (!connect(addrs, settings.getAsBoolean("autodiscover", false))) {
|
if (!connect(transportClient, addrs, settings.getAsBoolean("autodiscover", false))) {
|
||||||
throw new NoNodeAvailableException("no cluster nodes available, check settings "
|
throw new NoNodeAvailableException("no cluster nodes available, check settings "
|
||||||
+ settings.toString());
|
+ settings.toString());
|
||||||
}
|
}
|
||||||
} catch (IOException e) {
|
|
||||||
logger.error(e.getMessage(), e);
|
|
||||||
}
|
|
||||||
return this;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private Collection<TransportAddress> findAddresses(Settings settings) throws IOException {
|
private Collection<TransportAddress> findAddresses(Settings settings) throws IOException {
|
||||||
|
@ -112,11 +88,7 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
|
||||||
return addresses;
|
return addresses;
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean connect(Collection<TransportAddress> addresses, boolean autodiscover) {
|
private boolean connect(TransportClient transportClient, Collection<TransportAddress> addresses, boolean autodiscover) {
|
||||||
if (getClient() == null) {
|
|
||||||
throw new IllegalStateException("no client present");
|
|
||||||
}
|
|
||||||
TransportClient transportClient = (TransportClient) getClient();
|
|
||||||
for (TransportAddress address : addresses) {
|
for (TransportAddress address : addresses) {
|
||||||
transportClient.addTransportAddresses(address);
|
transportClient.addTransportAddresses(address);
|
||||||
}
|
}
|
||||||
|
@ -126,7 +98,7 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
|
||||||
if (autodiscover) {
|
if (autodiscover) {
|
||||||
logger.debug("trying to auto-discover all nodes...");
|
logger.debug("trying to auto-discover all nodes...");
|
||||||
ClusterStateRequestBuilder clusterStateRequestBuilder =
|
ClusterStateRequestBuilder clusterStateRequestBuilder =
|
||||||
new ClusterStateRequestBuilder(getClient(), ClusterStateAction.INSTANCE);
|
new ClusterStateRequestBuilder(transportClient, ClusterStateAction.INSTANCE);
|
||||||
ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet();
|
ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet();
|
||||||
DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes();
|
DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes();
|
||||||
addDiscoveryNodes(transportClient, discoveryNodes);
|
addDiscoveryNodes(transportClient, discoveryNodes);
|
|
@ -0,0 +1,39 @@
|
||||||
|
package org.xbib.elx.transport;
|
||||||
|
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.client.transport.TransportClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.xbib.elx.common.AbstractSearchClient;
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Transport search client with additional methods.
|
||||||
|
*/
|
||||||
|
public class TransportSearchClient extends AbstractSearchClient {
|
||||||
|
|
||||||
|
private final TransportClientHelper helper;
|
||||||
|
|
||||||
|
public TransportSearchClient() {
|
||||||
|
this.helper = new TransportClientHelper();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||||
|
return helper.createClient(settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() {
|
||||||
|
if (getClient() != null) {
|
||||||
|
TransportClient client = (TransportClient) getClient();
|
||||||
|
client.close();
|
||||||
|
client.threadPool().shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) throws IOException {
|
||||||
|
super.init(settings);
|
||||||
|
helper.init((TransportClient) getClient(), settings);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,11 @@
|
||||||
|
package org.xbib.elx.transport;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.SearchClientProvider;
|
||||||
|
|
||||||
|
public class TransportSearchClientProvider implements SearchClientProvider<TransportSearchClient> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public TransportSearchClient getClient() {
|
||||||
|
return new TransportSearchClient();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1 @@
|
||||||
|
org.xbib.elx.transport.TransportAdminClientProvider
|
|
@ -0,0 +1 @@
|
||||||
|
org.xbib.elx.transport.TransportBulkClientProvider
|
|
@ -1 +0,0 @@
|
||||||
org.xbib.elx.transport.ExtendedTransportClientProvider
|
|
|
@ -0,0 +1 @@
|
||||||
|
org.xbib.elx.transport.TransportSearchClientProvider
|
|
@ -0,0 +1,188 @@
|
||||||
|
package org.xbib.elx.transport.test;
|
||||||
|
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.common.unit.TimeValue;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
|
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.xbib.elx.common.ClientBuilder;
|
||||||
|
import org.xbib.elx.common.Parameters;
|
||||||
|
import org.xbib.elx.transport.TransportBulkClient;
|
||||||
|
import org.xbib.elx.transport.TransportBulkClientProvider;
|
||||||
|
|
||||||
|
import java.util.concurrent.CountDownLatch;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
@ExtendWith(TestExtension.class)
|
||||||
|
class BulkClientTest {
|
||||||
|
|
||||||
|
private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getName());
|
||||||
|
|
||||||
|
private static final Long ACTIONS = 10000L;
|
||||||
|
|
||||||
|
private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
|
||||||
|
|
||||||
|
private final TestExtension.Helper helper;
|
||||||
|
|
||||||
|
BulkClientTest(TestExtension.Helper helper) {
|
||||||
|
this.helper = helper;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testSingleDoc() throws Exception {
|
||||||
|
final TransportBulkClient bulkClient = ClientBuilder.builder()
|
||||||
|
.setBulkClientProvider(TransportBulkClientProvider.class)
|
||||||
|
.put(helper.getTransportSettings())
|
||||||
|
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||||
|
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
|
||||||
|
.build();
|
||||||
|
try {
|
||||||
|
bulkClient.newIndex("test");
|
||||||
|
bulkClient.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
||||||
|
bulkClient.flush();
|
||||||
|
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
|
} finally {
|
||||||
|
assertEquals(1, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||||
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
|
}
|
||||||
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
|
bulkClient.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testNewIndex() throws Exception {
|
||||||
|
final TransportBulkClient bulkClient = ClientBuilder.builder()
|
||||||
|
.setBulkClientProvider(TransportBulkClientProvider.class)
|
||||||
|
.put(helper.getTransportSettings())
|
||||||
|
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
||||||
|
.build();
|
||||||
|
bulkClient.newIndex("test");
|
||||||
|
bulkClient.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testMapping() throws Exception {
|
||||||
|
final TransportBulkClient bulkClient = ClientBuilder.builder()
|
||||||
|
.setBulkClientProvider(TransportBulkClientProvider.class)
|
||||||
|
.put(helper.getTransportSettings())
|
||||||
|
.build();
|
||||||
|
XContentBuilder builder = JsonXContent.contentBuilder()
|
||||||
|
.startObject()
|
||||||
|
.startObject("doc")
|
||||||
|
.startObject("properties")
|
||||||
|
.startObject("location")
|
||||||
|
.field("type", "geo_point")
|
||||||
|
.endObject()
|
||||||
|
.endObject()
|
||||||
|
.endObject()
|
||||||
|
.endObject();
|
||||||
|
bulkClient.newIndex("test", Settings.EMPTY, builder);
|
||||||
|
assertTrue(bulkClient.getMapping("test", "doc").containsKey("properties"));
|
||||||
|
bulkClient.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testRandomDocs() throws Exception {
|
||||||
|
long numactions = ACTIONS;
|
||||||
|
final TransportBulkClient bulkClient = ClientBuilder.builder()
|
||||||
|
.setBulkClientProvider(TransportBulkClientProvider.class)
|
||||||
|
.put(helper.getTransportSettings())
|
||||||
|
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||||
|
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
||||||
|
.build();
|
||||||
|
try {
|
||||||
|
bulkClient.newIndex("test");
|
||||||
|
for (int i = 0; i < ACTIONS; i++) {
|
||||||
|
bulkClient.index("test", null, false,
|
||||||
|
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||||
|
}
|
||||||
|
bulkClient.flush();
|
||||||
|
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error(e.getMessage(), e);
|
||||||
|
} finally {
|
||||||
|
assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||||
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
|
}
|
||||||
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
|
bulkClient.refreshIndex("test");
|
||||||
|
assertEquals(numactions, bulkClient.getSearchableDocs("test"));
|
||||||
|
bulkClient.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testThreadedRandomDocs() throws Exception {
|
||||||
|
int maxthreads = Runtime.getRuntime().availableProcessors();
|
||||||
|
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
|
||||||
|
final long actions = ACTIONS;
|
||||||
|
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
|
||||||
|
final TransportBulkClient bulkClient = ClientBuilder.builder()
|
||||||
|
.setBulkClientProvider(TransportBulkClientProvider.class)
|
||||||
|
.put(helper.getTransportSettings())
|
||||||
|
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
|
||||||
|
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
|
||||||
|
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
||||||
|
.put(Parameters.ENABLE_BULK_LOGGING.name(), "true")
|
||||||
|
.build();
|
||||||
|
try {
|
||||||
|
Settings settings = Settings.builder()
|
||||||
|
.put("index.number_of_shards", 1)
|
||||||
|
.put("index.number_of_replicas", 0)
|
||||||
|
.build();
|
||||||
|
bulkClient.newIndex("test", settings);
|
||||||
|
bulkClient.startBulk("test", 0, 1000);
|
||||||
|
logger.info("index created");
|
||||||
|
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
|
||||||
|
final CountDownLatch latch = new CountDownLatch(maxthreads);
|
||||||
|
for (int i = 0; i < maxthreads; i++) {
|
||||||
|
executorService.execute(() -> {
|
||||||
|
for (int i1 = 0; i1 < actions; i1++) {
|
||||||
|
bulkClient.index("test", null, false,
|
||||||
|
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||||
|
}
|
||||||
|
latch.countDown();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
logger.info("waiting for latch...");
|
||||||
|
if (latch.await(30L, TimeUnit.SECONDS)) {
|
||||||
|
logger.info("flush...");
|
||||||
|
bulkClient.flush();
|
||||||
|
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
|
logger.info("got all responses, executor service shutdown...");
|
||||||
|
executorService.shutdown();
|
||||||
|
executorService.awaitTermination(30L, TimeUnit.SECONDS);
|
||||||
|
logger.info("pool is shut down");
|
||||||
|
} else {
|
||||||
|
logger.warn("latch timeout");
|
||||||
|
}
|
||||||
|
bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS);
|
||||||
|
assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||||
|
} catch (NoNodeAvailableException e) {
|
||||||
|
logger.warn("skipping, no node available");
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error(e.getMessage(), e);
|
||||||
|
} finally {
|
||||||
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
|
}
|
||||||
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
|
bulkClient.refreshIndex("test");
|
||||||
|
assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test"));
|
||||||
|
bulkClient.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,223 +0,0 @@
|
||||||
package org.xbib.elx.transport.test;
|
|
||||||
|
|
||||||
import org.apache.logging.log4j.Level;
|
|
||||||
import org.apache.logging.log4j.LogManager;
|
|
||||||
import org.apache.logging.log4j.Logger;
|
|
||||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
|
|
||||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
|
|
||||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
|
|
||||||
import org.elasticsearch.action.search.SearchAction;
|
|
||||||
import org.elasticsearch.action.search.SearchRequest;
|
|
||||||
import org.elasticsearch.action.search.SearchResponse;
|
|
||||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
|
||||||
import org.elasticsearch.common.Strings;
|
|
||||||
import org.elasticsearch.common.settings.Settings;
|
|
||||||
import org.elasticsearch.common.unit.TimeValue;
|
|
||||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
|
||||||
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
|
||||||
import org.elasticsearch.index.query.QueryBuilders;
|
|
||||||
import org.elasticsearch.search.builder.SearchSourceBuilder;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
|
||||||
import org.xbib.elx.common.ClientBuilder;
|
|
||||||
import org.xbib.elx.common.Parameters;
|
|
||||||
import org.xbib.elx.transport.ExtendedTransportClient;
|
|
||||||
import org.xbib.elx.transport.ExtendedTransportClientProvider;
|
|
||||||
|
|
||||||
import java.util.concurrent.CountDownLatch;
|
|
||||||
import java.util.concurrent.ExecutorService;
|
|
||||||
import java.util.concurrent.Executors;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
||||||
|
|
||||||
@ExtendWith(TestExtension.class)
|
|
||||||
class ClientTest {
|
|
||||||
|
|
||||||
private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
|
|
||||||
|
|
||||||
private static final Long ACTIONS = 10000L;
|
|
||||||
|
|
||||||
private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
|
|
||||||
|
|
||||||
private final TestExtension.Helper helper;
|
|
||||||
|
|
||||||
ClientTest(TestExtension.Helper helper) {
|
|
||||||
this.helper = helper;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testSingleDoc() throws Exception {
|
|
||||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
|
||||||
.provider(ExtendedTransportClientProvider.class)
|
|
||||||
.put(helper.getTransportSettings())
|
|
||||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
|
||||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
|
|
||||||
.build();
|
|
||||||
try {
|
|
||||||
client.newIndex("test");
|
|
||||||
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
|
||||||
client.flush();
|
|
||||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} finally {
|
|
||||||
assertEquals(1, client.getBulkMetric().getSucceeded().getCount());
|
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
|
||||||
}
|
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
|
||||||
client.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testNewIndex() throws Exception {
|
|
||||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
|
||||||
.provider(ExtendedTransportClientProvider.class)
|
|
||||||
.put(helper.getTransportSettings())
|
|
||||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
|
||||||
.build();
|
|
||||||
client.newIndex("test");
|
|
||||||
client.close();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testMapping() throws Exception {
|
|
||||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
|
||||||
.provider(ExtendedTransportClientProvider.class)
|
|
||||||
.put(helper.getTransportSettings())
|
|
||||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
|
||||||
.build();
|
|
||||||
XContentBuilder builder = JsonXContent.contentBuilder()
|
|
||||||
.startObject()
|
|
||||||
.startObject("doc")
|
|
||||||
.startObject("properties")
|
|
||||||
.startObject("location")
|
|
||||||
.field("type", "geo_point")
|
|
||||||
.endObject()
|
|
||||||
.endObject()
|
|
||||||
.endObject()
|
|
||||||
.endObject();
|
|
||||||
client.newIndex("test", Settings.EMPTY, Strings.toString(builder));
|
|
||||||
GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test");
|
|
||||||
GetMappingsResponse getMappingsResponse =
|
|
||||||
client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
|
|
||||||
logger.info("mappings={}", getMappingsResponse.getMappings());
|
|
||||||
assertTrue(getMappingsResponse.getMappings().get("test").containsKey("doc"));
|
|
||||||
client.close();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testRandomDocs() throws Exception {
|
|
||||||
long numactions = ACTIONS;
|
|
||||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
|
||||||
.provider(ExtendedTransportClientProvider.class)
|
|
||||||
.put(helper.getTransportSettings())
|
|
||||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
|
||||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
|
||||||
.build();
|
|
||||||
try {
|
|
||||||
client.newIndex("test");
|
|
||||||
for (int i = 0; i < ACTIONS; i++) {
|
|
||||||
client.index("test", null, false,
|
|
||||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
|
||||||
}
|
|
||||||
client.flush();
|
|
||||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error(e.getMessage(), e);
|
|
||||||
} finally {
|
|
||||||
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
|
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
|
||||||
}
|
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
|
||||||
client.refreshIndex("test");
|
|
||||||
SearchSourceBuilder builder = new SearchSourceBuilder();
|
|
||||||
builder.query(QueryBuilders.matchAllQuery());
|
|
||||||
builder.size(0);
|
|
||||||
SearchRequest searchRequest = new SearchRequest();
|
|
||||||
searchRequest.indices("test");
|
|
||||||
searchRequest.source(builder);
|
|
||||||
SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
|
|
||||||
logger.log(Level.INFO, searchResponse.toString());
|
|
||||||
assertEquals(numactions, searchResponse.getHits().getTotalHits());
|
|
||||||
client.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testThreadedRandomDocs() throws Exception {
|
|
||||||
int maxthreads = Runtime.getRuntime().availableProcessors();
|
|
||||||
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
|
|
||||||
final long actions = ACTIONS;
|
|
||||||
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
|
|
||||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
|
||||||
.provider(ExtendedTransportClientProvider.class)
|
|
||||||
.put(helper.getTransportSettings())
|
|
||||||
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
|
|
||||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
|
|
||||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
|
||||||
.put(Parameters.ENABLE_BULK_LOGGING.name(), "true")
|
|
||||||
.build();
|
|
||||||
try {
|
|
||||||
Settings settings = Settings.builder()
|
|
||||||
.put("index.number_of_shards", 1)
|
|
||||||
.put("index.number_of_replicas", 0)
|
|
||||||
.build();
|
|
||||||
client.newIndex("test", settings, (String)null)
|
|
||||||
.startBulk("test", 0, 1000);
|
|
||||||
logger.info("index created");
|
|
||||||
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
|
|
||||||
final CountDownLatch latch = new CountDownLatch(maxthreads);
|
|
||||||
for (int i = 0; i < maxthreads; i++) {
|
|
||||||
executorService.execute(() -> {
|
|
||||||
for (int i1 = 0; i1 < actions; i1++) {
|
|
||||||
client.index("test", null, false,
|
|
||||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
|
||||||
}
|
|
||||||
latch.countDown();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
logger.info("waiting for latch...");
|
|
||||||
if (latch.await(30L, TimeUnit.SECONDS)) {
|
|
||||||
logger.info("flush...");
|
|
||||||
client.flush();
|
|
||||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
|
||||||
logger.info("got all responses, executor service shutdown...");
|
|
||||||
executorService.shutdown();
|
|
||||||
executorService.awaitTermination(30L, TimeUnit.SECONDS);
|
|
||||||
logger.info("pool is shut down");
|
|
||||||
} else {
|
|
||||||
logger.warn("latch timeout");
|
|
||||||
}
|
|
||||||
client.stopBulk("test", 30L, TimeUnit.SECONDS);
|
|
||||||
assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount());
|
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error(e.getMessage(), e);
|
|
||||||
} finally {
|
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
|
||||||
}
|
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
|
||||||
client.refreshIndex("test");
|
|
||||||
SearchSourceBuilder builder = new SearchSourceBuilder();
|
|
||||||
builder.query(QueryBuilders.matchAllQuery());
|
|
||||||
builder.size(0);
|
|
||||||
SearchRequest searchRequest = new SearchRequest();
|
|
||||||
searchRequest.indices("test");
|
|
||||||
searchRequest.source(builder);
|
|
||||||
SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
|
|
||||||
logger.log(Level.INFO, searchResponse.toString());
|
|
||||||
assertEquals(maxthreads * actions, searchResponse.getHits().getTotalHits());
|
|
||||||
client.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -2,18 +2,12 @@ package org.xbib.elx.transport.test;
|
||||||
|
|
||||||
import org.apache.logging.log4j.LogManager;
|
import org.apache.logging.log4j.LogManager;
|
||||||
import org.apache.logging.log4j.Logger;
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.elasticsearch.action.search.SearchAction;
|
|
||||||
import org.elasticsearch.action.search.SearchRequest;
|
|
||||||
import org.elasticsearch.action.search.SearchResponse;
|
|
||||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
|
||||||
import org.elasticsearch.index.query.QueryBuilders;
|
|
||||||
import org.elasticsearch.search.builder.SearchSourceBuilder;
|
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
import org.xbib.elx.common.ClientBuilder;
|
import org.xbib.elx.common.ClientBuilder;
|
||||||
import org.xbib.elx.common.Parameters;
|
import org.xbib.elx.common.Parameters;
|
||||||
import org.xbib.elx.transport.ExtendedTransportClient;
|
import org.xbib.elx.transport.TransportBulkClient;
|
||||||
import org.xbib.elx.transport.ExtendedTransportClientProvider;
|
import org.xbib.elx.transport.TransportBulkClientProvider;
|
||||||
|
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
@ -26,9 +20,9 @@ class DuplicateIDTest {
|
||||||
|
|
||||||
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName());
|
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName());
|
||||||
|
|
||||||
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
|
private static final Long ACTIONS = 100L;
|
||||||
|
|
||||||
private static final Long ACTIONS = 5L;
|
private static final Long MAX_ACTIONS_PER_REQUEST = 5L;
|
||||||
|
|
||||||
private final TestExtension.Helper helper;
|
private final TestExtension.Helper helper;
|
||||||
|
|
||||||
|
@ -39,41 +33,28 @@ class DuplicateIDTest {
|
||||||
@Test
|
@Test
|
||||||
void testDuplicateDocIDs() throws Exception {
|
void testDuplicateDocIDs() throws Exception {
|
||||||
long numactions = ACTIONS;
|
long numactions = ACTIONS;
|
||||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
final TransportBulkClient bulkClient = ClientBuilder.builder()
|
||||||
.provider(ExtendedTransportClientProvider.class)
|
.setBulkClientProvider(TransportBulkClientProvider.class)
|
||||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||||
.put(helper.getTransportSettings())
|
.put(helper.getTransportSettings())
|
||||||
.build();
|
.build();
|
||||||
try {
|
try {
|
||||||
client.newIndex("test_dup");
|
bulkClient.newIndex("test_dup");
|
||||||
for (int i = 0; i < ACTIONS; i++) {
|
for (int i = 0; i < ACTIONS; i++) {
|
||||||
client.index("test_dup", helper.randomString(1), false,
|
bulkClient.index("test_dup", helper.randomString(1), false,
|
||||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||||
}
|
}
|
||||||
client.flush();
|
bulkClient.flush();
|
||||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
client.refreshIndex("test_dup");
|
bulkClient.refreshIndex("test_dup");
|
||||||
SearchSourceBuilder builder = new SearchSourceBuilder();
|
assertTrue(bulkClient.getSearchableDocs("test_dup") < ACTIONS);
|
||||||
builder.query(QueryBuilders.matchAllQuery());
|
|
||||||
builder.size(0);
|
|
||||||
SearchRequest searchRequest = new SearchRequest();
|
|
||||||
searchRequest.indices("test_dup");
|
|
||||||
searchRequest.types("test_dup");
|
|
||||||
searchRequest.source(builder);
|
|
||||||
SearchResponse searchResponse =
|
|
||||||
helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet();
|
|
||||||
long hits = searchResponse.getHits().getTotalHits();
|
|
||||||
logger.info("hits = {}", hits);
|
|
||||||
assertTrue(hits < ACTIONS);
|
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} finally {
|
} finally {
|
||||||
client.close();
|
bulkClient.close();
|
||||||
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
|
assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
}
|
}
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,17 +2,15 @@ package org.xbib.elx.transport.test;
|
||||||
|
|
||||||
import org.apache.logging.log4j.LogManager;
|
import org.apache.logging.log4j.LogManager;
|
||||||
import org.apache.logging.log4j.Logger;
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
|
|
||||||
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
|
|
||||||
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
|
|
||||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
import org.xbib.elx.api.IndexPruneResult;
|
import org.xbib.elx.api.IndexPruneResult;
|
||||||
import org.xbib.elx.common.ClientBuilder;
|
import org.xbib.elx.common.ClientBuilder;
|
||||||
import org.xbib.elx.transport.ExtendedTransportClient;
|
import org.xbib.elx.transport.TransportAdminClient;
|
||||||
import org.xbib.elx.transport.ExtendedTransportClientProvider;
|
import org.xbib.elx.transport.TransportAdminClientProvider;
|
||||||
|
import org.xbib.elx.transport.TransportBulkClient;
|
||||||
|
import org.xbib.elx.transport.TransportBulkClientProvider;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
@ -37,8 +35,12 @@ class IndexPruneTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testPrune() throws IOException {
|
void testPrune() throws IOException {
|
||||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
final TransportAdminClient adminClient = ClientBuilder.builder()
|
||||||
.provider(ExtendedTransportClientProvider.class)
|
.setAdminClientProvider(TransportAdminClientProvider.class)
|
||||||
|
.put(helper.getTransportSettings())
|
||||||
|
.build();
|
||||||
|
final TransportBulkClient bulkClient = ClientBuilder.builder()
|
||||||
|
.setBulkClientProvider(TransportBulkClientProvider.class)
|
||||||
.put(helper.getTransportSettings())
|
.put(helper.getTransportSettings())
|
||||||
.build();
|
.build();
|
||||||
try {
|
try {
|
||||||
|
@ -46,41 +48,36 @@ class IndexPruneTest {
|
||||||
.put("index.number_of_shards", 1)
|
.put("index.number_of_shards", 1)
|
||||||
.put("index.number_of_replicas", 0)
|
.put("index.number_of_replicas", 0)
|
||||||
.build();
|
.build();
|
||||||
client.newIndex("test_prune1", settings);
|
bulkClient.newIndex("test_prune1", settings);
|
||||||
client.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
|
adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
|
||||||
client.newIndex("test_prune2", settings);
|
bulkClient.newIndex("test_prune2", settings);
|
||||||
client.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
|
adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
|
||||||
client.newIndex("test_prune3", settings);
|
bulkClient.newIndex("test_prune3", settings);
|
||||||
client.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
|
adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
|
||||||
client.newIndex("test_prune4", settings);
|
bulkClient.newIndex("test_prune4", settings);
|
||||||
client.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
|
adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
|
||||||
IndexPruneResult indexPruneResult =
|
IndexPruneResult indexPruneResult =
|
||||||
client.pruneIndex("test_prune", "test_prune4", 2, 2, true);
|
adminClient.pruneIndex("test_prune", "test_prune4", 2, 2, true);
|
||||||
assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1"));
|
assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1"));
|
||||||
assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2"));
|
assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2"));
|
||||||
assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3"));
|
assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3"));
|
||||||
assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune4"));
|
assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune4"));
|
||||||
List<Boolean> list = new ArrayList<>();
|
List<Boolean> list = new ArrayList<>();
|
||||||
for (String index : Arrays.asList("test_prune1", "test_prune2", "test_prune3", "test_prune4")) {
|
for (String index : Arrays.asList("test_prune1", "test_prune2", "test_prune3", "test_prune4")) {
|
||||||
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
|
list.add(bulkClient.isIndexExists(index));
|
||||||
indicesExistsRequest.indices(index);
|
|
||||||
IndicesExistsResponse indicesExistsResponse =
|
|
||||||
client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
|
|
||||||
list.add(indicesExistsResponse.isExists());
|
|
||||||
}
|
}
|
||||||
logger.info(list);
|
logger.info(list);
|
||||||
assertFalse(list.get(0));
|
assertFalse(list.get(0));
|
||||||
assertFalse(list.get(1));
|
assertFalse(list.get(1));
|
||||||
assertTrue(list.get(2));
|
assertTrue(list.get(2));
|
||||||
assertTrue(list.get(3));
|
assertTrue(list.get(3));
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} finally {
|
} finally {
|
||||||
client.close();
|
adminClient.close();
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
bulkClient.close();
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
}
|
}
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,15 +3,16 @@ package org.xbib.elx.transport.test;
|
||||||
import org.apache.logging.log4j.LogManager;
|
import org.apache.logging.log4j.LogManager;
|
||||||
import org.apache.logging.log4j.Logger;
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
|
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
|
||||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.index.query.QueryBuilders;
|
import org.elasticsearch.index.query.QueryBuilders;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
import org.xbib.elx.api.IndexShiftResult;
|
import org.xbib.elx.api.IndexShiftResult;
|
||||||
import org.xbib.elx.common.ClientBuilder;
|
import org.xbib.elx.common.ClientBuilder;
|
||||||
import org.xbib.elx.transport.ExtendedTransportClient;
|
import org.xbib.elx.transport.TransportAdminClient;
|
||||||
import org.xbib.elx.transport.ExtendedTransportClientProvider;
|
import org.xbib.elx.transport.TransportAdminClientProvider;
|
||||||
|
import org.xbib.elx.transport.TransportBulkClient;
|
||||||
|
import org.xbib.elx.transport.TransportBulkClientProvider;
|
||||||
|
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
@ -33,8 +34,12 @@ class IndexShiftTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testIndexShift() throws Exception {
|
void testIndexShift() throws Exception {
|
||||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
final TransportAdminClient adminClient = ClientBuilder.builder()
|
||||||
.provider(ExtendedTransportClientProvider.class)
|
.setAdminClientProvider(TransportAdminClientProvider.class)
|
||||||
|
.put(helper.getTransportSettings())
|
||||||
|
.build();
|
||||||
|
final TransportBulkClient bulkClient = ClientBuilder.builder()
|
||||||
|
.setBulkClientProvider(TransportBulkClientProvider.class)
|
||||||
.put(helper.getTransportSettings())
|
.put(helper.getTransportSettings())
|
||||||
.build();
|
.build();
|
||||||
try {
|
try {
|
||||||
|
@ -42,38 +47,38 @@ class IndexShiftTest {
|
||||||
.put("index.number_of_shards", 1)
|
.put("index.number_of_shards", 1)
|
||||||
.put("index.number_of_replicas", 0)
|
.put("index.number_of_replicas", 0)
|
||||||
.build();
|
.build();
|
||||||
client.newIndex("test_shift1234", settings);
|
bulkClient.newIndex("test_shift1234", settings);
|
||||||
for (int i = 0; i < 1; i++) {
|
for (int i = 0; i < 1; i++) {
|
||||||
client.index("test_shift1234", helper.randomString(1), false,
|
bulkClient.index("test_shift1234", helper.randomString(1), false,
|
||||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||||
}
|
}
|
||||||
client.flush();
|
bulkClient.flush();
|
||||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
IndexShiftResult indexShiftResult =
|
IndexShiftResult indexShiftResult =
|
||||||
client.shiftIndex("test_shift", "test_shift1234", Arrays.asList("a", "b", "c"));
|
adminClient.shiftIndex("test_shift", "test_shift1234", Arrays.asList("a", "b", "c"));
|
||||||
assertTrue(indexShiftResult.getNewAliases().contains("a"));
|
assertTrue(indexShiftResult.getNewAliases().contains("a"));
|
||||||
assertTrue(indexShiftResult.getNewAliases().contains("b"));
|
assertTrue(indexShiftResult.getNewAliases().contains("b"));
|
||||||
assertTrue(indexShiftResult.getNewAliases().contains("c"));
|
assertTrue(indexShiftResult.getNewAliases().contains("c"));
|
||||||
assertTrue(indexShiftResult.getMovedAliases().isEmpty());
|
assertTrue(indexShiftResult.getMovedAliases().isEmpty());
|
||||||
Map<String, String> aliases = client.getAliases("test_shift1234");
|
Map<String, String> aliases = adminClient.getAliases("test_shift1234");
|
||||||
assertTrue(aliases.containsKey("a"));
|
assertTrue(aliases.containsKey("a"));
|
||||||
assertTrue(aliases.containsKey("b"));
|
assertTrue(aliases.containsKey("b"));
|
||||||
assertTrue(aliases.containsKey("c"));
|
assertTrue(aliases.containsKey("c"));
|
||||||
assertTrue(aliases.containsKey("test_shift"));
|
assertTrue(aliases.containsKey("test_shift"));
|
||||||
String resolved = client.resolveAlias("test_shift");
|
String resolved = adminClient.resolveAlias("test_shift");
|
||||||
aliases = client.getAliases(resolved);
|
aliases = adminClient.getAliases(resolved);
|
||||||
assertTrue(aliases.containsKey("a"));
|
assertTrue(aliases.containsKey("a"));
|
||||||
assertTrue(aliases.containsKey("b"));
|
assertTrue(aliases.containsKey("b"));
|
||||||
assertTrue(aliases.containsKey("c"));
|
assertTrue(aliases.containsKey("c"));
|
||||||
assertTrue(aliases.containsKey("test_shift"));
|
assertTrue(aliases.containsKey("test_shift"));
|
||||||
client.newIndex("test_shift5678", settings);
|
bulkClient.newIndex("test_shift5678", settings);
|
||||||
for (int i = 0; i < 1; i++) {
|
for (int i = 0; i < 1; i++) {
|
||||||
client.index("test_shift5678", helper.randomString(1), false,
|
bulkClient.index("test_shift5678", helper.randomString(1), false,
|
||||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||||
}
|
}
|
||||||
client.flush();
|
bulkClient.flush();
|
||||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
indexShiftResult = client.shiftIndex("test_shift", "test_shift5678", Arrays.asList("d", "e", "f"),
|
indexShiftResult = adminClient.shiftIndex("test_shift", "test_shift5678", Arrays.asList("d", "e", "f"),
|
||||||
(request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
|
(request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
|
||||||
.index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
|
.index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
|
||||||
);
|
);
|
||||||
|
@ -83,29 +88,28 @@ class IndexShiftTest {
|
||||||
assertTrue(indexShiftResult.getMovedAliases().contains("a"));
|
assertTrue(indexShiftResult.getMovedAliases().contains("a"));
|
||||||
assertTrue(indexShiftResult.getMovedAliases().contains("b"));
|
assertTrue(indexShiftResult.getMovedAliases().contains("b"));
|
||||||
assertTrue(indexShiftResult.getMovedAliases().contains("c"));
|
assertTrue(indexShiftResult.getMovedAliases().contains("c"));
|
||||||
aliases = client.getAliases("test_shift5678");
|
aliases = adminClient.getAliases("test_shift5678");
|
||||||
assertTrue(aliases.containsKey("a"));
|
assertTrue(aliases.containsKey("a"));
|
||||||
assertTrue(aliases.containsKey("b"));
|
assertTrue(aliases.containsKey("b"));
|
||||||
assertTrue(aliases.containsKey("c"));
|
assertTrue(aliases.containsKey("c"));
|
||||||
assertTrue(aliases.containsKey("d"));
|
assertTrue(aliases.containsKey("d"));
|
||||||
assertTrue(aliases.containsKey("e"));
|
assertTrue(aliases.containsKey("e"));
|
||||||
assertTrue(aliases.containsKey("f"));
|
assertTrue(aliases.containsKey("f"));
|
||||||
resolved = client.resolveAlias("test_shift");
|
resolved = adminClient.resolveAlias("test_shift");
|
||||||
aliases = client.getAliases(resolved);
|
aliases = adminClient.getAliases(resolved);
|
||||||
assertTrue(aliases.containsKey("a"));
|
assertTrue(aliases.containsKey("a"));
|
||||||
assertTrue(aliases.containsKey("b"));
|
assertTrue(aliases.containsKey("b"));
|
||||||
assertTrue(aliases.containsKey("c"));
|
assertTrue(aliases.containsKey("c"));
|
||||||
assertTrue(aliases.containsKey("d"));
|
assertTrue(aliases.containsKey("d"));
|
||||||
assertTrue(aliases.containsKey("e"));
|
assertTrue(aliases.containsKey("e"));
|
||||||
assertTrue(aliases.containsKey("f"));
|
assertTrue(aliases.containsKey("f"));
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} finally {
|
} finally {
|
||||||
client.close();
|
adminClient.close();
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
bulkClient.close();
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
}
|
}
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,14 +2,15 @@ package org.xbib.elx.transport.test;
|
||||||
|
|
||||||
import org.apache.logging.log4j.LogManager;
|
import org.apache.logging.log4j.LogManager;
|
||||||
import org.apache.logging.log4j.Logger;
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
import org.xbib.elx.api.IndexDefinition;
|
import org.xbib.elx.api.IndexDefinition;
|
||||||
import org.xbib.elx.common.ClientBuilder;
|
import org.xbib.elx.common.ClientBuilder;
|
||||||
import org.xbib.elx.transport.ExtendedTransportClient;
|
import org.xbib.elx.transport.TransportAdminClient;
|
||||||
import org.xbib.elx.transport.ExtendedTransportClientProvider;
|
import org.xbib.elx.transport.TransportAdminClientProvider;
|
||||||
|
import org.xbib.elx.transport.TransportBulkClient;
|
||||||
|
import org.xbib.elx.transport.TransportBulkClientProvider;
|
||||||
|
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
@ -29,44 +30,48 @@ class SmokeTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void smokeTest() throws Exception {
|
void smokeTest() throws Exception {
|
||||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
final TransportAdminClient adminClient = ClientBuilder.builder()
|
||||||
.provider(ExtendedTransportClientProvider.class)
|
.setAdminClientProvider(TransportAdminClientProvider.class)
|
||||||
.put(helper.getTransportSettings())
|
.put(helper.getTransportSettings())
|
||||||
.build();
|
.build();
|
||||||
|
final TransportBulkClient bulkClient = ClientBuilder.builder()
|
||||||
|
.setBulkClientProvider(TransportBulkClientProvider.class)
|
||||||
|
.put(helper.getTransportSettings())
|
||||||
|
.build();
|
||||||
|
IndexDefinition indexDefinition =
|
||||||
|
adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.EMPTY);
|
||||||
try {
|
try {
|
||||||
assertEquals(helper.getClusterName(), client.getClusterName());
|
assertEquals(helper.getClusterName(), adminClient.getClusterName());
|
||||||
client.newIndex("test_smoke");
|
bulkClient.newIndex("test_smoke");
|
||||||
client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
||||||
client.flush();
|
bulkClient.flush();
|
||||||
client.waitForResponses(30, TimeUnit.SECONDS);
|
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
|
||||||
client.checkMapping("test_smoke");
|
adminClient.checkMapping("test_smoke");
|
||||||
client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
|
bulkClient.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
|
||||||
client.delete("test_smoke", "1");
|
bulkClient.delete("test_smoke", "1");
|
||||||
client.flush();
|
bulkClient.flush();
|
||||||
client.waitForResponses(30, TimeUnit.SECONDS);
|
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
|
||||||
client.checkMapping("test_smoke");
|
adminClient.checkMapping("test_smoke");
|
||||||
client.deleteIndex("test_smoke");
|
adminClient.deleteIndex("test_smoke");
|
||||||
IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder()
|
|
||||||
.build());
|
|
||||||
assertEquals(0, indexDefinition.getReplicaLevel());
|
assertEquals(0, indexDefinition.getReplicaLevel());
|
||||||
client.newIndex(indexDefinition);
|
bulkClient.newIndex(indexDefinition);
|
||||||
client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
|
bulkClient.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
|
||||||
client.flush();
|
bulkClient.flush();
|
||||||
client.waitForResponses(30, TimeUnit.SECONDS);
|
bulkClient.waitForResponses(30, TimeUnit.SECONDS);
|
||||||
client.updateReplicaLevel(indexDefinition, 2);
|
adminClient.updateReplicaLevel(indexDefinition, 2);
|
||||||
int replica = client.getReplicaLevel(indexDefinition);
|
int replica = adminClient.getReplicaLevel(indexDefinition);
|
||||||
assertEquals(2, replica);
|
assertEquals(2, replica);
|
||||||
client.deleteIndex(indexDefinition);
|
|
||||||
assertEquals(0, client.getBulkMetric().getFailed().getCount());
|
|
||||||
assertEquals(4, client.getBulkMetric().getSucceeded().getCount());
|
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} finally {
|
} finally {
|
||||||
client.close();
|
bulkClient.close();
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount());
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
assertEquals(4, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||||
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
}
|
}
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
|
// close admin after bulk
|
||||||
|
adminClient.deleteIndex(indexDefinition);
|
||||||
|
adminClient.close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue