split into admin, bulk, search client, fix HTTP client
parent
c8c8846bb2
commit
f7509bcecd
@ -0,0 +1,194 @@
|
|||||||
|
package org.xbib.elx.api;
|
||||||
|
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Interface for extended managing and indexing methods of an Elasticsearch client.
|
||||||
|
*/
|
||||||
|
public interface AdminClient extends NativeClient {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build index definition from settings.
|
||||||
|
*
|
||||||
|
* @param index the index name
|
||||||
|
* @param settings the settings for the index
|
||||||
|
* @return index definition
|
||||||
|
* @throws IOException if settings/mapping URL is invalid/malformed
|
||||||
|
*/
|
||||||
|
IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete an index.
|
||||||
|
* @param indexDefinition the index definition
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
AdminClient deleteIndex(IndexDefinition indexDefinition);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete an index.
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
AdminClient deleteIndex(String index);
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update replica level.
|
||||||
|
* @param indexDefinition the index definition
|
||||||
|
* @param level the replica level
|
||||||
|
* @return this
|
||||||
|
* @throws IOException if replica setting could not be updated
|
||||||
|
*/
|
||||||
|
AdminClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update replica level.
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
* @param level the replica level
|
||||||
|
* @param maxWaitTime maximum wait time
|
||||||
|
* @param timeUnit time unit
|
||||||
|
* @return this
|
||||||
|
* @throws IOException if replica setting could not be updated
|
||||||
|
*/
|
||||||
|
AdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get replica level.
|
||||||
|
* @param indexDefinition the index name
|
||||||
|
* @return the replica level of the index
|
||||||
|
*/
|
||||||
|
int getReplicaLevel(IndexDefinition indexDefinition);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get replica level.
|
||||||
|
* @param index the index name
|
||||||
|
* @return the replica level of the index
|
||||||
|
*/
|
||||||
|
int getReplicaLevel(String index);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Force segment merge of an index.
|
||||||
|
* @param indexDefinition the index definition
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
boolean forceMerge(IndexDefinition indexDefinition);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Force segment merge of an index.
|
||||||
|
* @param index the index
|
||||||
|
* @param maxWaitTime maximum wait time
|
||||||
|
* @param timeUnit time unit
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit);
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wait for index recovery (after replica change).
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
* @param maxWaitTime maximum wait time
|
||||||
|
* @param timeUnit time unit
|
||||||
|
* @return true if wait succeeded, false if wait timed out
|
||||||
|
*/
|
||||||
|
boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve alias.
|
||||||
|
*
|
||||||
|
* @param alias the alias
|
||||||
|
* @return this index name behind the alias or the alias if there is no index
|
||||||
|
*/
|
||||||
|
String resolveAlias(String alias);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index
|
||||||
|
* name.
|
||||||
|
*
|
||||||
|
* @param alias the alias
|
||||||
|
* @return the most recent index name pointing to the alias
|
||||||
|
*/
|
||||||
|
String resolveMostRecentIndex(String alias);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get all index aliases.
|
||||||
|
* @param index the index
|
||||||
|
* @return map of index aliases
|
||||||
|
*/
|
||||||
|
Map<String, String> getAliases(String index);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Shift from one index to another.
|
||||||
|
* @param indexDefinition the index definition
|
||||||
|
* @param additionalAliases new aliases
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Shift from one index to another.
|
||||||
|
* @param indexDefinition the index definition
|
||||||
|
* @param additionalAliases new aliases
|
||||||
|
* @param indexAliasAdder method to add aliases
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases,
|
||||||
|
IndexAliasAdder indexAliasAdder);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Shift from one index to another.
|
||||||
|
* @param index the index name
|
||||||
|
* @param fullIndexName the index name with timestamp
|
||||||
|
* @param additionalAliases a list of names that should be set as index aliases
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Shift from one index to another.
|
||||||
|
* @param index the index name
|
||||||
|
* @param fullIndexName the index name with timestamp
|
||||||
|
* @param additionalAliases a list of names that should be set as index aliases
|
||||||
|
* @param adder an adder method to create alias term queries
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases,
|
||||||
|
IndexAliasAdder adder);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Prune index.
|
||||||
|
* @param indexDefinition the index definition
|
||||||
|
* @return the index prune result
|
||||||
|
*/
|
||||||
|
IndexPruneResult pruneIndex(IndexDefinition indexDefinition);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Apply retention policy to prune indices. All indices before delta should be deleted,
|
||||||
|
* but the number of mintokeep indices must be kept.
|
||||||
|
*
|
||||||
|
* @param index index name
|
||||||
|
* @param fullIndexName index name with timestamp
|
||||||
|
* @param delta timestamp delta (for index timestamps)
|
||||||
|
* @param mintokeep minimum number of indices to keep
|
||||||
|
* @param perform true if pruning should be executed, false if not
|
||||||
|
* @return the index prune result
|
||||||
|
*/
|
||||||
|
IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find the timestamp of the most recently indexed document in the index.
|
||||||
|
*
|
||||||
|
* @param index the index name
|
||||||
|
* @param timestampfieldname the timestamp field name
|
||||||
|
* @return millis UTC millis of the most recent document
|
||||||
|
* @throws IOException if most rcent document can not be found
|
||||||
|
*/
|
||||||
|
Long mostRecentDocument(String index, String timestampfieldname) throws IOException;
|
||||||
|
}
|
@ -0,0 +1,7 @@
|
|||||||
|
package org.xbib.elx.api;
|
||||||
|
|
||||||
|
@FunctionalInterface
|
||||||
|
public interface AdminClientProvider<C extends AdminClient> {
|
||||||
|
|
||||||
|
C getClient();
|
||||||
|
}
|
@ -0,0 +1,227 @@
|
|||||||
|
package org.xbib.elx.api;
|
||||||
|
|
||||||
|
import org.elasticsearch.action.delete.DeleteRequest;
|
||||||
|
import org.elasticsearch.action.index.IndexRequest;
|
||||||
|
import org.elasticsearch.action.update.UpdateRequest;
|
||||||
|
import org.elasticsearch.common.bytes.BytesReference;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
|
import java.io.Flushable;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
public interface BulkClient extends NativeClient, Flushable {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get bulk metric.
|
||||||
|
* @return the bulk metric
|
||||||
|
*/
|
||||||
|
BulkMetric getBulkMetric();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get buulk control.
|
||||||
|
* @return the bulk control
|
||||||
|
*/
|
||||||
|
BulkController getBulkController();
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new index.
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
* @throws IOException if new index creation fails
|
||||||
|
*/
|
||||||
|
void newIndex(String index) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new index.
|
||||||
|
* @param indexDefinition the index definition
|
||||||
|
* @throws IOException if settings/mapping is invalid or index creation fails
|
||||||
|
*/
|
||||||
|
void newIndex(IndexDefinition indexDefinition) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new index.
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
* @param settings settings
|
||||||
|
* @throws IOException if settings is invalid or index creation fails
|
||||||
|
*/
|
||||||
|
void newIndex(String index, Settings settings) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new index.
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
* @param settings settings
|
||||||
|
* @param mapping mapping
|
||||||
|
* @throws IOException if settings/mapping is invalid or index creation fails
|
||||||
|
*/
|
||||||
|
void newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new index.
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
* @param settings settings
|
||||||
|
* @param mapping mapping
|
||||||
|
* @throws IOException if settings/mapping is invalid or index creation fails
|
||||||
|
*/
|
||||||
|
void newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add index request. Each request will be added to a queue for bulking requests.
|
||||||
|
* Submitting request will be done when limits are exceeded.
|
||||||
|
*
|
||||||
|
* @param index the index
|
||||||
|
* @param id the id
|
||||||
|
* @param create true if document must be created
|
||||||
|
* @param source the source
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
BulkClient index(String index, String id, boolean create, BytesReference source);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Index request. Each request will be added to a queue for bulking requests.
|
||||||
|
* Submitting request will be done when limits are exceeded.
|
||||||
|
*
|
||||||
|
* @param index the index
|
||||||
|
* @param id the id
|
||||||
|
* @param create true if document is to be created, false otherwise
|
||||||
|
* @param source the source
|
||||||
|
* @return this client methods
|
||||||
|
*/
|
||||||
|
BulkClient index(String index, String id, boolean create, String source);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Index request. Each request will be added to a queue for bulking requests.
|
||||||
|
* Submitting request will be done when bulk limits are exceeded.
|
||||||
|
*
|
||||||
|
* @param indexRequest the index request to add
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
BulkClient index(IndexRequest indexRequest);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete request.
|
||||||
|
*
|
||||||
|
* @param index the index
|
||||||
|
* @param id the id
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
BulkClient delete(String index, String id);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete request. Each request will be added to a queue for bulking requests.
|
||||||
|
* Submitting request will be done when bulk limits are exceeded.
|
||||||
|
*
|
||||||
|
* @param deleteRequest the delete request to add
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
BulkClient delete(DeleteRequest deleteRequest);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Bulked update request. Each request will be added to a queue for bulking requests.
|
||||||
|
* Submitting request will be done when bulk limits are exceeded.
|
||||||
|
* Note that updates only work correctly when all operations between nodes are synchronized.
|
||||||
|
*
|
||||||
|
* @param index the index
|
||||||
|
* @param id the id
|
||||||
|
* @param source the source
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
BulkClient update(String index, String id, BytesReference source);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update document. Use with precaution! Does not work in all cases.
|
||||||
|
*
|
||||||
|
* @param index the index
|
||||||
|
* @param id the id
|
||||||
|
* @param source the source
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
BulkClient update(String index, String id, String source);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Bulked update request. Each request will be added to a queue for bulking requests.
|
||||||
|
* Submitting request will be done when bulk limits are exceeded.
|
||||||
|
* Note that updates only work correctly when all operations between nodes are synchronized.
|
||||||
|
*
|
||||||
|
* @param updateRequest the update request to add
|
||||||
|
* @return this
|
||||||
|
*/
|
||||||
|
BulkClient update(UpdateRequest updateRequest);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start bulk mode for indexes.
|
||||||
|
* @param indexDefinition index definition
|
||||||
|
* @throws IOException if bulk could not be started
|
||||||
|
*/
|
||||||
|
void startBulk(IndexDefinition indexDefinition) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start bulk mode.
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
* @param startRefreshIntervalSeconds refresh interval before bulk
|
||||||
|
* @param stopRefreshIntervalSeconds refresh interval after bulk
|
||||||
|
* @throws IOException if bulk could not be started
|
||||||
|
*/
|
||||||
|
void startBulk(String index, long startRefreshIntervalSeconds,
|
||||||
|
long stopRefreshIntervalSeconds) throws IOException;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stop bulk mode.
|
||||||
|
*
|
||||||
|
* @param indexDefinition index definition
|
||||||
|
* @throws IOException if bulk could not be startet
|
||||||
|
*/
|
||||||
|
void stopBulk(IndexDefinition indexDefinition) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stops bulk mode.
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
* @param timeout maximum wait time
|
||||||
|
* @param timeUnit time unit for timeout
|
||||||
|
* @throws IOException if bulk could not be stopped
|
||||||
|
*/
|
||||||
|
void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wait for all outstanding bulk responses.
|
||||||
|
*
|
||||||
|
* @param timeout maximum wait time
|
||||||
|
* @param timeUnit unit of timeout value
|
||||||
|
* @return true if wait succeeded, false if wait timed out
|
||||||
|
*/
|
||||||
|
boolean waitForResponses(long timeout, TimeUnit timeUnit);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update index setting.
|
||||||
|
* @param index the index
|
||||||
|
* @param key the key of the value to be updated
|
||||||
|
* @param value the new value
|
||||||
|
* @param timeout timeout
|
||||||
|
* @param timeUnit time unit
|
||||||
|
* @throws IOException if update index setting failed
|
||||||
|
*/
|
||||||
|
void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Refresh the index.
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
*/
|
||||||
|
void refreshIndex(String index);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Flush the index. The cluster clears cache and completes indexing.
|
||||||
|
*
|
||||||
|
* @param index index
|
||||||
|
*/
|
||||||
|
void flushIndex(String index);
|
||||||
|
|
||||||
|
}
|
@ -0,0 +1,7 @@
|
|||||||
|
package org.xbib.elx.api;
|
||||||
|
|
||||||
|
@FunctionalInterface
|
||||||
|
public interface BulkClientProvider<C extends BulkClient> {
|
||||||
|
|
||||||
|
C getClient();
|
||||||
|
}
|
@ -1,487 +0,0 @@
|
|||||||
package org.xbib.elx.api;
|
|
||||||
|
|
||||||
import org.elasticsearch.action.delete.DeleteRequest;
|
|
||||||
import org.elasticsearch.action.index.IndexRequest;
|
|
||||||
import org.elasticsearch.action.update.UpdateRequest;
|
|
||||||
import org.elasticsearch.client.ElasticsearchClient;
|
|
||||||
import org.elasticsearch.common.bytes.BytesReference;
|
|
||||||
import org.elasticsearch.common.settings.Settings;
|
|
||||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
|
||||||
|
|
||||||
import java.io.Closeable;
|
|
||||||
import java.io.Flushable;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.InputStream;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Interface for extended managing and indexing methods of an Elasticsearch client.
|
|
||||||
*/
|
|
||||||
public interface ExtendedClient extends Flushable, Closeable {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Set an Elasticsearch client to extend from it. May be null for TransportClient.
|
|
||||||
* @param client client
|
|
||||||
* @return this client
|
|
||||||
*/
|
|
||||||
ExtendedClient setClient(ElasticsearchClient client);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Return Elasticsearch client.
|
|
||||||
*
|
|
||||||
* @return Elasticsearch client
|
|
||||||
*/
|
|
||||||
ElasticsearchClient getClient();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get buulk control.
|
|
||||||
* @return the bulk control
|
|
||||||
*/
|
|
||||||
BulkController getBulkController();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initiative the extended client, the bulk metric and bulk controller,
|
|
||||||
* creates instances and connect to cluster, if required.
|
|
||||||
*
|
|
||||||
* @param settings settings
|
|
||||||
* @return this client
|
|
||||||
* @throws IOException if init fails
|
|
||||||
*/
|
|
||||||
ExtendedClient init(Settings settings) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Build index definition from settings.
|
|
||||||
*
|
|
||||||
* @param index the index name
|
|
||||||
* @param settings the settings for the index
|
|
||||||
* @return index definition
|
|
||||||
* @throws IOException if settings/mapping URL is invalid/malformed
|
|
||||||
*/
|
|
||||||
IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Add index request. Each request will be added to a queue for bulking requests.
|
|
||||||
* Submitting request will be done when limits are exceeded.
|
|
||||||
*
|
|
||||||
* @param index the index
|
|
||||||
* @param id the id
|
|
||||||
* @param create true if document must be created
|
|
||||||
* @param source the source
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient index(String index, String id, boolean create, BytesReference source);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Index request. Each request will be added to a queue for bulking requests.
|
|
||||||
* Submitting request will be done when limits are exceeded.
|
|
||||||
*
|
|
||||||
* @param index the index
|
|
||||||
* @param id the id
|
|
||||||
* @param create true if document is to be created, false otherwise
|
|
||||||
* @param source the source
|
|
||||||
* @return this client methods
|
|
||||||
*/
|
|
||||||
ExtendedClient index(String index, String id, boolean create, String source);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Index request. Each request will be added to a queue for bulking requests.
|
|
||||||
* Submitting request will be done when bulk limits are exceeded.
|
|
||||||
*
|
|
||||||
* @param indexRequest the index request to add
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient index(IndexRequest indexRequest);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Delete request.
|
|
||||||
*
|
|
||||||
* @param index the index
|
|
||||||
* @param id the id
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient delete(String index, String id);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Delete request. Each request will be added to a queue for bulking requests.
|
|
||||||
* Submitting request will be done when bulk limits are exceeded.
|
|
||||||
*
|
|
||||||
* @param deleteRequest the delete request to add
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient delete(DeleteRequest deleteRequest);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Bulked update request. Each request will be added to a queue for bulking requests.
|
|
||||||
* Submitting request will be done when bulk limits are exceeded.
|
|
||||||
* Note that updates only work correctly when all operations between nodes are synchronized.
|
|
||||||
*
|
|
||||||
* @param index the index
|
|
||||||
* @param id the id
|
|
||||||
* @param source the source
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if update fails
|
|
||||||
*/
|
|
||||||
ExtendedClient update(String index, String id, BytesReference source) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Update document. Use with precaution! Does not work in all cases.
|
|
||||||
*
|
|
||||||
* @param index the index
|
|
||||||
* @param id the id
|
|
||||||
* @param source the source
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient update(String index, String id, String source);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Bulked update request. Each request will be added to a queue for bulking requests.
|
|
||||||
* Submitting request will be done when bulk limits are exceeded.
|
|
||||||
* Note that updates only work correctly when all operations between nodes are synchronized.
|
|
||||||
*
|
|
||||||
* @param updateRequest the update request to add
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient update(UpdateRequest updateRequest);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a new index.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if new index creation fails
|
|
||||||
*/
|
|
||||||
ExtendedClient newIndex(String index) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a new index.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @param settings settings
|
|
||||||
* @param mapping mapping
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if settings/mapping is invalid or index creation fails
|
|
||||||
*/
|
|
||||||
ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a new index.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @param settings settings
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if settings is invalid or index creation fails
|
|
||||||
*/
|
|
||||||
ExtendedClient newIndex(String index, Settings settings) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a new index.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @param settings settings
|
|
||||||
* @param mapping mapping
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if settings/mapping is invalid or index creation fails
|
|
||||||
*/
|
|
||||||
ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a new index.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @param settings settings
|
|
||||||
* @param mapping mapping
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if settings/mapping is invalid or index creation fails
|
|
||||||
*/
|
|
||||||
ExtendedClient newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a new index.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @param settings settings
|
|
||||||
* @param mapping mapping
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if settings/mapping is invalid or index creation fails
|
|
||||||
*/
|
|
||||||
ExtendedClient newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a new index.
|
|
||||||
* @param indexDefinition the index definition
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if settings/mapping is invalid or index creation fails
|
|
||||||
*/
|
|
||||||
ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Delete an index.
|
|
||||||
* @param indexDefinition the index definition
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient deleteIndex(IndexDefinition indexDefinition);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Delete an index.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient deleteIndex(String index);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Start bulk mode for indexes.
|
|
||||||
* @param indexDefinition index definition
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if bulk could not be started
|
|
||||||
*/
|
|
||||||
ExtendedClient startBulk(IndexDefinition indexDefinition) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Start bulk mode.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @param startRefreshIntervalSeconds refresh interval before bulk
|
|
||||||
* @param stopRefreshIntervalSeconds refresh interval after bulk
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if bulk could not be started
|
|
||||||
*/
|
|
||||||
ExtendedClient startBulk(String index, long startRefreshIntervalSeconds,
|
|
||||||
long stopRefreshIntervalSeconds) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Stop bulk mode.
|
|
||||||
*
|
|
||||||
* @param indexDefinition index definition
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if bulk could not be startet
|
|
||||||
*/
|
|
||||||
ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Stops bulk mode.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @param timeout maximum wait time
|
|
||||||
* @param timeUnit time unit for timeout
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if bulk could not be stopped
|
|
||||||
*/
|
|
||||||
ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Update replica level.
|
|
||||||
* @param indexDefinition the index definition
|
|
||||||
* @param level the replica level
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if replica setting could not be updated
|
|
||||||
*/
|
|
||||||
ExtendedClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Update replica level.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @param level the replica level
|
|
||||||
* @param maxWaitTime maximum wait time
|
|
||||||
* @param timeUnit time unit
|
|
||||||
* @return this
|
|
||||||
* @throws IOException if replica setting could not be updated
|
|
||||||
*/
|
|
||||||
ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get replica level.
|
|
||||||
* @param indexDefinition the index name
|
|
||||||
* @return the replica level of the index
|
|
||||||
*/
|
|
||||||
int getReplicaLevel(IndexDefinition indexDefinition);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get replica level.
|
|
||||||
* @param index the index name
|
|
||||||
* @return the replica level of the index
|
|
||||||
*/
|
|
||||||
int getReplicaLevel(String index);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Refresh the index.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient refreshIndex(String index);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Flush the index. The cluster clears cache and completes indexing.
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
ExtendedClient flushIndex(String index);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Force segment merge of an index.
|
|
||||||
* @param indexDefinition the index definition
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
boolean forceMerge(IndexDefinition indexDefinition);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Force segment merge of an index.
|
|
||||||
* @param index the index
|
|
||||||
* @param maxWaitTime maximum wait time
|
|
||||||
* @param timeUnit time unit
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Wait for all outstanding bulk responses.
|
|
||||||
*
|
|
||||||
* @param timeout maximum wait time
|
|
||||||
* @param timeUnit unit of timeout value
|
|
||||||
* @return true if wait succeeded, false if wait timed out
|
|
||||||
*/
|
|
||||||
boolean waitForResponses(long timeout, TimeUnit timeUnit);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Wait for cluster being healthy.
|
|
||||||
*
|
|
||||||
* @param healthColor cluster health color to wait for
|
|
||||||
* @param maxWaitTime time value
|
|
||||||
* @param timeUnit time unit
|
|
||||||
* @return true if wait succeeded, false if wait timed out
|
|
||||||
*/
|
|
||||||
boolean waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get current health color.
|
|
||||||
*
|
|
||||||
* @param maxWaitTime maximum wait time
|
|
||||||
* @param timeUnit time unit
|
|
||||||
* @return the cluster health color
|
|
||||||
*/
|
|
||||||
String getHealthColor(long maxWaitTime, TimeUnit timeUnit);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Wait for index recovery (after replica change).
|
|
||||||
*
|
|
||||||
* @param index index
|
|
||||||
* @param maxWaitTime maximum wait time
|
|
||||||
* @param timeUnit time unit
|
|
||||||
* @return true if wait succeeded, false if wait timed out
|
|
||||||
*/
|
|
||||||
boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Update index setting.
|
|
||||||
* @param index the index
|
|
||||||
* @param key the key of the value to be updated
|
|
||||||
* @param value the new value
|
|
||||||
* @param timeout timeout
|
|
||||||
* @param timeUnit time unit
|
|
||||||
* @throws IOException if update index setting failed
|
|
||||||
*/
|
|
||||||
void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Resolve alias.
|
|
||||||
*
|
|
||||||
* @param alias the alias
|
|
||||||
* @return this index name behind the alias or the alias if there is no index
|
|
||||||
*/
|
|
||||||
String resolveAlias(String alias);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index
|
|
||||||
* name.
|
|
||||||
*
|
|
||||||
* @param alias the alias
|
|
||||||
* @return the most recent index name pointing to the alias
|
|
||||||
*/
|
|
||||||
String resolveMostRecentIndex(String alias);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all index aliases.
|
|
||||||
* @param index the index
|
|
||||||
* @return map of index aliases
|
|
||||||
*/
|
|
||||||
Map<String, String> getAliases(String index);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Shift from one index to another.
|
|
||||||
* @param indexDefinition the index definition
|
|
||||||
* @param additionalAliases new aliases
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Shift from one index to another.
|
|
||||||
* @param indexDefinition the index definition
|
|
||||||
* @param additionalAliases new aliases
|
|
||||||
* @param indexAliasAdder method to add aliases
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases,
|
|
||||||
IndexAliasAdder indexAliasAdder);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Shift from one index to another.
|
|
||||||
* @param index the index name
|
|
||||||
* @param fullIndexName the index name with timestamp
|
|
||||||
* @param additionalAliases a list of names that should be set as index aliases
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Shift from one index to another.
|
|
||||||
* @param index the index name
|
|
||||||
* @param fullIndexName the index name with timestamp
|
|
||||||
* @param additionalAliases a list of names that should be set as index aliases
|
|
||||||
* @param adder an adder method to create alias term queries
|
|
||||||
* @return this
|
|
||||||
*/
|
|
||||||
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases,
|
|
||||||
IndexAliasAdder adder);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Prune index.
|
|
||||||
* @param indexDefinition the index definition
|
|
||||||
* @return the index prune result
|
|
||||||
*/
|
|
||||||
IndexPruneResult pruneIndex(IndexDefinition indexDefinition);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Apply retention policy to prune indices. All indices before delta should be deleted,
|
|
||||||
* but the number of mintokeep indices must be kept.
|
|
||||||
*
|
|
||||||
* @param index index name
|
|
||||||
* @param fullIndexName index name with timestamp
|
|
||||||
* @param delta timestamp delta (for index timestamps)
|
|
||||||
* @param mintokeep minimum number of indices to keep
|
|
||||||
* @param perform true if pruning should be executed, false if not
|
|
||||||
* @return the index prune result
|
|
||||||
*/
|
|
||||||
IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Find the timestamp of the most recently indexed document in the index.
|
|
||||||
*
|
|
||||||
* @param index the index name
|
|
||||||
* @param timestampfieldname the timestamp field name
|
|
||||||
* @return millis UTC millis of the most recent document
|
|
||||||
* @throws IOException if most rcent document can not be found
|
|
||||||
*/
|
|
||||||
Long mostRecentDocument(String index, String timestampfieldname) throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get cluster name.
|
|
||||||
* @return the cluster name
|
|
||||||
*/
|
|
||||||
String getClusterName();
|
|
||||||
}
|
|
@ -1,7 +0,0 @@
|
|||||||
package org.xbib.elx.api;
|
|
||||||
|
|
||||||
@FunctionalInterface
|
|
||||||
public interface ExtendedClientProvider<C extends ExtendedClient> {
|
|
||||||
|
|
||||||
C getExtendedClient();
|
|
||||||
}
|
|
@ -1,10 +1,10 @@
|
|||||||
package org.xbib.elx.api;
|
package org.xbib.elx.api;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.Collection;
|
||||||
|
|
||||||
public interface IndexShiftResult {
|
public interface IndexShiftResult {
|
||||||
|
|
||||||
List<String> getMovedAliases();
|
Collection<String> getMovedAliases();
|
||||||
|
|
||||||
List<String> getNewAliases();
|
Collection<String> getNewAliases();
|
||||||
}
|
}
|
||||||
|
@ -0,0 +1,63 @@
|
|||||||
|
package org.xbib.elx.api;
|
||||||
|
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import java.io.Closeable;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
public interface NativeClient extends Closeable {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set an Elasticsearch client to extend from it. May be null for TransportClient.
|
||||||
|
* @param client the Elasticsearch client
|
||||||
|
*/
|
||||||
|
void setClient(ElasticsearchClient client);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return Elasticsearch client.
|
||||||
|
*
|
||||||
|
* @return Elasticsearch client
|
||||||
|
*/
|
||||||
|
ElasticsearchClient getClient();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initiative the extended client, the bulk metric and bulk controller,
|
||||||
|
* creates instances and connect to cluster, if required.
|
||||||
|
*
|
||||||
|
* @param settings settings
|
||||||
|
* @throws IOException if init fails
|
||||||
|
*/
|
||||||
|
void init(Settings settings) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get cluster name.
|
||||||
|
* @return the cluster name
|
||||||
|
*/
|
||||||
|
String getClusterName();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get current health color.
|
||||||
|
*
|
||||||
|
* @param maxWaitTime maximum wait time
|
||||||
|
* @param timeUnit time unit
|
||||||
|
* @return the cluster health color
|
||||||
|
*/
|
||||||
|
String getHealthColor(long maxWaitTime, TimeUnit timeUnit);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wait for cluster being healthy.
|
||||||
|
*
|
||||||
|
* @param healthColor cluster health color to wait for
|
||||||
|
* @param maxWaitTime time value
|
||||||
|
* @param timeUnit time unit
|
||||||
|
*/
|
||||||
|
void waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit);
|
||||||
|
|
||||||
|
Map<String, ?> getMapping(String index, String mapping);
|
||||||
|
|
||||||
|
long getSearchableDocs(String index);
|
||||||
|
|
||||||
|
boolean isIndexExists(String index);
|
||||||
|
}
|
@ -1,25 +1,27 @@
|
|||||||
package org.xbib.elx.api;
|
package org.xbib.elx.api;
|
||||||
|
|
||||||
import org.elasticsearch.action.ActionFuture;
|
import org.elasticsearch.action.get.GetRequestBuilder;
|
||||||
import org.elasticsearch.action.ActionListener;
|
|
||||||
import org.elasticsearch.action.get.GetRequest;
|
|
||||||
import org.elasticsearch.action.get.GetResponse;
|
import org.elasticsearch.action.get.GetResponse;
|
||||||
import org.elasticsearch.action.get.MultiGetRequest;
|
import org.elasticsearch.action.get.MultiGetRequestBuilder;
|
||||||
import org.elasticsearch.action.get.MultiGetResponse;
|
import org.elasticsearch.action.get.MultiGetResponse;
|
||||||
import org.elasticsearch.action.search.SearchRequest;
|
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||||
import org.elasticsearch.action.search.SearchResponse;
|
import org.elasticsearch.action.search.SearchResponse;
|
||||||
|
import org.elasticsearch.common.unit.TimeValue;
|
||||||
|
import org.elasticsearch.search.SearchHit;
|
||||||
|
import java.util.Optional;
|
||||||
|
import java.util.function.Consumer;
|
||||||
|
import java.util.stream.Stream;
|
||||||
|
|
||||||
public interface ReadClient {
|
public interface SearchClient {
|
||||||
|
|
||||||
ActionFuture<GetResponse> get(GetRequest getRequest);
|
Optional<GetResponse> get(Consumer<GetRequestBuilder> getRequestBuilder);
|
||||||
|
|
||||||
void get(GetRequest request, ActionListener<GetResponse> listener);
|
Optional<MultiGetResponse> multiGet(Consumer<MultiGetRequestBuilder> multiGetRequestBuilder);
|
||||||
|
|
||||||
ActionFuture<MultiGetResponse> multiGet(MultiGetRequest request);
|
Optional<SearchResponse> search(Consumer<SearchRequestBuilder> searchRequestBuilder);
|
||||||
|
|
||||||
void multiGet(MultiGetRequest request, ActionListener<MultiGetResponse> listener);
|
Stream<SearchHit> search(Consumer<SearchRequestBuilder> searchRequestBuilder,
|
||||||
|
TimeValue scrollTime, int scrollSize);
|
||||||
|
|
||||||
ActionFuture<SearchResponse> search(SearchRequest request);
|
Stream<String> getIds(Consumer<SearchRequestBuilder> queryBuilder);
|
||||||
|
|
||||||
void search(SearchRequest request, ActionListener<SearchResponse> listener);
|
|
||||||
}
|
}
|
||||||
|
@ -0,0 +1,7 @@
|
|||||||
|
package org.xbib.elx.api;
|
||||||
|
|
||||||
|
@FunctionalInterface
|
||||||
|
public interface SearchClientProvider<C extends SearchClient> {
|
||||||
|
|
||||||
|
C getClient();
|
||||||
|
}
|
@ -0,0 +1,246 @@
|
|||||||
|
package org.xbib.elx.common;
|
||||||
|
|
||||||
|
import org.apache.logging.log4j.Level;
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
|
||||||
|
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
|
||||||
|
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
|
||||||
|
import org.elasticsearch.action.admin.indices.flush.FlushAction;
|
||||||
|
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
|
||||||
|
import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
|
||||||
|
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
|
||||||
|
import org.elasticsearch.action.delete.DeleteRequest;
|
||||||
|
import org.elasticsearch.action.index.IndexRequest;
|
||||||
|
import org.elasticsearch.action.update.UpdateRequest;
|
||||||
|
import org.elasticsearch.common.Strings;
|
||||||
|
import org.elasticsearch.common.bytes.BytesReference;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.common.xcontent.DeprecationHandler;
|
||||||
|
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
|
||||||
|
import org.elasticsearch.common.xcontent.ToXContent;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentType;
|
||||||
|
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||||
|
import org.xbib.elx.api.BulkClient;
|
||||||
|
import org.xbib.elx.api.BulkController;
|
||||||
|
import org.xbib.elx.api.BulkMetric;
|
||||||
|
import org.xbib.elx.api.IndexDefinition;
|
||||||
|
import java.io.ByteArrayOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
public abstract class AbstractBulkClient extends AbstractNativeClient implements BulkClient {
|
||||||
|
|
||||||
|
private static final Logger logger = LogManager.getLogger(AbstractBulkClient.class.getName());
|
||||||
|
|
||||||
|
private BulkMetric bulkMetric;
|
||||||
|
|
||||||
|
private BulkController bulkController;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) throws IOException {
|
||||||
|
logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(','));
|
||||||
|
super.init(settings);
|
||||||
|
if (bulkMetric == null) {
|
||||||
|
bulkMetric = new DefaultBulkMetric();
|
||||||
|
bulkMetric.init(settings);
|
||||||
|
}
|
||||||
|
if (bulkController == null) {
|
||||||
|
bulkController = new DefaultBulkController(this, bulkMetric);
|
||||||
|
bulkController.init(settings);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BulkMetric getBulkMetric() {
|
||||||
|
return bulkMetric;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BulkController getBulkController() {
|
||||||
|
return bulkController;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void flush() throws IOException {
|
||||||
|
if (bulkController != null) {
|
||||||
|
bulkController.flush();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() throws IOException {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
if (closed.compareAndSet(false, true)) {
|
||||||
|
if (bulkMetric != null) {
|
||||||
|
logger.info("closing bulk metric");
|
||||||
|
bulkMetric.close();
|
||||||
|
}
|
||||||
|
if (bulkController != null) {
|
||||||
|
logger.info("closing bulk controller");
|
||||||
|
bulkController.close();
|
||||||
|
}
|
||||||
|
closeClient();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void newIndex(IndexDefinition indexDefinition) throws IOException {
|
||||||
|
Settings settings = indexDefinition.getSettings() == null ? null :
|
||||||
|
Settings.builder().loadFromSource(indexDefinition.getSettings(), XContentType.JSON).build();
|
||||||
|
Map<String, ?> mappings = indexDefinition.getMappings() == null ? null :
|
||||||
|
JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
|
||||||
|
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, indexDefinition.getMappings()).mapOrdered();
|
||||||
|
newIndex(indexDefinition.getFullIndexName(), settings, mappings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void newIndex(String index) throws IOException {
|
||||||
|
newIndex(index, Settings.EMPTY, (Map<String, ?>) null);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void newIndex(String index, Settings settings) throws IOException {
|
||||||
|
newIndex(index, settings, (Map<String, ?>) null);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void newIndex(String index, Settings settings, XContentBuilder builder) throws IOException {
|
||||||
|
String mappingString = Strings.toString(builder);
|
||||||
|
Map<String, ?> mappings = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
|
||||||
|
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mappingString).mapOrdered();
|
||||||
|
newIndex(index, settings, mappings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException {
|
||||||
|
if (index == null) {
|
||||||
|
logger.warn("no index name given to create index");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
ensureClientIsPresent();
|
||||||
|
waitForCluster("YELLOW", 30L, TimeUnit.SECONDS);
|
||||||
|
CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index);
|
||||||
|
if (settings != null) {
|
||||||
|
createIndexRequest.settings(settings);
|
||||||
|
}
|
||||||
|
if (mapping != null) {
|
||||||
|
createIndexRequest.mapping(TYPE_NAME, mapping);
|
||||||
|
}
|
||||||
|
CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
|
||||||
|
XContentBuilder builder = XContentFactory.jsonBuilder();
|
||||||
|
logger.info("index {} created: {}", index,
|
||||||
|
Strings.toString(createIndexResponse.toXContent(builder, ToXContent.EMPTY_PARAMS)));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void startBulk(IndexDefinition indexDefinition) throws IOException {
|
||||||
|
startBulk(indexDefinition.getFullIndexName(), -1, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds)
|
||||||
|
throws IOException {
|
||||||
|
if (bulkController != null) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void stopBulk(IndexDefinition indexDefinition) throws IOException {
|
||||||
|
if (bulkController != null) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
bulkController.stopBulkMode(indexDefinition);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException {
|
||||||
|
if (bulkController != null) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
bulkController.stopBulkMode(index, timeout, timeUnit);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BulkClient index(String index, String id, boolean create, String source) {
|
||||||
|
return index(new IndexRequest(index, TYPE_NAME, id).create(create)
|
||||||
|
.source(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BulkClient index(String index, String id, boolean create, BytesReference source) {
|
||||||
|
return index(new IndexRequest(index, TYPE_NAME, id).create(create)
|
||||||
|
.source(source, XContentType.JSON));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BulkClient index(IndexRequest indexRequest) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
bulkController.index(indexRequest);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BulkClient delete(String index, String id) {
|
||||||
|
return delete(new DeleteRequest(index, TYPE_NAME, id));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BulkClient delete(DeleteRequest deleteRequest) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
bulkController.delete(deleteRequest);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BulkClient update(String index, String id, BytesReference source) {
|
||||||
|
return update(new UpdateRequest(index, TYPE_NAME, id)
|
||||||
|
.doc(source, XContentType.JSON));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BulkClient update(String index, String id, String source) {
|
||||||
|
return update(new UpdateRequest(index, TYPE_NAME, id)
|
||||||
|
.doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BulkClient update(UpdateRequest updateRequest) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
bulkController.update(updateRequest);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean waitForResponses(long timeout, TimeUnit timeUnit) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
return bulkController.waitForResponses(timeout, timeUnit);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException {
|
||||||
|
super.updateIndexSetting(index, key, value, timeout, timeUnit);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void flushIndex(String index) {
|
||||||
|
if (index != null) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void refreshIndex(String index) {
|
||||||
|
if (index != null) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,218 @@
|
|||||||
|
package org.xbib.elx.common;
|
||||||
|
|
||||||
|
import org.apache.logging.log4j.Level;
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
import org.elasticsearch.ElasticsearchTimeoutException;
|
||||||
|
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
|
||||||
|
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
|
||||||
|
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
|
||||||
|
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
|
||||||
|
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
|
||||||
|
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
|
||||||
|
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
|
||||||
|
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
|
||||||
|
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
|
||||||
|
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
|
||||||
|
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequestBuilder;
|
||||||
|
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
|
||||||
|
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
|
||||||
|
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
|
||||||
|
import org.elasticsearch.action.search.SearchAction;
|
||||||
|
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||||
|
import org.elasticsearch.cluster.health.ClusterHealthStatus;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.common.unit.TimeValue;
|
||||||
|
import org.elasticsearch.index.query.QueryBuilders;
|
||||||
|
import org.xbib.elx.api.NativeClient;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
|
|
||||||
|
public abstract class AbstractNativeClient implements NativeClient {
|
||||||
|
|
||||||
|
private static final Logger logger = LogManager.getLogger(AbstractNativeClient.class.getName());
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The one and only index type name used in the extended client.
|
||||||
|
* Notr that all Elasticsearch version < 6.2.0 do not allow a prepending "_".
|
||||||
|
*/
|
||||||
|
protected static final String TYPE_NAME = "doc";
|
||||||
|
|
||||||
|
protected ElasticsearchClient client;
|
||||||
|
|
||||||
|
protected final AtomicBoolean closed;
|
||||||
|
|
||||||
|
protected AbstractNativeClient() {
|
||||||
|
closed = new AtomicBoolean(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setClient(ElasticsearchClient client) {
|
||||||
|
this.client = client;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ElasticsearchClient getClient() {
|
||||||
|
return client;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected abstract ElasticsearchClient createClient(Settings settings) throws IOException;
|
||||||
|
|
||||||
|
protected abstract void closeClient() throws IOException;
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) throws IOException {
|
||||||
|
logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(','));
|
||||||
|
if (client == null) {
|
||||||
|
client = createClient(settings);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getClusterName() {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
try {
|
||||||
|
ClusterStateRequest clusterStateRequest = new ClusterStateRequest().clear();
|
||||||
|
ClusterStateResponse clusterStateResponse =
|
||||||
|
getClient().execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
|
||||||
|
return clusterStateResponse.getClusterName().value();
|
||||||
|
} catch (ElasticsearchTimeoutException e) {
|
||||||
|
logger.warn(e.getMessage(), e);
|
||||||
|
return "TIMEOUT";
|
||||||
|
} catch (NoNodeAvailableException e) {
|
||||||
|
logger.warn(e.getMessage(), e);
|
||||||
|
return "DISCONNECTED";
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.warn(e.getMessage(), e);
|
||||||
|
return "[" + e.getMessage() + "]";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString);
|
||||||
|
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
|
||||||
|
ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
|
||||||
|
new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet();
|
||||||
|
if (healthResponse != null && healthResponse.isTimedOut()) {
|
||||||
|
String message = "timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name();
|
||||||
|
if (logger.isErrorEnabled()) {
|
||||||
|
logger.error(message);
|
||||||
|
}
|
||||||
|
throw new IllegalStateException(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
try {
|
||||||
|
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
|
||||||
|
ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
|
||||||
|
new ClusterHealthRequest().timeout(timeout)).actionGet();
|
||||||
|
ClusterHealthStatus status = healthResponse.getStatus();
|
||||||
|
return status.name();
|
||||||
|
} catch (ElasticsearchTimeoutException e) {
|
||||||
|
logger.warn(e.getMessage(), e);
|
||||||
|
return "TIMEOUT";
|
||||||
|
} catch (NoNodeAvailableException e) {
|
||||||
|
logger.warn(e.getMessage(), e);
|
||||||
|
return "DISCONNECTED";
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.warn(e.getMessage(), e);
|
||||||
|
return "[" + e.getMessage() + "]";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<String, ?> getMapping(String index, String mapping) {
|
||||||
|
GetMappingsRequestBuilder getMappingsRequestBuilder = new GetMappingsRequestBuilder(client, GetMappingsAction.INSTANCE)
|
||||||
|
.setIndices(index)
|
||||||
|
.setTypes(mapping);
|
||||||
|
GetMappingsResponse getMappingsResponse = getMappingsRequestBuilder.execute().actionGet();
|
||||||
|
logger.info("get mappings response = {}", getMappingsResponse.getMappings().get(index).get(mapping).getSourceAsMap());
|
||||||
|
return getMappingsResponse.getMappings().get(index).get(mapping).getSourceAsMap();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long getSearchableDocs(String index) {
|
||||||
|
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE)
|
||||||
|
.setIndices(index)
|
||||||
|
.setQuery(QueryBuilders.matchAllQuery())
|
||||||
|
.setSize(0);
|
||||||
|
return searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isIndexExists(String index) {
|
||||||
|
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
|
||||||
|
indicesExistsRequest.indices(index);
|
||||||
|
IndicesExistsResponse indicesExistsResponse =
|
||||||
|
client.execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
|
||||||
|
return indicesExistsResponse.isExists();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() throws IOException {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
if (closed.compareAndSet(false, true)) {
|
||||||
|
closeClient();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException {
|
||||||
|
ensureClientIsPresent();
|
||||||
|
if (index == null) {
|
||||||
|
throw new IOException("no index name given");
|
||||||
|
}
|
||||||
|
if (key == null) {
|
||||||
|
throw new IOException("no key given");
|
||||||
|
}
|
||||||
|
if (value == null) {
|
||||||
|
throw new IOException("no value given");
|
||||||
|
}
|
||||||
|
Settings.Builder updateSettingsBuilder = Settings.builder();
|
||||||
|
updateSettingsBuilder.put(key, value.toString());
|
||||||
|
UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index)
|
||||||
|
.settings(updateSettingsBuilder).timeout(toTimeValue(timeout, timeUnit));
|
||||||
|
client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void ensureClientIsPresent() {
|
||||||
|
if (this instanceof MockAdminClient) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (client == null) {
|
||||||
|
throw new IllegalStateException("no client");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected static TimeValue toTimeValue(long timeValue, TimeUnit timeUnit) {
|
||||||
|
switch (timeUnit) {
|
||||||
|
case DAYS:
|
||||||
|
return TimeValue.timeValueHours(24 * timeValue);
|
||||||
|
case HOURS:
|
||||||
|
return TimeValue.timeValueHours(timeValue);
|
||||||
|
case MINUTES:
|
||||||
|
return TimeValue.timeValueMinutes(timeValue);
|
||||||
|
case SECONDS:
|
||||||
|
return TimeValue.timeValueSeconds(timeValue);
|
||||||
|
case MILLISECONDS:
|
||||||
|
return TimeValue.timeValueMillis(timeValue);
|
||||||
|
case MICROSECONDS:
|
||||||
|
return TimeValue.timeValueNanos(1000 * timeValue);
|
||||||
|
case NANOSECONDS:
|
||||||
|
return TimeValue.timeValueNanos(timeValue);
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException("unknown time unit: " + timeUnit);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -0,0 +1,158 @@
|
|||||||
|
package org.xbib.elx.common;
|
||||||
|
|
||||||
|
import org.elasticsearch.ElasticsearchException;
|
||||||
|
import org.elasticsearch.action.get.GetAction;
|
||||||
|
import org.elasticsearch.action.get.GetRequestBuilder;
|
||||||
|
import org.elasticsearch.action.get.GetResponse;
|
||||||
|
import org.elasticsearch.action.get.MultiGetAction;
|
||||||
|
import org.elasticsearch.action.get.MultiGetRequestBuilder;
|
||||||
|
import org.elasticsearch.action.get.MultiGetResponse;
|
||||||
|
import org.elasticsearch.action.search.ClearScrollAction;
|
||||||
|
import org.elasticsearch.action.search.ClearScrollRequestBuilder;
|
||||||
|
import org.elasticsearch.action.search.SearchAction;
|
||||||
|
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||||
|
import org.elasticsearch.action.search.SearchResponse;
|
||||||
|
import org.elasticsearch.action.search.SearchScrollAction;
|
||||||
|
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
|
||||||
|
import org.elasticsearch.action.search.ShardSearchFailure;
|
||||||
|
import org.elasticsearch.common.unit.TimeValue;
|
||||||
|
import org.elasticsearch.search.SearchHit;
|
||||||
|
import org.xbib.elx.api.SearchClient;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Comparator;
|
||||||
|
import java.util.Optional;
|
||||||
|
import java.util.Spliterator;
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
|
import java.util.function.Consumer;
|
||||||
|
import java.util.function.Predicate;
|
||||||
|
import java.util.stream.Stream;
|
||||||
|
import java.util.stream.StreamSupport;
|
||||||
|
|
||||||
|
public abstract class AbstractSearchClient extends AbstractNativeClient implements SearchClient {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Optional<GetResponse> get(Consumer<GetRequestBuilder> getRequestBuilderConsumer) {
|
||||||
|
GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE);
|
||||||
|
getRequestBuilderConsumer.accept(getRequestBuilder);
|
||||||
|
GetResponse getResponse = getRequestBuilder.execute().actionGet();
|
||||||
|
return getResponse.isExists() ? Optional.of(getResponse) : Optional.empty();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Optional<MultiGetResponse> multiGet(Consumer<MultiGetRequestBuilder> multiGetRequestBuilderConsumer) {
|
||||||
|
MultiGetRequestBuilder multiGetRequestBuilder = new MultiGetRequestBuilder(client, MultiGetAction.INSTANCE);
|
||||||
|
multiGetRequestBuilderConsumer.accept(multiGetRequestBuilder);
|
||||||
|
MultiGetResponse multiGetItemResponse = multiGetRequestBuilder.execute().actionGet();
|
||||||
|
return multiGetItemResponse.getResponses().length == 0 ? Optional.empty() : Optional.of(multiGetItemResponse);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Optional<SearchResponse> search(Consumer<SearchRequestBuilder> queryBuilder) {
|
||||||
|
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
|
||||||
|
queryBuilder.accept(searchRequestBuilder);
|
||||||
|
SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
|
||||||
|
if (searchResponse.getFailedShards() > 0) {
|
||||||
|
StringBuilder sb = new StringBuilder("Search failed:");
|
||||||
|
for (ShardSearchFailure failure : searchResponse.getShardFailures()) {
|
||||||
|
sb.append("\n").append(failure.reason());
|
||||||
|
}
|
||||||
|
throw new ElasticsearchException(sb.toString());
|
||||||
|
}
|
||||||
|
return searchResponse.getHits().getHits().length == 0 ? Optional.empty() : Optional.of(searchResponse);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Stream<SearchHit> search(Consumer<SearchRequestBuilder> queryBuilder,
|
||||||
|
TimeValue scrollTime, int scrollSize) {
|
||||||
|
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
|
||||||
|
queryBuilder.accept(searchRequestBuilder);
|
||||||
|
searchRequestBuilder.setScroll(scrollTime).setSize(scrollSize);
|
||||||
|
SearchResponse originalSearchResponse = searchRequestBuilder.execute().actionGet();
|
||||||
|
Stream<SearchResponse> infiniteResponses = Stream.iterate(originalSearchResponse,
|
||||||
|
searchResponse -> new SearchScrollRequestBuilder(client, SearchScrollAction.INSTANCE)
|
||||||
|
.setScrollId(searchResponse.getScrollId())
|
||||||
|
.setScroll(scrollTime)
|
||||||
|
.execute().actionGet());
|
||||||
|
Predicate<SearchResponse> condition = searchResponse -> searchResponse.getHits().getHits().length > 0;
|
||||||
|
Consumer<SearchResponse> lastAction = searchResponse -> {
|
||||||
|
ClearScrollRequestBuilder clearScrollRequestBuilder =
|
||||||
|
new ClearScrollRequestBuilder(client, ClearScrollAction.INSTANCE)
|
||||||
|
.addScrollId(searchResponse.getScrollId());
|
||||||
|
clearScrollRequestBuilder.execute().actionGet();
|
||||||
|
};
|
||||||
|
return StreamSupport.stream(TakeWhileSpliterator.over(infiniteResponses.spliterator(),
|
||||||
|
condition, lastAction), false)
|
||||||
|
.onClose(infiniteResponses::close)
|
||||||
|
.flatMap(searchResponse -> Arrays.stream(searchResponse.getHits().getHits()));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Stream<String> getIds(Consumer<SearchRequestBuilder> queryBuilder) {
|
||||||
|
return search(queryBuilder, TimeValue.timeValueMinutes(1), 1000).map(SearchHit::getId);
|
||||||
|
}
|
||||||
|
|
||||||
|
static class TakeWhileSpliterator<T> implements Spliterator<T> {
|
||||||
|
|
||||||
|
private final Spliterator<T> source;
|
||||||
|
|
||||||
|
private final Predicate<T> condition;
|
||||||
|
|
||||||
|
private final Consumer<T> lastElement;
|
||||||
|
|
||||||
|
private final boolean inclusive;
|
||||||
|
|
||||||
|
private final AtomicBoolean checked;
|
||||||
|
|
||||||
|
static <T> TakeWhileSpliterator<T> over(Spliterator<T> source,
|
||||||
|
Predicate<T> condition,
|
||||||
|
Consumer<T> lastElement) {
|
||||||
|
return new TakeWhileSpliterator<>(source, condition, lastElement, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
private TakeWhileSpliterator(Spliterator<T> source,
|
||||||
|
Predicate<T> condition,
|
||||||
|
Consumer<T> lastElement,
|
||||||
|
boolean inclusive) {
|
||||||
|
this.source = source;
|
||||||
|
this.condition = condition;
|
||||||
|
this.lastElement = lastElement;
|
||||||
|
this.inclusive = inclusive;
|
||||||
|
this.checked = new AtomicBoolean(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean tryAdvance(Consumer<? super T> action) {
|
||||||
|
return checked.get() && source.tryAdvance(e -> {
|
||||||
|
if (condition.test(e)) {
|
||||||
|
action.accept(e);
|
||||||
|
} else {
|
||||||
|
if (inclusive && checked.get()) {
|
||||||
|
action.accept(e);
|
||||||
|
}
|
||||||
|
lastElement.accept(e);
|
||||||
|
checked.set(false);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Spliterator<T> trySplit() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long estimateSize() {
|
||||||
|
return checked.get() ? source.estimateSize() : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int characteristics() {
|
||||||
|
return source.characteristics() &~ Spliterator.SIZED;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Comparator<? super T> getComparator() {
|
||||||
|
return source.getComparator();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,59 @@
|
|||||||
|
package org.xbib.elx.common;
|
||||||
|
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A mocked client, it does not perform any actions on a cluster. Useful for testing.
|
||||||
|
*/
|
||||||
|
public class MockAdminClient extends AbstractAdminClient {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ElasticsearchClient getClient() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MockAdminClient deleteIndex(String index) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MockAdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() {
|
||||||
|
// nothing to do
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,10 @@
|
|||||||
|
package org.xbib.elx.common;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.AdminClientProvider;
|
||||||
|
|
||||||
|
public class MockAdminClientProvider implements AdminClientProvider<MockAdminClient> {
|
||||||
|
@Override
|
||||||
|
public MockAdminClient getClient() {
|
||||||
|
return new MockAdminClient();
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,99 @@
|
|||||||
|
package org.xbib.elx.common;
|
||||||
|
|
||||||
|
import org.elasticsearch.action.delete.DeleteRequest;
|
||||||
|
import org.elasticsearch.action.index.IndexRequest;
|
||||||
|
import org.elasticsearch.action.update.UpdateRequest;
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A mocked client, it does not perform any actions on a cluster. Useful for testing.
|
||||||
|
*/
|
||||||
|
public class MockBulkClient extends AbstractBulkClient {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ElasticsearchClient getClient() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getClusterName() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MockBulkClient index(String index, String id, boolean create, String source) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MockBulkClient delete(String index, String id) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MockBulkClient update(String index, String id, String source) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MockBulkClient index(IndexRequest indexRequest) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MockBulkClient delete(DeleteRequest deleteRequest) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MockBulkClient update(UpdateRequest updateRequest) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void startBulk(String index, long startRefreshInterval, long stopRefreshIterval) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void refreshIndex(String index) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void flushIndex(String index) {
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void flush() {
|
||||||
|
// nothing to do
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() {
|
||||||
|
// nothing to do
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,11 @@
|
|||||||
|
package org.xbib.elx.common;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.BulkClientProvider;
|
||||||
|
|
||||||
|
public class MockBulkClientProvider implements BulkClientProvider<MockBulkClient> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MockBulkClient getClient() {
|
||||||
|
return new MockBulkClient();
|
||||||
|
}
|
||||||
|
}
|
@ -1,129 +0,0 @@
|
|||||||
package org.xbib.elx.common;
|
|
||||||
|
|
||||||
import org.elasticsearch.action.delete.DeleteRequest;
|
|
||||||
import org.elasticsearch.action.index.IndexRequest;
|
|
||||||
import org.elasticsearch.action.update.UpdateRequest;
|
|
||||||
import org.elasticsearch.client.ElasticsearchClient;
|
|
||||||
import org.elasticsearch.common.settings.Settings;
|
|
||||||
|
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A mocked client, it does not perform any actions on a cluster. Useful for testing.
|
|
||||||
*/
|
|
||||||
public class MockExtendedClient extends AbstractExtendedClient {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ElasticsearchClient getClient() {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient init(Settings settings) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected ElasticsearchClient createClient(Settings settings) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void closeClient() {
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient index(String index, String id, boolean create, String source) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient delete(String index, String id) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient update(String index, String id, String source) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient index(IndexRequest indexRequest) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient delete(DeleteRequest deleteRequest) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient update(UpdateRequest updateRequest) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient startBulk(String index, long startRefreshInterval, long stopRefreshIterval) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient newIndex(String index) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient deleteIndex(String index) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient refreshIndex(String index) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient flushIndex(String index) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void flush() {
|
|
||||||
// nothing to do
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void close() {
|
|
||||||
// nothing to do
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,10 +0,0 @@
|
|||||||
package org.xbib.elx.common;
|
|
||||||
|
|
||||||
import org.xbib.elx.api.ExtendedClientProvider;
|
|
||||||
|
|
||||||
public class MockExtendedClientProvider implements ExtendedClientProvider<MockExtendedClient> {
|
|
||||||
@Override
|
|
||||||
public MockExtendedClient getExtendedClient() {
|
|
||||||
return new MockExtendedClient();
|
|
||||||
}
|
|
||||||
}
|
|
@ -0,0 +1,38 @@
|
|||||||
|
package org.xbib.elx.common;
|
||||||
|
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A mocked client, it does not perform any actions on a cluster. Useful for testing.
|
||||||
|
*/
|
||||||
|
public class MockSearchClient extends AbstractSearchClient {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ElasticsearchClient getClient() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getClusterName() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() {
|
||||||
|
// nothing to do
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,11 @@
|
|||||||
|
package org.xbib.elx.common;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.SearchClientProvider;
|
||||||
|
|
||||||
|
public class MockSearchClientProvider implements SearchClientProvider<MockSearchClient> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public MockSearchClient getClient() {
|
||||||
|
return new MockSearchClient();
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1 @@
|
|||||||
|
org.xbib.elx.common.MockAdminClientProvider
|
@ -1 +0,0 @@
|
|||||||
org.xbib.elx.common.MockExtendedClientProvider
|
|
@ -0,0 +1,21 @@
|
|||||||
|
package org.xbib.elx.common.test;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.xbib.elx.common.ClientBuilder;
|
||||||
|
import org.xbib.elx.common.MockAdminClient;
|
||||||
|
import org.xbib.elx.common.MockAdminClientProvider;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
|
|
||||||
|
class MockAdminClientProviderTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testMockAdminProvider() throws IOException {
|
||||||
|
MockAdminClient client = ClientBuilder.builder()
|
||||||
|
.setAdminClientProvider(MockAdminClientProvider.class)
|
||||||
|
.build();
|
||||||
|
assertNotNull(client);
|
||||||
|
}
|
||||||
|
}
|
@ -1,19 +0,0 @@
|
|||||||
package org.xbib.elx.common.test;
|
|
||||||
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.xbib.elx.common.ClientBuilder;
|
|
||||||
import org.xbib.elx.common.MockExtendedClient;
|
|
||||||
import org.xbib.elx.common.MockExtendedClientProvider;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
|
||||||
|
|
||||||
class MockExtendedClientProviderTest {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testMockExtendedProvider() throws IOException {
|
|
||||||
MockExtendedClient client = ClientBuilder.builder().provider(MockExtendedClientProvider.class).build();
|
|
||||||
assertNotNull(client);
|
|
||||||
}
|
|
||||||
}
|
|
@ -0,0 +1,155 @@
|
|||||||
|
package org.elasticsearch.action.admin.indices.get;
|
||||||
|
|
||||||
|
import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
|
||||||
|
import org.apache.lucene.util.CollectionUtil;
|
||||||
|
import org.elasticsearch.cluster.metadata.AliasMetaData;
|
||||||
|
import org.elasticsearch.cluster.metadata.MappingMetaData;
|
||||||
|
import org.elasticsearch.common.CheckedFunction;
|
||||||
|
import org.elasticsearch.common.collect.ImmutableOpenMap;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentParser;
|
||||||
|
import org.xbib.elx.http.HttpAction;
|
||||||
|
import org.xbib.netty.http.client.api.Request;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.Comparator;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class HttpGetIndexAction extends HttpAction<GetIndexRequest, GetIndexResponse> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetIndexAction getActionInstance() {
|
||||||
|
return GetIndexAction.INSTANCE;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Request.Builder createHttpRequest(String url, GetIndexRequest getIndexRequest) throws IOException {
|
||||||
|
List<String> list = getIndexRequest.indices().length == 0 ?
|
||||||
|
List.of("*") : Arrays.asList(getIndexRequest.indices());
|
||||||
|
String command = "/" + String.join(",", list);
|
||||||
|
logger.info("command = " + command);
|
||||||
|
return newGetRequest(url, command);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected CheckedFunction<XContentParser, GetIndexResponse, IOException> entityParser() {
|
||||||
|
return this::fromXContent;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected GetIndexResponse emptyResponse() {
|
||||||
|
return new GetIndexResponse();
|
||||||
|
}
|
||||||
|
|
||||||
|
private GetIndexResponse fromXContent(XContentParser parser) throws IOException {
|
||||||
|
ImmutableOpenMap.Builder<String, List<AliasMetaData>> aliases = ImmutableOpenMap.builder();
|
||||||
|
ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> mappings = ImmutableOpenMap.builder();
|
||||||
|
ImmutableOpenMap.Builder<String, Settings> settings = ImmutableOpenMap.builder();
|
||||||
|
List<String> indices = new ArrayList<>();
|
||||||
|
if (parser.currentToken() == null) {
|
||||||
|
parser.nextToken();
|
||||||
|
}
|
||||||
|
ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
|
||||||
|
parser.nextToken();
|
||||||
|
while (!parser.isClosed()) {
|
||||||
|
if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
|
||||||
|
String indexName = parser.currentName();
|
||||||
|
indices.add(indexName);
|
||||||
|
IndexEntry indexEntry = parseIndexEntry(parser);
|
||||||
|
CollectionUtil.timSort(indexEntry.indexAliases, Comparator.comparing(AliasMetaData::alias));
|
||||||
|
aliases.put(indexName, Collections.unmodifiableList(indexEntry.indexAliases));
|
||||||
|
mappings.put(indexName, indexEntry.indexMappings);
|
||||||
|
settings.put(indexName, indexEntry.indexSettings);
|
||||||
|
} else if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
|
||||||
|
parser.skipChildren();
|
||||||
|
} else {
|
||||||
|
parser.nextToken();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return new GetIndexResponse(indices.toArray(new String[0]),
|
||||||
|
mappings.build(),
|
||||||
|
aliases.build(),
|
||||||
|
settings.build());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static IndexEntry parseIndexEntry(XContentParser parser) throws IOException {
|
||||||
|
List<AliasMetaData> indexAliases = null;
|
||||||
|
ImmutableOpenMap<String, MappingMetaData> indexMappings = null;
|
||||||
|
Settings indexSettings = null;
|
||||||
|
Settings indexDefaultSettings = null;
|
||||||
|
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
|
||||||
|
ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
|
||||||
|
parser.nextToken();
|
||||||
|
if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
|
||||||
|
switch (parser.currentName()) {
|
||||||
|
case "aliases":
|
||||||
|
indexAliases = parseAliases(parser);
|
||||||
|
break;
|
||||||
|
case "mappings":
|
||||||
|
indexMappings = parseMappings(parser);
|
||||||
|
break;
|
||||||
|
case "settings":
|
||||||
|
indexSettings = Settings.fromXContent(parser);
|
||||||
|
break;
|
||||||
|
case "defaults":
|
||||||
|
indexDefaultSettings = Settings.fromXContent(parser);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
parser.skipChildren();
|
||||||
|
}
|
||||||
|
} else if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
|
||||||
|
parser.skipChildren();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return new IndexEntry(indexAliases, indexMappings, indexSettings, indexDefaultSettings);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static List<AliasMetaData> parseAliases(XContentParser parser) throws IOException {
|
||||||
|
List<AliasMetaData> indexAliases = new ArrayList<>();
|
||||||
|
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
|
||||||
|
ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
|
||||||
|
indexAliases.add(AliasMetaData.Builder.fromXContent(parser));
|
||||||
|
}
|
||||||
|
return indexAliases;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ImmutableOpenMap<String, MappingMetaData> parseMappings(XContentParser parser) throws IOException {
|
||||||
|
ImmutableOpenMap.Builder<String, MappingMetaData> indexMappings = ImmutableOpenMap.builder();
|
||||||
|
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
|
||||||
|
ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
|
||||||
|
parser.nextToken();
|
||||||
|
if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
|
||||||
|
String mappingType = parser.currentName();
|
||||||
|
indexMappings.put(mappingType, new MappingMetaData(mappingType, parser.map()));
|
||||||
|
} else if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
|
||||||
|
parser.skipChildren();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return indexMappings.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static class IndexEntry {
|
||||||
|
List<AliasMetaData> indexAliases = new ArrayList<>();
|
||||||
|
ImmutableOpenMap<String, MappingMetaData> indexMappings = ImmutableOpenMap.of();
|
||||||
|
Settings indexSettings = Settings.EMPTY;
|
||||||
|
Settings indexDefaultSettings = Settings.EMPTY;
|
||||||
|
|
||||||
|
IndexEntry(List<AliasMetaData> indexAliases, ImmutableOpenMap<String, MappingMetaData> indexMappings,
|
||||||
|
Settings indexSettings, Settings indexDefaultSettings) {
|
||||||
|
if (indexAliases != null) {
|
||||||
|
this.indexAliases = indexAliases;
|
||||||
|
}
|
||||||
|
if (indexMappings != null) {
|
||||||
|
this.indexMappings = indexMappings;
|
||||||
|
}
|
||||||
|
if (indexSettings != null) {
|
||||||
|
this.indexSettings = indexSettings;
|
||||||
|
}
|
||||||
|
if (indexDefaultSettings != null) {
|
||||||
|
this.indexDefaultSettings = indexDefaultSettings;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -1,10 +0,0 @@
|
|||||||
package org.xbib.elx.http;
|
|
||||||
|
|
||||||
import org.xbib.elx.api.ExtendedClientProvider;
|
|
||||||
|
|
||||||
public class ExtendedHttpClientProvider implements ExtendedClientProvider<ExtendedHttpClient> {
|
|
||||||
@Override
|
|
||||||
public ExtendedHttpClient getExtendedClient() {
|
|
||||||
return new ExtendedHttpClient();
|
|
||||||
}
|
|
||||||
}
|
|
@ -0,0 +1,61 @@
|
|||||||
|
package org.xbib.elx.http;
|
||||||
|
|
||||||
|
import org.elasticsearch.action.Action;
|
||||||
|
import org.elasticsearch.action.ActionFuture;
|
||||||
|
import org.elasticsearch.action.ActionListener;
|
||||||
|
import org.elasticsearch.action.ActionRequest;
|
||||||
|
import org.elasticsearch.action.ActionRequestBuilder;
|
||||||
|
import org.elasticsearch.action.ActionResponse;
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.threadpool.ThreadPool;
|
||||||
|
import org.xbib.elx.common.AbstractAdminClient;
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Elasticsearch HTTP admin client.
|
||||||
|
*/
|
||||||
|
public class HttpAdminClient extends AbstractAdminClient implements ElasticsearchClient {
|
||||||
|
|
||||||
|
private final HttpClientHelper helper;
|
||||||
|
|
||||||
|
public HttpAdminClient() {
|
||||||
|
this.helper = new HttpClientHelper();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) throws IOException {
|
||||||
|
super.init(settings);
|
||||||
|
helper.init(settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||||
|
return helper.createClient(settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() throws IOException {
|
||||||
|
helper.closeClient();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(Action<Request, Response, RequestBuilder> action, Request request) {
|
||||||
|
return helper.execute(action, request);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
|
||||||
|
helper.execute(action, request, listener);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(Action<Request, Response, RequestBuilder> action) {
|
||||||
|
return helper.prepareExecute(action);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ThreadPool threadPool() {
|
||||||
|
return helper.threadPool();
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,10 @@
|
|||||||
|
package org.xbib.elx.http;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.AdminClientProvider;
|
||||||
|
|
||||||
|
public class HttpAdminClientProvider implements AdminClientProvider<HttpAdminClient> {
|
||||||
|
@Override
|
||||||
|
public HttpAdminClient getClient() {
|
||||||
|
return new HttpAdminClient();
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,61 @@
|
|||||||
|
package org.xbib.elx.http;
|
||||||
|
|
||||||
|
import org.elasticsearch.action.Action;
|
||||||
|
import org.elasticsearch.action.ActionFuture;
|
||||||
|
import org.elasticsearch.action.ActionListener;
|
||||||
|
import org.elasticsearch.action.ActionRequest;
|
||||||
|
import org.elasticsearch.action.ActionRequestBuilder;
|
||||||
|
import org.elasticsearch.action.ActionResponse;
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.threadpool.ThreadPool;
|
||||||
|
import org.xbib.elx.common.AbstractBulkClient;
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Elasticsearch HTTP bulk client.
|
||||||
|
*/
|
||||||
|
public class HttpBulkClient extends AbstractBulkClient implements ElasticsearchClient {
|
||||||
|
|
||||||
|
private final HttpClientHelper helper;
|
||||||
|
|
||||||
|
public HttpBulkClient() {
|
||||||
|
this.helper = new HttpClientHelper();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) throws IOException {
|
||||||
|
super.init(settings);
|
||||||
|
helper.init(settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||||
|
return helper.createClient(settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() throws IOException {
|
||||||
|
helper.closeClient();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(Action<Request, Response, RequestBuilder> action, Request request) {
|
||||||
|
return helper.execute(action, request);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
|
||||||
|
helper.execute(action, request, listener);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(Action<Request, Response, RequestBuilder> action) {
|
||||||
|
return helper.prepareExecute(action);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ThreadPool threadPool() {
|
||||||
|
return helper.threadPool();
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,11 @@
|
|||||||
|
package org.xbib.elx.http;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.BulkClientProvider;
|
||||||
|
|
||||||
|
public class HttpBulkClientProvider implements BulkClientProvider<HttpBulkClient> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public HttpBulkClient getClient() {
|
||||||
|
return new HttpBulkClient();
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,61 @@
|
|||||||
|
package org.xbib.elx.http;
|
||||||
|
|
||||||
|
import org.elasticsearch.action.Action;
|
||||||
|
import org.elasticsearch.action.ActionFuture;
|
||||||
|
import org.elasticsearch.action.ActionListener;
|
||||||
|
import org.elasticsearch.action.ActionRequest;
|
||||||
|
import org.elasticsearch.action.ActionRequestBuilder;
|
||||||
|
import org.elasticsearch.action.ActionResponse;
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.threadpool.ThreadPool;
|
||||||
|
import org.xbib.elx.common.AbstractSearchClient;
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Elasticsearch HTTP search client.
|
||||||
|
*/
|
||||||
|
public class HttpSearchClient extends AbstractSearchClient implements ElasticsearchClient {
|
||||||
|
|
||||||
|
private final HttpClientHelper helper;
|
||||||
|
|
||||||
|
public HttpSearchClient() {
|
||||||
|
this.helper = new HttpClientHelper();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) throws IOException {
|
||||||
|
super.init(settings);
|
||||||
|
helper.init(settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||||
|
return helper.createClient(settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() throws IOException {
|
||||||
|
helper.closeClient();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(Action<Request, Response, RequestBuilder> action, Request request) {
|
||||||
|
return helper.execute(action, request);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
|
||||||
|
helper.execute(action, request, listener);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(Action<Request, Response, RequestBuilder> action) {
|
||||||
|
return helper.prepareExecute(action);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ThreadPool threadPool() {
|
||||||
|
return helper.threadPool();
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,11 @@
|
|||||||
|
package org.xbib.elx.http;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.SearchClientProvider;
|
||||||
|
|
||||||
|
public class HttpSearchClientProvider implements SearchClientProvider<HttpSearchClient> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public HttpSearchClient getClient() {
|
||||||
|
return new HttpSearchClient();
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1 @@
|
|||||||
|
org.xbib.elx.http.HttpAdminClientProvider
|
@ -0,0 +1 @@
|
|||||||
|
org.xbib.elx.http.HttpBulkClientProvider
|
@ -1 +0,0 @@
|
|||||||
org.xbib.elx.http.ExtendedHttpClientProvider
|
|
@ -0,0 +1 @@
|
|||||||
|
org.xbib.elx.http.HttpSearchClientProvider
|
@ -1,3 +1,18 @@
|
|||||||
|
org.elasticsearch.action.admin.indices.mapping.get.HttpGetMappingsAction
|
||||||
|
org.xbib.elx.http.action.admin.cluster.health.HttpClusterHealthAction
|
||||||
|
org.xbib.elx.http.action.admin.cluster.node.info.HttpNodesInfoAction
|
||||||
|
org.xbib.elx.http.action.admin.cluster.settings.HttpClusterUpdateSettingsAction
|
||||||
|
org.xbib.elx.http.action.admin.cluster.state.HttpClusterStateAction
|
||||||
|
org.xbib.elx.http.action.admin.indices.alias.HttpIndicesAliasesAction
|
||||||
|
org.xbib.elx.http.action.admin.indices.alias.get.HttpGetAliasAction
|
||||||
|
org.xbib.elx.http.action.admin.indices.create.HttpCreateIndexAction
|
||||||
|
org.xbib.elx.http.action.admin.indices.delete.HttpDeleteIndexAction
|
||||||
|
org.xbib.elx.http.action.admin.indices.exists.indices.HttpIndicesExistsAction
|
||||||
|
org.xbib.elx.http.action.admin.indices.refresh.HttpRefreshIndexAction
|
||||||
|
org.xbib.elx.http.action.admin.indices.settings.get.HttpGetSettingsAction
|
||||||
|
org.xbib.elx.http.action.admin.indices.settings.put.HttpUpdateSettingsAction
|
||||||
|
org.xbib.elx.http.action.bulk.HttpBulkAction
|
||||||
|
org.xbib.elx.http.action.index.HttpIndexAction
|
||||||
org.xbib.elx.http.action.search.HttpSearchAction
|
org.xbib.elx.http.action.search.HttpSearchAction
|
||||||
org.xbib.elx.http.action.get.HttpGetAction
|
org.xbib.elx.http.action.get.HttpGetAction
|
||||||
org.xbib.elx.http.action.get.HttpMultiGetAction
|
org.xbib.elx.http.action.get.HttpMultiGetAction
|
||||||
|
@ -1,122 +0,0 @@
|
|||||||
package org.xbib.elx.http.test;
|
|
||||||
|
|
||||||
import org.apache.logging.log4j.LogManager;
|
|
||||||
import org.apache.logging.log4j.Logger;
|
|
||||||
import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
|
|
||||||
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
|
|
||||||
import org.elasticsearch.action.get.GetAction;
|
|
||||||
import org.elasticsearch.action.get.GetRequest;
|
|
||||||
import org.elasticsearch.action.get.GetResponse;
|
|
||||||
import org.elasticsearch.action.get.MultiGetAction;
|
|
||||||
import org.elasticsearch.action.get.MultiGetRequest;
|
|
||||||
import org.elasticsearch.action.get.MultiGetResponse;
|
|
||||||
import org.elasticsearch.action.index.IndexAction;
|
|
||||||
import org.elasticsearch.action.index.IndexRequest;
|
|
||||||
import org.elasticsearch.action.index.IndexResponse;
|
|
||||||
import org.elasticsearch.action.search.SearchAction;
|
|
||||||
import org.elasticsearch.action.search.SearchRequest;
|
|
||||||
import org.elasticsearch.action.search.SearchResponse;
|
|
||||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
|
||||||
import org.elasticsearch.index.query.QueryBuilders;
|
|
||||||
import org.elasticsearch.search.builder.SearchSourceBuilder;
|
|
||||||
import org.junit.Ignore;
|
|
||||||
import org.junit.Test;
|
|
||||||
import org.xbib.elx.common.ClientBuilder;
|
|
||||||
import org.xbib.elx.http.ExtendedHttpClient;
|
|
||||||
import org.xbib.elx.http.ExtendedHttpClientProvider;
|
|
||||||
|
|
||||||
import static org.junit.Assert.assertEquals;
|
|
||||||
import static org.junit.Assert.assertTrue;
|
|
||||||
|
|
||||||
public class ClientTest extends TestBase {
|
|
||||||
|
|
||||||
private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
|
|
||||||
|
|
||||||
@Ignore
|
|
||||||
@Test
|
|
||||||
public void testGet() throws Exception {
|
|
||||||
try (ExtendedHttpClient client = ClientBuilder.builder()
|
|
||||||
.provider(ExtendedHttpClientProvider.class)
|
|
||||||
.put("url", "http://" + host + ":" + httpPort)
|
|
||||||
.build()) {
|
|
||||||
IndexRequest indexRequest = new IndexRequest();
|
|
||||||
indexRequest.index("test");
|
|
||||||
indexRequest.type("test");
|
|
||||||
indexRequest.id("1");
|
|
||||||
indexRequest.source("test", "Hello Jörg");
|
|
||||||
IndexResponse indexResponse = client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet();
|
|
||||||
client("1").execute(RefreshAction.INSTANCE, new RefreshRequest());
|
|
||||||
|
|
||||||
GetRequest getRequest = new GetRequest();
|
|
||||||
getRequest.index("test");
|
|
||||||
getRequest.type("test");
|
|
||||||
getRequest.id("1");
|
|
||||||
|
|
||||||
GetResponse getResponse = client.execute(GetAction.INSTANCE, getRequest).actionGet();
|
|
||||||
|
|
||||||
assertTrue(getResponse.isExists());
|
|
||||||
assertEquals("{\"test\":\"Hello Jörg\"}", getResponse.getSourceAsString());
|
|
||||||
|
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Ignore
|
|
||||||
@Test
|
|
||||||
public void testMultiGet() throws Exception {
|
|
||||||
try (ExtendedHttpClient client = ClientBuilder.builder()
|
|
||||||
.provider(ExtendedHttpClientProvider.class)
|
|
||||||
.put("url", "http://" + host + ":" + httpPort)
|
|
||||||
.build()) {
|
|
||||||
IndexRequest indexRequest = new IndexRequest();
|
|
||||||
indexRequest.index("test");
|
|
||||||
indexRequest.type("test");
|
|
||||||
indexRequest.id("1");
|
|
||||||
indexRequest.source("test", "Hello Jörg");
|
|
||||||
IndexResponse indexResponse = client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet();
|
|
||||||
client("1").execute(RefreshAction.INSTANCE, new RefreshRequest());
|
|
||||||
|
|
||||||
MultiGetRequest multiGetRequest = new MultiGetRequest();
|
|
||||||
multiGetRequest.add("test", "test", "1");
|
|
||||||
|
|
||||||
MultiGetResponse multiGetResponse = client.execute(MultiGetAction.INSTANCE, multiGetRequest).actionGet();
|
|
||||||
|
|
||||||
assertEquals(1, multiGetResponse.getResponses().length);
|
|
||||||
assertEquals("{\"test\":\"Hello Jörg\"}", multiGetResponse.getResponses()[0].getResponse().getSourceAsString());
|
|
||||||
|
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testSearchDoc() throws Exception {
|
|
||||||
try (ExtendedHttpClient client = ClientBuilder.builder()
|
|
||||||
.provider(ExtendedHttpClientProvider.class)
|
|
||||||
.put("url", "http://" + host + ":" + httpPort)
|
|
||||||
.build()) {
|
|
||||||
IndexRequest indexRequest = new IndexRequest();
|
|
||||||
indexRequest.index("test");
|
|
||||||
indexRequest.type("test");
|
|
||||||
indexRequest.id("1");
|
|
||||||
indexRequest.source("test", "Hello Jörg");
|
|
||||||
IndexResponse indexResponse = client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet();
|
|
||||||
client("1").execute(RefreshAction.INSTANCE, new RefreshRequest());
|
|
||||||
|
|
||||||
SearchSourceBuilder builder = new SearchSourceBuilder();
|
|
||||||
builder.query(QueryBuilders.matchAllQuery());
|
|
||||||
SearchRequest searchRequest = new SearchRequest();
|
|
||||||
searchRequest.indices("test");
|
|
||||||
searchRequest.types("test");
|
|
||||||
searchRequest.source(builder);
|
|
||||||
SearchResponse searchResponse = client.execute(SearchAction.INSTANCE, searchRequest).actionGet();
|
|
||||||
long hits = searchResponse.getHits().getTotalHits();
|
|
||||||
assertEquals(1, hits);
|
|
||||||
logger.info("hits = {} source = {}", hits, searchResponse.getHits().getHits()[0].getSourceAsString());
|
|
||||||
assertEquals("{\"test\":\"Hello Jörg\"}", searchResponse.getHits().getHits()[0].getSourceAsString());
|
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,10 +0,0 @@
|
|||||||
package org.xbib.elx.node;
|
|
||||||
|
|
||||||
import org.xbib.elx.api.ExtendedClientProvider;
|
|
||||||
|
|
||||||
public class ExtendedNodeClientProvider implements ExtendedClientProvider<ExtendedNodeClient> {
|
|
||||||
@Override
|
|
||||||
public ExtendedNodeClient getExtendedClient() {
|
|
||||||
return new ExtendedNodeClient();
|
|
||||||
}
|
|
||||||
}
|
|
@ -0,0 +1,11 @@
|
|||||||
|
package org.xbib.elx.node;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.AdminClientProvider;
|
||||||
|
|
||||||
|
public class NodeAdminClientProvider implements AdminClientProvider<NodeAdminClient> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public NodeAdminClient getClient() {
|
||||||
|
return new NodeAdminClient();
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,68 @@
|
|||||||
|
package org.xbib.elx.node;
|
||||||
|
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.common.Strings;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.common.xcontent.ToXContent;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||||
|
import org.elasticsearch.env.Environment;
|
||||||
|
import org.elasticsearch.node.Node;
|
||||||
|
import org.elasticsearch.plugins.Plugin;
|
||||||
|
import org.xbib.elx.common.AbstractBulkClient;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.Collections;
|
||||||
|
|
||||||
|
public class NodeBulkClient extends AbstractBulkClient {
|
||||||
|
|
||||||
|
private static final Logger logger = LogManager.getLogger(NodeBulkClient.class.getName());
|
||||||
|
|
||||||
|
private Node node;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||||
|
if (settings == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String version = System.getProperty("os.name")
|
||||||
|
+ " " + System.getProperty("java.vm.name")
|
||||||
|
+ " " + System.getProperty("java.vm.vendor")
|
||||||
|
+ " " + System.getProperty("java.runtime.version")
|
||||||
|
+ " " + System.getProperty("java.vm.version");
|
||||||
|
Settings effectiveSettings = Settings.builder().put(settings)
|
||||||
|
.put("node.client", true)
|
||||||
|
.put("node.master", false)
|
||||||
|
.put("node.data", false)
|
||||||
|
.build();
|
||||||
|
XContentBuilder builder = XContentFactory.jsonBuilder();
|
||||||
|
effectiveSettings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true")));
|
||||||
|
logger.info("creating node client on {} with effective settings {}",
|
||||||
|
version, Strings.toString(builder));
|
||||||
|
Collection<Class<? extends Plugin>> plugins = Collections.emptyList();
|
||||||
|
this.node = new BulkNode(new Environment(effectiveSettings, null), plugins);
|
||||||
|
try {
|
||||||
|
node.start();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
return node.client();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() throws IOException {
|
||||||
|
if (node != null) {
|
||||||
|
logger.debug("closing node client");
|
||||||
|
node.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static class BulkNode extends Node {
|
||||||
|
|
||||||
|
BulkNode(Environment env, Collection<Class<? extends Plugin>> classpathPlugins) {
|
||||||
|
super(env, classpathPlugins);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,11 @@
|
|||||||
|
package org.xbib.elx.node;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.BulkClientProvider;
|
||||||
|
|
||||||
|
public class NodeBulkClientProvider implements BulkClientProvider<NodeBulkClient> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public NodeBulkClient getClient() {
|
||||||
|
return new NodeBulkClient();
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,68 @@
|
|||||||
|
package org.xbib.elx.node;
|
||||||
|
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.common.Strings;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.common.xcontent.ToXContent;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||||
|
import org.elasticsearch.env.Environment;
|
||||||
|
import org.elasticsearch.node.Node;
|
||||||
|
import org.elasticsearch.plugins.Plugin;
|
||||||
|
import org.xbib.elx.common.AbstractSearchClient;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.Collections;
|
||||||
|
|
||||||
|
public class NodeSearchClient extends AbstractSearchClient {
|
||||||
|
|
||||||
|
private static final Logger logger = LogManager.getLogger(NodeSearchClient.class.getName());
|
||||||
|
|
||||||
|
private Node node;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||||
|
if (settings == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String version = System.getProperty("os.name")
|
||||||
|
+ " " + System.getProperty("java.vm.name")
|
||||||
|
+ " " + System.getProperty("java.vm.vendor")
|
||||||
|
+ " " + System.getProperty("java.runtime.version")
|
||||||
|
+ " " + System.getProperty("java.vm.version");
|
||||||
|
Settings effectiveSettings = Settings.builder().put(settings)
|
||||||
|
.put("node.client", true)
|
||||||
|
.put("node.master", false)
|
||||||
|
.put("node.data", false)
|
||||||
|
.build();
|
||||||
|
XContentBuilder builder = XContentFactory.jsonBuilder();
|
||||||
|
effectiveSettings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true")));
|
||||||
|
logger.info("creating node client on {} with effective settings {}",
|
||||||
|
version, Strings.toString(builder));
|
||||||
|
Collection<Class<? extends Plugin>> plugins = Collections.emptyList();
|
||||||
|
this.node = new BulkNode(new Environment(effectiveSettings, null), plugins);
|
||||||
|
try {
|
||||||
|
node.start();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
return node.client();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() throws IOException {
|
||||||
|
if (node != null) {
|
||||||
|
logger.debug("closing node client");
|
||||||
|
node.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static class BulkNode extends Node {
|
||||||
|
|
||||||
|
BulkNode(Environment env, Collection<Class<? extends Plugin>> classpathPlugins) {
|
||||||
|
super(env, classpathPlugins);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,11 @@
|
|||||||
|
package org.xbib.elx.node;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.SearchClientProvider;
|
||||||
|
|
||||||
|
public class NodeSearchClientProvider implements SearchClientProvider<NodeSearchClient> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public NodeSearchClient getClient() {
|
||||||
|
return new NodeSearchClient();
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1 @@
|
|||||||
|
org.xbib.elx.node.NodeAdminClientProvider
|
@ -0,0 +1 @@
|
|||||||
|
org.xbib.elx.node.NodeBulkClientProvider
|
@ -1 +0,0 @@
|
|||||||
org.xbib.elx.node.ExtendedNodeClientProvider
|
|
@ -0,0 +1 @@
|
|||||||
|
org.xbib.elx.node.NodeSearchClientProvider
|
@ -0,0 +1,174 @@
|
|||||||
|
package org.xbib.elx.node.test;
|
||||||
|
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.common.unit.TimeValue;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
|
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.xbib.elx.common.ClientBuilder;
|
||||||
|
import org.xbib.elx.common.Parameters;
|
||||||
|
import org.xbib.elx.node.NodeBulkClient;
|
||||||
|
import org.xbib.elx.node.NodeBulkClientProvider;
|
||||||
|
|
||||||
|
import java.util.concurrent.CountDownLatch;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
@ExtendWith(TestExtension.class)
|
||||||
|
class BulkClientTest {
|
||||||
|
|
||||||
|
private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getName());
|
||||||
|
|
||||||
|
private static final Long ACTIONS = 10000L;
|
||||||
|
|
||||||
|
private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
|
||||||
|
|
||||||
|
private final TestExtension.Helper helper;
|
||||||
|
|
||||||
|
BulkClientTest(TestExtension.Helper helper) {
|
||||||
|
this.helper = helper;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testSingleDoc() throws Exception {
|
||||||
|
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||||
|
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||||
|
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||||
|
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
|
||||||
|
.build();
|
||||||
|
try {
|
||||||
|
bulkClient.newIndex("test");
|
||||||
|
bulkClient.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
||||||
|
bulkClient.flush();
|
||||||
|
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
|
} finally {
|
||||||
|
assertEquals(1, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||||
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
|
}
|
||||||
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
|
bulkClient.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testNewIndex() throws Exception {
|
||||||
|
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||||
|
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||||
|
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
||||||
|
.build();
|
||||||
|
bulkClient.newIndex("test");
|
||||||
|
bulkClient.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testMapping() throws Exception {
|
||||||
|
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||||
|
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||||
|
.build();
|
||||||
|
XContentBuilder builder = JsonXContent.contentBuilder()
|
||||||
|
.startObject()
|
||||||
|
.startObject("doc")
|
||||||
|
.startObject("properties")
|
||||||
|
.startObject("location")
|
||||||
|
.field("type", "geo_point")
|
||||||
|
.endObject()
|
||||||
|
.endObject()
|
||||||
|
.endObject()
|
||||||
|
.endObject();
|
||||||
|
bulkClient.newIndex("test", Settings.EMPTY, builder);
|
||||||
|
assertTrue(bulkClient.getMapping("test", "doc").containsKey("properties"));
|
||||||
|
bulkClient.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testRandomDocs() throws Exception {
|
||||||
|
long numactions = ACTIONS;
|
||||||
|
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||||
|
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||||
|
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||||
|
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
||||||
|
.build();
|
||||||
|
try {
|
||||||
|
bulkClient.newIndex("test");
|
||||||
|
for (int i = 0; i < ACTIONS; i++) {
|
||||||
|
bulkClient.index("test", null, false,
|
||||||
|
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||||
|
}
|
||||||
|
bulkClient.flush();
|
||||||
|
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
|
} finally {
|
||||||
|
assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||||
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
|
}
|
||||||
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
|
bulkClient.refreshIndex("test");
|
||||||
|
assertEquals(numactions, bulkClient.getSearchableDocs("test"));
|
||||||
|
bulkClient.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testThreadedRandomDocs() throws Exception {
|
||||||
|
int maxthreads = Runtime.getRuntime().availableProcessors();
|
||||||
|
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
|
||||||
|
final long actions = ACTIONS;
|
||||||
|
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
|
||||||
|
final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
|
||||||
|
.setBulkClientProvider(NodeBulkClientProvider.class)
|
||||||
|
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads)
|
||||||
|
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
|
||||||
|
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
||||||
|
.build();
|
||||||
|
try {
|
||||||
|
Settings settings = Settings.builder()
|
||||||
|
.put("index.number_of_shards", 1)
|
||||||
|
.put("index.number_of_replicas", 0)
|
||||||
|
.build();
|
||||||
|
bulkClient.newIndex("test", settings);
|
||||||
|
bulkClient.startBulk("test", 0, 1000);
|
||||||
|
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
|
||||||
|
final CountDownLatch latch = new CountDownLatch(maxthreads);
|
||||||
|
for (int i = 0; i < maxthreads; i++) {
|
||||||
|
executorService.execute(() -> {
|
||||||
|
for (int i1 = 0; i1 < actions; i1++) {
|
||||||
|
bulkClient.index("test", null, false,
|
||||||
|
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||||
|
}
|
||||||
|
latch.countDown();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
logger.info("waiting for latch...");
|
||||||
|
if (latch.await(30L, TimeUnit.SECONDS)) {
|
||||||
|
logger.info("flush...");
|
||||||
|
bulkClient.flush();
|
||||||
|
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
|
logger.info("got all responses, executor service shutdown...");
|
||||||
|
executorService.shutdown();
|
||||||
|
executorService.awaitTermination(30L, TimeUnit.SECONDS);
|
||||||
|
logger.info("pool is shut down");
|
||||||
|
} else {
|
||||||
|
logger.warn("latch timeout");
|
||||||
|
}
|
||||||
|
bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS);
|
||||||
|
assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||||
|
} finally {
|
||||||
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
|
}
|
||||||
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
|
bulkClient.refreshIndex("test");
|
||||||
|
assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test"));
|
||||||
|
bulkClient.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -1,134 +0,0 @@
|
|||||||
package org.xbib.elx.transport;
|
|
||||||
|
|
||||||
import org.apache.logging.log4j.LogManager;
|
|
||||||
import org.apache.logging.log4j.Logger;
|
|
||||||
import org.elasticsearch.Version;
|
|
||||||
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
|
|
||||||
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder;
|
|
||||||
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
|
|
||||||
import org.elasticsearch.client.ElasticsearchClient;
|
|
||||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
|
||||||
import org.elasticsearch.cluster.node.DiscoveryNode;
|
|
||||||
import org.elasticsearch.cluster.node.DiscoveryNodes;
|
|
||||||
import org.elasticsearch.common.settings.Settings;
|
|
||||||
import org.elasticsearch.common.transport.InetSocketTransportAddress;
|
|
||||||
import org.jboss.netty.channel.DefaultChannelFuture;
|
|
||||||
import org.xbib.elx.common.AbstractExtendedClient;
|
|
||||||
import org.xbib.elx.common.util.NetworkUtils;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.net.InetAddress;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Collection;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Transport client with additional methods using the BulkProcessor.
|
|
||||||
*/
|
|
||||||
public class ExtendedTransportClient extends AbstractExtendedClient {
|
|
||||||
|
|
||||||
private static final Logger logger = LogManager.getLogger(ExtendedTransportClient.class.getName());
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected ElasticsearchClient createClient(Settings settings) {
|
|
||||||
if (settings != null) {
|
|
||||||
String systemIdentifier = System.getProperty("os.name")
|
|
||||||
+ " " + System.getProperty("java.vm.name")
|
|
||||||
+ " " + System.getProperty("java.vm.vendor")
|
|
||||||
+ " " + System.getProperty("java.vm.version")
|
|
||||||
+ " Elasticsearch " + Version.CURRENT.toString();
|
|
||||||
Settings effectiveSettings = Settings.builder()
|
|
||||||
// for thread pool size
|
|
||||||
.put("processors",
|
|
||||||
settings.getAsInt("processors", Runtime.getRuntime().availableProcessors()))
|
|
||||||
.put("client.transport.sniff", false) // do not sniff
|
|
||||||
.put("client.transport.nodes_sampler_interval", "1m") // do not ping
|
|
||||||
.put("client.transport.ping_timeout", "1m") // wait for unresponsive nodes a very long time before disconnect
|
|
||||||
.put("client.transport.ignore_cluster_name", true) // connect to any cluster
|
|
||||||
// custom settings may override defaults
|
|
||||||
.put(settings)
|
|
||||||
.build();
|
|
||||||
logger.info("creating transport client on {} with custom settings {} and effective settings {}",
|
|
||||||
systemIdentifier, settings.getAsMap(), effectiveSettings.getAsMap());
|
|
||||||
// we need to disable dead lock check because we may have mixed node/transport clients
|
|
||||||
DefaultChannelFuture.setUseDeadLockChecker(false);
|
|
||||||
return TransportClient.builder().settings(effectiveSettings).build();
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void closeClient() {
|
|
||||||
if (getClient() != null) {
|
|
||||||
TransportClient client = (TransportClient) getClient();
|
|
||||||
client.close();
|
|
||||||
client.threadPool().shutdown();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedTransportClient init(Settings settings) throws IOException {
|
|
||||||
super.init(settings);
|
|
||||||
// additional auto-connect
|
|
||||||
try {
|
|
||||||
Collection<InetSocketTransportAddress> addrs = findAddresses(settings);
|
|
||||||
if (!connect(addrs, settings.getAsBoolean("autodiscover", false))) {
|
|
||||||
throw new NoNodeAvailableException("no cluster nodes available, check settings "
|
|
||||||
+ settings.toString());
|
|
||||||
}
|
|
||||||
} catch (IOException e) {
|
|
||||||
logger.error(e.getMessage(), e);
|
|
||||||
}
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Collection<InetSocketTransportAddress> findAddresses(Settings settings) throws IOException {
|
|
||||||
final int defaultPort = settings.getAsInt("port", 9300);
|
|
||||||
Collection<InetSocketTransportAddress> addresses = new ArrayList<>();
|
|
||||||
for (String hostname : settings.getAsArray("host")) {
|
|
||||||
String[] splitHost = hostname.split(":", 2);
|
|
||||||
if (splitHost.length == 2) {
|
|
||||||
try {
|
|
||||||
String host = splitHost[0];
|
|
||||||
InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null);
|
|
||||||
int port = Integer.parseInt(splitHost[1]);
|
|
||||||
InetSocketTransportAddress address = new InetSocketTransportAddress(inetAddress, port);
|
|
||||||
addresses.add(address);
|
|
||||||
} catch (NumberFormatException e) {
|
|
||||||
logger.warn(e.getMessage(), e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (splitHost.length == 1) {
|
|
||||||
String host = splitHost[0];
|
|
||||||
InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null);
|
|
||||||
InetSocketTransportAddress address = new InetSocketTransportAddress(inetAddress, defaultPort);
|
|
||||||
addresses.add(address);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return addresses;
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean connect(Collection<InetSocketTransportAddress> addresses, boolean autodiscover) {
|
|
||||||
if (getClient() == null) {
|
|
||||||
throw new IllegalStateException("no client present");
|
|
||||||
}
|
|
||||||
logger.debug("trying to connect to {}", addresses);
|
|
||||||
TransportClient transportClient = (TransportClient) getClient();
|
|
||||||
transportClient.addTransportAddresses(addresses);
|
|
||||||
List<DiscoveryNode> nodes = transportClient.connectedNodes();
|
|
||||||
logger.info("connected to nodes = {}", nodes);
|
|
||||||
if (nodes != null && !nodes.isEmpty()) {
|
|
||||||
if (autodiscover) {
|
|
||||||
logger.debug("trying to auto-discover all nodes...");
|
|
||||||
ClusterStateRequestBuilder clusterStateRequestBuilder =
|
|
||||||
new ClusterStateRequestBuilder(getClient(), ClusterStateAction.INSTANCE);
|
|
||||||
ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet();
|
|
||||||
DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes();
|
|
||||||
transportClient.addDiscoveryNodes(discoveryNodes);
|
|
||||||
logger.info("after auto-discovery: connected to {}", transportClient.connectedNodes());
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,11 +0,0 @@
|
|||||||
package org.xbib.elx.transport;
|
|
||||||
|
|
||||||
import org.xbib.elx.api.ExtendedClientProvider;
|
|
||||||
|
|
||||||
public class ExtendedTransportClientProvider implements ExtendedClientProvider<ExtendedTransportClient> {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ExtendedTransportClient getExtendedClient() {
|
|
||||||
return new ExtendedTransportClient();
|
|
||||||
}
|
|
||||||
}
|
|
@ -0,0 +1,40 @@
|
|||||||
|
package org.xbib.elx.transport;
|
||||||
|
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.client.transport.TransportClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.xbib.elx.common.AbstractAdminClient;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Transport admin client.
|
||||||
|
*/
|
||||||
|
public class TransportAdminClient extends AbstractAdminClient {
|
||||||
|
|
||||||
|
private final TransportClientHelper helper;
|
||||||
|
|
||||||
|
public TransportAdminClient() {
|
||||||
|
this.helper = new TransportClientHelper();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||||
|
return helper.createClient(settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() {
|
||||||
|
if (getClient() != null) {
|
||||||
|
TransportClient client = (TransportClient) getClient();
|
||||||
|
client.close();
|
||||||
|
client.threadPool().shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) throws IOException {
|
||||||
|
super.init(settings);
|
||||||
|
helper.init((TransportClient) getClient(), settings);
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,11 @@
|
|||||||
|
package org.xbib.elx.transport;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.AdminClientProvider;
|
||||||
|
|
||||||
|
public class TransportAdminClientProvider implements AdminClientProvider<TransportAdminClient> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public TransportAdminClient getClient() {
|
||||||
|
return new TransportAdminClient();
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,39 @@
|
|||||||
|
package org.xbib.elx.transport;
|
||||||
|
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.client.transport.TransportClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.xbib.elx.common.AbstractBulkClient;
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Transport search client with additional methods.
|
||||||
|
*/
|
||||||
|
public class TransportBulkClient extends AbstractBulkClient {
|
||||||
|
|
||||||
|
private final TransportClientHelper helper;
|
||||||
|
|
||||||
|
public TransportBulkClient() {
|
||||||
|
this.helper = new TransportClientHelper();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||||
|
return helper.createClient(settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() {
|
||||||
|
if (getClient() != null) {
|
||||||
|
TransportClient client = (TransportClient) getClient();
|
||||||
|
client.close();
|
||||||
|
client.threadPool().shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) throws IOException {
|
||||||
|
super.init(settings);
|
||||||
|
helper.init((TransportClient) getClient(), settings);
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,11 @@
|
|||||||
|
package org.xbib.elx.transport;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.BulkClientProvider;
|
||||||
|
|
||||||
|
public class TransportBulkClientProvider implements BulkClientProvider<TransportBulkClient> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public TransportBulkClient getClient() {
|
||||||
|
return new TransportBulkClient();
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,161 @@
|
|||||||
|
package org.xbib.elx.transport;
|
||||||
|
|
||||||
|
import org.apache.logging.log4j.Level;
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
import org.elasticsearch.Version;
|
||||||
|
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
|
||||||
|
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder;
|
||||||
|
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||||
|
import org.elasticsearch.client.transport.TransportClient;
|
||||||
|
import org.elasticsearch.cluster.ClusterName;
|
||||||
|
import org.elasticsearch.cluster.node.DiscoveryNode;
|
||||||
|
import org.elasticsearch.cluster.node.DiscoveryNodes;
|
||||||
|
import org.elasticsearch.common.Strings;
|
||||||
|
import org.elasticsearch.common.network.NetworkModule;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.common.transport.TransportAddress;
|
||||||
|
import org.elasticsearch.common.util.concurrent.EsExecutors;
|
||||||
|
import org.elasticsearch.common.xcontent.ToXContent;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||||
|
import org.elasticsearch.plugins.Plugin;
|
||||||
|
import org.elasticsearch.transport.netty4.Netty4Plugin;
|
||||||
|
import org.xbib.elx.common.util.NetworkUtils;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.InetAddress;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class TransportClientHelper {
|
||||||
|
|
||||||
|
|
||||||
|
private static final Logger logger = LogManager.getLogger(TransportAdminClient.class.getName());
|
||||||
|
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||||
|
if (settings != null) {
|
||||||
|
String systemIdentifier = System.getProperty("os.name")
|
||||||
|
+ " " + System.getProperty("java.vm.name")
|
||||||
|
+ " " + System.getProperty("java.vm.vendor")
|
||||||
|
+ " " + System.getProperty("java.vm.version")
|
||||||
|
+ " Elasticsearch " + Version.CURRENT.toString();
|
||||||
|
Settings transportClientSettings = getTransportClientSettings(settings);
|
||||||
|
//XContentBuilder settingsBuilder = XContentFactory.jsonBuilder().startObject();
|
||||||
|
XContentBuilder effectiveSettingsBuilder = XContentFactory.jsonBuilder().startObject();
|
||||||
|
logger.log(Level.INFO, "creating transport client on {} with settings {}",
|
||||||
|
systemIdentifier,
|
||||||
|
//Strings.toString(settings.toXContent(settingsBuilder, ToXContent.EMPTY_PARAMS).endObject()),
|
||||||
|
Strings.toString(transportClientSettings.toXContent(effectiveSettingsBuilder,
|
||||||
|
ToXContent.EMPTY_PARAMS).endObject()));
|
||||||
|
return new MyTransportClient(transportClientSettings, Collections.singletonList(Netty4Plugin.class));
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void init(TransportClient transportClient, Settings settings) throws IOException {
|
||||||
|
Collection<TransportAddress> addrs = findAddresses(settings);
|
||||||
|
if (!connect(transportClient, addrs, settings.getAsBoolean("autodiscover", false))) {
|
||||||
|
throw new NoNodeAvailableException("no cluster nodes available, check settings "
|
||||||
|
+ settings.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ExtendedTransportClient init(Settings settings) throws IOException {
|
||||||
|
super.init(settings);
|
||||||
|
// additional auto-connect
|
||||||
|
try {
|
||||||
|
Collection<TransportAddress> addrs = findAddresses(settings);
|
||||||
|
if (!connect(addrs, settings.getAsBoolean("autodiscover", false))) {
|
||||||
|
throw new NoNodeAvailableException("no cluster nodes available, check settings "
|
||||||
|
+ settings.toString());
|
||||||
|
}
|
||||||
|
} catch (IOException e) {
|
||||||
|
logger.error(e.getMessage(), e);
|
||||||
|
}
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Collection<TransportAddress> findAddresses(Settings settings) throws IOException {
|
||||||
|
final int defaultPort = settings.getAsInt("port", 9300);
|
||||||
|
Collection<TransportAddress> addresses = new ArrayList<>();
|
||||||
|
for (String hostname : settings.getAsList("host")) {
|
||||||
|
String[] splitHost = hostname.split(":", 2);
|
||||||
|
if (splitHost.length == 2) {
|
||||||
|
try {
|
||||||
|
String host = splitHost[0];
|
||||||
|
InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null);
|
||||||
|
int port = Integer.parseInt(splitHost[1]);
|
||||||
|
TransportAddress address = new TransportAddress(inetAddress, port);
|
||||||
|
addresses.add(address);
|
||||||
|
} catch (NumberFormatException e) {
|
||||||
|
logger.warn(e.getMessage(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (splitHost.length == 1) {
|
||||||
|
String host = splitHost[0];
|
||||||
|
InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null);
|
||||||
|
TransportAddress address = new TransportAddress(inetAddress, defaultPort);
|
||||||
|
addresses.add(address);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return addresses;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean connect(Collection<TransportAddress> addresses, boolean autodiscover) {
|
||||||
|
if (getClient() == null) {
|
||||||
|
throw new IllegalStateException("no client present");
|
||||||
|
}
|
||||||
|
TransportClient transportClient = (TransportClient) getClient();
|
||||||
|
for (TransportAddress address : addresses) {
|
||||||
|
transportClient.addTransportAddresses(address);
|
||||||
|
}
|
||||||
|
List<DiscoveryNode> nodes = transportClient.connectedNodes();
|
||||||
|
logger.info("connected to nodes = {}", nodes);
|
||||||
|
if (nodes != null && !nodes.isEmpty()) {
|
||||||
|
if (autodiscover) {
|
||||||
|
logger.debug("trying to auto-discover all nodes...");
|
||||||
|
ClusterStateRequestBuilder clusterStateRequestBuilder =
|
||||||
|
new ClusterStateRequestBuilder(transportClient, ClusterStateAction.INSTANCE);
|
||||||
|
ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet();
|
||||||
|
DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes();
|
||||||
|
addDiscoveryNodes(transportClient, discoveryNodes);
|
||||||
|
logger.info("after auto-discovery: connected to {}", transportClient.connectedNodes());
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Settings getTransportClientSettings(Settings settings) {
|
||||||
|
return Settings.builder()
|
||||||
|
// "cluster.name"
|
||||||
|
.put(ClusterName.CLUSTER_NAME_SETTING.getKey(),
|
||||||
|
settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey()))
|
||||||
|
// "processors"
|
||||||
|
.put(EsExecutors.PROCESSORS_SETTING.getKey(),
|
||||||
|
settings.get(EsExecutors.PROCESSORS_SETTING.getKey(),
|
||||||
|
String.valueOf(Runtime.getRuntime().availableProcessors())))
|
||||||
|
// "transport.type"
|
||||||
|
.put(NetworkModule.TRANSPORT_TYPE_KEY,
|
||||||
|
Netty4Plugin.NETTY_TRANSPORT_NAME)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addDiscoveryNodes(TransportClient transportClient, DiscoveryNodes discoveryNodes) {
|
||||||
|
for (DiscoveryNode discoveryNode : discoveryNodes) {
|
||||||
|
transportClient.addTransportAddress(discoveryNode.getAddress());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static class MyTransportClient extends TransportClient {
|
||||||
|
|
||||||
|
MyTransportClient(Settings settings, Collection<Class<? extends Plugin>> plugins) {
|
||||||
|
super(settings, plugins);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,39 @@
|
|||||||
|
package org.xbib.elx.transport;
|
||||||
|
|
||||||
|
import org.elasticsearch.client.ElasticsearchClient;
|
||||||
|
import org.elasticsearch.client.transport.TransportClient;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.xbib.elx.common.AbstractSearchClient;
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Transport search client with additional methods.
|
||||||
|
*/
|
||||||
|
public class TransportSearchClient extends AbstractSearchClient {
|
||||||
|
|
||||||
|
private final TransportClientHelper helper;
|
||||||
|
|
||||||
|
public TransportSearchClient() {
|
||||||
|
this.helper = new TransportClientHelper();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||||
|
return helper.createClient(settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void closeClient() {
|
||||||
|
if (getClient() != null) {
|
||||||
|
TransportClient client = (TransportClient) getClient();
|
||||||
|
client.close();
|
||||||
|
client.threadPool().shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(Settings settings) throws IOException {
|
||||||
|
super.init(settings);
|
||||||
|
helper.init((TransportClient) getClient(), settings);
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,11 @@
|
|||||||
|
package org.xbib.elx.transport;
|
||||||
|
|
||||||
|
import org.xbib.elx.api.SearchClientProvider;
|
||||||
|
|
||||||
|
public class TransportSearchClientProvider implements SearchClientProvider<TransportSearchClient> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public TransportSearchClient getClient() {
|
||||||
|
return new TransportSearchClient();
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1 @@
|
|||||||
|
org.xbib.elx.transport.TransportAdminClientProvider
|
@ -0,0 +1 @@
|
|||||||
|
org.xbib.elx.transport.TransportBulkClientProvider
|
@ -1 +0,0 @@
|
|||||||
org.xbib.elx.transport.ExtendedTransportClientProvider
|
|
@ -0,0 +1 @@
|
|||||||
|
org.xbib.elx.transport.TransportSearchClientProvider
|
@ -0,0 +1,188 @@
|
|||||||
|
package org.xbib.elx.transport.test;
|
||||||
|
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.common.unit.TimeValue;
|
||||||
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
|
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.xbib.elx.common.ClientBuilder;
|
||||||
|
import org.xbib.elx.common.Parameters;
|
||||||
|
import org.xbib.elx.transport.TransportBulkClient;
|
||||||
|
import org.xbib.elx.transport.TransportBulkClientProvider;
|
||||||
|
|
||||||
|
import java.util.concurrent.CountDownLatch;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
@ExtendWith(TestExtension.class)
|
||||||
|
class BulkClientTest {
|
||||||
|
|
||||||
|
private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getName());
|
||||||
|
|
||||||
|
private static final Long ACTIONS = 10000L;
|
||||||
|
|
||||||
|
private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
|
||||||
|
|
||||||
|
private final TestExtension.Helper helper;
|
||||||
|
|
||||||
|
BulkClientTest(TestExtension.Helper helper) {
|
||||||
|
this.helper = helper;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testSingleDoc() throws Exception {
|
||||||
|
final TransportBulkClient bulkClient = ClientBuilder.builder()
|
||||||
|
.setBulkClientProvider(TransportBulkClientProvider.class)
|
||||||
|
.put(helper.getTransportSettings())
|
||||||
|
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||||
|
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
|
||||||
|
.build();
|
||||||
|
try {
|
||||||
|
bulkClient.newIndex("test");
|
||||||
|
bulkClient.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
||||||
|
bulkClient.flush();
|
||||||
|
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
|
} finally {
|
||||||
|
assertEquals(1, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||||
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
|
}
|
||||||
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
|
bulkClient.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testNewIndex() throws Exception {
|
||||||
|
final TransportBulkClient bulkClient = ClientBuilder.builder()
|
||||||
|
.setBulkClientProvider(TransportBulkClientProvider.class)
|
||||||
|
.put(helper.getTransportSettings())
|
||||||
|
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
||||||
|
.build();
|
||||||
|
bulkClient.newIndex("test");
|
||||||
|
bulkClient.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testMapping() throws Exception {
|
||||||
|
final TransportBulkClient bulkClient = ClientBuilder.builder()
|
||||||
|
.setBulkClientProvider(TransportBulkClientProvider.class)
|
||||||
|
.put(helper.getTransportSettings())
|
||||||
|
.build();
|
||||||
|
XContentBuilder builder = JsonXContent.contentBuilder()
|
||||||
|
.startObject()
|
||||||
|
.startObject("doc")
|
||||||
|
.startObject("properties")
|
||||||
|
.startObject("location")
|
||||||
|
.field("type", "geo_point")
|
||||||
|
.endObject()
|
||||||
|
.endObject()
|
||||||
|
.endObject()
|
||||||
|
.endObject();
|
||||||
|
bulkClient.newIndex("test", Settings.EMPTY, builder);
|
||||||
|
assertTrue(bulkClient.getMapping("test", "doc").containsKey("properties"));
|
||||||
|
bulkClient.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testRandomDocs() throws Exception {
|
||||||
|
long numactions = ACTIONS;
|
||||||
|
final TransportBulkClient bulkClient = ClientBuilder.builder()
|
||||||
|
.setBulkClientProvider(TransportBulkClientProvider.class)
|
||||||
|
.put(helper.getTransportSettings())
|
||||||
|
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||||
|
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
||||||
|
.build();
|
||||||
|
try {
|
||||||
|
bulkClient.newIndex("test");
|
||||||
|
for (int i = 0; i < ACTIONS; i++) {
|
||||||
|
bulkClient.index("test", null, false,
|
||||||
|
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||||
|
}
|
||||||
|
bulkClient.flush();
|
||||||
|
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error(e.getMessage(), e);
|
||||||
|
} finally {
|
||||||
|
assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||||
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
|
}
|
||||||
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
|
bulkClient.refreshIndex("test");
|
||||||
|
assertEquals(numactions, bulkClient.getSearchableDocs("test"));
|
||||||
|
bulkClient.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testThreadedRandomDocs() throws Exception {
|
||||||
|
int maxthreads = Runtime.getRuntime().availableProcessors();
|
||||||
|
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
|
||||||
|
final long actions = ACTIONS;
|
||||||
|
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
|
||||||
|
final TransportBulkClient bulkClient = ClientBuilder.builder()
|
||||||
|
.setBulkClientProvider(TransportBulkClientProvider.class)
|
||||||
|
.put(helper.getTransportSettings())
|
||||||
|
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
|
||||||
|
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
|
||||||
|
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
||||||
|
.put(Parameters.ENABLE_BULK_LOGGING.name(), "true")
|
||||||
|
.build();
|
||||||
|
try {
|
||||||
|
Settings settings = Settings.builder()
|
||||||
|
.put("index.number_of_shards", 1)
|
||||||
|
.put("index.number_of_replicas", 0)
|
||||||
|
.build();
|
||||||
|
bulkClient.newIndex("test", settings);
|
||||||
|
bulkClient.startBulk("test", 0, 1000);
|
||||||
|
logger.info("index created");
|
||||||
|
ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
|
||||||
|
final CountDownLatch latch = new CountDownLatch(maxthreads);
|
||||||
|
for (int i = 0; i < maxthreads; i++) {
|
||||||
|
executorService.execute(() -> {
|
||||||
|
for (int i1 = 0; i1 < actions; i1++) {
|
||||||
|
bulkClient.index("test", null, false,
|
||||||
|
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
||||||
|
}
|
||||||
|
latch.countDown();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
logger.info("waiting for latch...");
|
||||||
|
if (latch.await(30L, TimeUnit.SECONDS)) {
|
||||||
|
logger.info("flush...");
|
||||||
|
bulkClient.flush();
|
||||||
|
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
|
||||||
|
logger.info("got all responses, executor service shutdown...");
|
||||||
|
executorService.shutdown();
|
||||||
|
executorService.awaitTermination(30L, TimeUnit.SECONDS);
|
||||||
|
logger.info("pool is shut down");
|
||||||
|
} else {
|
||||||
|
logger.warn("latch timeout");
|
||||||
|
}
|
||||||
|
bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS);
|
||||||
|
assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount());
|
||||||
|
} catch (NoNodeAvailableException e) {
|
||||||
|
logger.warn("skipping, no node available");
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error(e.getMessage(), e);
|
||||||
|
} finally {
|
||||||
|
if (bulkClient.getBulkController().getLastBulkError() != null) {
|
||||||
|
logger.error("error", bulkClient.getBulkController().getLastBulkError());
|
||||||
|
}
|
||||||
|
assertNull(bulkClient.getBulkController().getLastBulkError());
|
||||||
|
bulkClient.refreshIndex("test");
|
||||||
|
assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test"));
|
||||||
|
bulkClient.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -1,201 +0,0 @@
|
|||||||
package org.xbib.elx.transport.test;
|
|
||||||
|
|
||||||
import org.apache.logging.log4j.LogManager;
|
|
||||||
import org.apache.logging.log4j.Logger;
|
|
||||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
|
|
||||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
|
|
||||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
|
|
||||||
import org.elasticsearch.action.search.SearchAction;
|
|
||||||
import org.elasticsearch.action.search.SearchRequestBuilder;
|
|
||||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
|
||||||
import org.elasticsearch.common.settings.Settings;
|
|
||||||
import org.elasticsearch.common.unit.TimeValue;
|
|
||||||
import org.elasticsearch.common.util.concurrent.EsExecutors;
|
|
||||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
|
||||||
import org.elasticsearch.index.query.QueryBuilders;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.junit.jupiter.api.extension.ExtendWith;
|
|
||||||
import org.xbib.elx.common.ClientBuilder;
|
|
||||||
import org.xbib.elx.common.Parameters;
|
|
||||||
import org.xbib.elx.transport.ExtendedTransportClient;
|
|
||||||
import org.xbib.elx.transport.ExtendedTransportClientProvider;
|
|
||||||
|
|
||||||
import java.util.concurrent.CountDownLatch;
|
|
||||||
import java.util.concurrent.ThreadPoolExecutor;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
|
|
||||||
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
||||||
|
|
||||||
@ExtendWith(TestExtension.class)
|
|
||||||
class ClientTest {
|
|
||||||
|
|
||||||
private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
|
|
||||||
|
|
||||||
private static final Long ACTIONS = 100L;
|
|
||||||
|
|
||||||
private static final Long MAX_ACTIONS_PER_REQUEST = 1000L;
|
|
||||||
|
|
||||||
private final TestExtension.Helper helper;
|
|
||||||
|
|
||||||
ClientTest(TestExtension.Helper helper) {
|
|
||||||
this.helper = helper;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testClientIndexOp() throws Exception {
|
|
||||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
|
||||||
.provider(ExtendedTransportClientProvider.class)
|
|
||||||
.put(helper.getTransportSettings())
|
|
||||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
|
||||||
.build();
|
|
||||||
client.newIndex("test1");
|
|
||||||
client.close();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testMapping() throws Exception {
|
|
||||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
|
||||||
.provider(ExtendedTransportClientProvider.class)
|
|
||||||
.put(helper.getTransportSettings())
|
|
||||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
|
|
||||||
.build();
|
|
||||||
XContentBuilder builder = jsonBuilder()
|
|
||||||
.startObject()
|
|
||||||
.startObject("doc")
|
|
||||||
.startObject("properties")
|
|
||||||
.startObject("location")
|
|
||||||
.field("type", "geo_point")
|
|
||||||
.endObject()
|
|
||||||
.endObject()
|
|
||||||
.endObject()
|
|
||||||
.endObject();
|
|
||||||
client.newIndex("test2", Settings.EMPTY, builder.string());
|
|
||||||
GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test2");
|
|
||||||
GetMappingsResponse getMappingsResponse =
|
|
||||||
client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
|
|
||||||
logger.info("mappings={}", getMappingsResponse.getMappings());
|
|
||||||
assertTrue(getMappingsResponse.getMappings().get("test2").containsKey("doc"));
|
|
||||||
client.close();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testSingleDoc() throws Exception {
|
|
||||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
|
||||||
.provider(ExtendedTransportClientProvider.class)
|
|
||||||
.put(helper.getTransportSettings())
|
|
||||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
|
||||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
|
||||||
.build();
|
|
||||||
try {
|
|
||||||
client.newIndex("test3");
|
|
||||||
client.index("test3", "1", true, "{ \"name\" : \"Hello World\"}");
|
|
||||||
client.flush();
|
|
||||||
client.waitForResponses(30L, TimeUnit.SECONDS);
|
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} finally {
|
|
||||||
assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount());
|
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
|
||||||
}
|
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
|
||||||
client.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testRandomDocs() throws Exception {
|
|
||||||
long numactions = ACTIONS;
|
|
||||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
|
||||||
.provider(ExtendedTransportClientProvider.class)
|
|
||||||
.put(helper.getTransportSettings())
|
|
||||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
|
||||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
|
||||||
.build();
|
|
||||||
try {
|
|
||||||
client.newIndex("test4");
|
|
||||||
for (int i = 0; i < ACTIONS; i++) {
|
|
||||||
client.index("test4", null, false,
|
|
||||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
|
||||||
}
|
|
||||||
client.flush();
|
|
||||||
client.waitForResponses(60L, TimeUnit.SECONDS);
|
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} finally {
|
|
||||||
assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
|
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
|
||||||
}
|
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
|
||||||
client.refreshIndex("test4");
|
|
||||||
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
|
|
||||||
.setIndices("test4")
|
|
||||||
.setQuery(QueryBuilders.matchAllQuery())
|
|
||||||
.setSize(0);
|
|
||||||
assertEquals(numactions,
|
|
||||||
searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
|
|
||||||
client.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testThreadedRandomDocs() throws Exception {
|
|
||||||
int maxthreads = Runtime.getRuntime().availableProcessors();
|
|
||||||
long maxactions = MAX_ACTIONS_PER_REQUEST;
|
|
||||||
final long maxloop = ACTIONS;
|
|
||||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
|
||||||
.provider(ExtendedTransportClientProvider.class)
|
|
||||||
.put(helper.getTransportSettings())
|
|
||||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxactions)
|
|
||||||
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
|
|
||||||
.build();
|
|
||||||
try {
|
|
||||||
client.newIndex("test5")
|
|
||||||
.startBulk("test5", -1, 1000);
|
|
||||||
ThreadPoolExecutor pool = EsExecutors.newFixed("transportclient-test", maxthreads, 30,
|
|
||||||
EsExecutors.daemonThreadFactory("transportclient-test"));
|
|
||||||
final CountDownLatch latch = new CountDownLatch(maxthreads);
|
|
||||||
for (int i = 0; i < maxthreads; i++) {
|
|
||||||
pool.execute(() -> {
|
|
||||||
for (int i1 = 0; i1 < maxloop; i1++) {
|
|
||||||
client.index("test5",null, false,
|
|
||||||
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
|
|
||||||
}
|
|
||||||
latch.countDown();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
logger.info("waiting for latch...");
|
|
||||||
if (latch.await(60, TimeUnit.SECONDS)) {
|
|
||||||
logger.info("flush ...");
|
|
||||||
client.flush();
|
|
||||||
client.waitForResponses(60L, TimeUnit.SECONDS);
|
|
||||||
logger.info("pool shutdown ...");
|
|
||||||
pool.shutdown();
|
|
||||||
logger.info("poot is shut down");
|
|
||||||
} else {
|
|
||||||
logger.warn("latch timeout");
|
|
||||||
}
|
|
||||||
} catch (NoNodeAvailableException e) {
|
|
||||||
logger.warn("skipping, no node available");
|
|
||||||
} finally {
|
|
||||||
client.stopBulk("test5", 60L, TimeUnit.SECONDS);
|
|
||||||
assertEquals(maxthreads * maxloop, client.getBulkController().getBulkMetric().getSucceeded().getCount());
|
|
||||||
if (client.getBulkController().getLastBulkError() != null) {
|
|
||||||
logger.error("error", client.getBulkController().getLastBulkError());
|
|
||||||
}
|
|
||||||
assertNull(client.getBulkController().getLastBulkError());
|
|
||||||
client.refreshIndex("test5");
|
|
||||||
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
|
|
||||||
.setIndices("test5")
|
|
||||||
.setQuery(QueryBuilders.matchAllQuery())
|
|
||||||
.setSize(0);
|
|
||||||
assertEquals(maxthreads * maxloop,
|
|
||||||
searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
|
|
||||||
client.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
Loading…
Reference in New Issue