From 2701c2b11d5afe37340e25e60c505b44baafeb5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Mon, 7 Jun 2021 11:37:02 +0200 Subject: [PATCH] update search API, now with wrapper API, add multi get tests, add HTTP MultiGet action --- .../org/xbib/elx/api/IndexAliasAdder.java | 5 +- .../java/org/xbib/elx/api/SearchClient.java | 14 ++---- .../xbib/elx/common/AbstractAdminClient.java | 8 +++- .../xbib/elx/common/AbstractSearchClient.java | 35 +++++++++----- .../xbib/elx/common/DefaultBulkListener.java | 3 -- .../elx/common/DefaultSearchDocument.java | 2 +- .../xbib/elx/common/DefaultSearchResult.java | 2 +- .../java/org/xbib/elx/common/GetDocument.java | 34 ++++++++++++++ .../org/xbib/elx/common/MultiGetDocument.java | 34 ++++++++++++++ .../org/xbib/elx/common/test/SearchTest.java | 2 +- .../http/action/get/HttpMultiGetAction.java | 34 ++++++++++++++ .../services/org.xbib.elx.http.HttpAction | 1 + .../xbib/elx/http/test/IndexShiftTest.java | 5 +- .../org/xbib/elx/http/test/SearchTest.java | 47 ++++++++++++++----- .../xbib/elx/node/test/IndexShiftTest.java | 5 +- .../org/xbib/elx/node/test/SearchTest.java | 19 +++++--- .../elx/transport/test/IndexShiftTest.java | 5 +- .../xbib/elx/transport/test/SearchTest.java | 18 ++++--- gradle.properties | 4 +- 19 files changed, 206 insertions(+), 71 deletions(-) create mode 100644 elx-common/src/main/java/org/xbib/elx/common/GetDocument.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/MultiGetDocument.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/action/get/HttpMultiGetAction.java diff --git a/elx-api/src/main/java/org/xbib/elx/api/IndexAliasAdder.java b/elx-api/src/main/java/org/xbib/elx/api/IndexAliasAdder.java index 4e2dca9..3776e25 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/IndexAliasAdder.java +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexAliasAdder.java @@ -1,9 +1,10 @@ package org.xbib.elx.api; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.elasticsearch.index.query.QueryBuilder; @FunctionalInterface public interface IndexAliasAdder { - void addIndexAlias(IndicesAliasesRequest request, String index, String alias); + QueryBuilder addAliasOnField(String index, String alias); + } diff --git a/elx-api/src/main/java/org/xbib/elx/api/SearchClient.java b/elx-api/src/main/java/org/xbib/elx/api/SearchClient.java index 666d56d..61fd4d6 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/SearchClient.java +++ b/elx-api/src/main/java/org/xbib/elx/api/SearchClient.java @@ -1,13 +1,9 @@ package org.xbib.elx.api; import org.elasticsearch.action.get.GetRequestBuilder; -import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.get.MultiGetRequestBuilder; -import org.elasticsearch.action.get.MultiGetResponse; import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.search.SearchHit; import java.util.Optional; import java.util.function.Consumer; import java.util.stream.Stream; @@ -18,14 +14,14 @@ public interface SearchClient extends BasicClient { SearchMetric getSearchMetric(); - Optional get(Consumer getRequestBuilder); + Optional get(Consumer getRequestBuilder); - Optional multiGet(Consumer multiGetRequestBuilder); + Stream multiGet(Consumer multiGetRequestBuilder); - Optional search(Consumer searchRequestBuilder); + Optional search(Consumer searchRequestBuilder); - Stream search(Consumer searchRequestBuilder, - TimeValue scrollTime, int scrollSize); + Stream search(Consumer searchRequestBuilder, + TimeValue scrollTime, int scrollSize); Stream getIds(Consumer queryBuilder); } diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java index 3c4cf31..b24d54b 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java @@ -282,10 +282,14 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements if (oldAliasMap == null || !oldAliasMap.containsKey(additionalAlias)) { // index alias adder only active on extra aliases, and if alias is new if (adder != null) { - adder.addIndexAlias(indicesAliasesRequest, fullIndexName, additionalAlias); + indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add() + .index(fullIndexName) + .alias(additionalAlias) + .filter(adder.addAliasOnField(fullIndexName, additionalAlias))); } else { indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add() - .index(fullIndexName).alias(additionalAlias)); + .index(fullIndexName) + .alias(additionalAlias)); } newAliases.add(additionalAlias); } else { diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java index 4928853..fa57cee 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java @@ -18,9 +18,10 @@ import org.elasticsearch.action.search.SearchScrollRequestBuilder; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.search.SearchHit; import org.xbib.elx.api.SearchClient; +import org.xbib.elx.api.SearchDocument; import org.xbib.elx.api.SearchMetric; +import org.xbib.elx.api.SearchResult; import java.io.IOException; import java.util.Arrays; import java.util.Comparator; @@ -72,7 +73,7 @@ public abstract class AbstractSearchClient extends AbstractBasicClient implement } @Override - public Optional get(Consumer getRequestBuilderConsumer) { + public Optional get(Consumer getRequestBuilderConsumer) { GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE); getRequestBuilderConsumer.accept(getRequestBuilder); ActionFuture actionFuture = getRequestBuilder.execute(); @@ -94,24 +95,24 @@ public abstract class AbstractSearchClient extends AbstractBasicClient implement searchMetric.getEmptyQueries().inc(); } } - return getResponse.isExists() ? Optional.of(getResponse) : Optional.empty(); + return getResponse.isExists() ? Optional.of(new GetDocument(getResponse)) : Optional.empty(); } @Override - public Optional multiGet(Consumer multiGetRequestBuilderConsumer) { + public Stream multiGet(Consumer multiGetRequestBuilderConsumer) { MultiGetRequestBuilder multiGetRequestBuilder = new MultiGetRequestBuilder(client, MultiGetAction.INSTANCE); multiGetRequestBuilderConsumer.accept(multiGetRequestBuilder); ActionFuture actionFuture = multiGetRequestBuilder.execute(); if (searchMetric != null) { searchMetric.getCurrentQueries().inc(); } - MultiGetResponse multiGetItemResponse = actionFuture.actionGet(); + MultiGetResponse multiGetResponse = actionFuture.actionGet(); if (searchMetric != null) { searchMetric.getCurrentQueries().dec(); searchMetric.getQueries().inc(); searchMetric.markTotalQueries(1); } - boolean isempty = multiGetItemResponse.getResponses().length == 0; + boolean isempty = multiGetResponse.getResponses().length == 0; if (isempty) { if (searchMetric != null) { searchMetric.getEmptyQueries().inc(); @@ -121,12 +122,16 @@ public abstract class AbstractSearchClient extends AbstractBasicClient implement searchMetric.getSucceededQueries().inc(); } } - return isempty ? Optional.empty() : Optional.of(multiGetItemResponse); + return isempty ? Stream.of() : Arrays.stream(multiGetResponse.getResponses()) + .filter(r -> !r.isFailed()) + .map(MultiGetDocument::new); } @Override - public Optional search(Consumer queryBuilder) { + public Optional search(Consumer queryBuilder) { SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); + searchRequestBuilder.setTrackTotalHits(true); + searchRequestBuilder.setTrackScores(true); queryBuilder.accept(searchRequestBuilder); ActionFuture actionFuture = searchRequestBuilder.execute(); if (searchMetric != null) { @@ -158,13 +163,17 @@ public abstract class AbstractSearchClient extends AbstractBasicClient implement searchMetric.getSucceededQueries().inc(); } } - return isempty ? Optional.empty() : Optional.of(searchResponse); + return isempty ? + Optional.empty() : + Optional.of(new DefaultSearchResult(searchResponse.getHits())); } @Override - public Stream search(Consumer queryBuilder, - TimeValue scrollTime, int scrollSize) { + public Stream search(Consumer queryBuilder, + TimeValue scrollTime, int scrollSize) { SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); + searchRequestBuilder.setTrackTotalHits(true); + searchRequestBuilder.setTrackScores(true); queryBuilder.accept(searchRequestBuilder); searchRequestBuilder.setScroll(scrollTime).setSize(scrollSize); ActionFuture actionFuture = searchRequestBuilder.execute(); @@ -239,12 +248,12 @@ public abstract class AbstractSearchClient extends AbstractBasicClient implement return StreamSupport.stream(TakeWhileSpliterator.over(responseStream.spliterator(), condition, lastAction), false) .onClose(responseStream::close) - .flatMap(searchResponse -> Arrays.stream(searchResponse.getHits().getHits())); + .flatMap(searchResponse -> new DefaultSearchResult(searchResponse.getHits()).getDocuments().stream()); } @Override public Stream getIds(Consumer queryBuilder) { - return search(queryBuilder, TimeValue.timeValueMinutes(1), 1000).map(SearchHit::getId); + return search(queryBuilder, TimeValue.timeValueMinutes(1), 1000).map(SearchDocument::getId); } private static class TakeWhileSpliterator implements Spliterator { diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkListener.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkListener.java index 68d3170..23fed98 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkListener.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkListener.java @@ -71,9 +71,6 @@ public class DefaultBulkListener implements BulkListener { } int n = 0; for (BulkItemResponse itemResponse : response.getItems()) { - if (bulkMetric != null) { - bulkMetric.getCurrentIngest().dec(itemResponse.getIndex(), itemResponse.getType(), itemResponse.getId()); - } if (itemResponse.isFailed()) { n++; if (bulkMetric != null) { diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultSearchDocument.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultSearchDocument.java index ef8ad6b..7185201 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultSearchDocument.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultSearchDocument.java @@ -30,6 +30,6 @@ public class DefaultSearchDocument implements SearchDocument { @Override public Map getFields() { - return searchHit.sourceAsMap(); + return searchHit.getSourceAsMap(); } } diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultSearchResult.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultSearchResult.java index c445085..4ab88ff 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultSearchResult.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultSearchResult.java @@ -17,7 +17,7 @@ public class DefaultSearchResult implements SearchResult { } @Override public long getTotal() { - return searchHits.getTotalHits(); + return searchHits.getTotalHits().value; } @Override diff --git a/elx-common/src/main/java/org/xbib/elx/common/GetDocument.java b/elx-common/src/main/java/org/xbib/elx/common/GetDocument.java new file mode 100644 index 0000000..4d47d6b --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/GetDocument.java @@ -0,0 +1,34 @@ +package org.xbib.elx.common; + +import org.elasticsearch.action.get.GetResponse; +import org.xbib.elx.api.SearchDocument; +import java.util.Map; + +public class GetDocument implements SearchDocument { + + private final GetResponse getResponse; + + public GetDocument(GetResponse getResponse) { + this.getResponse = getResponse; + } + + @Override + public String getIndex() { + return getResponse.getIndex(); + } + + @Override + public String getId() { + return getResponse.getId(); + } + + @Override + public float getScore() { + return -1f; + } + + @Override + public Map getFields() { + return getResponse.getSourceAsMap(); + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MultiGetDocument.java b/elx-common/src/main/java/org/xbib/elx/common/MultiGetDocument.java new file mode 100644 index 0000000..346fabc --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MultiGetDocument.java @@ -0,0 +1,34 @@ +package org.xbib.elx.common; + +import org.elasticsearch.action.get.MultiGetItemResponse; +import org.xbib.elx.api.SearchDocument; +import java.util.Map; + +public class MultiGetDocument implements SearchDocument { + + private final MultiGetItemResponse getResponse; + + public MultiGetDocument(MultiGetItemResponse getResponse) { + this.getResponse = getResponse; + } + + @Override + public String getIndex() { + return getResponse.getResponse().getIndex(); + } + + @Override + public String getId() { + return getResponse.getResponse().getId(); + } + + @Override + public float getScore() { + return -1f; + } + + @Override + public Map getFields() { + return getResponse.getResponse().getSourceAsMap(); + } +} diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java index 470f742..9473067 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java @@ -57,7 +57,7 @@ class SearchTest { .setIndices("pages") .setQuery(queryStringBuilder) .addSort("rowcount", SortOrder.DESC) - .setFrom(i * 10) + .setFrom(0) .setSize(10); SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); assertTrue(searchResponse.getHits().getTotalHits().value > 0); diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/get/HttpMultiGetAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/get/HttpMultiGetAction.java new file mode 100644 index 0000000..d6a5f69 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/action/get/HttpMultiGetAction.java @@ -0,0 +1,34 @@ +package org.xbib.elx.http.action.get; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.get.MultiGetAction; +import org.elasticsearch.action.get.MultiGetRequest; +import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.xbib.elx.http.HttpAction; +import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; +import java.io.IOException; + +public class HttpMultiGetAction extends HttpAction { + + @Override + public ActionType getActionInstance() { + return MultiGetAction.INSTANCE; + } + + @Override + protected Request.Builder createHttpRequest(String url, MultiGetRequest request) throws IOException { + BytesReference source = XContentHelper.toXContent(request, XContentType.JSON, false); + return newGetRequest(url, "/_mget", source); + } + + @Override + protected CheckedFunction entityParser(HttpResponse httpResponse) { + return MultiGetResponse::fromXContent; + } +} diff --git a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction index 1a16493..89df3d6 100644 --- a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction +++ b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction @@ -20,3 +20,4 @@ org.xbib.elx.http.action.search.HttpSearchScrollAction org.xbib.elx.http.action.main.HttpMainAction org.xbib.elx.http.action.get.HttpExistsAction org.xbib.elx.http.action.get.HttpGetAction +org.xbib.elx.http.action.get.HttpMultiGetAction \ No newline at end of file diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java index ee3ea3a..8541970 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java @@ -3,7 +3,6 @@ package org.xbib.elx.http.test; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.index.query.QueryBuilders; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -83,9 +82,7 @@ class IndexShiftTest { bulkClient.waitForResponses(30L, TimeUnit.SECONDS); indexDefinition.setShift(true); indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("d", "e", "f"), - (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add() - .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias))) - ); + (index, alias) -> QueryBuilders.termQuery("my_key", alias)); assertTrue(indexShiftResult.getNewAliases().contains("d")); assertTrue(indexShiftResult.getNewAliases().contains("e")); assertTrue(indexShiftResult.getNewAliases().contains("f")); diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/SearchTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/SearchTest.java index 4a205b1..1226f57 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/SearchTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/SearchTest.java @@ -6,18 +6,20 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexDefinition; +import org.xbib.elx.api.SearchDocument; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.DefaultIndexDefinition; import org.xbib.elx.http.HttpBulkClient; import org.xbib.elx.http.HttpBulkClientProvider; import org.xbib.elx.http.HttpSearchClient; import org.xbib.elx.http.HttpSearchClientProvider; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import java.util.stream.Stream; @ExtendWith(TestExtension.class) @@ -61,27 +63,48 @@ class SearchTest { .setSearchClientProvider(HttpSearchClientProvider.class) .put(helper.getClientSettings()) .build()) { - Stream stream = searchClient.search(qb -> qb + Stream stream = searchClient.search(qb -> qb .setIndices(indexDefinition.getFullIndexName()) .setQuery(QueryBuilders.matchAllQuery()), TimeValue.timeValueMillis(100), 579); long count = stream.count(); assertEquals(numactions, count); - Stream ids = searchClient.getIds(qb -> qb + if (searchClient.isSearchMetricEnabled()) { + assertEquals(0L, searchClient.getSearchMetric().getFailedQueries().getCount()); + assertEquals(0L, searchClient.getSearchMetric().getTimeoutQueries().getCount()); + assertEquals(1L, searchClient.getSearchMetric().getEmptyQueries().getCount()); + } + stream = searchClient.search(qb -> qb + .setIndices(indexDefinition.getFullIndexName()) + .setQuery(QueryBuilders.matchAllQuery()), + TimeValue.timeValueMillis(10), 79); + final AtomicInteger hitcount = new AtomicInteger(); + stream.forEach(hit -> hitcount.incrementAndGet()); + assertEquals(numactions, hitcount.get()); + if (searchClient.isSearchMetricEnabled()) { + assertEquals(0L, searchClient.getSearchMetric().getFailedQueries().getCount()); + assertEquals(0L, searchClient.getSearchMetric().getTimeoutQueries().getCount()); + assertEquals(2L, searchClient.getSearchMetric().getEmptyQueries().getCount()); + } + List ids = searchClient.getIds(qb -> qb .setIndices(indexDefinition.getFullIndexName()) - .setQuery(QueryBuilders.matchAllQuery())); + .setQuery(QueryBuilders.matchAllQuery())).collect(Collectors.toList()); final AtomicInteger idcount = new AtomicInteger(); - ids.forEach(id -> { - idcount.incrementAndGet(); - }); + ids.forEach(id -> idcount.incrementAndGet()); assertEquals(numactions, idcount.get()); if (searchClient.isSearchMetricEnabled()) { - assertEquals(275, searchClient.getSearchMetric().getQueries().getCount()); - assertEquals(273, searchClient.getSearchMetric().getSucceededQueries().getCount()); - assertEquals(2, searchClient.getSearchMetric().getEmptyQueries().getCount()); - assertEquals(0, searchClient.getSearchMetric().getFailedQueries().getCount()); - assertEquals(0, searchClient.getSearchMetric().getTimeoutQueries().getCount()); + assertEquals(1542L, searchClient.getSearchMetric().getQueries().getCount()); + assertEquals(1539L, searchClient.getSearchMetric().getSucceededQueries().getCount()); + assertEquals(3L, searchClient.getSearchMetric().getEmptyQueries().getCount()); + assertEquals(0L, searchClient.getSearchMetric().getFailedQueries().getCount()); + assertEquals(0L, searchClient.getSearchMetric().getTimeoutQueries().getCount()); } + stream = searchClient.multiGet(mgrb -> { + for (String id : ids) { + mgrb.add(indexDefinition.getFullIndexName(), indexDefinition.getType(), id); + } + }); + assertEquals(numactions, stream.count()); } } } diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java index 5f3fe68..9f35756 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java @@ -2,7 +2,6 @@ package org.xbib.elx.node.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.index.query.QueryBuilders; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -83,9 +82,7 @@ class IndexShiftTest { bulkClient.waitForResponses(30L, TimeUnit.SECONDS); indexDefinition.setShift(true); indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("d", "e", "f"), - (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add() - .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias))) - ); + (index, alias) -> QueryBuilders.termQuery("my_key", alias)); assertTrue(indexShiftResult.getNewAliases().contains("d")); assertTrue(indexShiftResult.getNewAliases().contains("e")); assertTrue(indexShiftResult.getNewAliases().contains("f")); diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/SearchTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/SearchTest.java index 0de8a28..a6f1deb 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/SearchTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/SearchTest.java @@ -6,10 +6,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexDefinition; +import org.xbib.elx.api.SearchDocument; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.DefaultIndexDefinition; import org.xbib.elx.node.NodeBulkClient; @@ -17,8 +17,10 @@ import org.xbib.elx.node.NodeBulkClientProvider; import org.xbib.elx.node.NodeSearchClient; import org.xbib.elx.node.NodeSearchClientProvider; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import java.util.stream.Stream; @ExtendWith(TestExtension.class) @@ -64,8 +66,7 @@ class SearchTest { .setSearchClientProvider(NodeSearchClientProvider.class) .put(helper.getClientSettings()) .build()) { - // test stream count - Stream stream = searchClient.search(qb -> qb + Stream stream = searchClient.search(qb -> qb .setIndices(indexDefinition.getFullIndexName()) .setQuery(QueryBuilders.matchAllQuery()), TimeValue.timeValueMillis(100), 579); @@ -76,7 +77,6 @@ class SearchTest { assertEquals(0L, searchClient.getSearchMetric().getTimeoutQueries().getCount()); assertEquals(1L, searchClient.getSearchMetric().getEmptyQueries().getCount()); } - // test stream docs stream = searchClient.search(qb -> qb .setIndices(indexDefinition.getFullIndexName()) .setQuery(QueryBuilders.matchAllQuery()), @@ -89,10 +89,9 @@ class SearchTest { assertEquals(0L, searchClient.getSearchMetric().getTimeoutQueries().getCount()); assertEquals(2L, searchClient.getSearchMetric().getEmptyQueries().getCount()); } - // test stream doc ids - Stream ids = searchClient.getIds(qb -> qb + List ids = searchClient.getIds(qb -> qb .setIndices(indexDefinition.getFullIndexName()) - .setQuery(QueryBuilders.matchAllQuery())); + .setQuery(QueryBuilders.matchAllQuery())).collect(Collectors.toList()); final AtomicInteger idcount = new AtomicInteger(); ids.forEach(id -> idcount.incrementAndGet()); assertEquals(numactions, idcount.get()); @@ -103,6 +102,12 @@ class SearchTest { assertEquals(0L, searchClient.getSearchMetric().getFailedQueries().getCount()); assertEquals(0L, searchClient.getSearchMetric().getTimeoutQueries().getCount()); } + stream = searchClient.multiGet(mgrb -> { + for (String id : ids) { + mgrb.add(indexDefinition.getFullIndexName(), indexDefinition.getType(), id); + } + }); + assertEquals(numactions, stream.count()); } } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java index 30b0013..7095f5f 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java @@ -2,7 +2,6 @@ package org.xbib.elx.transport.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.index.query.QueryBuilders; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -80,9 +79,7 @@ class IndexShiftTest { bulkClient.waitForResponses(30L, TimeUnit.SECONDS); indexDefinition.setShift(true); indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("d", "e", "f"), - (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add() - .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias))) - ); + (index, alias) -> QueryBuilders.termQuery("my_key", alias)); assertTrue(indexShiftResult.getNewAliases().contains("d")); assertTrue(indexShiftResult.getNewAliases().contains("e")); assertTrue(indexShiftResult.getNewAliases().contains("f")); diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/SearchTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/SearchTest.java index 7bf402c..cef0988 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/SearchTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/SearchTest.java @@ -6,10 +6,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexDefinition; +import org.xbib.elx.api.SearchDocument; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.DefaultIndexDefinition; import org.xbib.elx.transport.TransportBulkClient; @@ -17,8 +17,10 @@ import org.xbib.elx.transport.TransportBulkClientProvider; import org.xbib.elx.transport.TransportSearchClient; import org.xbib.elx.transport.TransportSearchClientProvider; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import java.util.stream.Stream; @ExtendWith(TestExtension.class) @@ -63,8 +65,7 @@ class SearchTest { .setSearchClientProvider(TransportSearchClientProvider.class) .put(helper.getClientSettings()) .build()) { - // test stream count - Stream stream = searchClient.search(qb -> qb + Stream stream = searchClient.search(qb -> qb .setIndices(indexDefinition.getFullIndexName()) .setQuery(QueryBuilders.matchAllQuery()), TimeValue.timeValueMillis(100), 579); @@ -75,7 +76,6 @@ class SearchTest { assertEquals(0L, searchClient.getSearchMetric().getTimeoutQueries().getCount()); assertEquals(1L, searchClient.getSearchMetric().getEmptyQueries().getCount()); } - // test stream docs stream = searchClient.search(qb -> qb .setIndices(indexDefinition.getFullIndexName()) .setQuery(QueryBuilders.matchAllQuery()), @@ -89,9 +89,9 @@ class SearchTest { assertEquals(2L, searchClient.getSearchMetric().getEmptyQueries().getCount()); } // test stream doc ids - Stream ids = searchClient.getIds(qb -> qb + List ids = searchClient.getIds(qb -> qb .setIndices(indexDefinition.getFullIndexName()) - .setQuery(QueryBuilders.matchAllQuery())); + .setQuery(QueryBuilders.matchAllQuery())).collect(Collectors.toList()); final AtomicInteger idcount = new AtomicInteger(); ids.forEach(id -> idcount.incrementAndGet()); assertEquals(numactions, idcount.get()); @@ -102,6 +102,12 @@ class SearchTest { assertEquals(0L, searchClient.getSearchMetric().getFailedQueries().getCount()); assertEquals(0L, searchClient.getSearchMetric().getTimeoutQueries().getCount()); } + stream = searchClient.multiGet(mgrb -> { + for (String id : ids) { + mgrb.add(indexDefinition.getFullIndexName(), indexDefinition.getType(), id); + } + }); + assertEquals(numactions, stream.count()); } } } diff --git a/gradle.properties b/gradle.properties index 5f08f21..56c5814 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,9 +1,9 @@ group = org.xbib name = elx -version = 7.10.2.12 +version = 7.10.2.13 gradle.wrapper.version = 6.6.1 -xbib-metrics.version = 2.1.0 +xbib-metrics.version = 2.2.0 xbib-time.version = 2.1.0 elasticsearch.version = 7.10.2 # ES 7.10.2 uses Jackson 2.10.4