From 83b0c51ac9884d173c26b56cc2567773e6c8e93f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Thu, 8 Apr 2021 17:12:02 +0200 Subject: [PATCH] fix index shift/prune, add failure counter in search metrics --- .../java/org/xbib/elx/api/AdminClient.java | 32 ++--------- .../java/org/xbib/elx/api/BulkClient.java | 2 - .../java/org/xbib/elx/api/BulkListener.java | 3 + .../org/xbib/elx/api/IndexDefinition.java | 2 +- .../java/org/xbib/elx/api/SearchMetric.java | 4 ++ .../xbib/elx/common/AbstractAdminClient.java | 56 +++++++++---------- .../xbib/elx/common/AbstractSearchClient.java | 17 +++--- .../elx/common/DefaultIndexDefinition.java | 6 ++ .../xbib/elx/common/DefaultSearchMetric.java | 16 ++++++ .../xbib/elx/node/test/IndexPruneTest.java | 24 ++++---- .../xbib/elx/node/test/IndexShiftTest.java | 12 +++- .../org/xbib/elx/node/test/SearchTest.java | 40 +++++++++---- .../elx/transport/test/IndexPruneTest.java | 21 ++++--- .../elx/transport/test/IndexShiftTest.java | 11 +++- .../xbib/elx/transport/test/SearchTest.java | 26 +++++++-- gradle.properties | 2 +- 16 files changed, 168 insertions(+), 106 deletions(-) diff --git a/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java b/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java index e00e241..d02c2f2 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java +++ b/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java @@ -123,7 +123,8 @@ public interface AdminClient extends BasicClient { * @param additionalAliases new aliases * @return this */ - IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List additionalAliases); + IndexShiftResult shiftIndex(IndexDefinition indexDefinition, + List additionalAliases); /** * Shift from one index to another. @@ -137,34 +138,9 @@ public interface AdminClient extends BasicClient { IndexAliasAdder indexAliasAdder); /** - * Shift from one index to another. - * @param index the index name - * @param fullIndexName the index name with timestamp - * @param additionalAliases a list of names that should be set as index aliases - * @return this - */ - IndexShiftResult shiftIndex(String index, - String fullIndexName, - List additionalAliases); - - /** - * Shift from one index to another. - * @param index the index name - * @param fullIndexName the index name with timestamp - * @param additionalAliases a list of names that should be set as index aliases - * @param adder an adder method to create alias term queries - * @return this - */ - IndexShiftResult shiftIndex(String index, - String fullIndexName, List additionalAliases, - IndexAliasAdder adder); - - - /** - * Apply retention policy to prune indices. All indices before delta should be deleted, - * but the number of mintokeep indices must be kept. + * Prune index. * - * @param indexDefinition index definition + * @param indexDefinition the index definition * @return the index prune result */ IndexPruneResult pruneIndex(IndexDefinition indexDefinition); diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkClient.java b/elx-api/src/main/java/org/xbib/elx/api/BulkClient.java index 7ba88f3..52a79c4 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/BulkClient.java +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkClient.java @@ -164,7 +164,6 @@ public interface BulkClient extends BasicClient, Flushable { void startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds) throws IOException; - /** * Stop bulk mode. * @@ -216,5 +215,4 @@ public interface BulkClient extends BasicClient, Flushable { * @param index index */ void flushIndex(String index); - } diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkListener.java b/elx-api/src/main/java/org/xbib/elx/api/BulkListener.java index 9caa339..f434ed4 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/BulkListener.java +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkListener.java @@ -3,6 +3,9 @@ package org.xbib.elx.api; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; +/** + * A bulk listener for following executions of bulk operations. + */ public interface BulkListener { /** diff --git a/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java b/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java index 4e24f5a..9d846e6 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java @@ -26,7 +26,7 @@ public interface IndexDefinition { DateTimeFormatter getDateTimeFormatter(); - IndexDefinition setDateTimePattern(Pattern timeWindow); + IndexDefinition setDateTimePattern(Pattern pattern); Pattern getDateTimePattern(); diff --git a/elx-api/src/main/java/org/xbib/elx/api/SearchMetric.java b/elx-api/src/main/java/org/xbib/elx/api/SearchMetric.java index e0cdb96..e190a51 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/SearchMetric.java +++ b/elx-api/src/main/java/org/xbib/elx/api/SearchMetric.java @@ -21,6 +21,10 @@ public interface SearchMetric extends Closeable { Count getEmptyQueries(); + Count getFailedQueries(); + + Count getTimeoutQueries(); + long elapsed(); void start(); diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java index f2c4b85..03fa256 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java @@ -100,19 +100,6 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements */ private static final String TYPE_NAME = "doc"; - private static final IndexShiftResult EMPTY_INDEX_SHIFT_RESULT = new IndexShiftResult() { - @Override - public List getMovedAliases() { - return Collections.emptyList(); - } - - @Override - public List getNewAliases() { - return Collections.emptyList(); - } - }; - - @Override public Map getMapping(String index) throws IOException { GetMappingsRequestBuilder getMappingsRequestBuilder = new GetMappingsRequestBuilder(client, GetMappingsAction.INSTANCE) @@ -138,7 +125,6 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest().indices(index); client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet(); waitForCluster("YELLOW", 30L, TimeUnit.SECONDS); - waitForShards(30L, TimeUnit.SECONDS); return this; } @@ -155,7 +141,6 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements return this; } updateIndexSetting(index, "number_of_replicas", level, maxWaitTime, timeUnit); - waitForShards(maxWaitTime, timeUnit); return this; } @@ -208,6 +193,9 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements @Override public List resolveAlias(String alias) { + if (alias == null) { + return Collections.emptyList(); + } ensureClientIsPresent(); ClusterStateRequest clusterStateRequest = new ClusterStateRequest(); clusterStateRequest.blocks(false); @@ -223,15 +211,17 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements } @Override - public IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List additionalAliases) { + public IndexShiftResult shiftIndex(IndexDefinition indexDefinition, + List additionalAliases) { return shiftIndex(indexDefinition, additionalAliases, null); } @Override public IndexShiftResult shiftIndex(IndexDefinition indexDefinition, - List additionalAliases, IndexAliasAdder indexAliasAdder) { + List additionalAliases, + IndexAliasAdder indexAliasAdder) { if (additionalAliases == null) { - return EMPTY_INDEX_SHIFT_RESULT; + return new EmptyIndexShiftResult(); } if (indexDefinition.isShiftEnabled()) { return shiftIndex(indexDefinition.getIndex(), @@ -239,30 +229,24 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements .filter(a -> a != null && !a.isEmpty()) .collect(Collectors.toList()), indexAliasAdder); } - return EMPTY_INDEX_SHIFT_RESULT; + return new EmptyIndexShiftResult(); } - @Override - public IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases) { - return shiftIndex(index, fullIndexName, additionalAliases, null); - } - - @Override - public IndexShiftResult shiftIndex(String index, String fullIndexName, + private IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases, IndexAliasAdder adder) { ensureClientIsPresent(); if (index == null) { - return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to + return new EmptyIndexShiftResult(); // nothing to shift to } if (index.equals(fullIndexName)) { - return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to + return new EmptyIndexShiftResult(); // nothing to shift to } waitForCluster("YELLOW", 30L, TimeUnit.SECONDS); // two situations: 1. a new alias 2. there is already an old index with the alias Optional oldIndex = resolveAlias(index).stream().sorted().findFirst(); Map oldAliasMap = oldIndex.map(this::getAliases).orElse(null); - logger.debug("old index = {} old alias map = {}", oldIndex.orElse(""), oldAliasMap); + logger.info("old index = {} old alias map = {}", oldIndex.orElse(""), oldAliasMap); final List newAliases = new ArrayList<>(); final List moveAliases = new ArrayList<>(); IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest(); @@ -670,6 +654,20 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements } } + + private static class EmptyIndexShiftResult implements IndexShiftResult { + + @Override + public List getMovedAliases() { + return Collections.emptyList(); + } + + @Override + public List getNewAliases() { + return Collections.emptyList(); + } + } + private static class SuccessPruneResult implements IndexPruneResult { Collection candidateIndices; diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java index 0be1268..7f18569 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java @@ -122,12 +122,12 @@ public abstract class AbstractSearchClient extends AbstractBasicClient implement @Override public Stream search(Consumer queryBuilder, - TimeValue scrollTime, int scrollSize) { + TimeValue scrollTime, int scrollSize) { SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); queryBuilder.accept(searchRequestBuilder); searchRequestBuilder.setScroll(scrollTime).setSize(scrollSize); SearchResponse originalSearchResponse = searchRequestBuilder.execute().actionGet(); - Stream infiniteResponses = Stream.iterate(originalSearchResponse, + Stream responseStream = Stream.iterate(originalSearchResponse, searchResponse -> { SearchScrollRequestBuilder searchScrollRequestBuilder = new SearchScrollRequestBuilder(client, SearchScrollAction.INSTANCE) @@ -139,8 +139,11 @@ public abstract class AbstractSearchClient extends AbstractBasicClient implement searchMetric.getCurrentQueries().dec(); searchMetric.getQueries().inc(); searchMetric.markTotalQueries(1); - boolean isempty = searchResponse1.getHits().getHits().length == 0; - if (isempty) { + if (searchResponse1.getFailedShards() > 0) { + searchMetric.getFailedQueries().inc(); + } else if (searchResponse1.isTimedOut()) { + searchMetric.getTimeoutQueries().inc(); + } else if (searchResponse1.getHits().getHits().length == 0) { searchMetric.getEmptyQueries().inc(); } else { searchMetric.getSucceededQueries().inc(); @@ -154,9 +157,9 @@ public abstract class AbstractSearchClient extends AbstractBasicClient implement .addScrollId(searchResponse.getScrollId()); clearScrollRequestBuilder.execute().actionGet(); }; - return StreamSupport.stream(TakeWhileSpliterator.over(infiniteResponses.spliterator(), + return StreamSupport.stream(TakeWhileSpliterator.over(responseStream.spliterator(), condition, lastAction), false) - .onClose(infiniteResponses::close) + .onClose(responseStream::close) .flatMap(searchResponse -> Arrays.stream(searchResponse.getHits().getHits())); } @@ -165,7 +168,7 @@ public abstract class AbstractSearchClient extends AbstractBasicClient implement return search(queryBuilder, TimeValue.timeValueMinutes(1), 1000).map(SearchHit::getId); } - static class TakeWhileSpliterator implements Spliterator { + private static class TakeWhileSpliterator implements Spliterator { private final Spliterator source; diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java index 0ef0556..a888d15 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java @@ -4,6 +4,7 @@ import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.api.IndexRetention; import java.time.format.DateTimeFormatter; +import java.util.Locale; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; @@ -43,6 +44,11 @@ public class DefaultIndexDefinition implements IndexDefinition { private long stopRefreshInterval; + public DefaultIndexDefinition() { + setDateTimeFormatter(DateTimeFormatter.ofPattern("yyyyMMdd", Locale.getDefault())); + setDateTimePattern(Pattern.compile("^(.*?)(\\d+)$")); + } + @Override public IndexDefinition setIndex(String index) { this.index = index; diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultSearchMetric.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultSearchMetric.java index 5f164b3..b57eee6 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultSearchMetric.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultSearchMetric.java @@ -24,6 +24,10 @@ public class DefaultSearchMetric implements SearchMetric { private final Count emptyQueries; + private final Count failedQueries; + + private final Count timeoutQueries; + private Long started; private Long stopped; @@ -34,6 +38,8 @@ public class DefaultSearchMetric implements SearchMetric { queries = new CountMetric(); succeededQueries = new CountMetric(); emptyQueries = new CountMetric(); + failedQueries = new CountMetric(); + timeoutQueries = new CountMetric(); } @Override @@ -72,6 +78,16 @@ public class DefaultSearchMetric implements SearchMetric { return emptyQueries; } + @Override + public Count getFailedQueries() { + return failedQueries; + } + + @Override + public Count getTimeoutQueries() { + return timeoutQueries; + } + @Override public long elapsed() { return started != null ? ((stopped != null ? stopped : System.nanoTime()) - started) : -1L; diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java index 0f07a7b..2d0efbd 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java @@ -16,13 +16,10 @@ import org.xbib.elx.node.NodeAdminClientProvider; import org.xbib.elx.node.NodeBulkClient; import org.xbib.elx.node.NodeBulkClientProvider; import java.io.IOException; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Locale; -import java.util.regex.Pattern; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; @@ -54,21 +51,28 @@ class IndexPruneTest { .put("index.number_of_replicas", 0) .build(); bulkClient.newIndex("test_prune1", settings); - adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList()); + IndexDefinition indexDefinition = new DefaultIndexDefinition(); + indexDefinition.setIndex("test_prune"); + indexDefinition.setFullIndexName("test_prune1"); + indexDefinition.setShift(true); + adminClient.shiftIndex(indexDefinition, Collections.emptyList()); bulkClient.newIndex("test_prune2", settings); - adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList()); + indexDefinition.setFullIndexName("test_prune2"); + indexDefinition.setShift(true); + adminClient.shiftIndex(indexDefinition, Collections.emptyList()); bulkClient.newIndex("test_prune3", settings); - adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList()); + indexDefinition.setFullIndexName("test_prune3"); + indexDefinition.setShift(true); + adminClient.shiftIndex(indexDefinition, Collections.emptyList()); bulkClient.newIndex("test_prune4", settings); - adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList()); + indexDefinition.setFullIndexName("test_prune4"); + indexDefinition.setShift(true); + adminClient.shiftIndex(indexDefinition, Collections.emptyList()); IndexRetention indexRetention = new DefaultIndexRetention(); indexRetention.setDelta(2); indexRetention.setMinToKeep(2); - IndexDefinition indexDefinition = new DefaultIndexDefinition(); indexDefinition.setIndex("test_prune"); indexDefinition.setFullIndexName("test_prune4"); - indexDefinition.setDateTimeFormatter(DateTimeFormatter.ofPattern("yyyyMMdd", Locale.getDefault())); - indexDefinition.setDateTimePattern(Pattern.compile("^(.*?)(\\d+)$")); indexDefinition.setRetention(indexRetention); indexDefinition.setEnabled(true); indexDefinition.setPrune(true); diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java index b192b3e..dbe206a 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java @@ -8,8 +8,10 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.DefaultIndexDefinition; import org.xbib.elx.node.NodeAdminClient; import org.xbib.elx.node.NodeAdminClientProvider; import org.xbib.elx.node.NodeBulkClient; @@ -54,8 +56,11 @@ class IndexShiftTest { } bulkClient.flush(); bulkClient.waitForResponses(30L, TimeUnit.SECONDS); - IndexShiftResult indexShiftResult = - adminClient.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c")); + IndexDefinition indexDefinition = new DefaultIndexDefinition(); + indexDefinition.setIndex("test"); + indexDefinition.setFullIndexName("test_shift"); + indexDefinition.setShift(true); + IndexShiftResult indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("a", "b", "c")); assertTrue(indexShiftResult.getNewAliases().contains("a")); assertTrue(indexShiftResult.getNewAliases().contains("b")); assertTrue(indexShiftResult.getNewAliases().contains("c")); @@ -78,7 +83,8 @@ class IndexShiftTest { } bulkClient.flush(); bulkClient.waitForResponses(30L, TimeUnit.SECONDS); - indexShiftResult = adminClient.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"), + indexDefinition.setFullIndexName("test_shift2"); + indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("d", "e", "f"), (request, index, alias) -> request.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, index, alias).filter(QueryBuilders.termQuery("my_key", alias))) ); diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/SearchTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/SearchTest.java index 0ca18f9..562aaaf 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/SearchTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/SearchTest.java @@ -24,9 +24,9 @@ class SearchTest { private static final Logger logger = LogManager.getLogger(SearchTest.class.getName()); - private static final Long ACTIONS = 100L; + private static final Long ACTIONS = 100000L; - private static final Long MAX_ACTIONS_PER_REQUEST = 10L; + private static final Long MAX_ACTIONS_PER_REQUEST = 100L; private final TestExtension.Helper helper; @@ -36,21 +36,22 @@ class SearchTest { @Test void testDocStream() throws Exception { + long numactions = ACTIONS; try (NodeBulkClient bulkClient = ClientBuilder.builder(helper.client) .setBulkClientProvider(NodeBulkClientProvider.class) .put(helper.getNodeSettings()) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .build()) { bulkClient.newIndex("test"); - for (int i = 0; i < ACTIONS; i++) { + for (int i = 0; i < numactions; i++) { bulkClient.index("test", null, false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } bulkClient.flush(); bulkClient.waitForResponses(30L, TimeUnit.SECONDS); bulkClient.refreshIndex("test"); - assertEquals(ACTIONS, bulkClient.getSearchableDocs("test")); - assertEquals(ACTIONS, bulkClient.getBulkController().getBulkMetric().getSucceeded().getCount()); + assertEquals(numactions, bulkClient.getSearchableDocs("test")); + assertEquals(numactions, bulkClient.getBulkController().getBulkMetric().getSucceeded().getCount()); if (bulkClient.getBulkController().getLastBulkError() != null) { logger.error("error", bulkClient.getBulkController().getLastBulkError()); } @@ -60,24 +61,39 @@ class SearchTest { .setSearchClientProvider(NodeSearchClientProvider.class) .put(helper.getNodeSettings()) .build()) { + // test stream count Stream stream = searchClient.search(qb -> qb .setIndices("test") .setQuery(QueryBuilders.matchAllQuery()), - TimeValue.timeValueMinutes(1), 10); + TimeValue.timeValueMillis(100), 570); long count = stream.count(); - assertEquals(ACTIONS, count); + assertEquals(numactions, count); + assertEquals(0L, searchClient.getSearchMetric().getFailedQueries().getCount()); + assertEquals(0L, searchClient.getSearchMetric().getTimeoutQueries().getCount()); + assertEquals(1L, searchClient.getSearchMetric().getEmptyQueries().getCount()); + // test stream docs + stream = searchClient.search(qb -> qb + .setIndices("test") + .setQuery(QueryBuilders.matchAllQuery()), + TimeValue.timeValueMillis(10), 79); + final AtomicInteger hitcount = new AtomicInteger(); + stream.forEach(hit -> hitcount.incrementAndGet()); + assertEquals(numactions, hitcount.get()); + assertEquals(0L, searchClient.getSearchMetric().getFailedQueries().getCount()); + assertEquals(0L, searchClient.getSearchMetric().getTimeoutQueries().getCount()); + assertEquals(2L, searchClient.getSearchMetric().getEmptyQueries().getCount()); + // test stream doc ids Stream ids = searchClient.getIds(qb -> qb .setIndices("test") .setQuery(QueryBuilders.matchAllQuery())); final AtomicInteger idcount = new AtomicInteger(); ids.forEach(id -> { - logger.info(id); idcount.incrementAndGet(); }); - assertEquals(ACTIONS, idcount.get()); - assertEquals(11, searchClient.getSearchMetric().getQueries().getCount()); - assertEquals(9, searchClient.getSearchMetric().getSucceededQueries().getCount()); - assertEquals(2, searchClient.getSearchMetric().getEmptyQueries().getCount()); + assertEquals(numactions, idcount.get()); + assertEquals(0L, searchClient.getSearchMetric().getFailedQueries().getCount()); + assertEquals(0L, searchClient.getSearchMetric().getTimeoutQueries().getCount()); + assertEquals(3L, searchClient.getSearchMetric().getEmptyQueries().getCount()); } } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java index 7383a80..ff8148b 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java @@ -55,21 +55,28 @@ class IndexPruneTest { .put("index.number_of_replicas", 0) .build(); bulkClient.newIndex("test_prune1", settings); - adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList()); + IndexDefinition indexDefinition = new DefaultIndexDefinition(); + indexDefinition.setIndex("test_prune"); + indexDefinition.setFullIndexName("test_prune1"); + indexDefinition.setShift(true); + adminClient.shiftIndex(indexDefinition, Collections.emptyList()); bulkClient.newIndex("test_prune2", settings); - adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList()); + indexDefinition.setFullIndexName("test_prune2"); + indexDefinition.setShift(true); + adminClient.shiftIndex(indexDefinition, Collections.emptyList()); bulkClient.newIndex("test_prune3", settings); - adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList()); + indexDefinition.setFullIndexName("test_prune3"); + indexDefinition.setShift(true); + adminClient.shiftIndex(indexDefinition, Collections.emptyList()); bulkClient.newIndex("test_prune4", settings); - adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList()); + indexDefinition.setFullIndexName("test_prune4"); + indexDefinition.setShift(true); + adminClient.shiftIndex(indexDefinition, Collections.emptyList()); IndexRetention indexRetention = new DefaultIndexRetention(); indexRetention.setDelta(2); indexRetention.setMinToKeep(2); - IndexDefinition indexDefinition = new DefaultIndexDefinition(); indexDefinition.setIndex("test_prune"); indexDefinition.setFullIndexName("test_prune4"); - indexDefinition.setDateTimeFormatter(DateTimeFormatter.ofPattern("yyyyMMdd", Locale.getDefault())); - indexDefinition.setDateTimePattern(Pattern.compile("^(.*?)(\\d+)$")); indexDefinition.setRetention(indexRetention); indexDefinition.setEnabled(true); indexDefinition.setPrune(true); diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java index 993d798..baa2a99 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java @@ -8,8 +8,10 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.DefaultIndexDefinition; import org.xbib.elx.transport.TransportAdminClient; import org.xbib.elx.transport.TransportAdminClientProvider; import org.xbib.elx.transport.TransportBulkClient; @@ -56,8 +58,12 @@ class IndexShiftTest { } bulkClient.flush(); bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + IndexDefinition indexDefinition = new DefaultIndexDefinition(); + indexDefinition.setIndex("test"); + indexDefinition.setFullIndexName("test_shift"); + indexDefinition.setShift(true); IndexShiftResult indexShiftResult = - adminClient.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c")); + adminClient.shiftIndex(indexDefinition, Arrays.asList("a", "b", "c")); assertTrue(indexShiftResult.getNewAliases().contains("a")); assertTrue(indexShiftResult.getNewAliases().contains("b")); assertTrue(indexShiftResult.getNewAliases().contains("c")); @@ -80,7 +86,8 @@ class IndexShiftTest { } bulkClient.flush(); bulkClient.waitForResponses(30L, TimeUnit.SECONDS); - indexShiftResult = adminClient.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"), + indexDefinition.setFullIndexName("test_shift2"); + indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("d", "e", "f"), (request, index, alias) -> request.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, index, alias).filter(QueryBuilders.termQuery("my_key", alias))) ); diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/SearchTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/SearchTest.java index 1aec9b1..c69a90d 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/SearchTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/SearchTest.java @@ -26,9 +26,9 @@ class SearchTest { private static final Logger logger = LogManager.getLogger(SearchTest.class.getName()); - private static final Long ACTIONS = 100L; + private static final Long ACTIONS = 100000L; - private static final Long MAX_ACTIONS_PER_REQUEST = 10L; + private static final Long MAX_ACTIONS_PER_REQUEST = 100L; private final TestExtension.Helper helper; @@ -63,21 +63,39 @@ class SearchTest { .setSearchClientProvider(TransportSearchClientProvider.class) .put(helper.getTransportSettings()) .build()) { + // test stream count Stream stream = searchClient.search(qb -> qb .setIndices("test") .setQuery(QueryBuilders.matchAllQuery()), - TimeValue.timeValueMinutes(1), 10); + TimeValue.timeValueMillis(100), 570); long count = stream.count(); assertEquals(numactions, count); + assertEquals(0L, searchClient.getSearchMetric().getFailedQueries().getCount()); + assertEquals(0L, searchClient.getSearchMetric().getTimeoutQueries().getCount()); + assertEquals(1L, searchClient.getSearchMetric().getEmptyQueries().getCount()); + // test stream docs + stream = searchClient.search(qb -> qb + .setIndices("test") + .setQuery(QueryBuilders.matchAllQuery()), + TimeValue.timeValueMillis(10), 79); + final AtomicInteger hitcount = new AtomicInteger(); + stream.forEach(hit -> hitcount.incrementAndGet()); + assertEquals(numactions, hitcount.get()); + assertEquals(0L, searchClient.getSearchMetric().getFailedQueries().getCount()); + assertEquals(0L, searchClient.getSearchMetric().getTimeoutQueries().getCount()); + assertEquals(2L, searchClient.getSearchMetric().getEmptyQueries().getCount()); + // test stream doc ids Stream ids = searchClient.getIds(qb -> qb .setIndices("test") .setQuery(QueryBuilders.matchAllQuery())); final AtomicInteger idcount = new AtomicInteger(); ids.forEach(id -> { - logger.info(id); idcount.incrementAndGet(); }); assertEquals(numactions, idcount.get()); + assertEquals(0L, searchClient.getSearchMetric().getFailedQueries().getCount()); + assertEquals(0L, searchClient.getSearchMetric().getTimeoutQueries().getCount()); + assertEquals(3L, searchClient.getSearchMetric().getEmptyQueries().getCount()); } } } diff --git a/gradle.properties b/gradle.properties index 211aa21..e912222 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ group = org.xbib name = elx -version = 2.2.1.28 +version = 2.2.1.29 gradle.wrapper.version = 6.6.1 xbib-metrics.version = 2.1.0