joerg
/
elx
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix index shift/prune, add failure counter in search metrics

Browse Source
2.2.1.29
Jörg Prante 3 years ago
parent 33baa0b683
commit 83b0c51ac9
16 changed files with 168 additions and 106 deletions
Show Stats Download Patch File Download Diff File

32
elx-api/src/main/java/org/xbib/elx/api/AdminClient.java
Unescape Escape View File

@ -123,7 +123,8 @@ public interface AdminClient extends BasicClient {
* @param additionalAliases new aliases
* @return this
*/
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases);
IndexShiftResult shiftIndex(IndexDefinition indexDefinition,
List<String> additionalAliases);
/**
* Shift from one index to another.
@ -137,34 +138,9 @@ public interface AdminClient extends BasicClient {
IndexAliasAdder indexAliasAdder);
/**
* Shift from one index to another.
* @param index the index name
* @param fullIndexName the index name with timestamp
* @param additionalAliases a list of names that should be set as index aliases
* @return this
*/
IndexShiftResult shiftIndex(String index,
String fullIndexName,
List<String> additionalAliases);
/**
* Shift from one index to another.
* @param index the index name
* @param fullIndexName the index name with timestamp
* @param additionalAliases a list of names that should be set as index aliases
* @param adder an adder method to create alias term queries
* @return this
*/
IndexShiftResult shiftIndex(String index,
String fullIndexName, List<String> additionalAliases,
IndexAliasAdder adder);
/**
* Apply retention policy to prune indices. All indices before delta should be deleted,
* but the number of mintokeep indices must be kept.
* Prune index.
*
* @param indexDefinition index definition
* @param indexDefinition the index definition
* @return the index prune result
*/
IndexPruneResult pruneIndex(IndexDefinition indexDefinition);

2
elx-api/src/main/java/org/xbib/elx/api/BulkClient.java
Unescape Escape View File

@ -164,7 +164,6 @@ public interface BulkClient extends BasicClient, Flushable {
void startBulk(String index, long startRefreshIntervalSeconds,
long stopRefreshIntervalSeconds) throws IOException;
/**
* Stop bulk mode.
*
@ -216,5 +215,4 @@ public interface BulkClient extends BasicClient, Flushable {
* @param index index
*/
void flushIndex(String index);
}

3
elx-api/src/main/java/org/xbib/elx/api/BulkListener.java
Unescape Escape View File

@ -3,6 +3,9 @@ package org.xbib.elx.api;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
/**
* A bulk listener for following executions of bulk operations.
*/
public interface BulkListener {
/**

2
elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java
Unescape Escape View File

@ -26,7 +26,7 @@ public interface IndexDefinition {
DateTimeFormatter getDateTimeFormatter();
IndexDefinition setDateTimePattern(Pattern timeWindow);
IndexDefinition setDateTimePattern(Pattern pattern);
Pattern getDateTimePattern();

4
elx-api/src/main/java/org/xbib/elx/api/SearchMetric.java
Unescape Escape View File

@ -21,6 +21,10 @@ public interface SearchMetric extends Closeable {
Count getEmptyQueries();
Count getFailedQueries();
Count getTimeoutQueries();
long elapsed();
void start();

56
elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java
Unescape Escape View File

@ -100,19 +100,6 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
*/
private static final String TYPE_NAME = "doc";
private static final IndexShiftResult EMPTY_INDEX_SHIFT_RESULT = new IndexShiftResult() {
@Override
public List<String> getMovedAliases() {
return Collections.emptyList();
}
@Override
public List<String> getNewAliases() {
return Collections.emptyList();
}
};
@Override
public Map<String, ?> getMapping(String index) throws IOException {
GetMappingsRequestBuilder getMappingsRequestBuilder = new GetMappingsRequestBuilder(client, GetMappingsAction.INSTANCE)
@ -138,7 +125,6 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest().indices(index);
client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet();
waitForCluster("YELLOW", 30L, TimeUnit.SECONDS);
waitForShards(30L, TimeUnit.SECONDS);
return this;
}
@ -155,7 +141,6 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
return this;
}
updateIndexSetting(index, "number_of_replicas", level, maxWaitTime, timeUnit);
waitForShards(maxWaitTime, timeUnit);
return this;
}
@ -208,6 +193,9 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
@Override
public List<String> resolveAlias(String alias) {
if (alias == null) {
return Collections.emptyList();
}
ensureClientIsPresent();
ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
clusterStateRequest.blocks(false);
@ -223,15 +211,17 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
}
@Override
public IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases) {
public IndexShiftResult shiftIndex(IndexDefinition indexDefinition,
List<String> additionalAliases) {
return shiftIndex(indexDefinition, additionalAliases, null);
}
@Override
public IndexShiftResult shiftIndex(IndexDefinition indexDefinition,
List<String> additionalAliases, IndexAliasAdder indexAliasAdder) {
List<String> additionalAliases,
IndexAliasAdder indexAliasAdder) {
if (additionalAliases == null) {
return EMPTY_INDEX_SHIFT_RESULT;
return new EmptyIndexShiftResult();
}
if (indexDefinition.isShiftEnabled()) {
return shiftIndex(indexDefinition.getIndex(),
@ -239,30 +229,24 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
.filter(a -> a != null && !a.isEmpty())
.collect(Collectors.toList()), indexAliasAdder);
}
return EMPTY_INDEX_SHIFT_RESULT;
return new EmptyIndexShiftResult();
}
@Override
public IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases) {
return shiftIndex(index, fullIndexName, additionalAliases, null);
}
@Override
public IndexShiftResult shiftIndex(String index, String fullIndexName,
private IndexShiftResult shiftIndex(String index, String fullIndexName,
List<String> additionalAliases,
IndexAliasAdder adder) {
ensureClientIsPresent();
if (index == null) {
return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to
return new EmptyIndexShiftResult(); // nothing to shift to
}
if (index.equals(fullIndexName)) {
return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to
return new EmptyIndexShiftResult(); // nothing to shift to
}
waitForCluster("YELLOW", 30L, TimeUnit.SECONDS);
// two situations: 1. a new alias 2. there is already an old index with the alias
Optional<String> oldIndex = resolveAlias(index).stream().sorted().findFirst();
Map<String, String> oldAliasMap = oldIndex.map(this::getAliases).orElse(null);
logger.debug("old index = {} old alias map = {}", oldIndex.orElse(""), oldAliasMap);
logger.info("old index = {} old alias map = {}", oldIndex.orElse(""), oldAliasMap);
final List<String> newAliases = new ArrayList<>();
final List<String> moveAliases = new ArrayList<>();
IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest();
@ -670,6 +654,20 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
}
}
private static class EmptyIndexShiftResult implements IndexShiftResult {
@Override
public List<String> getMovedAliases() {
return Collections.emptyList();
}
@Override
public List<String> getNewAliases() {
return Collections.emptyList();
}
}
private static class SuccessPruneResult implements IndexPruneResult {
Collection<String> candidateIndices;

17
elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java
Unescape Escape View File

@ -122,12 +122,12 @@ public abstract class AbstractSearchClient extends AbstractBasicClient implement
@Override
public Stream<SearchHit> search(Consumer<SearchRequestBuilder> queryBuilder,
TimeValue scrollTime, int scrollSize) {
TimeValue scrollTime, int scrollSize) {
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
queryBuilder.accept(searchRequestBuilder);
searchRequestBuilder.setScroll(scrollTime).setSize(scrollSize);
SearchResponse originalSearchResponse = searchRequestBuilder.execute().actionGet();
Stream<SearchResponse> infiniteResponses = Stream.iterate(originalSearchResponse,
Stream<SearchResponse> responseStream = Stream.iterate(originalSearchResponse,
searchResponse -> {
SearchScrollRequestBuilder searchScrollRequestBuilder =
new SearchScrollRequestBuilder(client, SearchScrollAction.INSTANCE)
@ -139,8 +139,11 @@ public abstract class AbstractSearchClient extends AbstractBasicClient implement
searchMetric.getCurrentQueries().dec();
searchMetric.getQueries().inc();
searchMetric.markTotalQueries(1);
boolean isempty = searchResponse1.getHits().getHits().length == 0;
if (isempty) {
if (searchResponse1.getFailedShards() > 0) {
searchMetric.getFailedQueries().inc();
} else if (searchResponse1.isTimedOut()) {
searchMetric.getTimeoutQueries().inc();
} else if (searchResponse1.getHits().getHits().length == 0) {
searchMetric.getEmptyQueries().inc();
} else {
searchMetric.getSucceededQueries().inc();
@ -154,9 +157,9 @@ public abstract class AbstractSearchClient extends AbstractBasicClient implement
.addScrollId(searchResponse.getScrollId());
clearScrollRequestBuilder.execute().actionGet();
};
return StreamSupport.stream(TakeWhileSpliterator.over(infiniteResponses.spliterator(),
return StreamSupport.stream(TakeWhileSpliterator.over(responseStream.spliterator(),
condition, lastAction), false)
.onClose(infiniteResponses::close)
.onClose(responseStream::close)
.flatMap(searchResponse -> Arrays.stream(searchResponse.getHits().getHits()));
}
@ -165,7 +168,7 @@ public abstract class AbstractSearchClient extends AbstractBasicClient implement
return search(queryBuilder, TimeValue.timeValueMinutes(1), 1000).map(SearchHit::getId);
}
static class TakeWhileSpliterator<T> implements Spliterator<T> {
private static class TakeWhileSpliterator<T> implements Spliterator<T> {
private final Spliterator<T> source;

6
elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java
Unescape Escape View File

@ -4,6 +4,7 @@ import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.api.IndexRetention;
import java.time.format.DateTimeFormatter;
import java.util.Locale;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
@ -43,6 +44,11 @@ public class DefaultIndexDefinition implements IndexDefinition {
private long stopRefreshInterval;
public DefaultIndexDefinition() {
setDateTimeFormatter(DateTimeFormatter.ofPattern("yyyyMMdd", Locale.getDefault()));
setDateTimePattern(Pattern.compile("^(.*?)(\\d+)$"));
}
@Override
public IndexDefinition setIndex(String index) {
this.index = index;

16
elx-common/src/main/java/org/xbib/elx/common/DefaultSearchMetric.java
Unescape Escape View File

@ -24,6 +24,10 @@ public class DefaultSearchMetric implements SearchMetric {
private final Count emptyQueries;
private final Count failedQueries;
private final Count timeoutQueries;
private Long started;
private Long stopped;
@ -34,6 +38,8 @@ public class DefaultSearchMetric implements SearchMetric {
queries = new CountMetric();
succeededQueries = new CountMetric();
emptyQueries = new CountMetric();
failedQueries = new CountMetric();
timeoutQueries = new CountMetric();
}
@Override
@ -72,6 +78,16 @@ public class DefaultSearchMetric implements SearchMetric {
return emptyQueries;
}
@Override
public Count getFailedQueries() {
return failedQueries;
}
@Override
public Count getTimeoutQueries() {
return timeoutQueries;
}
@Override
public long elapsed() {
return started != null ? ((stopped != null ? stopped : System.nanoTime()) - started) : -1L;

24
elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java
Unescape Escape View File

@ -16,13 +16,10 @@ import org.xbib.elx.node.NodeAdminClientProvider;
import org.xbib.elx.node.NodeBulkClient;
import org.xbib.elx.node.NodeBulkClientProvider;
import java.io.IOException;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.regex.Pattern;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
@ -54,21 +51,28 @@ class IndexPruneTest {
.put("index.number_of_replicas", 0)
.build();
bulkClient.newIndex("test_prune1", settings);
adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
IndexDefinition indexDefinition = new DefaultIndexDefinition();
indexDefinition.setIndex("test_prune");
indexDefinition.setFullIndexName("test_prune1");
indexDefinition.setShift(true);
adminClient.shiftIndex(indexDefinition, Collections.emptyList());
bulkClient.newIndex("test_prune2", settings);
adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
indexDefinition.setFullIndexName("test_prune2");
indexDefinition.setShift(true);
adminClient.shiftIndex(indexDefinition, Collections.emptyList());
bulkClient.newIndex("test_prune3", settings);
adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
indexDefinition.setFullIndexName("test_prune3");
indexDefinition.setShift(true);
adminClient.shiftIndex(indexDefinition, Collections.emptyList());
bulkClient.newIndex("test_prune4", settings);
adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
indexDefinition.setFullIndexName("test_prune4");
indexDefinition.setShift(true);
adminClient.shiftIndex(indexDefinition, Collections.emptyList());
IndexRetention indexRetention = new DefaultIndexRetention();
indexRetention.setDelta(2);
indexRetention.setMinToKeep(2);
IndexDefinition indexDefinition = new DefaultIndexDefinition();
indexDefinition.setIndex("test_prune");
indexDefinition.setFullIndexName("test_prune4");
indexDefinition.setDateTimeFormatter(DateTimeFormatter.ofPattern("yyyyMMdd", Locale.getDefault()));
indexDefinition.setDateTimePattern(Pattern.compile("^(.*?)(\\d+)$"));
indexDefinition.setRetention(indexRetention);
indexDefinition.setEnabled(true);
indexDefinition.setPrune(true);

12
elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java
Unescape Escape View File

@ -8,8 +8,10 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.api.IndexShiftResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.DefaultIndexDefinition;
import org.xbib.elx.node.NodeAdminClient;
import org.xbib.elx.node.NodeAdminClientProvider;
import org.xbib.elx.node.NodeBulkClient;
@ -54,8 +56,11 @@ class IndexShiftTest {
}
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
IndexShiftResult indexShiftResult =
adminClient.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c"));
IndexDefinition indexDefinition = new DefaultIndexDefinition();
indexDefinition.setIndex("test");
indexDefinition.setFullIndexName("test_shift");
indexDefinition.setShift(true);
IndexShiftResult indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("a", "b", "c"));
assertTrue(indexShiftResult.getNewAliases().contains("a"));
assertTrue(indexShiftResult.getNewAliases().contains("b"));
assertTrue(indexShiftResult.getNewAliases().contains("c"));
@ -78,7 +83,8 @@ class IndexShiftTest {
}
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
indexShiftResult = adminClient.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"),
indexDefinition.setFullIndexName("test_shift2");
indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("d", "e", "f"),
(request, index, alias) -> request.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD,
index, alias).filter(QueryBuilders.termQuery("my_key", alias)))
);

40
elx-node/src/test/java/org/xbib/elx/node/test/SearchTest.java
Unescape Escape View File

@ -24,9 +24,9 @@ class SearchTest {
private static final Logger logger = LogManager.getLogger(SearchTest.class.getName());
private static final Long ACTIONS = 100L;
private static final Long ACTIONS = 100000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
private static final Long MAX_ACTIONS_PER_REQUEST = 100L;
private final TestExtension.Helper helper;
@ -36,21 +36,22 @@ class SearchTest {
@Test
void testDocStream() throws Exception {
long numactions = ACTIONS;
try (NodeBulkClient bulkClient = ClientBuilder.builder(helper.client)
.setBulkClientProvider(NodeBulkClientProvider.class)
.put(helper.getNodeSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.build()) {
bulkClient.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
for (int i = 0; i < numactions; i++) {
bulkClient.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
bulkClient.refreshIndex("test");
assertEquals(ACTIONS, bulkClient.getSearchableDocs("test"));
assertEquals(ACTIONS, bulkClient.getBulkController().getBulkMetric().getSucceeded().getCount());
assertEquals(numactions, bulkClient.getSearchableDocs("test"));
assertEquals(numactions, bulkClient.getBulkController().getBulkMetric().getSucceeded().getCount());
if (bulkClient.getBulkController().getLastBulkError() != null) {
logger.error("error", bulkClient.getBulkController().getLastBulkError());
}
@ -60,24 +61,39 @@ class SearchTest {
.setSearchClientProvider(NodeSearchClientProvider.class)
.put(helper.getNodeSettings())
.build()) {
// test stream count
Stream<SearchHit> stream = searchClient.search(qb -> qb
.setIndices("test")
.setQuery(QueryBuilders.matchAllQuery()),
TimeValue.timeValueMinutes(1), 10);
TimeValue.timeValueMillis(100), 570);
long count = stream.count();
assertEquals(ACTIONS, count);
assertEquals(numactions, count);
assertEquals(0L, searchClient.getSearchMetric().getFailedQueries().getCount());
assertEquals(0L, searchClient.getSearchMetric().getTimeoutQueries().getCount());
assertEquals(1L, searchClient.getSearchMetric().getEmptyQueries().getCount());
// test stream docs
stream = searchClient.search(qb -> qb
.setIndices("test")
.setQuery(QueryBuilders.matchAllQuery()),
TimeValue.timeValueMillis(10), 79);
final AtomicInteger hitcount = new AtomicInteger();
stream.forEach(hit -> hitcount.incrementAndGet());
assertEquals(numactions, hitcount.get());
assertEquals(0L, searchClient.getSearchMetric().getFailedQueries().getCount());
assertEquals(0L, searchClient.getSearchMetric().getTimeoutQueries().getCount());
assertEquals(2L, searchClient.getSearchMetric().getEmptyQueries().getCount());
// test stream doc ids
Stream<String> ids = searchClient.getIds(qb -> qb
.setIndices("test")
.setQuery(QueryBuilders.matchAllQuery()));
final AtomicInteger idcount = new AtomicInteger();
ids.forEach(id -> {
logger.info(id);
idcount.incrementAndGet();
});
assertEquals(ACTIONS, idcount.get());
assertEquals(11, searchClient.getSearchMetric().getQueries().getCount());
assertEquals(9, searchClient.getSearchMetric().getSucceededQueries().getCount());
assertEquals(2, searchClient.getSearchMetric().getEmptyQueries().getCount());
assertEquals(numactions, idcount.get());
assertEquals(0L, searchClient.getSearchMetric().getFailedQueries().getCount());
assertEquals(0L, searchClient.getSearchMetric().getTimeoutQueries().getCount());
assertEquals(3L, searchClient.getSearchMetric().getEmptyQueries().getCount());
}
}
}

21
elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java
Unescape Escape View File

@ -55,21 +55,28 @@ class IndexPruneTest {
.put("index.number_of_replicas", 0)
.build();
bulkClient.newIndex("test_prune1", settings);
adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
IndexDefinition indexDefinition = new DefaultIndexDefinition();
indexDefinition.setIndex("test_prune");
indexDefinition.setFullIndexName("test_prune1");
indexDefinition.setShift(true);
adminClient.shiftIndex(indexDefinition, Collections.emptyList());
bulkClient.newIndex("test_prune2", settings);
adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
indexDefinition.setFullIndexName("test_prune2");
indexDefinition.setShift(true);
adminClient.shiftIndex(indexDefinition, Collections.emptyList());
bulkClient.newIndex("test_prune3", settings);
adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
indexDefinition.setFullIndexName("test_prune3");
indexDefinition.setShift(true);
adminClient.shiftIndex(indexDefinition, Collections.emptyList());
bulkClient.newIndex("test_prune4", settings);
adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
indexDefinition.setFullIndexName("test_prune4");
indexDefinition.setShift(true);
adminClient.shiftIndex(indexDefinition, Collections.emptyList());
IndexRetention indexRetention = new DefaultIndexRetention();
indexRetention.setDelta(2);
indexRetention.setMinToKeep(2);
IndexDefinition indexDefinition = new DefaultIndexDefinition();
indexDefinition.setIndex("test_prune");
indexDefinition.setFullIndexName("test_prune4");
indexDefinition.setDateTimeFormatter(DateTimeFormatter.ofPattern("yyyyMMdd", Locale.getDefault()));
indexDefinition.setDateTimePattern(Pattern.compile("^(.*?)(\\d+)$"));
indexDefinition.setRetention(indexRetention);
indexDefinition.setEnabled(true);
indexDefinition.setPrune(true);

11
elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java
Unescape Escape View File

@ -8,8 +8,10 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.api.IndexShiftResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.DefaultIndexDefinition;
import org.xbib.elx.transport.TransportAdminClient;
import org.xbib.elx.transport.TransportAdminClientProvider;
import org.xbib.elx.transport.TransportBulkClient;
@ -56,8 +58,12 @@ class IndexShiftTest {
}
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
IndexDefinition indexDefinition = new DefaultIndexDefinition();
indexDefinition.setIndex("test");
indexDefinition.setFullIndexName("test_shift");
indexDefinition.setShift(true);
IndexShiftResult indexShiftResult =
adminClient.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c"));
adminClient.shiftIndex(indexDefinition, Arrays.asList("a", "b", "c"));
assertTrue(indexShiftResult.getNewAliases().contains("a"));
assertTrue(indexShiftResult.getNewAliases().contains("b"));
assertTrue(indexShiftResult.getNewAliases().contains("c"));
@ -80,7 +86,8 @@ class IndexShiftTest {
}
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
indexShiftResult = adminClient.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"),
indexDefinition.setFullIndexName("test_shift2");
indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("d", "e", "f"),
(request, index, alias) -> request.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD,
index, alias).filter(QueryBuilders.termQuery("my_key", alias)))
);

26
elx-transport/src/test/java/org/xbib/elx/transport/test/SearchTest.java
Unescape Escape View File

@ -26,9 +26,9 @@ class SearchTest {
private static final Logger logger = LogManager.getLogger(SearchTest.class.getName());
private static final Long ACTIONS = 100L;
private static final Long ACTIONS = 100000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
private static final Long MAX_ACTIONS_PER_REQUEST = 100L;
private final TestExtension.Helper helper;
@ -63,21 +63,39 @@ class SearchTest {
.setSearchClientProvider(TransportSearchClientProvider.class)
.put(helper.getTransportSettings())
.build()) {
// test stream count
Stream<SearchHit> stream = searchClient.search(qb -> qb
.setIndices("test")
.setQuery(QueryBuilders.matchAllQuery()),
TimeValue.timeValueMinutes(1), 10);
TimeValue.timeValueMillis(100), 570);
long count = stream.count();
assertEquals(numactions, count);
assertEquals(0L, searchClient.getSearchMetric().getFailedQueries().getCount());
assertEquals(0L, searchClient.getSearchMetric().getTimeoutQueries().getCount());
assertEquals(1L, searchClient.getSearchMetric().getEmptyQueries().getCount());
// test stream docs
stream = searchClient.search(qb -> qb
.setIndices("test")
.setQuery(QueryBuilders.matchAllQuery()),
TimeValue.timeValueMillis(10), 79);
final AtomicInteger hitcount = new AtomicInteger();
stream.forEach(hit -> hitcount.incrementAndGet());
assertEquals(numactions, hitcount.get());
assertEquals(0L, searchClient.getSearchMetric().getFailedQueries().getCount());
assertEquals(0L, searchClient.getSearchMetric().getTimeoutQueries().getCount());
assertEquals(2L, searchClient.getSearchMetric().getEmptyQueries().getCount());
// test stream doc ids
Stream<String> ids = searchClient.getIds(qb -> qb
.setIndices("test")
.setQuery(QueryBuilders.matchAllQuery()));
final AtomicInteger idcount = new AtomicInteger();
ids.forEach(id -> {
logger.info(id);
idcount.incrementAndGet();
});
assertEquals(numactions, idcount.get());
assertEquals(0L, searchClient.getSearchMetric().getFailedQueries().getCount());
assertEquals(0L, searchClient.getSearchMetric().getTimeoutQueries().getCount());
assertEquals(3L, searchClient.getSearchMetric().getEmptyQueries().getCount());
}
}
}

2
gradle.properties
Unescape Escape View File

@ -1,6 +1,6 @@
group = org.xbib
name = elx
version = 2.2.1.28
version = 2.2.1.29
gradle.wrapper.version = 6.6.1
xbib-metrics.version = 2.1.0

Write Preview
Loading…
Cancel
Save