joerg/elx
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

update shift and prune

Browse source
This commit is contained in:
Jörg Prante 2021-03-29 17:02:25 +02:00
parent 94c4fcf02c
commit 95649e7950
19 changed files with 277 additions and 188 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
elx-api/src/main/java/org/xbib/elx/api/AdminClient.java
View file

@ -123,7 +123,8 @@ public interface AdminClient extends BasicClient {
* @param additionalAliases new aliases
* @return this
*/
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases);
IndexShiftResult shiftIndex(IndexDefinition indexDefinition,
List<String> additionalAliases);
/**
* Shift from one index to another.
@ -132,29 +133,10 @@ public interface AdminClient extends BasicClient {
* @param indexAliasAdder method to add aliases
* @return this
*/
IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases,
IndexShiftResult shiftIndex(IndexDefinition indexDefinition,
List<String> additionalAliases,
IndexAliasAdder indexAliasAdder);
/**
* Shift from one index to another.
* @param index the index name
* @param fullIndexName the index name with timestamp
* @param additionalAliases a list of names that should be set as index aliases
* @return this
*/
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases);
/**
* Shift from one index to another.
* @param index the index name
* @param fullIndexName the index name with timestamp
* @param additionalAliases a list of names that should be set as index aliases
* @param adder an adder method to create alias term queries
* @return this
*/
IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases,
IndexAliasAdder adder);
/**
* Prune index.
* @param indexDefinition the index definition
@ -162,19 +144,6 @@ public interface AdminClient extends BasicClient {
*/
IndexPruneResult pruneIndex(IndexDefinition indexDefinition);
/**
* Apply retention policy to prune indices. All indices before delta should be deleted,
* but the number of mintokeep indices must be kept.
*
* @param index index name
* @param fullIndexName index name with timestamp
* @param delta timestamp delta (for index timestamps)
* @param mintokeep minimum number of indices to keep
* @param perform true if pruning should be executed, false if not
* @return the index prune result
*/
IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform);
/**
* Find the timestamp of the most recently indexed document in the index.
*

4
elx-api/src/main/java/org/xbib/elx/api/BulkListener.java
View file

@ -39,8 +39,8 @@ public interface BulkListener {
void afterBulk(long executionId, BulkRequest request, Throwable failure);
/**
* Get the last bulk error
* @return the lst bulk error
* Get the last bulk error.
* @return the last bulk error
*/
Throwable getLastBulkError();
}

18
elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java
View file

@ -1,6 +1,8 @@
package org.xbib.elx.api;
import java.time.format.DateTimeFormatter;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
public interface IndexDefinition {
@ -26,9 +28,13 @@ public interface IndexDefinition {
String getMappings();
IndexDefinition setDateTimePattern(String timeWindow);
IndexDefinition setDateTimeFormatter(DateTimeFormatter formatter);
String getDateTimePattern();
DateTimeFormatter getDateTimeFormatter();
IndexDefinition setDateTimePattern(Pattern pattern);
Pattern getDateTimePattern();
IndexDefinition setEnabled(boolean enabled);
@ -42,9 +48,13 @@ public interface IndexDefinition {
boolean isShiftEnabled();
IndexDefinition setForceMerge(boolean hasForceMerge);
IndexDefinition setPrune(boolean prune);
boolean hasForceMerge();
boolean isPruneEnabled();
IndexDefinition setForceMerge(boolean forcemerge);
boolean isForceMergeEnabled();
IndexDefinition setReplicaLevel(int replicaLevel);

194
elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java
View file

@ -75,6 +75,7 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
@ -93,40 +94,6 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
private static final Logger logger = LogManager.getLogger(AbstractAdminClient.class.getName());
private static final IndexShiftResult EMPTY_INDEX_SHIFT_RESULT = new IndexShiftResult() {
@Override
public List<String> getMovedAliases() {
return Collections.emptyList();
}
@Override
public List<String> getNewAliases() {
return Collections.emptyList();
}
};
private static final IndexPruneResult EMPTY_INDEX_PRUNE_RESULT = new IndexPruneResult() {
@Override
public State getState() {
return State.NONE;
}
@Override
public List<String> getCandidateIndices() {
return Collections.emptyList();
}
@Override
public List<String> getDeletedIndices() {
return Collections.emptyList();
}
@Override
public boolean isAcknowledged() {
return false;
}
};
@Override
public Map<String, ?> getMapping(String index) {
GetMappingsRequestBuilder getMappingsRequestBuilder = new GetMappingsRequestBuilder(client, GetMappingsAction.INSTANCE)
@ -245,46 +212,42 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
}
@Override
public IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases) {
public IndexShiftResult shiftIndex(IndexDefinition indexDefinition,
List<String> additionalAliases) {
return shiftIndex(indexDefinition, additionalAliases, null);
}
@Override
public IndexShiftResult shiftIndex(IndexDefinition indexDefinition,
List<String> additionalAliases, IndexAliasAdder indexAliasAdder) {
List<String> additionalAliases,
IndexAliasAdder indexAliasAdder) {
if (additionalAliases == null) {
return EMPTY_INDEX_SHIFT_RESULT;
return new EmptyIndexShiftResult();
}
if (indexDefinition.isShiftEnabled()) {
return shiftIndex(indexDefinition.getIndex(),
indexDefinition.getFullIndexName(), additionalAliases.stream()
.filter(a -> a != null && !a.isEmpty())
return shiftIndex(indexDefinition.getIndex(), indexDefinition.getFullIndexName(),
additionalAliases.stream().filter(a -> a != null && !a.isEmpty())
.collect(Collectors.toList()), indexAliasAdder);
}
return EMPTY_INDEX_SHIFT_RESULT;
return new EmptyIndexShiftResult();
}
@Override
public IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases) {
return shiftIndex(index, fullIndexName, additionalAliases, null);
}
@Override
public IndexShiftResult shiftIndex(String index, String fullIndexName,
List<String> additionalAliases, IndexAliasAdder adder) {
private IndexShiftResult shiftIndex(String index,
String fullIndexName,
List<String> additionalAliases,
IndexAliasAdder adder) {
ensureClientIsPresent();
if (index == null) {
return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to
return new EmptyIndexShiftResult(); // nothing to shift to
}
if (index.equals(fullIndexName)) {
return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to
return new EmptyIndexShiftResult(); // nothing to shift to
}
waitForCluster("YELLOW", 30L, TimeUnit.SECONDS);
// two situations: 1. a new alias 2. there is already an old index with the alias
List<String> oldIndices = resolveAlias(index);
String oldIndex = oldIndices.stream().findFirst().orElse(null);
Map<String, String> oldAliasMap = getAliases(oldIndex);
logger.debug("old index = {} old alias map = {}", oldIndex, oldAliasMap);
Optional<String> oldIndex = resolveAlias(index).stream().sorted().findFirst();
Map<String, String> oldAliasMap = oldIndex.map(this::getAliases).orElse(null);
logger.info("old index = {} old alias map = {}", oldIndex.orElse("<not found>"), oldAliasMap);
final List<String> newAliases = new ArrayList<>();
final List<String> moveAliases = new ArrayList<>();
IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest();
@ -294,12 +257,12 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
newAliases.add(index);
}
// move existing aliases
if (oldAliasMap != null) {
if (oldIndex.isPresent() && oldAliasMap != null) {
for (Map.Entry<String, String> entry : oldAliasMap.entrySet()) {
String alias = entry.getKey();
String filter = entry.getValue();
indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove()
.index(oldIndex).alias(alias));
.index(oldIndex.get()).alias(alias));
if (filter != null) {
indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove()
.index(fullIndexName).alias(alias).filter(filter));
@ -325,7 +288,7 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
} else {
String filter = oldAliasMap.get(additionalAlias);
indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove()
.index(oldIndex).alias(additionalAlias));
.index(oldIndex.get()).alias(additionalAlias));
if (filter != null) {
indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add()
.index(fullIndexName).alias(additionalAlias).filter(filter));
@ -349,41 +312,50 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
@Override
public IndexPruneResult pruneIndex(IndexDefinition indexDefinition) {
return pruneIndex(indexDefinition.getIndex(), indexDefinition.getFullIndexName(),
indexDefinition.getRetention().getDelta(), indexDefinition.getRetention().getMinToKeep(), true);
return indexDefinition != null && indexDefinition.isPruneEnabled() && indexDefinition.getRetention() != null && indexDefinition.getDateTimePattern() != null?
pruneIndex(indexDefinition.getIndex(),
indexDefinition.getFullIndexName(),
indexDefinition.getDateTimePattern(),
indexDefinition.getRetention().getDelta(),
indexDefinition.getRetention().getMinToKeep()) : new EmptyPruneResult();
}
@Override
public IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform) {
private IndexPruneResult pruneIndex(String index,
String protectedIndexName,
Pattern pattern,
int delta,
int mintokeep) {
logger.info("before pruning: index = {} full index = {} delta = {} mintokeep = {} pattern = {}",
index, protectedIndexName, delta, mintokeep, pattern);
if (delta == 0 && mintokeep == 0) {
return EMPTY_INDEX_PRUNE_RESULT;
return new EmptyPruneResult();
}
if (index.equals(fullIndexName)) {
return EMPTY_INDEX_PRUNE_RESULT;
if (index.equals(protectedIndexName)) {
return new EmptyPruneResult();
}
ensureClientIsPresent();
GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE);
GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet();
Pattern pattern = Pattern.compile("^(.*?)(\\d+)$");
logger.info("found {} indices in the cluster", getIndexResponse.getIndices().length);
logger.info("before pruning: found total of {} indices", getIndexResponse.getIndices().length);
List<String> candidateIndices = new ArrayList<>();
for (String s : getIndexResponse.getIndices()) {
Matcher m = pattern.matcher(s);
if (m.matches() && index.equals(m.group(1)) && !s.equals(fullIndexName)) {
if (m.matches() && m.group(1).equals(index) && !s.equals(protectedIndexName)) {
candidateIndices.add(s);
} else {
logger.info("not a candidate: " + s);
}
}
logger.info("found {} as candidates for pruning", candidateIndices);
if (candidateIndices.isEmpty()) {
logger.info("empty pruning");
return EMPTY_INDEX_PRUNE_RESULT;
return new EmptyPruneResult();
}
if (mintokeep > 0 && candidateIndices.size() <= mintokeep) {
logger.info("nothing to prune, min to keep = " + mintokeep);
return new NothingToDoPruneResult(candidateIndices, Collections.emptyList());
}
Collections.sort(candidateIndices);
logger.info("found {} candidates", candidateIndices);
List<String> indicesToDelete = new ArrayList<>();
Matcher m1 = pattern.matcher(fullIndexName);
Matcher m1 = pattern.matcher(protectedIndexName);
if (m1.matches()) {
Integer i1 = Integer.parseInt(m1.group(2));
for (String s : candidateIndices) {
@ -397,15 +369,14 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
}
}
}
logger.warn("removing {}", indicesToDelete);
if (indicesToDelete.isEmpty()) {
return new NothingToDoPruneResult(candidateIndices, indicesToDelete);
}
logger.warn("deleting {}", indicesToDelete);
String[] s = new String[indicesToDelete.size()];
DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest()
.indices(indicesToDelete.toArray(s));
AcknowledgedResponse response = client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet();
logger.info("removed {}", indicesToDelete);
return new SuccessPruneResult(candidateIndices, indicesToDelete, response);
}
@ -435,7 +406,7 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
@Override
public boolean forceMerge(IndexDefinition indexDefinition) {
if (indexDefinition.hasForceMerge()) {
if (indexDefinition.isForceMergeEnabled()) {
return forceMerge(indexDefinition.getFullIndexName(), indexDefinition.getMaxWaitTime(),
indexDefinition.getMaxWaitTimeUnit());
}
@ -466,18 +437,13 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
throws IOException {
boolean isEnabled = settings.getAsBoolean("enabled", false);
String indexName = settings.get("name", index);
String fullIndexName;
String dateTimePattern = settings.get("dateTimePattern");
if (dateTimePattern != null) {
// check if index name with current date already exists, resolve to it
String dateAppendix = DateTimeFormatter.ofPattern(dateTimePattern)
.withZone(ZoneId.systemDefault()) // not GMT
.format(LocalDate.now());
fullIndexName = resolveAlias(indexName + dateAppendix).stream().findFirst().orElse(index);
} else {
// check if index name already exists, resolve to it
fullIndexName = resolveMostRecentIndex(indexName);
}
String dateTimePatternStr = settings.get("dateTimePattern", "^(.*?)(\\\\d+)$");
Pattern dateTimePattern = Pattern.compile(dateTimePatternStr);
String dateTimeFormat = settings.get("dateTimeFormat", "yyyyMMdd");
DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(dateTimeFormat)
.withZone(ZoneId.systemDefault());
String fullName = indexName + dateTimeFormatter.format(LocalDate.now());
String fullIndexName = resolveAlias(fullName).stream().findFirst().orElse(fullName);
IndexRetention indexRetention = new DefaultIndexRetention()
.setMinToKeep(settings.getAsInt("retention.mintokeep", 0))
.setDelta(settings.getAsInt("retention.delta", 0));
@ -487,9 +453,11 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
.setFullIndexName(fullIndexName)
.setSettings(findSettingsFrom(settings.get("settings")))
.setMappings(findMappingsFrom(settings.get("mapping")))
.setDateTimeFormatter(dateTimeFormatter)
.setDateTimePattern(dateTimePattern)
.setIgnoreErrors(settings.getAsBoolean("skiperrors", false))
.setShift(settings.getAsBoolean("shift", true))
.setShift(settings.getAsBoolean("prune", true))
.setReplicaLevel(settings.getAsInt("replica", 0))
.setMaxWaitTime(settings.getAsLong("timeout", 30L), TimeUnit.SECONDS)
.setRetention(indexRetention)
@ -699,6 +667,18 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
}
}
private static class EmptyIndexShiftResult implements IndexShiftResult {
@Override
public List<String> getMovedAliases() {
return Collections.emptyList();
}
@Override
public List<String> getNewAliases() {
return Collections.emptyList();
}
}
private static class SuccessPruneResult implements IndexPruneResult {
Collection<String> candidateIndices;
@ -734,6 +714,11 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
public boolean isAcknowledged() {
return response.isAcknowledged();
}
@Override
public String toString() {
return "PRUNED: " + indicesToDelete;
}
}
private static class NothingToDoPruneResult implements IndexPruneResult {
@ -766,5 +751,38 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
public boolean isAcknowledged() {
return false;
}
@Override
public String toString() {
return "NOTHING TO DO";
}
}
private static class EmptyPruneResult implements IndexPruneResult {
@Override
public State getState() {
return State.NONE;
}
@Override
public List<String> getCandidateIndices() {
return Collections.emptyList();
}
@Override
public List<String> getDeletedIndices() {
return Collections.emptyList();
}
@Override
public boolean isAcknowledged() {
return false;
}
@Override
public String toString() {
return "NOTHING TO DO";
}
}
}

5
elx-common/src/main/java/org/xbib/elx/common/AbstractBasicClient.java
View file

@ -92,12 +92,11 @@ public abstract class AbstractBasicClient implements BasicClient {
new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
String message = "timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name();
if (logger.isErrorEnabled()) {
logger.error(message);
}
logger.error(message);
throw new IllegalStateException(message);
}
}
@Override
public void waitForShards(long maxWaitTime, TimeUnit timeUnit) {
ensureClientIsPresent();

3
elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java
View file

@ -126,8 +126,7 @@ public abstract class AbstractBulkClient extends AbstractBasicClient implements
createIndexRequestBuilder.addMapping(TYPE_NAME,
JsonXContent.contentBuilder().startObject().startObject(TYPE_NAME).endObject().endObject());
logger.debug("empty mapping");
}
}
CreateIndexResponse createIndexResponse = createIndexRequestBuilder.execute().actionGet();
if (createIndexResponse.isAcknowledged()) {
logger.info("index {} created", index);

4
elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java
View file

@ -126,7 +126,7 @@ public abstract class AbstractSearchClient extends AbstractBasicClient implement
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
queryBuilder.accept(searchRequestBuilder);
searchRequestBuilder.setScroll(scrollTime).setSize(scrollSize);
ActionFuture<SearchResponse> actionFuture = searchRequestBuilder.execute();
ActionFuture<SearchResponse> actionFuture = searchRequestBuilder.execute();
searchMetric.getCurrentQueries().inc();
SearchResponse originalSearchResponse = actionFuture.actionGet();
searchMetric.getCurrentQueries().dec();
@ -149,7 +149,7 @@ public abstract class AbstractSearchClient extends AbstractBasicClient implement
searchMetric.getCurrentQueries().dec();
searchMetric.getQueries().inc();
searchMetric.markTotalQueries(1);
if ( searchResponse1.getHits().getHits().length == 0) {
if (searchResponse1.getHits().getHits().length == 0) {
searchMetric.getEmptyQueries().inc();
} else {
searchMetric.getSucceededQueries().inc();

2
elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java
View file

@ -223,7 +223,7 @@ public class DefaultBulkProcessor implements BulkProcessor {
* Creates a builder of bulk processor with the client to use and the listener that will be used
* to be notified on the completion of bulk requests.
*
* @param client the client
* @param client the client
* @param bulkListener the listener
*/
Builder(ElasticsearchClient client, BulkListener bulkListener) {

62
elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java
View file

@ -3,7 +3,10 @@ package org.xbib.elx.common;
import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.api.IndexRetention;
import java.time.format.DateTimeFormatter;
import java.util.Locale;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
public class DefaultIndexDefinition implements IndexDefinition {
@ -11,7 +14,9 @@ public class DefaultIndexDefinition implements IndexDefinition {
private String fullIndexName;
private String dateTimePattern;
private DateTimeFormatter formatter;
private Pattern pattern;
private String settings;
@ -21,9 +26,11 @@ public class DefaultIndexDefinition implements IndexDefinition {
private boolean ignoreErrors;
private boolean switchAliases;
private boolean shift;
private boolean hasForceMerge;
private boolean prune;
private boolean forcemerge;
private int replicaLevel;
@ -37,6 +44,11 @@ public class DefaultIndexDefinition implements IndexDefinition {
private long stopRefreshInterval;
public DefaultIndexDefinition() {
this.pattern = Pattern.compile("^(.*?)(\\d+)$");
this.formatter = DateTimeFormatter.ofPattern("yyyyMMdd", Locale.getDefault());
}
@Override
public IndexDefinition setIndex(String index) {
this.index = index;
@ -82,14 +94,25 @@ public class DefaultIndexDefinition implements IndexDefinition {
}
@Override
public IndexDefinition setDateTimePattern(String timeWindow) {
this.dateTimePattern = timeWindow;
public IndexDefinition setDateTimeFormatter(DateTimeFormatter formatter) {
this.formatter = formatter;
return this;
}
@Override
public String getDateTimePattern() {
return dateTimePattern;
public DateTimeFormatter getDateTimeFormatter() {
return formatter;
}
@Override
public IndexDefinition setDateTimePattern(Pattern pattern) {
this.pattern = pattern;
return this;
}
@Override
public Pattern getDateTimePattern() {
return pattern;
}
@Override
@ -115,25 +138,36 @@ public class DefaultIndexDefinition implements IndexDefinition {
}
@Override
public IndexDefinition setShift(boolean switchAliases) {
this.switchAliases = switchAliases;
public IndexDefinition setShift(boolean shift) {
this.shift = shift;
return this;
}
@Override
public boolean isShiftEnabled() {
return switchAliases;
return shift;
}
@Override
public IndexDefinition setForceMerge(boolean hasForceMerge) {
this.hasForceMerge = hasForceMerge;
public IndexDefinition setPrune(boolean prune) {
this.prune = prune;
return this;
}
@Override
public boolean hasForceMerge() {
return hasForceMerge;
public boolean isPruneEnabled() {
return prune;
}
@Override
public IndexDefinition setForceMerge(boolean forcemerge) {
this.forcemerge = forcemerge;
return this;
}
@Override
public boolean isForceMergeEnabled() {
return forcemerge;
}
@Override

7
elx-common/src/main/java/org/xbib/elx/common/DefaultIndexRetention.java
View file

@ -8,7 +8,12 @@ public class DefaultIndexRetention implements IndexRetention {
private int minToKeep;
@Override
public DefaultIndexRetention() {
this.delta = 2;
this.minToKeep = 2;
}
@Override
public IndexRetention setDelta(int delta) {
this.delta = delta;
return this;

6
elx-common/src/main/java/org/xbib/elx/common/Parameters.java
View file

@ -20,11 +20,11 @@ public enum Parameters {
FLUSH_INTERVAL("flush_interval");
private boolean flag = false;
boolean flag;
private int num = -1;
int num;
private String string;
String string;
Parameters(boolean flag) {
this.flag = flag;

4
elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java
View file

@ -31,7 +31,7 @@ class SearchTest {
void testSearch() throws Exception {
ElasticsearchClient client = helper.client("1");
BulkRequestBuilder builder = new BulkRequestBuilder(client, BulkAction.INSTANCE);
for (int i = 0; i < 1000; i++) {
for (int i = 0; i < 1; i++) {
IndexRequest indexRequest = new IndexRequest().index("pages")
.source(XContentFactory.jsonBuilder()
.startObject()
@ -51,7 +51,7 @@ class SearchTest {
}
client.execute(BulkAction.INSTANCE, builder.request()).actionGet();
client.execute(RefreshAction.INSTANCE, new RefreshRequest()).actionGet();
for (int i = 0; i < 100; i++) {
for (int i = 0; i < 1; i++) {
QueryBuilder queryStringBuilder = QueryBuilders.queryStringQuery("rs:" + 1234);
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE)
.setIndices("pages")

26
elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java
View file

@ -5,8 +5,12 @@ import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.settings.Settings;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.api.IndexPruneResult;
import org.xbib.elx.api.IndexRetention;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.DefaultIndexDefinition;
import org.xbib.elx.common.DefaultIndexRetention;
import org.xbib.elx.http.HttpAdminClient;
import org.xbib.elx.http.HttpAdminClientProvider;
import org.xbib.elx.http.HttpBulkClient;
@ -15,7 +19,6 @@ import org.xbib.elx.http.HttpBulkClientProvider;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertFalse;
@ -49,15 +52,24 @@ class IndexPruneTest {
.put("index.number_of_replicas", 0)
.build();
bulkClient.newIndex("test1", settings);
adminClient.shiftIndex("test", "test1", Collections.emptyList());
IndexDefinition indexDefinition = new DefaultIndexDefinition();
indexDefinition.setIndex("test");
indexDefinition.setFullIndexName("test1");
adminClient.shiftIndex(indexDefinition, List.of());
bulkClient.newIndex("test2", settings);
adminClient.shiftIndex("test", "test2", Collections.emptyList());
indexDefinition.setFullIndexName("test2");
adminClient.shiftIndex(indexDefinition, List.of());
bulkClient.newIndex("test3", settings);
adminClient.shiftIndex("test", "test3", Collections.emptyList());
indexDefinition.setFullIndexName("test3");
adminClient.shiftIndex(indexDefinition, List.of());
bulkClient.newIndex("test4", settings);
adminClient.shiftIndex("test", "test4", Collections.emptyList());
IndexPruneResult indexPruneResult =
adminClient.pruneIndex("test", "test4", 2, 2, true);
indexDefinition.setFullIndexName("test4");
adminClient.shiftIndex(indexDefinition, List.of());
indexDefinition.setPrune(true);
IndexRetention indexRetention = new DefaultIndexRetention();
indexDefinition.setRetention(indexRetention);
IndexPruneResult indexPruneResult = adminClient.pruneIndex(indexDefinition);
logger.info(indexPruneResult.toString());
assertTrue(indexPruneResult.getDeletedIndices().contains("test1"));
assertTrue(indexPruneResult.getDeletedIndices().contains("test2"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test3"));

13
elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java
View file

@ -8,8 +8,10 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.api.IndexShiftResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.DefaultIndexDefinition;
import org.xbib.elx.http.HttpAdminClient;
import org.xbib.elx.http.HttpAdminClientProvider;
import org.xbib.elx.http.HttpBulkClient;
@ -56,8 +58,11 @@ class IndexShiftTest {
}
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
IndexShiftResult indexShiftResult =
adminClient.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c"));
IndexDefinition indexDefinition = new DefaultIndexDefinition();
indexDefinition.setIndex("test");
indexDefinition.setFullIndexName("test1234");
indexDefinition.setShift(true);
IndexShiftResult indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("a", "b", "c"));
assertTrue(indexShiftResult.getNewAliases().contains("a"));
assertTrue(indexShiftResult.getNewAliases().contains("b"));
assertTrue(indexShiftResult.getNewAliases().contains("c"));
@ -83,7 +88,9 @@ class IndexShiftTest {
}
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
indexShiftResult = adminClient.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"),
indexDefinition.setFullIndexName("test5678");
indexDefinition.setShift(true);
indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("d", "e", "f"),
(request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
.index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
);

25
elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java
View file

@ -5,8 +5,12 @@ import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.settings.Settings;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.api.IndexPruneResult;
import org.xbib.elx.api.IndexRetention;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.DefaultIndexDefinition;
import org.xbib.elx.common.DefaultIndexRetention;
import org.xbib.elx.node.NodeAdminClient;
import org.xbib.elx.node.NodeAdminClientProvider;
import org.xbib.elx.node.NodeBulkClient;
@ -15,7 +19,6 @@ import org.xbib.elx.node.NodeBulkClientProvider;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertFalse;
@ -48,15 +51,23 @@ class IndexPruneTest {
.put("index.number_of_replicas", 0)
.build();
bulkClient.newIndex("test_prune1", settings);
adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
IndexDefinition indexDefinition = new DefaultIndexDefinition();
indexDefinition.setIndex("test_prune");
indexDefinition.setFullIndexName("test_prune1");
adminClient.shiftIndex(indexDefinition, List.of());
bulkClient.newIndex("test_prune2", settings);
adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
indexDefinition.setFullIndexName("test_prune2");
adminClient.shiftIndex(indexDefinition, List.of());
bulkClient.newIndex("test_prune3", settings);
adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
indexDefinition.setFullIndexName("test_prune3");
adminClient.shiftIndex(indexDefinition, List.of());
bulkClient.newIndex("test_prune4", settings);
adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
IndexPruneResult indexPruneResult =
adminClient.pruneIndex("test_prune", "test_prune4", 2, 2, true);
indexDefinition.setFullIndexName("test_prune4");
adminClient.shiftIndex(indexDefinition, List.of());
indexDefinition.setPrune(true);
IndexRetention indexRetention = new DefaultIndexRetention();
indexDefinition.setRetention(indexRetention);
IndexPruneResult indexPruneResult = adminClient.pruneIndex(indexDefinition);
assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1"));
assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3"));

13
elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java
View file

@ -7,8 +7,10 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.api.IndexShiftResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.DefaultIndexDefinition;
import org.xbib.elx.node.NodeAdminClient;
import org.xbib.elx.node.NodeAdminClientProvider;
import org.xbib.elx.node.NodeBulkClient;
@ -54,8 +56,11 @@ class IndexShiftTest {
}
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
IndexShiftResult indexShiftResult =
adminClient.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c"));
IndexDefinition indexDefinition = new DefaultIndexDefinition();
indexDefinition.setIndex("test");
indexDefinition.setFullIndexName("test_shift");
indexDefinition.setShift(true);
IndexShiftResult indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("a", "b", "c"));
assertTrue(indexShiftResult.getNewAliases().contains("a"));
assertTrue(indexShiftResult.getNewAliases().contains("b"));
assertTrue(indexShiftResult.getNewAliases().contains("c"));
@ -79,7 +84,9 @@ class IndexShiftTest {
}
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
indexShiftResult = adminClient.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"),
indexDefinition.setFullIndexName("test_shift2");
indexDefinition.setShift(true);
indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("d", "e", "f"),
(request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
.index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
);

25
elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java
View file

@ -5,8 +5,12 @@ import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.settings.Settings;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.api.IndexPruneResult;
import org.xbib.elx.api.IndexRetention;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.DefaultIndexDefinition;
import org.xbib.elx.common.DefaultIndexRetention;
import org.xbib.elx.transport.TransportAdminClient;
import org.xbib.elx.transport.TransportAdminClientProvider;
import org.xbib.elx.transport.TransportBulkClient;
@ -15,7 +19,6 @@ import org.xbib.elx.transport.TransportBulkClientProvider;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertFalse;
@ -48,16 +51,24 @@ class IndexPruneTest {
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
IndexDefinition indexDefinition = new DefaultIndexDefinition();
indexDefinition.setIndex("test_prune");
indexDefinition.setFullIndexName("test_prune1");
bulkClient.newIndex("test_prune1", settings);
adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
adminClient.shiftIndex(indexDefinition, List.of());
bulkClient.newIndex("test_prune2", settings);
adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
indexDefinition.setFullIndexName("test_prune2");
adminClient.shiftIndex(indexDefinition, List.of());
bulkClient.newIndex("test_prune3", settings);
adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
indexDefinition.setFullIndexName("test_prune3");
adminClient.shiftIndex(indexDefinition, List.of());
bulkClient.newIndex("test_prune4", settings);
adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
IndexPruneResult indexPruneResult =
adminClient.pruneIndex("test_prune", "test_prune4", 2, 2, true);
indexDefinition.setFullIndexName("test_prune4");
adminClient.shiftIndex(indexDefinition, List.of());
indexDefinition.setPrune(true);
IndexRetention indexRetention = new DefaultIndexRetention();
indexDefinition.setRetention(indexRetention);
IndexPruneResult indexPruneResult = adminClient.pruneIndex(indexDefinition);
assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1"));
assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3"));

13
elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java
View file

@ -7,8 +7,10 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.api.IndexShiftResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.DefaultIndexDefinition;
import org.xbib.elx.transport.TransportAdminClient;
import org.xbib.elx.transport.TransportAdminClientProvider;
import org.xbib.elx.transport.TransportBulkClient;
@ -55,8 +57,11 @@ class IndexShiftTest {
}
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
IndexShiftResult indexShiftResult =
adminClient.shiftIndex("test_shift", "test_shift1234", Arrays.asList("a", "b", "c"));
IndexDefinition indexDefinition = new DefaultIndexDefinition();
indexDefinition.setIndex("test_shift");
indexDefinition.setFullIndexName("test_shift1234");
indexDefinition.setShift(true);
IndexShiftResult indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("a", "b", "c"));
assertTrue(indexShiftResult.getNewAliases().contains("a"));
assertTrue(indexShiftResult.getNewAliases().contains("b"));
assertTrue(indexShiftResult.getNewAliases().contains("c"));
@ -80,7 +85,9 @@ class IndexShiftTest {
}
bulkClient.flush();
bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
indexShiftResult = adminClient.shiftIndex("test_shift", "test_shift5678", Arrays.asList("d", "e", "f"),
indexDefinition.setFullIndexName("test_shift5678");
indexDefinition.setShift(true);
indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("d", "e", "f"),
(request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
.index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
);

2
gradle.properties
View file

@ -4,7 +4,7 @@ version = 7.10.2.0
gradle.wrapper.version = 6.6.1
xbib-metrics.version = 2.1.0
xbib-netty-http.version = 4.1.58.0
xbib-netty-http.version = 4.1.60.0
elasticsearch.version = 7.10.2
# ES 7.10.2.1 uses Jackson 2.10.4
jackson.version = 2.12.1