From 95649e79509efdf3cc81016af3791e4e1acc7d7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Mon, 29 Mar 2021 17:02:25 +0200 Subject: [PATCH] update shift and prune --- .../java/org/xbib/elx/api/AdminClient.java | 39 +--- .../java/org/xbib/elx/api/BulkListener.java | 4 +- .../org/xbib/elx/api/IndexDefinition.java | 18 +- .../xbib/elx/common/AbstractAdminClient.java | 194 ++++++++++-------- .../xbib/elx/common/AbstractBasicClient.java | 5 +- .../xbib/elx/common/AbstractBulkClient.java | 3 +- .../xbib/elx/common/AbstractSearchClient.java | 4 +- .../xbib/elx/common/DefaultBulkProcessor.java | 2 +- .../elx/common/DefaultIndexDefinition.java | 62 ++++-- .../elx/common/DefaultIndexRetention.java | 7 +- .../java/org/xbib/elx/common/Parameters.java | 6 +- .../org/xbib/elx/common/test/SearchTest.java | 4 +- .../xbib/elx/http/test/IndexPruneTest.java | 26 ++- .../xbib/elx/http/test/IndexShiftTest.java | 13 +- .../xbib/elx/node/test/IndexPruneTest.java | 25 ++- .../xbib/elx/node/test/IndexShiftTest.java | 13 +- .../elx/transport/test/IndexPruneTest.java | 25 ++- .../elx/transport/test/IndexShiftTest.java | 13 +- gradle.properties | 2 +- 19 files changed, 277 insertions(+), 188 deletions(-) diff --git a/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java b/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java index 059430f..d97a044 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java +++ b/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java @@ -123,7 +123,8 @@ public interface AdminClient extends BasicClient { * @param additionalAliases new aliases * @return this */ - IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List additionalAliases); + IndexShiftResult shiftIndex(IndexDefinition indexDefinition, + List additionalAliases); /** * Shift from one index to another. @@ -132,29 +133,10 @@ public interface AdminClient extends BasicClient { * @param indexAliasAdder method to add aliases * @return this */ - IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List additionalAliases, + IndexShiftResult shiftIndex(IndexDefinition indexDefinition, + List additionalAliases, IndexAliasAdder indexAliasAdder); - /** - * Shift from one index to another. - * @param index the index name - * @param fullIndexName the index name with timestamp - * @param additionalAliases a list of names that should be set as index aliases - * @return this - */ - IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases); - - /** - * Shift from one index to another. - * @param index the index name - * @param fullIndexName the index name with timestamp - * @param additionalAliases a list of names that should be set as index aliases - * @param adder an adder method to create alias term queries - * @return this - */ - IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases, - IndexAliasAdder adder); - /** * Prune index. * @param indexDefinition the index definition @@ -162,19 +144,6 @@ public interface AdminClient extends BasicClient { */ IndexPruneResult pruneIndex(IndexDefinition indexDefinition); - /** - * Apply retention policy to prune indices. All indices before delta should be deleted, - * but the number of mintokeep indices must be kept. - * - * @param index index name - * @param fullIndexName index name with timestamp - * @param delta timestamp delta (for index timestamps) - * @param mintokeep minimum number of indices to keep - * @param perform true if pruning should be executed, false if not - * @return the index prune result - */ - IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform); - /** * Find the timestamp of the most recently indexed document in the index. * diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkListener.java b/elx-api/src/main/java/org/xbib/elx/api/BulkListener.java index da80393..f434ed4 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/BulkListener.java +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkListener.java @@ -39,8 +39,8 @@ public interface BulkListener { void afterBulk(long executionId, BulkRequest request, Throwable failure); /** - * Get the last bulk error - * @return the lst bulk error + * Get the last bulk error. + * @return the last bulk error */ Throwable getLastBulkError(); } diff --git a/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java b/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java index 02eda88..6ab4e8a 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java @@ -1,6 +1,8 @@ package org.xbib.elx.api; +import java.time.format.DateTimeFormatter; import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; public interface IndexDefinition { @@ -26,9 +28,13 @@ public interface IndexDefinition { String getMappings(); - IndexDefinition setDateTimePattern(String timeWindow); + IndexDefinition setDateTimeFormatter(DateTimeFormatter formatter); - String getDateTimePattern(); + DateTimeFormatter getDateTimeFormatter(); + + IndexDefinition setDateTimePattern(Pattern pattern); + + Pattern getDateTimePattern(); IndexDefinition setEnabled(boolean enabled); @@ -42,9 +48,13 @@ public interface IndexDefinition { boolean isShiftEnabled(); - IndexDefinition setForceMerge(boolean hasForceMerge); + IndexDefinition setPrune(boolean prune); - boolean hasForceMerge(); + boolean isPruneEnabled(); + + IndexDefinition setForceMerge(boolean forcemerge); + + boolean isForceMergeEnabled(); IndexDefinition setReplicaLevel(int replicaLevel); diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java index bb1aaa7..209f8d2 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java @@ -75,6 +75,7 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; @@ -93,40 +94,6 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements private static final Logger logger = LogManager.getLogger(AbstractAdminClient.class.getName()); - private static final IndexShiftResult EMPTY_INDEX_SHIFT_RESULT = new IndexShiftResult() { - @Override - public List getMovedAliases() { - return Collections.emptyList(); - } - - @Override - public List getNewAliases() { - return Collections.emptyList(); - } - }; - - private static final IndexPruneResult EMPTY_INDEX_PRUNE_RESULT = new IndexPruneResult() { - @Override - public State getState() { - return State.NONE; - } - - @Override - public List getCandidateIndices() { - return Collections.emptyList(); - } - - @Override - public List getDeletedIndices() { - return Collections.emptyList(); - } - - @Override - public boolean isAcknowledged() { - return false; - } - }; - @Override public Map getMapping(String index) { GetMappingsRequestBuilder getMappingsRequestBuilder = new GetMappingsRequestBuilder(client, GetMappingsAction.INSTANCE) @@ -245,46 +212,42 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements } @Override - public IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List additionalAliases) { + public IndexShiftResult shiftIndex(IndexDefinition indexDefinition, + List additionalAliases) { return shiftIndex(indexDefinition, additionalAliases, null); } @Override public IndexShiftResult shiftIndex(IndexDefinition indexDefinition, - List additionalAliases, IndexAliasAdder indexAliasAdder) { + List additionalAliases, + IndexAliasAdder indexAliasAdder) { if (additionalAliases == null) { - return EMPTY_INDEX_SHIFT_RESULT; + return new EmptyIndexShiftResult(); } if (indexDefinition.isShiftEnabled()) { - return shiftIndex(indexDefinition.getIndex(), - indexDefinition.getFullIndexName(), additionalAliases.stream() - .filter(a -> a != null && !a.isEmpty()) + return shiftIndex(indexDefinition.getIndex(), indexDefinition.getFullIndexName(), + additionalAliases.stream().filter(a -> a != null && !a.isEmpty()) .collect(Collectors.toList()), indexAliasAdder); } - return EMPTY_INDEX_SHIFT_RESULT; + return new EmptyIndexShiftResult(); } - @Override - public IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases) { - return shiftIndex(index, fullIndexName, additionalAliases, null); - } - - @Override - public IndexShiftResult shiftIndex(String index, String fullIndexName, - List additionalAliases, IndexAliasAdder adder) { + private IndexShiftResult shiftIndex(String index, + String fullIndexName, + List additionalAliases, + IndexAliasAdder adder) { ensureClientIsPresent(); if (index == null) { - return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to + return new EmptyIndexShiftResult(); // nothing to shift to } if (index.equals(fullIndexName)) { - return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to + return new EmptyIndexShiftResult(); // nothing to shift to } waitForCluster("YELLOW", 30L, TimeUnit.SECONDS); // two situations: 1. a new alias 2. there is already an old index with the alias - List oldIndices = resolveAlias(index); - String oldIndex = oldIndices.stream().findFirst().orElse(null); - Map oldAliasMap = getAliases(oldIndex); - logger.debug("old index = {} old alias map = {}", oldIndex, oldAliasMap); + Optional oldIndex = resolveAlias(index).stream().sorted().findFirst(); + Map oldAliasMap = oldIndex.map(this::getAliases).orElse(null); + logger.info("old index = {} old alias map = {}", oldIndex.orElse(""), oldAliasMap); final List newAliases = new ArrayList<>(); final List moveAliases = new ArrayList<>(); IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest(); @@ -294,12 +257,12 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements newAliases.add(index); } // move existing aliases - if (oldAliasMap != null) { + if (oldIndex.isPresent() && oldAliasMap != null) { for (Map.Entry entry : oldAliasMap.entrySet()) { String alias = entry.getKey(); String filter = entry.getValue(); indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove() - .index(oldIndex).alias(alias)); + .index(oldIndex.get()).alias(alias)); if (filter != null) { indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove() .index(fullIndexName).alias(alias).filter(filter)); @@ -325,7 +288,7 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements } else { String filter = oldAliasMap.get(additionalAlias); indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove() - .index(oldIndex).alias(additionalAlias)); + .index(oldIndex.get()).alias(additionalAlias)); if (filter != null) { indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add() .index(fullIndexName).alias(additionalAlias).filter(filter)); @@ -349,41 +312,50 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements @Override public IndexPruneResult pruneIndex(IndexDefinition indexDefinition) { - return pruneIndex(indexDefinition.getIndex(), indexDefinition.getFullIndexName(), - indexDefinition.getRetention().getDelta(), indexDefinition.getRetention().getMinToKeep(), true); + return indexDefinition != null && indexDefinition.isPruneEnabled() && indexDefinition.getRetention() != null && indexDefinition.getDateTimePattern() != null? + pruneIndex(indexDefinition.getIndex(), + indexDefinition.getFullIndexName(), + indexDefinition.getDateTimePattern(), + indexDefinition.getRetention().getDelta(), + indexDefinition.getRetention().getMinToKeep()) : new EmptyPruneResult(); } - @Override - public IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform) { + private IndexPruneResult pruneIndex(String index, + String protectedIndexName, + Pattern pattern, + int delta, + int mintokeep) { + logger.info("before pruning: index = {} full index = {} delta = {} mintokeep = {} pattern = {}", + index, protectedIndexName, delta, mintokeep, pattern); if (delta == 0 && mintokeep == 0) { - return EMPTY_INDEX_PRUNE_RESULT; + return new EmptyPruneResult(); } - if (index.equals(fullIndexName)) { - return EMPTY_INDEX_PRUNE_RESULT; + if (index.equals(protectedIndexName)) { + return new EmptyPruneResult(); } ensureClientIsPresent(); GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE); GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet(); - Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); - logger.info("found {} indices in the cluster", getIndexResponse.getIndices().length); + logger.info("before pruning: found total of {} indices", getIndexResponse.getIndices().length); List candidateIndices = new ArrayList<>(); for (String s : getIndexResponse.getIndices()) { Matcher m = pattern.matcher(s); - if (m.matches() && index.equals(m.group(1)) && !s.equals(fullIndexName)) { + if (m.matches() && m.group(1).equals(index) && !s.equals(protectedIndexName)) { candidateIndices.add(s); + } else { + logger.info("not a candidate: " + s); } } - logger.info("found {} as candidates for pruning", candidateIndices); if (candidateIndices.isEmpty()) { - logger.info("empty pruning"); - return EMPTY_INDEX_PRUNE_RESULT; + return new EmptyPruneResult(); } if (mintokeep > 0 && candidateIndices.size() <= mintokeep) { - logger.info("nothing to prune, min to keep = " + mintokeep); return new NothingToDoPruneResult(candidateIndices, Collections.emptyList()); } + Collections.sort(candidateIndices); + logger.info("found {} candidates", candidateIndices); List indicesToDelete = new ArrayList<>(); - Matcher m1 = pattern.matcher(fullIndexName); + Matcher m1 = pattern.matcher(protectedIndexName); if (m1.matches()) { Integer i1 = Integer.parseInt(m1.group(2)); for (String s : candidateIndices) { @@ -397,15 +369,14 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements } } } - logger.warn("removing {}", indicesToDelete); if (indicesToDelete.isEmpty()) { return new NothingToDoPruneResult(candidateIndices, indicesToDelete); } + logger.warn("deleting {}", indicesToDelete); String[] s = new String[indicesToDelete.size()]; DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest() .indices(indicesToDelete.toArray(s)); AcknowledgedResponse response = client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet(); - logger.info("removed {}", indicesToDelete); return new SuccessPruneResult(candidateIndices, indicesToDelete, response); } @@ -435,7 +406,7 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements @Override public boolean forceMerge(IndexDefinition indexDefinition) { - if (indexDefinition.hasForceMerge()) { + if (indexDefinition.isForceMergeEnabled()) { return forceMerge(indexDefinition.getFullIndexName(), indexDefinition.getMaxWaitTime(), indexDefinition.getMaxWaitTimeUnit()); } @@ -466,18 +437,13 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements throws IOException { boolean isEnabled = settings.getAsBoolean("enabled", false); String indexName = settings.get("name", index); - String fullIndexName; - String dateTimePattern = settings.get("dateTimePattern"); - if (dateTimePattern != null) { - // check if index name with current date already exists, resolve to it - String dateAppendix = DateTimeFormatter.ofPattern(dateTimePattern) - .withZone(ZoneId.systemDefault()) // not GMT - .format(LocalDate.now()); - fullIndexName = resolveAlias(indexName + dateAppendix).stream().findFirst().orElse(index); - } else { - // check if index name already exists, resolve to it - fullIndexName = resolveMostRecentIndex(indexName); - } + String dateTimePatternStr = settings.get("dateTimePattern", "^(.*?)(\\\\d+)$"); + Pattern dateTimePattern = Pattern.compile(dateTimePatternStr); + String dateTimeFormat = settings.get("dateTimeFormat", "yyyyMMdd"); + DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(dateTimeFormat) + .withZone(ZoneId.systemDefault()); + String fullName = indexName + dateTimeFormatter.format(LocalDate.now()); + String fullIndexName = resolveAlias(fullName).stream().findFirst().orElse(fullName); IndexRetention indexRetention = new DefaultIndexRetention() .setMinToKeep(settings.getAsInt("retention.mintokeep", 0)) .setDelta(settings.getAsInt("retention.delta", 0)); @@ -487,9 +453,11 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements .setFullIndexName(fullIndexName) .setSettings(findSettingsFrom(settings.get("settings"))) .setMappings(findMappingsFrom(settings.get("mapping"))) + .setDateTimeFormatter(dateTimeFormatter) .setDateTimePattern(dateTimePattern) .setIgnoreErrors(settings.getAsBoolean("skiperrors", false)) .setShift(settings.getAsBoolean("shift", true)) + .setShift(settings.getAsBoolean("prune", true)) .setReplicaLevel(settings.getAsInt("replica", 0)) .setMaxWaitTime(settings.getAsLong("timeout", 30L), TimeUnit.SECONDS) .setRetention(indexRetention) @@ -699,6 +667,18 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements } } + private static class EmptyIndexShiftResult implements IndexShiftResult { + @Override + public List getMovedAliases() { + return Collections.emptyList(); + } + + @Override + public List getNewAliases() { + return Collections.emptyList(); + } + } + private static class SuccessPruneResult implements IndexPruneResult { Collection candidateIndices; @@ -734,6 +714,11 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements public boolean isAcknowledged() { return response.isAcknowledged(); } + + @Override + public String toString() { + return "PRUNED: " + indicesToDelete; + } } private static class NothingToDoPruneResult implements IndexPruneResult { @@ -766,5 +751,38 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements public boolean isAcknowledged() { return false; } + + @Override + public String toString() { + return "NOTHING TO DO"; + } + } + + private static class EmptyPruneResult implements IndexPruneResult { + + @Override + public State getState() { + return State.NONE; + } + + @Override + public List getCandidateIndices() { + return Collections.emptyList(); + } + + @Override + public List getDeletedIndices() { + return Collections.emptyList(); + } + + @Override + public boolean isAcknowledged() { + return false; + } + + @Override + public String toString() { + return "NOTHING TO DO"; + } } } diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractBasicClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractBasicClient.java index f8883f4..f08191c 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractBasicClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractBasicClient.java @@ -92,12 +92,11 @@ public abstract class AbstractBasicClient implements BasicClient { new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet(); if (healthResponse != null && healthResponse.isTimedOut()) { String message = "timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name(); - if (logger.isErrorEnabled()) { - logger.error(message); - } + logger.error(message); throw new IllegalStateException(message); } } + @Override public void waitForShards(long maxWaitTime, TimeUnit timeUnit) { ensureClientIsPresent(); diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java index 7aad329..4adb845 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java @@ -126,8 +126,7 @@ public abstract class AbstractBulkClient extends AbstractBasicClient implements createIndexRequestBuilder.addMapping(TYPE_NAME, JsonXContent.contentBuilder().startObject().startObject(TYPE_NAME).endObject().endObject()); logger.debug("empty mapping"); - } - + } CreateIndexResponse createIndexResponse = createIndexRequestBuilder.execute().actionGet(); if (createIndexResponse.isAcknowledged()) { logger.info("index {} created", index); diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java index c0f544e..b899be8 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java @@ -126,7 +126,7 @@ public abstract class AbstractSearchClient extends AbstractBasicClient implement SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); queryBuilder.accept(searchRequestBuilder); searchRequestBuilder.setScroll(scrollTime).setSize(scrollSize); - ActionFuture actionFuture = searchRequestBuilder.execute(); + ActionFuture actionFuture = searchRequestBuilder.execute(); searchMetric.getCurrentQueries().inc(); SearchResponse originalSearchResponse = actionFuture.actionGet(); searchMetric.getCurrentQueries().dec(); @@ -149,7 +149,7 @@ public abstract class AbstractSearchClient extends AbstractBasicClient implement searchMetric.getCurrentQueries().dec(); searchMetric.getQueries().inc(); searchMetric.markTotalQueries(1); - if ( searchResponse1.getHits().getHits().length == 0) { + if (searchResponse1.getHits().getHits().length == 0) { searchMetric.getEmptyQueries().inc(); } else { searchMetric.getSucceededQueries().inc(); diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java index 5906d38..88ce3dc 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java @@ -223,7 +223,7 @@ public class DefaultBulkProcessor implements BulkProcessor { * Creates a builder of bulk processor with the client to use and the listener that will be used * to be notified on the completion of bulk requests. * - * @param client the client + * @param client the client * @param bulkListener the listener */ Builder(ElasticsearchClient client, BulkListener bulkListener) { diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java index 8d2d3fd..ace4d6a 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java @@ -3,7 +3,10 @@ package org.xbib.elx.common; import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.api.IndexRetention; +import java.time.format.DateTimeFormatter; +import java.util.Locale; import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; public class DefaultIndexDefinition implements IndexDefinition { @@ -11,7 +14,9 @@ public class DefaultIndexDefinition implements IndexDefinition { private String fullIndexName; - private String dateTimePattern; + private DateTimeFormatter formatter; + + private Pattern pattern; private String settings; @@ -21,9 +26,11 @@ public class DefaultIndexDefinition implements IndexDefinition { private boolean ignoreErrors; - private boolean switchAliases; + private boolean shift; - private boolean hasForceMerge; + private boolean prune; + + private boolean forcemerge; private int replicaLevel; @@ -37,6 +44,11 @@ public class DefaultIndexDefinition implements IndexDefinition { private long stopRefreshInterval; + public DefaultIndexDefinition() { + this.pattern = Pattern.compile("^(.*?)(\\d+)$"); + this.formatter = DateTimeFormatter.ofPattern("yyyyMMdd", Locale.getDefault()); + } + @Override public IndexDefinition setIndex(String index) { this.index = index; @@ -82,14 +94,25 @@ public class DefaultIndexDefinition implements IndexDefinition { } @Override - public IndexDefinition setDateTimePattern(String timeWindow) { - this.dateTimePattern = timeWindow; + public IndexDefinition setDateTimeFormatter(DateTimeFormatter formatter) { + this.formatter = formatter; return this; } @Override - public String getDateTimePattern() { - return dateTimePattern; + public DateTimeFormatter getDateTimeFormatter() { + return formatter; + } + + @Override + public IndexDefinition setDateTimePattern(Pattern pattern) { + this.pattern = pattern; + return this; + } + + @Override + public Pattern getDateTimePattern() { + return pattern; } @Override @@ -115,25 +138,36 @@ public class DefaultIndexDefinition implements IndexDefinition { } @Override - public IndexDefinition setShift(boolean switchAliases) { - this.switchAliases = switchAliases; + public IndexDefinition setShift(boolean shift) { + this.shift = shift; return this; } @Override public boolean isShiftEnabled() { - return switchAliases; + return shift; } @Override - public IndexDefinition setForceMerge(boolean hasForceMerge) { - this.hasForceMerge = hasForceMerge; + public IndexDefinition setPrune(boolean prune) { + this.prune = prune; return this; } @Override - public boolean hasForceMerge() { - return hasForceMerge; + public boolean isPruneEnabled() { + return prune; + } + + @Override + public IndexDefinition setForceMerge(boolean forcemerge) { + this.forcemerge = forcemerge; + return this; + } + + @Override + public boolean isForceMergeEnabled() { + return forcemerge; } @Override diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexRetention.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexRetention.java index 4e49be3..abc6307 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexRetention.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexRetention.java @@ -8,7 +8,12 @@ public class DefaultIndexRetention implements IndexRetention { private int minToKeep; - @Override + public DefaultIndexRetention() { + this.delta = 2; + this.minToKeep = 2; + } + + @Override public IndexRetention setDelta(int delta) { this.delta = delta; return this; diff --git a/elx-common/src/main/java/org/xbib/elx/common/Parameters.java b/elx-common/src/main/java/org/xbib/elx/common/Parameters.java index e53a56d..69df0a4 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/Parameters.java +++ b/elx-common/src/main/java/org/xbib/elx/common/Parameters.java @@ -20,11 +20,11 @@ public enum Parameters { FLUSH_INTERVAL("flush_interval"); - private boolean flag = false; + boolean flag; - private int num = -1; + int num; - private String string; + String string; Parameters(boolean flag) { this.flag = flag; diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java index b276a45..6bd4ba0 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java @@ -31,7 +31,7 @@ class SearchTest { void testSearch() throws Exception { ElasticsearchClient client = helper.client("1"); BulkRequestBuilder builder = new BulkRequestBuilder(client, BulkAction.INSTANCE); - for (int i = 0; i < 1000; i++) { + for (int i = 0; i < 1; i++) { IndexRequest indexRequest = new IndexRequest().index("pages") .source(XContentFactory.jsonBuilder() .startObject() @@ -51,7 +51,7 @@ class SearchTest { } client.execute(BulkAction.INSTANCE, builder.request()).actionGet(); client.execute(RefreshAction.INSTANCE, new RefreshRequest()).actionGet(); - for (int i = 0; i < 100; i++) { + for (int i = 0; i < 1; i++) { QueryBuilder queryStringBuilder = QueryBuilders.queryStringQuery("rs:" + 1234); SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE) .setIndices("pages") diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java index 207953f..65745bc 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java @@ -5,8 +5,12 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.common.settings.Settings; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.api.IndexPruneResult; +import org.xbib.elx.api.IndexRetention; import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.DefaultIndexDefinition; +import org.xbib.elx.common.DefaultIndexRetention; import org.xbib.elx.http.HttpAdminClient; import org.xbib.elx.http.HttpAdminClientProvider; import org.xbib.elx.http.HttpBulkClient; @@ -15,7 +19,6 @@ import org.xbib.elx.http.HttpBulkClientProvider; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -49,15 +52,24 @@ class IndexPruneTest { .put("index.number_of_replicas", 0) .build(); bulkClient.newIndex("test1", settings); - adminClient.shiftIndex("test", "test1", Collections.emptyList()); + IndexDefinition indexDefinition = new DefaultIndexDefinition(); + indexDefinition.setIndex("test"); + indexDefinition.setFullIndexName("test1"); + adminClient.shiftIndex(indexDefinition, List.of()); bulkClient.newIndex("test2", settings); - adminClient.shiftIndex("test", "test2", Collections.emptyList()); + indexDefinition.setFullIndexName("test2"); + adminClient.shiftIndex(indexDefinition, List.of()); bulkClient.newIndex("test3", settings); - adminClient.shiftIndex("test", "test3", Collections.emptyList()); + indexDefinition.setFullIndexName("test3"); + adminClient.shiftIndex(indexDefinition, List.of()); bulkClient.newIndex("test4", settings); - adminClient.shiftIndex("test", "test4", Collections.emptyList()); - IndexPruneResult indexPruneResult = - adminClient.pruneIndex("test", "test4", 2, 2, true); + indexDefinition.setFullIndexName("test4"); + adminClient.shiftIndex(indexDefinition, List.of()); + indexDefinition.setPrune(true); + IndexRetention indexRetention = new DefaultIndexRetention(); + indexDefinition.setRetention(indexRetention); + IndexPruneResult indexPruneResult = adminClient.pruneIndex(indexDefinition); + logger.info(indexPruneResult.toString()); assertTrue(indexPruneResult.getDeletedIndices().contains("test1")); assertTrue(indexPruneResult.getDeletedIndices().contains("test2")); assertFalse(indexPruneResult.getDeletedIndices().contains("test3")); diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java index 60db3c2..a56ea63 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java @@ -8,8 +8,10 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.DefaultIndexDefinition; import org.xbib.elx.http.HttpAdminClient; import org.xbib.elx.http.HttpAdminClientProvider; import org.xbib.elx.http.HttpBulkClient; @@ -56,8 +58,11 @@ class IndexShiftTest { } bulkClient.flush(); bulkClient.waitForResponses(30L, TimeUnit.SECONDS); - IndexShiftResult indexShiftResult = - adminClient.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c")); + IndexDefinition indexDefinition = new DefaultIndexDefinition(); + indexDefinition.setIndex("test"); + indexDefinition.setFullIndexName("test1234"); + indexDefinition.setShift(true); + IndexShiftResult indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("a", "b", "c")); assertTrue(indexShiftResult.getNewAliases().contains("a")); assertTrue(indexShiftResult.getNewAliases().contains("b")); assertTrue(indexShiftResult.getNewAliases().contains("c")); @@ -83,7 +88,9 @@ class IndexShiftTest { } bulkClient.flush(); bulkClient.waitForResponses(30L, TimeUnit.SECONDS); - indexShiftResult = adminClient.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"), + indexDefinition.setFullIndexName("test5678"); + indexDefinition.setShift(true); + indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("d", "e", "f"), (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add() .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias))) ); diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java index 27a4b5c..38b341c 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java @@ -5,8 +5,12 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.common.settings.Settings; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.api.IndexPruneResult; +import org.xbib.elx.api.IndexRetention; import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.DefaultIndexDefinition; +import org.xbib.elx.common.DefaultIndexRetention; import org.xbib.elx.node.NodeAdminClient; import org.xbib.elx.node.NodeAdminClientProvider; import org.xbib.elx.node.NodeBulkClient; @@ -15,7 +19,6 @@ import org.xbib.elx.node.NodeBulkClientProvider; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -48,15 +51,23 @@ class IndexPruneTest { .put("index.number_of_replicas", 0) .build(); bulkClient.newIndex("test_prune1", settings); - adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList()); + IndexDefinition indexDefinition = new DefaultIndexDefinition(); + indexDefinition.setIndex("test_prune"); + indexDefinition.setFullIndexName("test_prune1"); + adminClient.shiftIndex(indexDefinition, List.of()); bulkClient.newIndex("test_prune2", settings); - adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList()); + indexDefinition.setFullIndexName("test_prune2"); + adminClient.shiftIndex(indexDefinition, List.of()); bulkClient.newIndex("test_prune3", settings); - adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList()); + indexDefinition.setFullIndexName("test_prune3"); + adminClient.shiftIndex(indexDefinition, List.of()); bulkClient.newIndex("test_prune4", settings); - adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList()); - IndexPruneResult indexPruneResult = - adminClient.pruneIndex("test_prune", "test_prune4", 2, 2, true); + indexDefinition.setFullIndexName("test_prune4"); + adminClient.shiftIndex(indexDefinition, List.of()); + indexDefinition.setPrune(true); + IndexRetention indexRetention = new DefaultIndexRetention(); + indexDefinition.setRetention(indexRetention); + IndexPruneResult indexPruneResult = adminClient.pruneIndex(indexDefinition); assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1")); assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2")); assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3")); diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java index 66771dc..341a866 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java @@ -7,8 +7,10 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.DefaultIndexDefinition; import org.xbib.elx.node.NodeAdminClient; import org.xbib.elx.node.NodeAdminClientProvider; import org.xbib.elx.node.NodeBulkClient; @@ -54,8 +56,11 @@ class IndexShiftTest { } bulkClient.flush(); bulkClient.waitForResponses(30L, TimeUnit.SECONDS); - IndexShiftResult indexShiftResult = - adminClient.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c")); + IndexDefinition indexDefinition = new DefaultIndexDefinition(); + indexDefinition.setIndex("test"); + indexDefinition.setFullIndexName("test_shift"); + indexDefinition.setShift(true); + IndexShiftResult indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("a", "b", "c")); assertTrue(indexShiftResult.getNewAliases().contains("a")); assertTrue(indexShiftResult.getNewAliases().contains("b")); assertTrue(indexShiftResult.getNewAliases().contains("c")); @@ -79,7 +84,9 @@ class IndexShiftTest { } bulkClient.flush(); bulkClient.waitForResponses(30L, TimeUnit.SECONDS); - indexShiftResult = adminClient.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"), + indexDefinition.setFullIndexName("test_shift2"); + indexDefinition.setShift(true); + indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("d", "e", "f"), (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add() .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias))) ); diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java index 0640fbd..8708d70 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java @@ -5,8 +5,12 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.common.settings.Settings; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.api.IndexPruneResult; +import org.xbib.elx.api.IndexRetention; import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.DefaultIndexDefinition; +import org.xbib.elx.common.DefaultIndexRetention; import org.xbib.elx.transport.TransportAdminClient; import org.xbib.elx.transport.TransportAdminClientProvider; import org.xbib.elx.transport.TransportBulkClient; @@ -15,7 +19,6 @@ import org.xbib.elx.transport.TransportBulkClientProvider; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -48,16 +51,24 @@ class IndexPruneTest { .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); + IndexDefinition indexDefinition = new DefaultIndexDefinition(); + indexDefinition.setIndex("test_prune"); + indexDefinition.setFullIndexName("test_prune1"); bulkClient.newIndex("test_prune1", settings); - adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList()); + adminClient.shiftIndex(indexDefinition, List.of()); bulkClient.newIndex("test_prune2", settings); - adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList()); + indexDefinition.setFullIndexName("test_prune2"); + adminClient.shiftIndex(indexDefinition, List.of()); bulkClient.newIndex("test_prune3", settings); - adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList()); + indexDefinition.setFullIndexName("test_prune3"); + adminClient.shiftIndex(indexDefinition, List.of()); bulkClient.newIndex("test_prune4", settings); - adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList()); - IndexPruneResult indexPruneResult = - adminClient.pruneIndex("test_prune", "test_prune4", 2, 2, true); + indexDefinition.setFullIndexName("test_prune4"); + adminClient.shiftIndex(indexDefinition, List.of()); + indexDefinition.setPrune(true); + IndexRetention indexRetention = new DefaultIndexRetention(); + indexDefinition.setRetention(indexRetention); + IndexPruneResult indexPruneResult = adminClient.pruneIndex(indexDefinition); assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1")); assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2")); assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3")); diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java index 347bd42..88f9b15 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java @@ -7,8 +7,10 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.DefaultIndexDefinition; import org.xbib.elx.transport.TransportAdminClient; import org.xbib.elx.transport.TransportAdminClientProvider; import org.xbib.elx.transport.TransportBulkClient; @@ -55,8 +57,11 @@ class IndexShiftTest { } bulkClient.flush(); bulkClient.waitForResponses(30L, TimeUnit.SECONDS); - IndexShiftResult indexShiftResult = - adminClient.shiftIndex("test_shift", "test_shift1234", Arrays.asList("a", "b", "c")); + IndexDefinition indexDefinition = new DefaultIndexDefinition(); + indexDefinition.setIndex("test_shift"); + indexDefinition.setFullIndexName("test_shift1234"); + indexDefinition.setShift(true); + IndexShiftResult indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("a", "b", "c")); assertTrue(indexShiftResult.getNewAliases().contains("a")); assertTrue(indexShiftResult.getNewAliases().contains("b")); assertTrue(indexShiftResult.getNewAliases().contains("c")); @@ -80,7 +85,9 @@ class IndexShiftTest { } bulkClient.flush(); bulkClient.waitForResponses(30L, TimeUnit.SECONDS); - indexShiftResult = adminClient.shiftIndex("test_shift", "test_shift5678", Arrays.asList("d", "e", "f"), + indexDefinition.setFullIndexName("test_shift5678"); + indexDefinition.setShift(true); + indexShiftResult = adminClient.shiftIndex(indexDefinition, Arrays.asList("d", "e", "f"), (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add() .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias))) ); diff --git a/gradle.properties b/gradle.properties index c579ac0..2b4de0f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -4,7 +4,7 @@ version = 7.10.2.0 gradle.wrapper.version = 6.6.1 xbib-metrics.version = 2.1.0 -xbib-netty-http.version = 4.1.58.0 +xbib-netty-http.version = 4.1.60.0 elasticsearch.version = 7.10.2 # ES 7.10.2.1 uses Jackson 2.10.4 jackson.version = 2.12.1