diff --git a/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java b/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java index 4a3abed..5bc6865 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java +++ b/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java @@ -1,5 +1,6 @@ package org.xbib.elx.api; +import java.time.Instant; import java.util.Collection; import java.util.Map; @@ -8,6 +9,31 @@ import java.util.Map; */ public interface AdminClient extends BasicClient { + /** + * List all indices. + * @return the names of the indices. + */ + Collection allIndices(); + + /** + * List all closed indices. + * @return the names of the closed indices. + */ + Collection allClosedIndices(); + + /** + * List all closed indices which were created before a given instant. + * @param instant the instant + * @return the names of the closed indices + */ + Collection allClosedIndicesOlderThan(Instant instant); + + /** + * Delete all closed indices which were created before a given instant. + * @param instant the instant + */ + void purgeAllClosedIndicesOlderThan(Instant instant); + /** * Get the mapping of an index. * @@ -16,6 +42,10 @@ public interface AdminClient extends BasicClient { */ Map getMapping(IndexDefinition indexDefinition); + /** + * Check the mapping. + * @param indexDefinition the index definition + */ void checkMapping(IndexDefinition indexDefinition); /** diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java index ffa69db..ac38c3f 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java @@ -2,9 +2,11 @@ package org.xbib.elx.common; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import java.text.MessageFormat; +import java.time.Instant; +import java.util.SortedMap; +import java.util.logging.Level; +import java.util.logging.Logger; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; @@ -82,7 +84,54 @@ import static org.xbib.elx.api.IndexDefinition.TYPE_NAME; public abstract class AbstractAdminClient extends AbstractBasicClient implements AdminClient { - private static final Logger logger = LogManager.getLogger(AbstractAdminClient.class.getName()); + private static final Logger logger = Logger.getLogger(AbstractAdminClient.class.getName()); + + @Override + public Collection allIndices() { + ensureClientIsPresent(); + ClusterStateRequest clusterStateRequest = new ClusterStateRequest(); + clusterStateRequest.blocks(false); + clusterStateRequest.metadata(true); + clusterStateRequest.nodes(false); + clusterStateRequest.routingTable(false); + clusterStateRequest.customs(false); + ClusterStateResponse clusterStateResponse = + client.execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); + SortedMap indexAbstractions = clusterStateResponse.getState().getMetadata() + .getIndicesLookup(); + if (indexAbstractions == null) { + return Collections.emptyList(); + } + return indexAbstractions.keySet(); + } + + @Override + public Collection allClosedIndices() { + return allClosedIndicesOlderThan(Instant.now()); + } + + @Override + public Collection allClosedIndicesOlderThan(Instant instant) { + ensureClientIsPresent(); + ClusterStateRequest clusterStateRequest = new ClusterStateRequest(); + clusterStateRequest.blocks(false); + clusterStateRequest.metadata(true); + clusterStateRequest.nodes(false); + clusterStateRequest.routingTable(false); + clusterStateRequest.customs(false); + ClusterStateResponse clusterStateResponse = + client.execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); + return clusterStateResponse.getState().getMetadata() + .getIndicesLookup().values().stream() + .flatMap(ia -> ia.getIndices().stream().filter(i -> i.getState().equals(IndexMetadata.State.CLOSE) && i.getCreationDate() < instant.toEpochMilli())) + .map(im -> im.getIndex().getName()) + .collect(Collectors.toList()); + } + + @Override + public void purgeAllClosedIndicesOlderThan(Instant instant) { + allClosedIndicesOlderThan(instant).forEach(this::deleteIndex); + } @Override public Map getMapping(IndexDefinition indexDefinition) { @@ -110,7 +159,7 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements @Override public void deleteIndex(String indexName) { if (indexName == null) { - logger.warn("no index name given to delete index"); + logger.log(Level.WARNING, "no index name given to delete index"); return; } ensureClientIsPresent(); @@ -133,7 +182,7 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements @Override public void closeIndex(String indexName) { if (indexName == null) { - logger.warn("no index name given to close index"); + logger.log(Level.WARNING, "no index name given to close index"); return; } ensureClientIsPresent(); @@ -143,9 +192,9 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements List list = closeIndexResponse.getIndices(); list.forEach(result -> { if (result.hasFailures()) { - logger.warn("error when closing " + result.getIndex(), result.getException()); + logger.log(Level.WARNING, "error when closing " + result.getIndex(), result.getException()); } else { - logger.info("index " + result.getIndex() + " closed"); + logger.log(Level.INFO, "index " + result.getIndex() + " closed"); } }); } @@ -162,14 +211,14 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements @Override public void openIndex(String indexName) { if (indexName == null) { - logger.warn("no index name given to close index"); + logger.log(Level.WARNING, "no index name given to close index"); return; } ensureClientIsPresent(); OpenIndexRequest openIndexRequest = new OpenIndexRequest().indices(indexName); OpenIndexResponse openIndexResponse = client.execute(OpenIndexAction.INSTANCE, openIndexRequest).actionGet(); if (openIndexResponse.isAcknowledged()) { - logger.info("index " + indexName + " opened"); + logger.log(Level.INFO, "index " + indexName + " opened"); } } @@ -179,7 +228,7 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements return; } if (indexDefinition.getReplicaCount() < 0) { - logger.warn("invalid replica level defined for index " + logger.log(Level.WARNING, "invalid replica level defined for index " + indexDefinition.getIndex() + ": " + indexDefinition.getReplicaCount()); return; } @@ -324,7 +373,7 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements // two situations: 1. a new alias 2. there is already an old index with the alias Optional oldIndex = resolveAliasFromClusterState(index).stream().sorted().findFirst(); Map oldAliasMap = oldIndex.map(this::getAliases).orElse(null); - logger.info("old index = {} old alias map = {}", oldIndex.orElse(""), oldAliasMap); + logger.log(Level.INFO, "old index = " + oldIndex.orElse("") + " old alias map = " + oldAliasMap); final List newAliases = new ArrayList<>(); final List moveAliases = new ArrayList<>(); IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest(); @@ -405,20 +454,20 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements Pattern pattern, int delta, int mintokeep) { - logger.info("before pruning: index = {} full index = {} delta = {} mintokeep = {} pattern = {}", - index, protectedIndexName, delta, mintokeep, pattern); + logger.log(Level.INFO, MessageFormat.format("before pruning: index = {0} full index = {1} delta = {2} mintokeep = {3} pattern = {4}", + index, protectedIndexName, delta, mintokeep, pattern)); if (delta == 0 && mintokeep == 0) { - logger.warn("no candidates found, delta is 0 and mintokeep is 0"); + logger.log(Level.INFO, "no candidates found, delta is 0 and mintokeep is 0"); return new NonePruneResult(); } if (index.equals(protectedIndexName)) { - logger.warn("no candidates found, only protected index name is given"); + logger.log(Level.INFO, "no candidates found, only protected index name is given"); return new NonePruneResult(); } ensureClientIsPresent(); GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE); GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet(); - logger.info("before pruning: found total of {} indices", getIndexResponse.getIndices().length); + logger.log(Level.INFO, "before pruning: found total of " + getIndexResponse.getIndices().length + " indices"); List candidateIndices = new ArrayList<>(); for (String s : getIndexResponse.getIndices()) { Matcher m = pattern.matcher(s); @@ -434,7 +483,7 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements return new NothingToDoPruneResult(candidateIndices, Collections.emptyList()); } Collections.sort(candidateIndices); - logger.info("found {} candidates", candidateIndices); + logger.log(Level.INFO, "found candidates: " + candidateIndices); List indicesToDelete = new ArrayList<>(); Matcher m1 = pattern.matcher(protectedIndexName); if (m1.matches()) { @@ -453,7 +502,7 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements if (indicesToDelete.isEmpty()) { return new NothingToDoPruneResult(candidateIndices, indicesToDelete); } - logger.warn("deleting {}", indicesToDelete); + logger.log(Level.INFO, "deleting " + indicesToDelete); String[] s = new String[indicesToDelete.size()]; DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest() .indices(indicesToDelete.toArray(s)); @@ -578,19 +627,19 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements AtomicInteger empty = new AtomicInteger(); Map map = sortByValue(fields); map.forEach((key, value) -> { - logger.info("{} {} {}", + logger.log(Level.INFO, MessageFormat.format("{0} {1} {2}", key, value, - (double) value * 100 / total); + (double) value * 100 / total)); if (value == 0) { empty.incrementAndGet(); } }); - logger.info("index={} numfields={} fieldsnotused={}", - index, map.size(), empty.get()); + logger.log(Level.INFO, MessageFormat.format("index = {0} numfields = {1} fieldsnotused = {2}", + index, map.size(), empty.get())); } } catch (Exception e) { - logger.error(e.getMessage(), e); + logger.log(Level.SEVERE, e.getMessage(), e); } } diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractBasicClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractBasicClient.java index ae870d8..e273ab6 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractBasicClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractBasicClient.java @@ -1,8 +1,7 @@ package org.xbib.elx.common; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import java.util.logging.Level; +import java.util.logging.Logger; import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; @@ -40,7 +39,7 @@ import java.util.concurrent.atomic.AtomicBoolean; public abstract class AbstractBasicClient implements BasicClient { - private static final Logger logger = LogManager.getLogger(AbstractBasicClient.class.getName()); + private static final Logger logger = Logger.getLogger(AbstractBasicClient.class.getName()); protected ElasticsearchClient client; @@ -111,13 +110,13 @@ public abstract class AbstractBasicClient implements BasicClient { getClient().execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); return clusterStateResponse.getClusterName().value(); } catch (ElasticsearchTimeoutException e) { - logger.warn(e.getMessage(), e); + logger.log(Level.WARNING, e.getMessage(), e); return "TIMEOUT"; } catch (NoNodeAvailableException e) { - logger.warn(e.getMessage(), e); + logger.log(Level.WARNING, e.getMessage(), e); return "DISCONNECTED"; } catch (Exception e) { - logger.warn(e.getMessage(), e); + logger.log(Level.WARNING, e.getMessage(), e); return "[" + e.getMessage() + "]"; } } @@ -163,7 +162,7 @@ public abstract class AbstractBasicClient implements BasicClient { logger.info("got cluster status " + healthResponse.getStatus().name()); if (healthResponse.isTimedOut()) { String message = "timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name(); - logger.error(message); + logger.log(Level.SEVERE, message); throw new IllegalStateException(message); } } @@ -178,13 +177,13 @@ public abstract class AbstractBasicClient implements BasicClient { ClusterHealthStatus status = healthResponse.getStatus(); return status.name(); } catch (ElasticsearchTimeoutException e) { - logger.warn(e.getMessage(), e); + logger.log(Level.WARNING, e.getMessage(), e); return "TIMEOUT"; } catch (NoNodeAvailableException e) { - logger.warn(e.getMessage(), e); + logger.log(Level.WARNING, e.getMessage(), e); return "DISCONNECTED"; } catch (Exception e) { - logger.warn(e.getMessage(), e); + logger.log(Level.WARNING, e.getMessage(), e); return "[" + e.getMessage() + "]"; } } @@ -241,14 +240,14 @@ public abstract class AbstractBasicClient implements BasicClient { @Override public boolean isIndexClosed(IndexDefinition indexDefinition) { String state = getIndexState(indexDefinition); - logger.log(Level.DEBUG, "index " + indexDefinition.getFullIndexName() + " is " + state); + logger.log(Level.FINE, "index " + indexDefinition.getFullIndexName() + " is " + state); return "CLOSE".equals(state); } @Override public boolean isIndexOpen(IndexDefinition indexDefinition) { String state = getIndexState(indexDefinition); - logger.log(Level.DEBUG, "index " + indexDefinition.getFullIndexName() + " is " + state); + logger.log(Level.FINE, "index " + indexDefinition.getFullIndexName() + " is " + state); return "OPEN".equals(state); } @@ -282,7 +281,7 @@ public abstract class AbstractBasicClient implements BasicClient { protected boolean isIndexDefinitionDisabled(IndexDefinition indexDefinition) { if (!indexDefinition.isEnabled()) { - logger.warn("index " + indexDefinition.getFullIndexName() + " is disabled"); + logger.log(Level.WARNING, "index " + indexDefinition.getFullIndexName() + " is disabled"); return true; } return false; diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java index 2792b54..5964fe0 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java @@ -1,8 +1,7 @@ package org.xbib.elx.common; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import java.util.logging.Level; +import java.util.logging.Logger; import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; @@ -31,7 +30,7 @@ import static org.xbib.elx.api.IndexDefinition.TYPE_NAME; public abstract class AbstractBulkClient extends AbstractBasicClient implements BulkClient { - private static final Logger logger = LogManager.getLogger(AbstractBulkClient.class.getName()); + private static final Logger logger = Logger.getLogger(AbstractBulkClient.class.getName()); private BulkProcessor bulkProcessor; @@ -94,7 +93,7 @@ public abstract class AbstractBulkClient extends AbstractBasicClient implements .endObject(); indexDefinition.setSettings(Strings.toString(builder)); } catch (IOException e) { - logger.log(Level.WARN, e.getMessage(), e); + logger.log(Level.WARNING, e.getMessage(), e); } } Settings settings = Settings.builder() @@ -112,13 +111,13 @@ public abstract class AbstractBulkClient extends AbstractBasicClient implements createIndexRequestBuilder.addMapping(TYPE_NAME, builder); } } catch (IOException e) { - logger.log(Level.WARN, e.getMessage(), e); + logger.log(Level.WARNING, e.getMessage(), e); } CreateIndexResponse createIndexResponse = createIndexRequestBuilder.execute().actionGet(); if (createIndexResponse.isAcknowledged()) { - logger.info("index {} created", index); + logger.log(Level.INFO, "index created: " + index); } else { - logger.warn("index creation of {} not acknowledged", index); + logger.log(Level.WARNING, "index creation of {} not acknowledged", index); return; } // we really need state GREEN. If yellow, we may trigger shard write errors and queue will exceed quickly. @@ -134,13 +133,13 @@ public abstract class AbstractBulkClient extends AbstractBasicClient implements String indexName = indexDefinition.getFullIndexName(); int interval = indexDefinition.getStartBulkRefreshSeconds(); if (interval != 0) { - logger.info("starting bulk on " + indexName + " with new refresh interval " + interval); + logger.log(Level.INFO, "starting bulk on " + indexName + " with new refresh interval " + interval); updateIndexSetting(indexName, "refresh_interval", interval >=0 ? interval + "s" : interval, 30L, TimeUnit.SECONDS); updateIndexSetting(indexName, "index.translog.durability", "async", 30L, TimeUnit.SECONDS); } else { - logger.warn("ignoring starting bulk on " + indexName + " with refresh interval " + interval); + logger.log(Level.WARNING, "ignoring starting bulk on " + indexName + " with refresh interval " + interval); } } @@ -167,7 +166,7 @@ public abstract class AbstractBulkClient extends AbstractBasicClient implements updateIndexSetting(indexName, "index.translog.durability", "request", 30L, TimeUnit.SECONDS); } else { - logger.warn("ignoring stopping bulk on " + indexName + " with refresh interval " + interval); + logger.log(Level.WARNING, "ignoring stopping bulk on " + indexName + " with refresh interval " + interval); } } } diff --git a/elx-common/src/test/resources/logging.properties b/elx-common/src/test/resources/logging.properties new file mode 100644 index 0000000..eabbc03 --- /dev/null +++ b/elx-common/src/test/resources/logging.properties @@ -0,0 +1,9 @@ +handlers=java.util.logging.FileHandler, java.util.logging.ConsoleHandler +.level=ALL +java.util.logging.SimpleFormatter.format=%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS.%1$tL %4$-7s [%3$s] %5$s %6$s%n +java.util.logging.ConsoleHandler.level=ALL +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.FileHandler.level=ALL +java.util.logging.FileHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.FileHandler.pattern=build/elx.log +jdk.event.security.level=INFO diff --git a/elx-http/build.gradle b/elx-http/build.gradle index 127ba7e..f572634 100644 --- a/elx-http/build.gradle +++ b/elx-http/build.gradle @@ -2,4 +2,8 @@ dependencies{ api project(':elx-common') api libs.net.http.netty.client api libs.es.plugin.transport.netty4 + implementation libs.netty.codec.http + implementation libs.netty.handler + implementation libs.netty.buffer + implementation libs.netty.transport } diff --git a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction index 086ac3b..33b2fbb 100644 --- a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction +++ b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction @@ -1,26 +1,27 @@ -org.xbib.elx.http.action.admin.indices.mapping.get.HttpGetMappingsAction org.xbib.elx.http.action.admin.cluster.health.HttpClusterHealthAction org.xbib.elx.http.action.admin.cluster.node.info.HttpNodesInfoAction org.xbib.elx.http.action.admin.cluster.settings.HttpClusterUpdateSettingsAction org.xbib.elx.http.action.admin.cluster.state.HttpClusterStateAction org.xbib.elx.http.action.admin.indices.alias.HttpIndicesAliasesAction org.xbib.elx.http.action.admin.indices.alias.get.HttpGetAliasAction +org.xbib.elx.http.action.admin.indices.close.HttpCloseIndexAction org.xbib.elx.http.action.admin.indices.create.HttpCreateIndexAction org.xbib.elx.http.action.admin.indices.delete.HttpDeleteIndexAction -org.xbib.elx.http.action.admin.indices.close.HttpCloseIndexAction org.xbib.elx.http.action.admin.indices.exists.indices.HttpIndicesExistsAction org.xbib.elx.http.action.admin.indices.forcemerge.HttpForceMergeAction org.xbib.elx.http.action.admin.indices.get.HttpGetIndexAction +org.xbib.elx.http.action.admin.indices.mapping.get.HttpGetMappingsAction +org.xbib.elx.http.action.admin.indices.open.HttpOpenIndexAction org.xbib.elx.http.action.admin.indices.refresh.HttpRefreshIndexAction org.xbib.elx.http.action.admin.indices.resolve.HttpResolveIndexAction org.xbib.elx.http.action.admin.indices.settings.get.HttpGetSettingsAction org.xbib.elx.http.action.admin.indices.settings.put.HttpUpdateSettingsAction org.xbib.elx.http.action.bulk.HttpBulkAction -org.xbib.elx.http.action.index.HttpIndexAction -org.xbib.elx.http.action.search.HttpClearScrollAction -org.xbib.elx.http.action.search.HttpSearchAction -org.xbib.elx.http.action.search.HttpSearchScrollAction -org.xbib.elx.http.action.main.HttpMainAction org.xbib.elx.http.action.get.HttpExistsAction org.xbib.elx.http.action.get.HttpGetAction org.xbib.elx.http.action.get.HttpMultiGetAction +org.xbib.elx.http.action.index.HttpIndexAction +org.xbib.elx.http.action.main.HttpMainAction +org.xbib.elx.http.action.search.HttpClearScrollAction +org.xbib.elx.http.action.search.HttpSearchAction +org.xbib.elx.http.action.search.HttpSearchScrollAction diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java index 141922f..0d4e2c3 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java @@ -1,8 +1,8 @@ package org.xbib.elx.http.test; +import java.time.Instant; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexDefinition; @@ -20,6 +20,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; @@ -36,6 +37,32 @@ class IndexPruneTest { this.helper = helper; } + @Test + void testOpenClose() throws IOException { + try (HttpAdminClient adminClient = ClientBuilder.builder() + .setAdminClientProvider(HttpAdminClientProvider.class) + .put(helper.getClientSettings()) + .build(); + HttpBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(HttpBulkClientProvider.class) + .put(helper.getClientSettings()) + .build()) { + Instant instant = Instant.now(); + IndexDefinition indexDefinition = new DefaultIndexDefinition("test", "_doc"); + indexDefinition.setIndex("test_openclose"); + indexDefinition.setFullIndexName("test_openclose"); + bulkClient.newIndex(indexDefinition); + assertEquals(List.of("test_openclose").toString(), adminClient.allIndices().toString()); + adminClient.closeIndex(indexDefinition); + assertEquals(List.of("test_openclose").toString(), adminClient.allClosedIndices().toString()); + assertEquals(List.of().toString(), adminClient.allClosedIndicesOlderThan(instant).toString()); + adminClient.openIndex(indexDefinition); + assertEquals(List.of("test_openclose").toString(), adminClient.allIndices().toString()); + assertEquals(List.of().toString(), adminClient.allClosedIndices().toString()); + assertEquals(List.of().toString(), adminClient.allClosedIndicesOlderThan(instant).toString()); + } + } + @Test void testPrune() throws IOException { try (HttpAdminClient adminClient = ClientBuilder.builder() diff --git a/elx-http/src/test/resources/logging.properties b/elx-http/src/test/resources/logging.properties new file mode 100644 index 0000000..eabbc03 --- /dev/null +++ b/elx-http/src/test/resources/logging.properties @@ -0,0 +1,9 @@ +handlers=java.util.logging.FileHandler, java.util.logging.ConsoleHandler +.level=ALL +java.util.logging.SimpleFormatter.format=%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS.%1$tL %4$-7s [%3$s] %5$s %6$s%n +java.util.logging.ConsoleHandler.level=ALL +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.FileHandler.level=ALL +java.util.logging.FileHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.FileHandler.pattern=build/elx.log +jdk.event.security.level=INFO diff --git a/elx-node/build.gradle b/elx-node/build.gradle index aa78967..75381ce 100644 --- a/elx-node/build.gradle +++ b/elx-node/build.gradle @@ -1,4 +1,8 @@ dependencies { api project(':elx-common') api libs.es.plugin.transport.netty4 + implementation libs.netty.codec.http + implementation libs.netty.handler + implementation libs.netty.buffer + implementation libs.netty.transport } diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java index 90ff7e7..80b9286 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java @@ -1,5 +1,6 @@ package org.xbib.elx.node.test; +import java.time.Instant; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.junit.jupiter.api.Test; @@ -19,6 +20,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; @@ -35,6 +37,32 @@ class IndexPruneTest { this.helper = helper; } + @Test + void testOpenClose() throws IOException { + try (NodeAdminClient adminClient = ClientBuilder.builder() + .setAdminClientProvider(NodeAdminClientProvider.class) + .put(helper.getClientSettings()) + .build(); + NodeBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(NodeBulkClientProvider.class) + .put(helper.getClientSettings()) + .build()) { + Instant instant = Instant.now(); + IndexDefinition indexDefinition = new DefaultIndexDefinition("test", "_doc"); + indexDefinition.setIndex("test_openclose"); + indexDefinition.setFullIndexName("test_openclose"); + bulkClient.newIndex(indexDefinition); + assertEquals(List.of("test_openclose").toString(), adminClient.allIndices().toString()); + adminClient.closeIndex(indexDefinition); + assertEquals(List.of("test_openclose").toString(), adminClient.allClosedIndices().toString()); + assertEquals(List.of().toString(), adminClient.allClosedIndicesOlderThan(instant).toString()); + adminClient.openIndex(indexDefinition); + assertEquals(List.of("test_openclose").toString(), adminClient.allIndices().toString()); + assertEquals(List.of().toString(), adminClient.allClosedIndices().toString()); + assertEquals(List.of().toString(), adminClient.allClosedIndicesOlderThan(instant).toString()); + } + } + @Test void testPrune() throws IOException { try (NodeAdminClient adminClient = ClientBuilder.builder(helper.client()) diff --git a/elx-node/src/test/resources/logging.properties b/elx-node/src/test/resources/logging.properties new file mode 100644 index 0000000..eabbc03 --- /dev/null +++ b/elx-node/src/test/resources/logging.properties @@ -0,0 +1,9 @@ +handlers=java.util.logging.FileHandler, java.util.logging.ConsoleHandler +.level=ALL +java.util.logging.SimpleFormatter.format=%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS.%1$tL %4$-7s [%3$s] %5$s %6$s%n +java.util.logging.ConsoleHandler.level=ALL +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.FileHandler.level=ALL +java.util.logging.FileHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.FileHandler.pattern=build/elx.log +jdk.event.security.level=INFO diff --git a/elx-transport/build.gradle b/elx-transport/build.gradle index aa78967..75381ce 100644 --- a/elx-transport/build.gradle +++ b/elx-transport/build.gradle @@ -1,4 +1,8 @@ dependencies { api project(':elx-common') api libs.es.plugin.transport.netty4 + implementation libs.netty.codec.http + implementation libs.netty.handler + implementation libs.netty.buffer + implementation libs.netty.transport } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java index 1a9fea1..f3a0be1 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java @@ -1,5 +1,6 @@ package org.xbib.elx.transport.test; +import java.time.Instant; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.junit.jupiter.api.Test; @@ -19,6 +20,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; @@ -35,6 +37,32 @@ class IndexPruneTest { this.helper = helper; } + @Test + void testOpenClose() throws IOException { + try (TransportAdminClient adminClient = ClientBuilder.builder() + .setAdminClientProvider(TransportAdminClientProvider.class) + .put(helper.getClientSettings()) + .build(); + TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getClientSettings()) + .build()) { + Instant instant = Instant.now(); + IndexDefinition indexDefinition = new DefaultIndexDefinition("test", "_doc"); + indexDefinition.setIndex("test_openclose"); + indexDefinition.setFullIndexName("test_openclose"); + bulkClient.newIndex(indexDefinition); + assertEquals(List.of("test_openclose").toString(), adminClient.allIndices().toString()); + adminClient.closeIndex(indexDefinition); + assertEquals(List.of("test_openclose").toString(), adminClient.allClosedIndices().toString()); + assertEquals(List.of().toString(), adminClient.allClosedIndicesOlderThan(instant).toString()); + adminClient.openIndex(indexDefinition); + assertEquals(List.of("test_openclose").toString(), adminClient.allIndices().toString()); + assertEquals(List.of().toString(), adminClient.allClosedIndices().toString()); + assertEquals(List.of().toString(), adminClient.allClosedIndicesOlderThan(instant).toString()); + } + } + @Test void testPruneWithoutClose() throws IOException { try (TransportAdminClient adminClient = ClientBuilder.builder() diff --git a/elx-transport/src/test/resources/logging.properties b/elx-transport/src/test/resources/logging.properties new file mode 100644 index 0000000..eabbc03 --- /dev/null +++ b/elx-transport/src/test/resources/logging.properties @@ -0,0 +1,9 @@ +handlers=java.util.logging.FileHandler, java.util.logging.ConsoleHandler +.level=ALL +java.util.logging.SimpleFormatter.format=%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS.%1$tL %4$-7s [%3$s] %5$s %6$s%n +java.util.logging.ConsoleHandler.level=ALL +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.FileHandler.level=ALL +java.util.logging.FileHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.FileHandler.pattern=build/elx.log +jdk.event.security.level=INFO diff --git a/gradle.properties b/gradle.properties index 040e4c2..b338ed9 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ group = org.xbib name = elx -version = 7.10.2.28 +version = 7.10.2.29 org.gradle.warning.mode = ALL diff --git a/gradle/test/junit5.gradle b/gradle/test/junit5.gradle index 6c107cb..6e6ae7e 100644 --- a/gradle/test/junit5.gradle +++ b/gradle/test/junit5.gradle @@ -17,6 +17,7 @@ test { '--add-opens=java.base/java.nio=ALL-UNNAMED' ] systemProperty 'java.util.logging.manager', 'org.apache.logging.log4j.jul.LogManager' + systemProperty 'io.netty.tryReflectionSetAccessible', 'true' systemProperty 'jna.debug_load', 'true' systemProperty 'path.home', "${project.buildDir}/" failFast = false diff --git a/settings.gradle b/settings.gradle index 547080b..84e366d 100644 --- a/settings.gradle +++ b/settings.gradle @@ -6,6 +6,7 @@ dependencyResolutionManagement { version('elasticsearch', '7.10.2') version('lucene', '8.7.0') version('log4j', '2.17.1') // ES 7.10.2 uses log4j2 2.11.1 + version('netty', '4.1.87.Final') library('junit-jupiter-api', 'org.junit.jupiter', 'junit-jupiter-api').versionRef('junit') library('junit-jupiter-params', 'org.junit.jupiter', 'junit-jupiter-params').versionRef('junit') library('junit-jupiter-engine', 'org.junit.jupiter', 'junit-jupiter-engine').versionRef('junit') @@ -33,6 +34,10 @@ dependencyResolutionManagement { library('net-http-netty-client', 'org.xbib', 'net-http-client-netty').version('3.0.4') library('metrics', 'org.xbib', 'metrics-common').version('3.0.0') library('time', 'org.xbib', 'time').version('2.1.0') + library('netty-codec-http', 'io.netty', 'netty-codec-http').versionRef('netty') + library('netty-handler', 'io.netty', 'netty-handler').versionRef('netty') + library('netty-buffer', 'io.netty', 'netty-buffer').versionRef('netty') + library('netty-transport', 'io.netty', 'netty-transport').versionRef('netty') } } }