joerg/elx
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

enforce newer netty, add HTTP open index, add admin methods for closed indices, clean up log4j/jul logging

Browse source
This commit is contained in:
Jörg Prante 2023-02-23 18:51:18 +01:00
parent f4c25b6430
commit 833418ca57
18 changed files with 273 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
elx-api/src/main/java/org/xbib/elx/api/AdminClient.java
View file

@ -1,5 +1,6 @@
package org.xbib.elx.api;
import java.time.Instant;
import java.util.Collection;
import java.util.Map;
@ -8,6 +9,31 @@ import java.util.Map;
*/
public interface AdminClient extends BasicClient {
/**
* List all indices.
* @return the names of the indices.
*/
Collection<String> allIndices();
/**
* List all closed indices.
* @return the names of the closed indices.
*/
Collection<String> allClosedIndices();
/**
* List all closed indices which were created before a given instant.
* @param instant the instant
* @return the names of the closed indices
*/
Collection<String> allClosedIndicesOlderThan(Instant instant);
/**
* Delete all closed indices which were created before a given instant.
* @param instant the instant
*/
void purgeAllClosedIndicesOlderThan(Instant instant);
/**
* Get the mapping of an index.
*
@ -16,6 +42,10 @@ public interface AdminClient extends BasicClient {
*/
Map<String, Object> getMapping(IndexDefinition indexDefinition);
/**
* Check the mapping.
* @param indexDefinition the index definition
*/
void checkMapping(IndexDefinition indexDefinition);
/**

97
elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java
View file

@ -2,9 +2,11 @@ package org.xbib.elx.common;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.text.MessageFormat;
import java.time.Instant;
import java.util.SortedMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
@ -82,7 +84,54 @@ import static org.xbib.elx.api.IndexDefinition.TYPE_NAME;
public abstract class AbstractAdminClient extends AbstractBasicClient implements AdminClient {
private static final Logger logger = LogManager.getLogger(AbstractAdminClient.class.getName());
private static final Logger logger = Logger.getLogger(AbstractAdminClient.class.getName());
@Override
public Collection<String> allIndices() {
ensureClientIsPresent();
ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
clusterStateRequest.blocks(false);
clusterStateRequest.metadata(true);
clusterStateRequest.nodes(false);
clusterStateRequest.routingTable(false);
clusterStateRequest.customs(false);
ClusterStateResponse clusterStateResponse =
client.execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
SortedMap<String, IndexAbstraction> indexAbstractions = clusterStateResponse.getState().getMetadata()
.getIndicesLookup();
if (indexAbstractions == null) {
return Collections.emptyList();
}
return indexAbstractions.keySet();
}
@Override
public Collection<String> allClosedIndices() {
return allClosedIndicesOlderThan(Instant.now());
}
@Override
public Collection<String> allClosedIndicesOlderThan(Instant instant) {
ensureClientIsPresent();
ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
clusterStateRequest.blocks(false);
clusterStateRequest.metadata(true);
clusterStateRequest.nodes(false);
clusterStateRequest.routingTable(false);
clusterStateRequest.customs(false);
ClusterStateResponse clusterStateResponse =
client.execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
return clusterStateResponse.getState().getMetadata()
.getIndicesLookup().values().stream()
.flatMap(ia -> ia.getIndices().stream().filter(i -> i.getState().equals(IndexMetadata.State.CLOSE) && i.getCreationDate() < instant.toEpochMilli()))
.map(im -> im.getIndex().getName())
.collect(Collectors.toList());
}
@Override
public void purgeAllClosedIndicesOlderThan(Instant instant) {
allClosedIndicesOlderThan(instant).forEach(this::deleteIndex);
}
@Override
public Map<String, Object> getMapping(IndexDefinition indexDefinition) {
@ -110,7 +159,7 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
@Override
public void deleteIndex(String indexName) {
if (indexName == null) {
logger.warn("no index name given to delete index");
logger.log(Level.WARNING, "no index name given to delete index");
return;
}
ensureClientIsPresent();
@ -133,7 +182,7 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
@Override
public void closeIndex(String indexName) {
if (indexName == null) {
logger.warn("no index name given to close index");
logger.log(Level.WARNING, "no index name given to close index");
return;
}
ensureClientIsPresent();
@ -143,9 +192,9 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
List<CloseIndexResponse.IndexResult> list = closeIndexResponse.getIndices();
list.forEach(result -> {
if (result.hasFailures()) {
logger.warn("error when closing " + result.getIndex(), result.getException());
logger.log(Level.WARNING, "error when closing " + result.getIndex(), result.getException());
} else {
logger.info("index " + result.getIndex() + " closed");
logger.log(Level.INFO, "index " + result.getIndex() + " closed");
}
});
}
@ -162,14 +211,14 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
@Override
public void openIndex(String indexName) {
if (indexName == null) {
logger.warn("no index name given to close index");
logger.log(Level.WARNING, "no index name given to close index");
return;
}
ensureClientIsPresent();
OpenIndexRequest openIndexRequest = new OpenIndexRequest().indices(indexName);
OpenIndexResponse openIndexResponse = client.execute(OpenIndexAction.INSTANCE, openIndexRequest).actionGet();
if (openIndexResponse.isAcknowledged()) {
logger.info("index " + indexName + " opened");
logger.log(Level.INFO, "index " + indexName + " opened");
}
}
@ -179,7 +228,7 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
return;
}
if (indexDefinition.getReplicaCount() < 0) {
logger.warn("invalid replica level defined for index "
logger.log(Level.WARNING, "invalid replica level defined for index "
+ indexDefinition.getIndex() + ": " + indexDefinition.getReplicaCount());
return;
}
@ -324,7 +373,7 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
// two situations: 1. a new alias 2. there is already an old index with the alias
Optional<String> oldIndex = resolveAliasFromClusterState(index).stream().sorted().findFirst();
Map<String, String> oldAliasMap = oldIndex.map(this::getAliases).orElse(null);
logger.info("old index = {} old alias map = {}", oldIndex.orElse(""), oldAliasMap);
logger.log(Level.INFO, "old index = " + oldIndex.orElse("") + " old alias map = " + oldAliasMap);
final List<String> newAliases = new ArrayList<>();
final List<String> moveAliases = new ArrayList<>();
IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest();
@ -405,20 +454,20 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
Pattern pattern,
int delta,
int mintokeep) {
logger.info("before pruning: index = {} full index = {} delta = {} mintokeep = {} pattern = {}",
index, protectedIndexName, delta, mintokeep, pattern);
logger.log(Level.INFO, MessageFormat.format("before pruning: index = {0} full index = {1} delta = {2} mintokeep = {3} pattern = {4}",
index, protectedIndexName, delta, mintokeep, pattern));
if (delta == 0 && mintokeep == 0) {
logger.warn("no candidates found, delta is 0 and mintokeep is 0");
logger.log(Level.INFO, "no candidates found, delta is 0 and mintokeep is 0");
return new NonePruneResult();
}
if (index.equals(protectedIndexName)) {
logger.warn("no candidates found, only protected index name is given");
logger.log(Level.INFO, "no candidates found, only protected index name is given");
return new NonePruneResult();
}
ensureClientIsPresent();
GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE);
GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet();
logger.info("before pruning: found total of {} indices", getIndexResponse.getIndices().length);
logger.log(Level.INFO, "before pruning: found total of " + getIndexResponse.getIndices().length + " indices");
List<String> candidateIndices = new ArrayList<>();
for (String s : getIndexResponse.getIndices()) {
Matcher m = pattern.matcher(s);
@ -434,7 +483,7 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
return new NothingToDoPruneResult(candidateIndices, Collections.emptyList());
}
Collections.sort(candidateIndices);
logger.info("found {} candidates", candidateIndices);
logger.log(Level.INFO, "found candidates: " + candidateIndices);
List<String> indicesToDelete = new ArrayList<>();
Matcher m1 = pattern.matcher(protectedIndexName);
if (m1.matches()) {
@ -453,7 +502,7 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
if (indicesToDelete.isEmpty()) {
return new NothingToDoPruneResult(candidateIndices, indicesToDelete);
}
logger.warn("deleting {}", indicesToDelete);
logger.log(Level.INFO, "deleting " + indicesToDelete);
String[] s = new String[indicesToDelete.size()];
DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest()
.indices(indicesToDelete.toArray(s));
@ -578,19 +627,19 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
AtomicInteger empty = new AtomicInteger();
Map<String, Long> map = sortByValue(fields);
map.forEach((key, value) -> {
logger.info("{} {} {}",
logger.log(Level.INFO, MessageFormat.format("{0} {1} {2}",
key,
value,
(double) value * 100 / total);
(double) value * 100 / total));
if (value == 0) {
empty.incrementAndGet();
}
});
logger.info("index={} numfields={} fieldsnotused={}",
index, map.size(), empty.get());
logger.log(Level.INFO, MessageFormat.format("index = {0} numfields = {1} fieldsnotused = {2}",
index, map.size(), empty.get()));
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
logger.log(Level.SEVERE, e.getMessage(), e);
}
}

27
elx-common/src/main/java/org/xbib/elx/common/AbstractBasicClient.java
View file

@ -1,8 +1,7 @@
package org.xbib.elx.common;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
@ -40,7 +39,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
public abstract class AbstractBasicClient implements BasicClient {
private static final Logger logger = LogManager.getLogger(AbstractBasicClient.class.getName());
private static final Logger logger = Logger.getLogger(AbstractBasicClient.class.getName());
protected ElasticsearchClient client;
@ -111,13 +110,13 @@ public abstract class AbstractBasicClient implements BasicClient {
getClient().execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
return clusterStateResponse.getClusterName().value();
} catch (ElasticsearchTimeoutException e) {
logger.warn(e.getMessage(), e);
logger.log(Level.WARNING, e.getMessage(), e);
return "TIMEOUT";
} catch (NoNodeAvailableException e) {
logger.warn(e.getMessage(), e);
logger.log(Level.WARNING, e.getMessage(), e);
return "DISCONNECTED";
} catch (Exception e) {
logger.warn(e.getMessage(), e);
logger.log(Level.WARNING, e.getMessage(), e);
return "[" + e.getMessage() + "]";
}
}
@ -163,7 +162,7 @@ public abstract class AbstractBasicClient implements BasicClient {
logger.info("got cluster status " + healthResponse.getStatus().name());
if (healthResponse.isTimedOut()) {
String message = "timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name();
logger.error(message);
logger.log(Level.SEVERE, message);
throw new IllegalStateException(message);
}
}
@ -178,13 +177,13 @@ public abstract class AbstractBasicClient implements BasicClient {
ClusterHealthStatus status = healthResponse.getStatus();
return status.name();
} catch (ElasticsearchTimeoutException e) {
logger.warn(e.getMessage(), e);
logger.log(Level.WARNING, e.getMessage(), e);
return "TIMEOUT";
} catch (NoNodeAvailableException e) {
logger.warn(e.getMessage(), e);
logger.log(Level.WARNING, e.getMessage(), e);
return "DISCONNECTED";
} catch (Exception e) {
logger.warn(e.getMessage(), e);
logger.log(Level.WARNING, e.getMessage(), e);
return "[" + e.getMessage() + "]";
}
}
@ -241,14 +240,14 @@ public abstract class AbstractBasicClient implements BasicClient {
@Override
public boolean isIndexClosed(IndexDefinition indexDefinition) {
String state = getIndexState(indexDefinition);
logger.log(Level.DEBUG, "index " + indexDefinition.getFullIndexName() + " is " + state);
logger.log(Level.FINE, "index " + indexDefinition.getFullIndexName() + " is " + state);
return "CLOSE".equals(state);
}
@Override
public boolean isIndexOpen(IndexDefinition indexDefinition) {
String state = getIndexState(indexDefinition);
logger.log(Level.DEBUG, "index " + indexDefinition.getFullIndexName() + " is " + state);
logger.log(Level.FINE, "index " + indexDefinition.getFullIndexName() + " is " + state);
return "OPEN".equals(state);
}
@ -282,7 +281,7 @@ public abstract class AbstractBasicClient implements BasicClient {
protected boolean isIndexDefinitionDisabled(IndexDefinition indexDefinition) {
if (!indexDefinition.isEnabled()) {
logger.warn("index " + indexDefinition.getFullIndexName() + " is disabled");
logger.log(Level.WARNING, "index " + indexDefinition.getFullIndexName() + " is disabled");
return true;
}
return false;

21
elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java
View file

@ -1,8 +1,7 @@
package org.xbib.elx.common;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
@ -31,7 +30,7 @@ import static org.xbib.elx.api.IndexDefinition.TYPE_NAME;
public abstract class AbstractBulkClient extends AbstractBasicClient implements BulkClient {
private static final Logger logger = LogManager.getLogger(AbstractBulkClient.class.getName());
private static final Logger logger = Logger.getLogger(AbstractBulkClient.class.getName());
private BulkProcessor bulkProcessor;
@ -94,7 +93,7 @@ public abstract class AbstractBulkClient extends AbstractBasicClient implements
.endObject();
indexDefinition.setSettings(Strings.toString(builder));
} catch (IOException e) {
logger.log(Level.WARN, e.getMessage(), e);
logger.log(Level.WARNING, e.getMessage(), e);
}
}
Settings settings = Settings.builder()
@ -112,13 +111,13 @@ public abstract class AbstractBulkClient extends AbstractBasicClient implements
createIndexRequestBuilder.addMapping(TYPE_NAME, builder);
}
} catch (IOException e) {
logger.log(Level.WARN, e.getMessage(), e);
logger.log(Level.WARNING, e.getMessage(), e);
}
CreateIndexResponse createIndexResponse = createIndexRequestBuilder.execute().actionGet();
if (createIndexResponse.isAcknowledged()) {
logger.info("index {} created", index);
logger.log(Level.INFO, "index created: " + index);
} else {
logger.warn("index creation of {} not acknowledged", index);
logger.log(Level.WARNING, "index creation of {} not acknowledged", index);
return;
}
// we really need state GREEN. If yellow, we may trigger shard write errors and queue will exceed quickly.
@ -134,13 +133,13 @@ public abstract class AbstractBulkClient extends AbstractBasicClient implements
String indexName = indexDefinition.getFullIndexName();
int interval = indexDefinition.getStartBulkRefreshSeconds();
if (interval != 0) {
logger.info("starting bulk on " + indexName + " with new refresh interval " + interval);
logger.log(Level.INFO, "starting bulk on " + indexName + " with new refresh interval " + interval);
updateIndexSetting(indexName,
"refresh_interval", interval >=0 ? interval + "s" : interval, 30L, TimeUnit.SECONDS);
updateIndexSetting(indexName,
"index.translog.durability", "async", 30L, TimeUnit.SECONDS);
} else {
logger.warn("ignoring starting bulk on " + indexName + " with refresh interval " + interval);
logger.log(Level.WARNING, "ignoring starting bulk on " + indexName + " with refresh interval " + interval);
}
}
@ -167,7 +166,7 @@ public abstract class AbstractBulkClient extends AbstractBasicClient implements
updateIndexSetting(indexName,
"index.translog.durability", "request", 30L, TimeUnit.SECONDS);
} else {
logger.warn("ignoring stopping bulk on " + indexName + " with refresh interval " + interval);
logger.log(Level.WARNING, "ignoring stopping bulk on " + indexName + " with refresh interval " + interval);
}
}
}

9
elx-common/src/test/resources/logging.properties Normal file
View file

@ -0,0 +1,9 @@
handlers=java.util.logging.FileHandler, java.util.logging.ConsoleHandler
.level=ALL
java.util.logging.SimpleFormatter.format=%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS.%1$tL %4$-7s [%3$s] %5$s %6$s%n
java.util.logging.ConsoleHandler.level=ALL
java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter
java.util.logging.FileHandler.level=ALL
java.util.logging.FileHandler.formatter=java.util.logging.SimpleFormatter
java.util.logging.FileHandler.pattern=build/elx.log
jdk.event.security.level=INFO

4
elx-http/build.gradle
View file

@ -2,4 +2,8 @@ dependencies{
api project(':elx-common')
api libs.net.http.netty.client
api libs.es.plugin.transport.netty4
implementation libs.netty.codec.http
implementation libs.netty.handler
implementation libs.netty.buffer
implementation libs.netty.transport
}

15
elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction
View file

@ -1,26 +1,27 @@
org.xbib.elx.http.action.admin.indices.mapping.get.HttpGetMappingsAction
org.xbib.elx.http.action.admin.cluster.health.HttpClusterHealthAction
org.xbib.elx.http.action.admin.cluster.node.info.HttpNodesInfoAction
org.xbib.elx.http.action.admin.cluster.settings.HttpClusterUpdateSettingsAction
org.xbib.elx.http.action.admin.cluster.state.HttpClusterStateAction
org.xbib.elx.http.action.admin.indices.alias.HttpIndicesAliasesAction
org.xbib.elx.http.action.admin.indices.alias.get.HttpGetAliasAction
org.xbib.elx.http.action.admin.indices.close.HttpCloseIndexAction
org.xbib.elx.http.action.admin.indices.create.HttpCreateIndexAction
org.xbib.elx.http.action.admin.indices.delete.HttpDeleteIndexAction
org.xbib.elx.http.action.admin.indices.close.HttpCloseIndexAction
org.xbib.elx.http.action.admin.indices.exists.indices.HttpIndicesExistsAction
org.xbib.elx.http.action.admin.indices.forcemerge.HttpForceMergeAction
org.xbib.elx.http.action.admin.indices.get.HttpGetIndexAction
org.xbib.elx.http.action.admin.indices.mapping.get.HttpGetMappingsAction
org.xbib.elx.http.action.admin.indices.open.HttpOpenIndexAction
org.xbib.elx.http.action.admin.indices.refresh.HttpRefreshIndexAction
org.xbib.elx.http.action.admin.indices.resolve.HttpResolveIndexAction
org.xbib.elx.http.action.admin.indices.settings.get.HttpGetSettingsAction
org.xbib.elx.http.action.admin.indices.settings.put.HttpUpdateSettingsAction
org.xbib.elx.http.action.bulk.HttpBulkAction
org.xbib.elx.http.action.index.HttpIndexAction
org.xbib.elx.http.action.search.HttpClearScrollAction
org.xbib.elx.http.action.search.HttpSearchAction
org.xbib.elx.http.action.search.HttpSearchScrollAction
org.xbib.elx.http.action.main.HttpMainAction
org.xbib.elx.http.action.get.HttpExistsAction
org.xbib.elx.http.action.get.HttpGetAction
org.xbib.elx.http.action.get.HttpMultiGetAction
org.xbib.elx.http.action.index.HttpIndexAction
org.xbib.elx.http.action.main.HttpMainAction
org.xbib.elx.http.action.search.HttpClearScrollAction
org.xbib.elx.http.action.search.HttpSearchAction
org.xbib.elx.http.action.search.HttpSearchScrollAction

29
elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java
View file

@ -1,8 +1,8 @@
package org.xbib.elx.http.test;
import java.time.Instant;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexDefinition;
@ -20,6 +20,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
@ -36,6 +37,32 @@ class IndexPruneTest {
this.helper = helper;
}
@Test
void testOpenClose() throws IOException {
try (HttpAdminClient adminClient = ClientBuilder.builder()
.setAdminClientProvider(HttpAdminClientProvider.class)
.put(helper.getClientSettings())
.build();
HttpBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(HttpBulkClientProvider.class)
.put(helper.getClientSettings())
.build()) {
Instant instant = Instant.now();
IndexDefinition indexDefinition = new DefaultIndexDefinition("test", "_doc");
indexDefinition.setIndex("test_openclose");
indexDefinition.setFullIndexName("test_openclose");
bulkClient.newIndex(indexDefinition);
assertEquals(List.of("test_openclose").toString(), adminClient.allIndices().toString());
adminClient.closeIndex(indexDefinition);
assertEquals(List.of("test_openclose").toString(), adminClient.allClosedIndices().toString());
assertEquals(List.of().toString(), adminClient.allClosedIndicesOlderThan(instant).toString());
adminClient.openIndex(indexDefinition);
assertEquals(List.of("test_openclose").toString(), adminClient.allIndices().toString());
assertEquals(List.of().toString(), adminClient.allClosedIndices().toString());
assertEquals(List.of().toString(), adminClient.allClosedIndicesOlderThan(instant).toString());
}
}
@Test
void testPrune() throws IOException {
try (HttpAdminClient adminClient = ClientBuilder.builder()

9
elx-http/src/test/resources/logging.properties Normal file
View file

@ -0,0 +1,9 @@
handlers=java.util.logging.FileHandler, java.util.logging.ConsoleHandler
.level=ALL
java.util.logging.SimpleFormatter.format=%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS.%1$tL %4$-7s [%3$s] %5$s %6$s%n
java.util.logging.ConsoleHandler.level=ALL
java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter
java.util.logging.FileHandler.level=ALL
java.util.logging.FileHandler.formatter=java.util.logging.SimpleFormatter
java.util.logging.FileHandler.pattern=build/elx.log
jdk.event.security.level=INFO

4
elx-node/build.gradle
View file

@ -1,4 +1,8 @@
dependencies {
api project(':elx-common')
api libs.es.plugin.transport.netty4
implementation libs.netty.codec.http
implementation libs.netty.handler
implementation libs.netty.buffer
implementation libs.netty.transport
}

28
elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java
View file

@ -1,5 +1,6 @@
package org.xbib.elx.node.test;
import java.time.Instant;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.Test;
@ -19,6 +20,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
@ -35,6 +37,32 @@ class IndexPruneTest {
this.helper = helper;
}
@Test
void testOpenClose() throws IOException {
try (NodeAdminClient adminClient = ClientBuilder.builder()
.setAdminClientProvider(NodeAdminClientProvider.class)
.put(helper.getClientSettings())
.build();
NodeBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(NodeBulkClientProvider.class)
.put(helper.getClientSettings())
.build()) {
Instant instant = Instant.now();
IndexDefinition indexDefinition = new DefaultIndexDefinition("test", "_doc");
indexDefinition.setIndex("test_openclose");
indexDefinition.setFullIndexName("test_openclose");
bulkClient.newIndex(indexDefinition);
assertEquals(List.of("test_openclose").toString(), adminClient.allIndices().toString());
adminClient.closeIndex(indexDefinition);
assertEquals(List.of("test_openclose").toString(), adminClient.allClosedIndices().toString());
assertEquals(List.of().toString(), adminClient.allClosedIndicesOlderThan(instant).toString());
adminClient.openIndex(indexDefinition);
assertEquals(List.of("test_openclose").toString(), adminClient.allIndices().toString());
assertEquals(List.of().toString(), adminClient.allClosedIndices().toString());
assertEquals(List.of().toString(), adminClient.allClosedIndicesOlderThan(instant).toString());
}
}
@Test
void testPrune() throws IOException {
try (NodeAdminClient adminClient = ClientBuilder.builder(helper.client())

9
elx-node/src/test/resources/logging.properties Normal file
View file

@ -0,0 +1,9 @@
handlers=java.util.logging.FileHandler, java.util.logging.ConsoleHandler
.level=ALL
java.util.logging.SimpleFormatter.format=%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS.%1$tL %4$-7s [%3$s] %5$s %6$s%n
java.util.logging.ConsoleHandler.level=ALL
java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter
java.util.logging.FileHandler.level=ALL
java.util.logging.FileHandler.formatter=java.util.logging.SimpleFormatter
java.util.logging.FileHandler.pattern=build/elx.log
jdk.event.security.level=INFO

4
elx-transport/build.gradle
View file

@ -1,4 +1,8 @@
dependencies {
api project(':elx-common')
api libs.es.plugin.transport.netty4
implementation libs.netty.codec.http
implementation libs.netty.handler
implementation libs.netty.buffer
implementation libs.netty.transport
}

28
elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java
View file

@ -1,5 +1,6 @@
package org.xbib.elx.transport.test;
import java.time.Instant;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.Test;
@ -19,6 +20,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
@ -35,6 +37,32 @@ class IndexPruneTest {
this.helper = helper;
}
@Test
void testOpenClose() throws IOException {
try (TransportAdminClient adminClient = ClientBuilder.builder()
.setAdminClientProvider(TransportAdminClientProvider.class)
.put(helper.getClientSettings())
.build();
TransportBulkClient bulkClient = ClientBuilder.builder()
.setBulkClientProvider(TransportBulkClientProvider.class)
.put(helper.getClientSettings())
.build()) {
Instant instant = Instant.now();
IndexDefinition indexDefinition = new DefaultIndexDefinition("test", "_doc");
indexDefinition.setIndex("test_openclose");
indexDefinition.setFullIndexName("test_openclose");
bulkClient.newIndex(indexDefinition);
assertEquals(List.of("test_openclose").toString(), adminClient.allIndices().toString());
adminClient.closeIndex(indexDefinition);
assertEquals(List.of("test_openclose").toString(), adminClient.allClosedIndices().toString());
assertEquals(List.of().toString(), adminClient.allClosedIndicesOlderThan(instant).toString());
adminClient.openIndex(indexDefinition);
assertEquals(List.of("test_openclose").toString(), adminClient.allIndices().toString());
assertEquals(List.of().toString(), adminClient.allClosedIndices().toString());
assertEquals(List.of().toString(), adminClient.allClosedIndicesOlderThan(instant).toString());
}
}
@Test
void testPruneWithoutClose() throws IOException {
try (TransportAdminClient adminClient = ClientBuilder.builder()

9
elx-transport/src/test/resources/logging.properties Normal file
View file

@ -0,0 +1,9 @@
handlers=java.util.logging.FileHandler, java.util.logging.ConsoleHandler
.level=ALL
java.util.logging.SimpleFormatter.format=%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS.%1$tL %4$-7s [%3$s] %5$s %6$s%n
java.util.logging.ConsoleHandler.level=ALL
java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter
java.util.logging.FileHandler.level=ALL
java.util.logging.FileHandler.formatter=java.util.logging.SimpleFormatter
java.util.logging.FileHandler.pattern=build/elx.log
jdk.event.security.level=INFO

2
gradle.properties
View file

@ -1,5 +1,5 @@
group = org.xbib
name = elx
version = 7.10.2.28
version = 7.10.2.29
org.gradle.warning.mode = ALL

1
gradle/test/junit5.gradle
View file

@ -17,6 +17,7 @@ test {
'--add-opens=java.base/java.nio=ALL-UNNAMED'
]
systemProperty 'java.util.logging.manager', 'org.apache.logging.log4j.jul.LogManager'
systemProperty 'io.netty.tryReflectionSetAccessible', 'true'
systemProperty 'jna.debug_load', 'true'
systemProperty 'path.home', "${project.buildDir}/"
failFast = false

5
settings.gradle
View file

@ -6,6 +6,7 @@ dependencyResolutionManagement {
version('elasticsearch', '7.10.2')
version('lucene', '8.7.0')
version('log4j', '2.17.1') // ES 7.10.2 uses log4j2 2.11.1
version('netty', '4.1.87.Final')
library('junit-jupiter-api', 'org.junit.jupiter', 'junit-jupiter-api').versionRef('junit')
library('junit-jupiter-params', 'org.junit.jupiter', 'junit-jupiter-params').versionRef('junit')
library('junit-jupiter-engine', 'org.junit.jupiter', 'junit-jupiter-engine').versionRef('junit')
@ -33,6 +34,10 @@ dependencyResolutionManagement {
library('net-http-netty-client', 'org.xbib', 'net-http-client-netty').version('3.0.4')
library('metrics', 'org.xbib', 'metrics-common').version('3.0.0')
library('time', 'org.xbib', 'time').version('2.1.0')
library('netty-codec-http', 'io.netty', 'netty-codec-http').versionRef('netty')
library('netty-handler', 'io.netty', 'netty-handler').versionRef('netty')
library('netty-buffer', 'io.netty', 'netty-buffer').versionRef('netty')
library('netty-transport', 'io.netty', 'netty-transport').versionRef('netty')
}
}
}