Compare commits

...

10 commits

16 changed files with 133 additions and 63 deletions

View file

@ -1,18 +1,14 @@
plugins { plugins {
id "checkstyle"
id "pmd"
id 'maven-publish' id 'maven-publish'
id 'signing' id 'signing'
id "io.github.gradle-nexus.publish-plugin" version "2.0.0-rc-1" id "io.github.gradle-nexus.publish-plugin" version "2.0.0-rc-1"
id "com.github.spotbugs" version "6.0.0-beta.3"
id "org.cyclonedx.bom" version "1.7.4"
id "org.xbib.gradle.plugin.asciidoctor" version "3.0.0" id "org.xbib.gradle.plugin.asciidoctor" version "3.0.0"
} }
wrapper { wrapper {
gradleVersion = libs.versions.gradle.get() gradleVersion = libs.versions.gradle.get()
distributionType = Wrapper.DistributionType.ALL distributionType = Wrapper.DistributionType.BIN
} }
ext { ext {
@ -34,9 +30,7 @@ subprojects {
apply from: rootProject.file('gradle/repositories/maven.gradle') apply from: rootProject.file('gradle/repositories/maven.gradle')
apply from: rootProject.file('gradle/compile/java.gradle') apply from: rootProject.file('gradle/compile/java.gradle')
apply from: rootProject.file('gradle/test/junit5.gradle') apply from: rootProject.file('gradle/test/junit5.gradle')
apply from: rootProject.file('gradle/quality/pmd.gradle')
apply from: rootProject.file('gradle/publish/maven.gradle') apply from: rootProject.file('gradle/publish/maven.gradle')
} }
apply from: rootProject.file('gradle/publish/sonatype.gradle') apply from: rootProject.file('gradle/publish/sonatype.gradle')
apply from: rootProject.file('gradle/publish/forgejo.gradle') apply from: rootProject.file('gradle/publish/forgejo.gradle')
apply from: rootProject.file('gradle/quality/cyclonedx.gradle')

View file

@ -47,11 +47,27 @@ public interface BasicClient extends Closeable {
boolean isIndexExists(IndexDefinition indexDefinition); boolean isIndexExists(IndexDefinition indexDefinition);
boolean isIndexEmpty(IndexDefinition indexDefinition);
String getIndexState(IndexDefinition indexDefinition); String getIndexState(IndexDefinition indexDefinition);
boolean isIndexOpen(IndexDefinition indexDefinition); boolean isIndexOpen(IndexDefinition indexDefinition);
boolean isIndexClosed(IndexDefinition indexDefinition); boolean isIndexClosed(IndexDefinition indexDefinition);
/**
* Flush the index. The cluster clears cache and completes indexing.
*
* @param indexDefinition index definition
*/
void flushIndex(IndexDefinition indexDefinition);
/**
* Refresh the index. The cluster will flush the index and prepare for search.
*
* @param indexDefinition index definition
*/
void refreshIndex(IndexDefinition indexDefinition);
ScheduledExecutorService getScheduler(); ScheduledExecutorService getScheduler();
} }

View file

@ -130,19 +130,5 @@ public interface BulkClient extends BasicClient, Flushable {
*/ */
void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit); void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit);
/**
* Refresh the index.
*
* @param indexDefinition index definition
*/
void refreshIndex(IndexDefinition indexDefinition);
/**
* Flush the index. The cluster clears cache and completes indexing.
*
* @param indexDefinition index definition
*/
void flushIndex(IndexDefinition indexDefinition);
BulkProcessor getBulkProcessor(); BulkProcessor getBulkProcessor();
} }

View file

@ -59,6 +59,10 @@ public interface IndexDefinition {
boolean isShiftEnabled(); boolean isShiftEnabled();
void setShiftNotEmpty(boolean shiftNotEmpty);
boolean isShiftNotEmpty();
void setPrune(boolean prune); void setPrune(boolean prune);
boolean isPruneEnabled(); boolean isPruneEnabled();

View file

@ -351,7 +351,20 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
return new EmptyIndexShiftResult(); return new EmptyIndexShiftResult();
} }
if (indexDefinition.isShiftEnabled()) { if (indexDefinition.isShiftEnabled()) {
logger.log(Level.INFO, "before shift, flushing index " + indexDefinition);
flushIndex(indexDefinition);
logger.log(Level.INFO, "before shift, refreshing index " + indexDefinition);
refreshIndex(indexDefinition);
if (indexDefinition.isShiftNotEmpty() && isIndexEmpty(indexDefinition)) {
logger.log(Level.WARNING, "something is wrong, the index is empty. Deleting index, disabling definition, rejecting to continue shifting: " +
indexDefinition);
deleteIndex(indexDefinition);
indexDefinition.setEnabled(false);
indexDefinition.setPrune(false);
return new EmptyIndexShiftResult();
}
if (indexDefinition.isCloseShifted()) { if (indexDefinition.isCloseShifted()) {
logger.log(Level.INFO, "before shift, closing all previous indices of " + indexDefinition);
getAlias(indexDefinition.getIndex()).stream() getAlias(indexDefinition.getIndex()).stream()
.filter(s -> !s.equals(indexDefinition.getFullIndexName())) .filter(s -> !s.equals(indexDefinition.getFullIndexName()))
.forEach(this::closeIndex); .forEach(this::closeIndex);

View file

@ -16,6 +16,10 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.flush.FlushAction;
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchAction;
@ -212,6 +216,11 @@ public abstract class AbstractBasicClient implements BasicClient {
return indicesExistsResponse.isExists(); return indicesExistsResponse.isExists();
} }
@Override
public boolean isIndexEmpty(IndexDefinition indexDefinition) {
return getSearchableDocs(indexDefinition) == 0L;
}
@Override @Override
public String getIndexState(IndexDefinition indexDefinition) { public String getIndexState(IndexDefinition indexDefinition) {
ClusterStateRequest clusterStateRequest = new ClusterStateRequest(); ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
@ -251,6 +260,24 @@ public abstract class AbstractBasicClient implements BasicClient {
return "OPEN".equals(state); return "OPEN".equals(state);
} }
@Override
public void flushIndex(IndexDefinition indexDefinition) {
if (isIndexDefinitionDisabled(indexDefinition)) {
return;
}
ensureClientIsPresent();
client.execute(FlushAction.INSTANCE, new FlushRequest(indexDefinition.getFullIndexName())).actionGet();
}
@Override
public void refreshIndex(IndexDefinition indexDefinition) {
if (isIndexDefinitionDisabled(indexDefinition)) {
return;
}
ensureClientIsPresent();
client.execute(RefreshAction.INSTANCE, new RefreshRequest(indexDefinition.getFullIndexName())).actionGet();
}
protected abstract ElasticsearchClient createClient(Settings settings); protected abstract ElasticsearchClient createClient(Settings settings);
protected abstract void closeClient(Settings settings); protected abstract void closeClient(Settings settings);

View file

@ -251,21 +251,4 @@ public abstract class AbstractBulkClient extends AbstractBasicClient implements
super.updateIndexSetting(index, key, value, timeout, timeUnit); super.updateIndexSetting(index, key, value, timeout, timeUnit);
} }
@Override
public void flushIndex(IndexDefinition indexDefinition) {
if (isIndexDefinitionDisabled(indexDefinition)) {
return;
}
ensureClientIsPresent();
client.execute(FlushAction.INSTANCE, new FlushRequest(indexDefinition.getFullIndexName())).actionGet();
}
@Override
public void refreshIndex(IndexDefinition indexDefinition) {
if (isIndexDefinitionDisabled(indexDefinition)) {
return;
}
ensureClientIsPresent();
client.execute(RefreshAction.INSTANCE, new RefreshRequest(indexDefinition.getFullIndexName())).actionGet();
}
} }

View file

@ -44,6 +44,8 @@ public class DefaultIndexDefinition implements IndexDefinition {
private boolean shift; private boolean shift;
private boolean shiftNotEmpty;
private boolean prune; private boolean prune;
private boolean forcemerge; private boolean forcemerge;
@ -74,6 +76,7 @@ public class DefaultIndexDefinition implements IndexDefinition {
setFullIndexName(index + getDateTimeFormatter().format(LocalDateTime.now())); setFullIndexName(index + getDateTimeFormatter().format(LocalDateTime.now()));
setShardCount(1); setShardCount(1);
setShift(false); setShift(false);
setShiftNotEmpty(false);
setPrune(false); setPrune(false);
setForceMerge(false); setForceMerge(false);
setCloseShifted(false); setCloseShifted(false);
@ -265,6 +268,16 @@ public class DefaultIndexDefinition implements IndexDefinition {
return shift; return shift;
} }
@Override
public void setShiftNotEmpty(boolean shiftNotEmpty) {
this.shiftNotEmpty = shiftNotEmpty;
}
@Override
public boolean isShiftNotEmpty() {
return shiftNotEmpty;
}
@Override @Override
public void setPrune(boolean prune) { public void setPrune(boolean prune) {
this.prune = prune; this.prune = prune;

View file

@ -6,7 +6,8 @@ public enum Parameters {
PORT("port", Integer.class, 9300), PORT("port", Integer.class, 9300),
CLUSTER_TARGET_HEALTH("cluster.target_health", String.class, "GREEN"), // yellow and not green, because there may be parallel indexing
CLUSTER_TARGET_HEALTH("cluster.target_health", String.class, "YELLOW"),
CLUSTER_TARGET_HEALTH_TIMEOUT("cluster.target_health_timeout", String.class, "30m"), CLUSTER_TARGET_HEALTH_TIMEOUT("cluster.target_health_timeout", String.class, "30m"),

View file

@ -0,0 +1,31 @@
package org.xbib.elx.http.action.admin.indices.flush;
import org.elasticsearch.action.admin.indices.flush.FlushAction;
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
import org.elasticsearch.action.admin.indices.flush.FlushResponse;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.xcontent.XContentParser;
import org.xbib.elx.http.HttpAction;
import org.xbib.net.http.client.HttpResponse;
import org.xbib.net.http.client.netty.HttpRequestBuilder;
import java.io.IOException;
public class HttpFlushIndexAction extends HttpAction<FlushRequest, FlushResponse> {
@Override
public FlushAction getActionInstance() {
return FlushAction.INSTANCE;
}
@Override
protected HttpRequestBuilder createHttpRequest(String url, FlushRequest request) {
String index = request.indices() != null ? String.join(",", request.indices()) + "/" : "";
return newPostRequest(url, "/" + index + "_flush");
}
@Override
protected CheckedFunction<XContentParser, FlushResponse, IOException> entityParser(HttpResponse httpResponse) {
return FlushResponse::fromXContent;
}
}

View file

@ -8,6 +8,7 @@ org.xbib.elx.http.action.admin.indices.close.HttpCloseIndexAction
org.xbib.elx.http.action.admin.indices.create.HttpCreateIndexAction org.xbib.elx.http.action.admin.indices.create.HttpCreateIndexAction
org.xbib.elx.http.action.admin.indices.delete.HttpDeleteIndexAction org.xbib.elx.http.action.admin.indices.delete.HttpDeleteIndexAction
org.xbib.elx.http.action.admin.indices.exists.indices.HttpIndicesExistsAction org.xbib.elx.http.action.admin.indices.exists.indices.HttpIndicesExistsAction
org.xbib.elx.http.action.admin.indices.flush.HttpFlushIndexAction
org.xbib.elx.http.action.admin.indices.forcemerge.HttpForceMergeAction org.xbib.elx.http.action.admin.indices.forcemerge.HttpForceMergeAction
org.xbib.elx.http.action.admin.indices.get.HttpGetIndexAction org.xbib.elx.http.action.admin.indices.get.HttpGetIndexAction
org.xbib.elx.http.action.admin.indices.mapping.get.HttpGetMappingsAction org.xbib.elx.http.action.admin.indices.mapping.get.HttpGetMappingsAction

View file

@ -1,3 +1,3 @@
group = org.xbib group = org.xbib
name = elx name = elx
version = 7.10.2.39 version = 7.10.2.48

Binary file not shown.

View file

@ -1,6 +1,6 @@
distributionBase=GRADLE_USER_HOME distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-all.zip distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip
networkTimeout=10000 networkTimeout=10000
validateDistributionUrl=true validateDistributionUrl=true
zipStoreBase=GRADLE_USER_HOME zipStoreBase=GRADLE_USER_HOME

20
gradlew.bat vendored
View file

@ -43,11 +43,11 @@ set JAVA_EXE=java.exe
%JAVA_EXE% -version >NUL 2>&1 %JAVA_EXE% -version >NUL 2>&1
if %ERRORLEVEL% equ 0 goto execute if %ERRORLEVEL% equ 0 goto execute
echo. echo. 1>&2
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
echo. echo. 1>&2
echo Please set the JAVA_HOME variable in your environment to match the echo Please set the JAVA_HOME variable in your environment to match the 1>&2
echo location of your Java installation. echo location of your Java installation. 1>&2
goto fail goto fail
@ -57,11 +57,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe
if exist "%JAVA_EXE%" goto execute if exist "%JAVA_EXE%" goto execute
echo. echo. 1>&2
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
echo. echo. 1>&2
echo Please set the JAVA_HOME variable in your environment to match the echo Please set the JAVA_HOME variable in your environment to match the 1>&2
echo location of your Java installation. echo location of your Java installation. 1>&2
goto fail goto fail

View file

@ -1,17 +1,17 @@
dependencyResolutionManagement { dependencyResolutionManagement {
versionCatalogs { versionCatalogs {
libs { libs {
version('gradle', '8.5') version('gradle', '8.7')
version('elasticsearch', '7.10.2') version('elasticsearch', '7.10.2')
// ES 7.10.2 uses lucene 8.7.0 // ES 7.10.2 uses lucene 8.7.0
version('lucene', '8.7.0') version('lucene', '8.7.0')
// ES 7.10.2 uses log4j2 2.11.1 // ES 7.10.2 uses log4j2 2.11.1
version('log4j', '2.22.0') version('log4j', '2.23.1')
// ES 7.10.2 uses netty 4.1.49 // ES 7.10.2 uses netty 4.1.49
version('netty', '4.1.107.Final') version('netty', '4.1.111.Final')
// ES 7.10.2 uses Jackson 2.10.4 // ES 7.10.2 uses Jackson 2.10.4
version('jackson', '2.16.0') version('jackson', '2.17.1')
version('net-http', '4.2.0') // beware, net-http must match netty version version('net-http', '4.8.1')
library('log4j-api', 'org.apache.logging.log4j', 'log4j-api').versionRef('log4j') library('log4j-api', 'org.apache.logging.log4j', 'log4j-api').versionRef('log4j')
library('log4j-core', 'org.apache.logging.log4j', 'log4j-core').versionRef('log4j') library('log4j-core', 'org.apache.logging.log4j', 'log4j-core').versionRef('log4j')
library('log4j-slf4j', 'org.apache.logging.log4j', 'log4j-slf4j-impl').versionRef('log4j') library('log4j-slf4j', 'org.apache.logging.log4j', 'log4j-slf4j-impl').versionRef('log4j')
@ -24,9 +24,6 @@ dependencyResolutionManagement {
library('lucene-queryparser', 'org.apache.lucene', 'lucene-queryparser').versionRef('lucene') library('lucene-queryparser', 'org.apache.lucene', 'lucene-queryparser').versionRef('lucene')
library('lucene-grouping', 'org.apache.lucene', 'lucene-grouping').versionRef('lucene') library('lucene-grouping', 'org.apache.lucene', 'lucene-grouping').versionRef('lucene')
library('lucene-misc', 'org.apache.lucene', 'lucene-misc').versionRef('lucene') library('lucene-misc', 'org.apache.lucene', 'lucene-misc').versionRef('lucene')
library('hppc', 'com.carrotsearch', 'hppc').version('0.8.1')
library('joda', 'joda-time', 'joda-time').version('2.10.4')
library('tdigest', 'com.tdunning', 't-digest').version('3.2')
library('es-plugin-transport-netty4', 'org.elasticsearch.plugin', 'transport-netty4-client').versionRef('elasticsearch') library('es-plugin-transport-netty4', 'org.elasticsearch.plugin', 'transport-netty4-client').versionRef('elasticsearch')
library('jackson', 'com.fasterxml.jackson.core', 'jackson-core').versionRef('jackson') library('jackson', 'com.fasterxml.jackson.core', 'jackson-core').versionRef('jackson')
library('jackson.smile', 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-smile').versionRef('jackson') library('jackson.smile', 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-smile').versionRef('jackson')
@ -36,14 +33,18 @@ dependencyResolutionManagement {
library('netty-handler', 'io.netty', 'netty-handler').versionRef('netty') library('netty-handler', 'io.netty', 'netty-handler').versionRef('netty')
library('netty-buffer', 'io.netty', 'netty-buffer').versionRef('netty') library('netty-buffer', 'io.netty', 'netty-buffer').versionRef('netty')
library('netty-transport', 'io.netty', 'netty-transport').versionRef('netty') library('netty-transport', 'io.netty', 'netty-transport').versionRef('netty')
library('net-http-netty-client', 'org.xbib', 'net-http-client-netty').versionRef('net-http')
library('jna', 'net.java.dev.jna', 'jna').version('5.14.0') library('jna', 'net.java.dev.jna', 'jna').version('5.14.0')
library('snakeyaml', 'org.yaml', 'snakeyaml').version('2.2') library('snakeyaml', 'org.yaml', 'snakeyaml').version('2.2')
library('metrics', 'org.xbib', 'metrics-common').version('3.0.0') library('hppc', 'com.carrotsearch', 'hppc').version('0.8.1')
library('time', 'org.xbib', 'time').version('3.0.0') library('joda', 'joda-time', 'joda-time').version('2.10.4')
library('tdigest', 'com.tdunning', 't-digest').version('3.2')
// xbib
library('net-http-netty-client', 'org.xbib', 'net-http-client-netty').versionRef('net-http')
library('metrics', 'org.xbib', 'metrics-common').version('4.0.0')
library('time', 'org.xbib', 'time').version('4.0.0')
} }
testLibs { testLibs {
version('junit', '5.10.1') version('junit', '5.10.2')
library('junit-jupiter-api', 'org.junit.jupiter', 'junit-jupiter-api').versionRef('junit') library('junit-jupiter-api', 'org.junit.jupiter', 'junit-jupiter-api').versionRef('junit')
library('junit-jupiter-params', 'org.junit.jupiter', 'junit-jupiter-params').versionRef('junit') library('junit-jupiter-params', 'org.junit.jupiter', 'junit-jupiter-params').versionRef('junit')
library('junit-jupiter-engine', 'org.junit.jupiter', 'junit-jupiter-engine').versionRef('junit') library('junit-jupiter-engine', 'org.junit.jupiter', 'junit-jupiter-engine').versionRef('junit')