add flush and refresh action to index shift operation

build.gradle

@@ -1,18 +1,14 @@
 
 plugins {
-    id "checkstyle"
-    id "pmd"
     id 'maven-publish'
     id 'signing'
     id "io.github.gradle-nexus.publish-plugin" version "2.0.0-rc-1"
-    id "com.github.spotbugs" version "6.0.0-beta.3"
-    id "org.cyclonedx.bom" version "1.7.4"
     id "org.xbib.gradle.plugin.asciidoctor" version "3.0.0"
 }
 
 wrapper {
     gradleVersion = libs.versions.gradle.get()
-    distributionType = Wrapper.DistributionType.ALL
+    distributionType = Wrapper.DistributionType.BIN
 }
 
 ext {
@@ -34,9 +30,7 @@ subprojects {
     apply from: rootProject.file('gradle/repositories/maven.gradle')
     apply from: rootProject.file('gradle/compile/java.gradle')
     apply from: rootProject.file('gradle/test/junit5.gradle')
-    apply from: rootProject.file('gradle/quality/pmd.gradle')
     apply from: rootProject.file('gradle/publish/maven.gradle')
 }
 apply from: rootProject.file('gradle/publish/sonatype.gradle')
 apply from: rootProject.file('gradle/publish/forgejo.gradle')
-apply from: rootProject.file('gradle/quality/cyclonedx.gradle')

elx-api/src/main/java/org/xbib/elx/api/BasicClient.java

@@ -47,11 +47,27 @@ public interface BasicClient extends Closeable {
 
     boolean isIndexExists(IndexDefinition indexDefinition);
 
+    boolean isIndexEmpty(IndexDefinition indexDefinition);
+
     String getIndexState(IndexDefinition indexDefinition);
 
     boolean isIndexOpen(IndexDefinition indexDefinition);
 
     boolean isIndexClosed(IndexDefinition indexDefinition);
 
+    /**
+     * Flush the index. The cluster clears cache and completes indexing.
+     *
+     * @param indexDefinition index definition
+     */
+    void flushIndex(IndexDefinition indexDefinition);
+
+    /**
+     * Refresh the index. The cluster will flush the index and prepare for search.
+     *
+     * @param indexDefinition index definition
+     */
+    void refreshIndex(IndexDefinition indexDefinition);
+
     ScheduledExecutorService getScheduler();
 }

elx-api/src/main/java/org/xbib/elx/api/BulkClient.java

@@ -130,19 +130,5 @@ public interface BulkClient extends BasicClient, Flushable {
      */
     void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit);
 
-    /**
-     * Refresh the index.
-     *
-     * @param indexDefinition index definition
-     */
-    void refreshIndex(IndexDefinition indexDefinition);
-
-    /**
-     * Flush the index. The cluster clears cache and completes indexing.
-     *
-     * @param indexDefinition index definition
-     */
-    void flushIndex(IndexDefinition indexDefinition);
-
     BulkProcessor getBulkProcessor();
 }

elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java

@@ -59,6 +59,10 @@ public interface IndexDefinition {
 
     boolean isShiftEnabled();
 
+    void setShiftNotEmpty(boolean shiftNotEmpty);
+
+    boolean isShiftNotEmpty();
+
     void setPrune(boolean prune);
 
     boolean isPruneEnabled();

elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java

@@ -351,7 +351,20 @@ public abstract class AbstractAdminClient extends AbstractBasicClient implements
             return new EmptyIndexShiftResult();
         }
         if (indexDefinition.isShiftEnabled()) {
+            logger.log(Level.INFO, "before shift, flushing index " + indexDefinition);
+            flushIndex(indexDefinition);
+            logger.log(Level.INFO, "before shift, refreshing index " + indexDefinition);
+            refreshIndex(indexDefinition);
+            if (indexDefinition.isShiftNotEmpty() && isIndexEmpty(indexDefinition)) {
+                logger.log(Level.WARNING, "something is wrong, the index is empty. Deleting index, disabling definition, rejecting to continue shifting: " +
+                        indexDefinition);
+                deleteIndex(indexDefinition);
+                indexDefinition.setEnabled(false);
+                indexDefinition.setPrune(false);
+                return new EmptyIndexShiftResult();
+            }
             if (indexDefinition.isCloseShifted()) {
+                logger.log(Level.INFO, "before shift, closing all previous indices of " + indexDefinition);
                 getAlias(indexDefinition.getIndex()).stream()
                         .filter(s -> !s.equals(indexDefinition.getFullIndexName()))
                         .forEach(this::closeIndex);

elx-common/src/main/java/org/xbib/elx/common/AbstractBasicClient.java

@@ -16,6 +16,10 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
 import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
 import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
+import org.elasticsearch.action.admin.indices.flush.FlushAction;
+import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
 import org.elasticsearch.action.search.SearchAction;
@@ -212,6 +216,11 @@ public abstract class AbstractBasicClient implements BasicClient {
         return indicesExistsResponse.isExists();
     }
 
+    @Override
+    public boolean isIndexEmpty(IndexDefinition indexDefinition) {
+        return getSearchableDocs(indexDefinition) == 0L;
+    }
+
     @Override
     public String getIndexState(IndexDefinition indexDefinition) {
         ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
@@ -251,6 +260,24 @@ public abstract class AbstractBasicClient implements BasicClient {
         return "OPEN".equals(state);
     }
 
+    @Override
+    public void flushIndex(IndexDefinition indexDefinition) {
+        if (isIndexDefinitionDisabled(indexDefinition)) {
+            return;
+        }
+        ensureClientIsPresent();
+        client.execute(FlushAction.INSTANCE, new FlushRequest(indexDefinition.getFullIndexName())).actionGet();
+    }
+
+    @Override
+    public void refreshIndex(IndexDefinition indexDefinition) {
+        if (isIndexDefinitionDisabled(indexDefinition)) {
+            return;
+        }
+        ensureClientIsPresent();
+        client.execute(RefreshAction.INSTANCE, new RefreshRequest(indexDefinition.getFullIndexName())).actionGet();
+    }
+
     protected abstract ElasticsearchClient createClient(Settings settings);
 
     protected abstract void closeClient(Settings settings);

elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java

@@ -251,21 +251,4 @@ public abstract class AbstractBulkClient extends AbstractBasicClient implements
         super.updateIndexSetting(index, key, value, timeout, timeUnit);
     }
 
-    @Override
-    public void flushIndex(IndexDefinition indexDefinition) {
-        if (isIndexDefinitionDisabled(indexDefinition)) {
-            return;
-        }
-        ensureClientIsPresent();
-        client.execute(FlushAction.INSTANCE, new FlushRequest(indexDefinition.getFullIndexName())).actionGet();
-    }
-
-    @Override
-    public void refreshIndex(IndexDefinition indexDefinition) {
-        if (isIndexDefinitionDisabled(indexDefinition)) {
-            return;
-        }
-        ensureClientIsPresent();
-        client.execute(RefreshAction.INSTANCE, new RefreshRequest(indexDefinition.getFullIndexName())).actionGet();
-    }
 }

elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java

@@ -44,6 +44,8 @@ public class DefaultIndexDefinition implements IndexDefinition {
 
     private boolean shift;
 
+    private boolean shiftNotEmpty;
+
     private boolean prune;
 
     private boolean forcemerge;
@@ -74,6 +76,7 @@ public class DefaultIndexDefinition implements IndexDefinition {
         setFullIndexName(index + getDateTimeFormatter().format(LocalDateTime.now()));
         setShardCount(1);
         setShift(false);
+        setShiftNotEmpty(false);
         setPrune(false);
         setForceMerge(false);
         setCloseShifted(false);
@@ -265,6 +268,16 @@ public class DefaultIndexDefinition implements IndexDefinition {
         return shift;
     }
 
+    @Override
+    public void setShiftNotEmpty(boolean shiftNotEmpty) {
+        this.shiftNotEmpty = shiftNotEmpty;
+    }
+
+    @Override
+    public boolean isShiftNotEmpty() {
+        return shiftNotEmpty;
+    }
+
     @Override
     public void setPrune(boolean prune) {
         this.prune = prune;

elx-common/src/main/java/org/xbib/elx/common/Parameters.java

@@ -6,7 +6,8 @@ public enum Parameters {
 
     PORT("port", Integer.class, 9300),
 
-    CLUSTER_TARGET_HEALTH("cluster.target_health", String.class, "GREEN"),
+    // yellow and not green, because there may be parallel indexing
+    CLUSTER_TARGET_HEALTH("cluster.target_health", String.class, "YELLOW"),
 
     CLUSTER_TARGET_HEALTH_TIMEOUT("cluster.target_health_timeout", String.class, "30m"),
 

elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/flush/HttpFlushIndexAction.java

@@ -0,0 +1,31 @@
+package org.xbib.elx.http.action.admin.indices.flush;
+
+import org.elasticsearch.action.admin.indices.flush.FlushAction;
+import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.admin.indices.flush.FlushResponse;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.xbib.elx.http.HttpAction;
+import org.xbib.net.http.client.HttpResponse;
+import org.xbib.net.http.client.netty.HttpRequestBuilder;
+
+import java.io.IOException;
+
+public class HttpFlushIndexAction extends HttpAction<FlushRequest, FlushResponse> {
+
+    @Override
+    public FlushAction getActionInstance() {
+        return FlushAction.INSTANCE;
+    }
+
+    @Override
+    protected HttpRequestBuilder createHttpRequest(String url, FlushRequest request) {
+        String index = request.indices() != null ? String.join(",", request.indices()) + "/" : "";
+        return newPostRequest(url, "/" + index + "_flush");
+    }
+
+    @Override
+    protected CheckedFunction<XContentParser, FlushResponse, IOException> entityParser(HttpResponse httpResponse) {
+        return FlushResponse::fromXContent;
+    }
+}

elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction

@@ -8,6 +8,7 @@ org.xbib.elx.http.action.admin.indices.close.HttpCloseIndexAction
 org.xbib.elx.http.action.admin.indices.create.HttpCreateIndexAction
 org.xbib.elx.http.action.admin.indices.delete.HttpDeleteIndexAction
 org.xbib.elx.http.action.admin.indices.exists.indices.HttpIndicesExistsAction
+org.xbib.elx.http.action.admin.indices.flush.HttpFlushIndexAction
 org.xbib.elx.http.action.admin.indices.forcemerge.HttpForceMergeAction
 org.xbib.elx.http.action.admin.indices.get.HttpGetIndexAction
 org.xbib.elx.http.action.admin.indices.mapping.get.HttpGetMappingsAction

gradle.properties

@@ -1,3 +1,3 @@
 group = org.xbib
 name = elx
-version = 7.10.2.39
+version = 7.10.2.48

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

gradlew.bat

@@ -43,11 +43,11 @@ set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
 if %ERRORLEVEL% equ 0 goto execute
 
-echo.
+echo. 1>&2
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
-echo.
+echo. 1>&2
-echo Please set the JAVA_HOME variable in your environment to match the
+echo Please set the JAVA_HOME variable in your environment to match the 1>&2
-echo location of your Java installation.
+echo location of your Java installation. 1>&2
 
 goto fail
 
@@ -57,11 +57,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe
 
 if exist "%JAVA_EXE%" goto execute
 
-echo.
+echo. 1>&2
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
-echo.
+echo. 1>&2
-echo Please set the JAVA_HOME variable in your environment to match the
+echo Please set the JAVA_HOME variable in your environment to match the 1>&2
-echo location of your Java installation.
+echo location of your Java installation. 1>&2
 
 goto fail
 

settings.gradle

@@ -1,17 +1,17 @@
 dependencyResolutionManagement {
     versionCatalogs {
         libs {
-            version('gradle', '8.5')
+            version('gradle', '8.7')
             version('elasticsearch', '7.10.2')
             // ES 7.10.2 uses lucene 8.7.0
             version('lucene', '8.7.0')
             // ES 7.10.2 uses log4j2 2.11.1
-            version('log4j', '2.22.0')
+            version('log4j', '2.23.1')
             // ES 7.10.2 uses netty 4.1.49
-            version('netty', '4.1.107.Final')
+            version('netty', '4.1.111.Final')
             // ES 7.10.2 uses Jackson 2.10.4
-            version('jackson', '2.16.0')
+            version('jackson', '2.17.1')
-            version('net-http', '4.2.0') // beware, net-http must match netty version
+            version('net-http', '4.8.1')
             library('log4j-api', 'org.apache.logging.log4j', 'log4j-api').versionRef('log4j')
             library('log4j-core', 'org.apache.logging.log4j', 'log4j-core').versionRef('log4j')
             library('log4j-slf4j', 'org.apache.logging.log4j', 'log4j-slf4j-impl').versionRef('log4j')
@@ -24,9 +24,6 @@ dependencyResolutionManagement {
             library('lucene-queryparser', 'org.apache.lucene', 'lucene-queryparser').versionRef('lucene')
             library('lucene-grouping', 'org.apache.lucene', 'lucene-grouping').versionRef('lucene')
             library('lucene-misc', 'org.apache.lucene', 'lucene-misc').versionRef('lucene')
-            library('hppc', 'com.carrotsearch', 'hppc').version('0.8.1')
-            library('joda', 'joda-time', 'joda-time').version('2.10.4')
-            library('tdigest', 'com.tdunning', 't-digest').version('3.2')
             library('es-plugin-transport-netty4', 'org.elasticsearch.plugin', 'transport-netty4-client').versionRef('elasticsearch')
             library('jackson', 'com.fasterxml.jackson.core', 'jackson-core').versionRef('jackson')
             library('jackson.smile', 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-smile').versionRef('jackson')
@@ -36,14 +33,18 @@ dependencyResolutionManagement {
             library('netty-handler', 'io.netty', 'netty-handler').versionRef('netty')
             library('netty-buffer', 'io.netty', 'netty-buffer').versionRef('netty')
             library('netty-transport', 'io.netty', 'netty-transport').versionRef('netty')
-            library('net-http-netty-client', 'org.xbib', 'net-http-client-netty').versionRef('net-http')
             library('jna', 'net.java.dev.jna', 'jna').version('5.14.0')
             library('snakeyaml', 'org.yaml', 'snakeyaml').version('2.2')
-            library('metrics', 'org.xbib', 'metrics-common').version('3.0.0')
+            library('hppc', 'com.carrotsearch', 'hppc').version('0.8.1')
-            library('time', 'org.xbib', 'time').version('3.0.0')
+            library('joda', 'joda-time', 'joda-time').version('2.10.4')
+            library('tdigest', 'com.tdunning', 't-digest').version('3.2')
+            // xbib
+            library('net-http-netty-client', 'org.xbib', 'net-http-client-netty').versionRef('net-http')
+            library('metrics', 'org.xbib', 'metrics-common').version('4.0.0')
+            library('time', 'org.xbib', 'time').version('4.0.0')
         }
         testLibs {
-            version('junit', '5.10.1')
+            version('junit', '5.10.2')
             library('junit-jupiter-api', 'org.junit.jupiter', 'junit-jupiter-api').versionRef('junit')
             library('junit-jupiter-params', 'org.junit.jupiter', 'junit-jupiter-params').versionRef('junit')
             library('junit-jupiter-engine', 'org.junit.jupiter', 'junit-jupiter-engine').versionRef('junit')