diff --git a/build.gradle b/build.gradle index 800b799..292a912 100644 --- a/build.gradle +++ b/build.gradle @@ -5,20 +5,6 @@ plugins { id "org.xbib.gradle.plugin.asciidoctor" version "1.5.6.0.1" } -printf "Host: %s\nOS: %s %s %s\nJVM: %s %s %s %s\nGradle: %s Groovy: %s Java: %s\n" + - "Build: group: ${project.group} name: ${project.name} version: ${project.version}\n", - InetAddress.getLocalHost(), - System.getProperty("os.name"), - System.getProperty("os.arch"), - System.getProperty("os.version"), - System.getProperty("java.version"), - System.getProperty("java.vm.version"), - System.getProperty("java.vm.vendor"), - System.getProperty("java.vm.name"), - gradle.gradleVersion, - GroovySystem.getVersion(), - JavaVersion.current() - if (JavaVersion.current() < JavaVersion.VERSION_11) { throw new GradleException("The build must be run with Java 11") } @@ -34,7 +20,6 @@ subprojects { configurations { asciidoclet - wagon } dependencies { @@ -44,7 +29,6 @@ subprojects { testImplementation "org.apache.logging.log4j:log4j-jul:${project.property('log4j.version')}" testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:${project.property('log4j.version')}" asciidoclet "org.xbib:asciidoclet:${project.property('asciidoclet.version')}" - wagon "org.apache.maven.wagon:wagon-ssh:${project.property('wagon.version')}" } compileJava { @@ -110,8 +94,6 @@ subprojects { archives javadocJar, sourcesJar } - apply from: "${rootProject.projectDir}/gradle/publish.gradle" - spotbugs { effort = "max" reportLevel = "low" @@ -162,4 +144,68 @@ subprojects { property "sonar.junit.reportsPath", "build/test-results/test/" } } + + ext { + description = 'Extensions for Elasticsearch clients (node and transport)' + scmUrl = 'https://github.com/jprante/elx' + scmConnection = 'scm:git:git://github.com/jprante/elx.git' + scmDeveloperConnection = 'scm:git:git://github.com/jprante/elx.git' + } + + task sonaTypeUpload(type: Upload, dependsOn: build) { + group = 'publish' + configuration = configurations.archives + uploadDescriptor = true + repositories { + if (project.hasProperty('ossrhUsername')) { + mavenDeployer { + beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } + repository(url: 'https://oss.sonatype.org/service/local/staging/deploy/maven2') { + authentication(userName: ossrhUsername, password: ossrhPassword) + } + snapshotRepository(url: 'https://oss.sonatype.org/content/repositories/snapshots') { + authentication(userName: ossrhUsername, password: ossrhPassword) + } + pom.project { + groupId project.group + artifactId project.name + version project.version + name project.name + description description + packaging 'jar' + inceptionYear '2019' + url scmUrl + organization { + name 'xbib' + url 'http://xbib.org' + } + developers { + developer { + id 'xbib' + name 'Jörg Prante' + email 'joergprante@gmail.com' + url 'https://github.com/jprante' + } + } + scm { + url scmUrl + connection scmConnection + developerConnection scmDeveloperConnection + } + licenses { + license { + name 'The Apache License, Version 2.0' + url 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + } + } + } + } + } + + nexusStaging { + packageGroup = "org.xbib" + } + } \ No newline at end of file diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java b/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java index 52b6d04..cb994e0 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java @@ -31,7 +31,7 @@ public interface BulkProcessor extends Closeable, Flushable { /** * A listener for the execution. */ - public interface Listener { + interface Listener { /** * Callback before the bulk is executed. diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java index f6af929..dbd7fa6 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java @@ -49,6 +49,8 @@ public class DefaultBulkController implements BulkController { private AtomicBoolean active; + private boolean enableBulkLogging; + public DefaultBulkController(ExtendedClient client, BulkMetric bulkMetric) { this.client = client; this.bulkMetric = bulkMetric; @@ -76,11 +78,8 @@ public class DefaultBulkController implements BulkController { ByteSizeValue maxVolumePerRequest = settings.getAsBytesSize(Parameters.MAX_VOLUME_PER_REQUEST.name(), ByteSizeValue.parseBytesSizeValue(Parameters.DEFAULT_MAX_VOLUME_PER_REQUEST.getString(), "maxVolumePerRequest")); - if (logger.isInfoEnabled()) { - logger.info("bulk processor up with maxActionsPerRequest = {} maxConcurrentRequests = {} " + - "flushIngestInterval = {} maxVolumePerRequest = {}", - maxActionsPerRequest, maxConcurrentRequests, flushIngestInterval, maxVolumePerRequest); - } + this.enableBulkLogging = settings.getAsBoolean(Parameters.ENABLE_BULK_LOGGING.name(), + Parameters.ENABLE_BULK_LOGGING.getValue()); this.bulkListener = new BulkListener(); this.bulkProcessor = DefaultBulkProcessor.builder(client.getClient(), bulkListener) .setBulkActions(maxActionsPerRequest) @@ -89,6 +88,12 @@ public class DefaultBulkController implements BulkController { .setBulkSize(maxVolumePerRequest) .build(); this.active.set(true); + if (logger.isInfoEnabled()) { + logger.info("bulk processor set up with maxActionsPerRequest = {} maxConcurrentRequests = {} " + + "flushIngestInterval = {} maxVolumePerRequest = {}, bulk logging = {}", + maxActionsPerRequest, maxConcurrentRequests, flushIngestInterval, maxVolumePerRequest, + enableBulkLogging); + } } @Override @@ -240,7 +245,7 @@ public class DefaultBulkController implements BulkController { private class BulkListener implements DefaultBulkProcessor.Listener { - private final Logger logger = LogManager.getLogger("org.xbib.elx.BulkProcessor.Listener"); + private final Logger logger = LogManager.getLogger(BulkListener.class.getName()); private Throwable lastBulkError = null; @@ -255,7 +260,7 @@ public class DefaultBulkController implements BulkController { bulkMetric.getCurrentIngestNumDocs().inc(n); bulkMetric.getTotalIngestSizeInBytes().inc(request.estimatedSizeInBytes()); } - if (logger.isDebugEnabled()) { + if (enableBulkLogging && logger.isDebugEnabled()) { logger.debug("before bulk [{}] [actions={}] [bytes={}] [concurrent requests={}]", executionId, request.numberOfActions(), @@ -285,7 +290,7 @@ public class DefaultBulkController implements BulkController { } } } - if (bulkMetric != null && logger.isDebugEnabled()) { + if (enableBulkLogging && bulkMetric != null && logger.isDebugEnabled()) { logger.debug("after bulk [{}] [succeeded={}] [failed={}] [{}ms] {} concurrent requests", executionId, bulkMetric.getSucceeded().getCount(), @@ -294,7 +299,7 @@ public class DefaultBulkController implements BulkController { l); } if (n > 0) { - if (logger.isErrorEnabled()) { + if (enableBulkLogging && logger.isErrorEnabled()) { logger.error("bulk [{}] failed with {} failed items, failure message = {}", executionId, n, response.buildFailureMessage()); } @@ -312,7 +317,7 @@ public class DefaultBulkController implements BulkController { } lastBulkError = failure; active.set(false); - if (logger.isErrorEnabled()) { + if (enableBulkLogging && logger.isErrorEnabled()) { logger.error("after bulk [" + executionId + "] error", failure); } } diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java index 5cc6cdd..4f91c14 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java @@ -6,7 +6,6 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; -import java.io.IOException; import java.util.concurrent.TimeUnit; /** diff --git a/elx-common/src/main/java/org/xbib/elx/common/Parameters.java b/elx-common/src/main/java/org/xbib/elx/common/Parameters.java index 28d10d7..73819e1 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/Parameters.java +++ b/elx-common/src/main/java/org/xbib/elx/common/Parameters.java @@ -2,6 +2,8 @@ package org.xbib.elx.common; public enum Parameters { + ENABLE_BULK_LOGGING(false), + DEFAULT_MAX_ACTIONS_PER_REQUEST(1000), DEFAULT_MAX_CONCURRENT_REQUESTS(Runtime.getRuntime().availableProcessors()), @@ -18,10 +20,16 @@ public enum Parameters { FLUSH_INTERVAL("flush_interval"); + boolean flag; + int num; String string; + Parameters(boolean flag) { + this.flag = flag; + } + Parameters(int num) { this.num = num; } @@ -30,6 +38,10 @@ public enum Parameters { this.string = string; } + boolean getValue() { + return flag; + } + int getNum() { return num; } diff --git a/gradle.properties b/gradle.properties index 7e22f6c..f941f54 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,10 +1,10 @@ group = org.xbib name = elx -version = 2.2.1.9 +version = 2.2.1.10 xbib-metrics.version = 1.2.0 xbib-guice.version = 4.0.4 -xbib-netty-http.version = 4.1.35.0 +xbib-netty-http.version = 4.1.36.7 elasticsearch.version = 2.2.1 jackson.version = 2.6.7 @@ -13,8 +13,10 @@ log4j.version = 2.11.1 mustache.version = 0.9.5 jts.version = 1.13 +# test junit.version = 5.4.2 -wagon.version = 3.0.0 + +# docs asciidoclet.version = 1.5.4 org.gradle.warning.mode = all diff --git a/gradle/publish.gradle b/gradle/publish.gradle index 4b652e4..e69de29 100644 --- a/gradle/publish.gradle +++ b/gradle/publish.gradle @@ -1,62 +0,0 @@ -ext { - description = 'Extensions for Elasticsearch clients (node and transport)' - scmUrl = 'https://github.com/jprante/elx' - scmConnection = 'scm:git:git://github.com/jprante/elx.git' - scmDeveloperConnection = 'scm:git:git://github.com/jprante/elx.git' -} - -task sonaTypeUpload(type: Upload, dependsOn: build) { - group = 'publish' - configuration = configurations.archives - uploadDescriptor = true - repositories { - if (project.hasProperty('ossrhUsername')) { - mavenDeployer { - beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } - repository(url: 'https://oss.sonatype.org/service/local/staging/deploy/maven2') { - authentication(userName: ossrhUsername, password: ossrhPassword) - } - snapshotRepository(url: 'https://oss.sonatype.org/content/repositories/snapshots') { - authentication(userName: ossrhUsername, password: ossrhPassword) - } - pom.project { - groupId project.group - artifactId project.name - version project.version - name project.name - description description - packaging 'jar' - inceptionYear '2019' - url scmUrl - organization { - name 'xbib' - url 'http://xbib.org' - } - developers { - developer { - id 'xbib' - name 'Jörg Prante' - email 'joergprante@gmail.com' - url 'https://github.com/jprante' - } - } - scm { - url scmUrl - connection scmConnection - developerConnection scmDeveloperConnection - } - licenses { - license { - name 'The Apache License, Version 2.0' - url 'http://www.apache.org/licenses/LICENSE-2.0.txt' - } - } - } - } - } - } -} - -nexusStaging { - packageGroup = "org.xbib" -} diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index c5cbb11..f3a4f28 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Thu May 02 09:39:03 CEST 2019 +#Tue Jul 23 11:33:24 CEST 2019 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.3-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-5.3.1-all.zip