From c0dfb9a617e3c4eaa737fa6f34a75c430411d81c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Mon, 18 Feb 2019 17:21:57 +0100 Subject: [PATCH 1/3] large refactoring, new subpackage structure, Gradle 5.2.1, Java 11 --- .gitignore | 6 +- build.gradle | 227 +- config/pmd/category/java/bestpractices.xml | 1636 ++++++++ .../pmd/category/java/categories.properties | 13 + config/pmd/category/java/codestyle.xml | 2176 +++++++++++ config/pmd/category/java/design.xml | 1657 ++++++++ config/pmd/category/java/documentation.xml | 144 + config/pmd/category/java/errorprone.xml | 3383 +++++++++++++++++ config/pmd/category/java/multithreading.xml | 393 ++ config/pmd/category/java/performance.xml | 1006 +++++ config/pmd/category/java/security.xml | 65 + elx-api/build.gradle | 19 + elx-api/build.gradle~ | 18 + .../java/org/xbib/elx/api}/BulkControl.java | 5 +- .../java/org/xbib/elx/api}/BulkMetric.java | 6 +- .../java/org/xbib/elx/api/ExtendedClient.java | 178 +- .../xbib/elx/api/ExtendedClientProvider.java | 7 + .../org/xbib/elx/api}/IndexAliasAdder.java | 5 +- .../java/org/xbib/elx/api/package-info.java | 4 + elx-common/build.gradle | 9 + elx-common/build.gradle~ | 65 + .../elx/common/AbstractExtendedClient.java | 1144 ++++++ .../org/xbib/elx/common}/BulkProcessor.java | 34 +- .../org/xbib/elx/common/ClientBuilder.java | 124 + .../xbib/elx/common/MockExtendedClient.java | 146 + .../common/MockExtendedClientProvider.java | 10 + .../java/org/xbib/elx/common/Parameters.java | 40 + .../xbib/elx/common}/SimpleBulkControl.java | 4 +- .../xbib/elx/common}/SimpleBulkMetric.java | 40 +- .../common/io/ClasspathURLStreamHandler.java | 25 + .../io/ClasspathURLStreamHandlerFactory.java | 12 + .../org/xbib/elx/common/io/package-info.java | 1 + .../common/management/IndexDefinition.java | 139 + .../elx/common/management/IndexRetention.java | 27 + .../elx/common/management/package-info.java | 1 + .../org/xbib/elx/common/package-info.java | 4 + .../xbib/elx/common/util}/NetworkUtils.java | 23 +- .../xbib/elx/common/util/package-info.java | 1 + .../services/java.net.URLStreamHandlerFactory | 1 + .../org.xbib.elx.api.ExtendedClientProvider | 1 + .../java/org/elasticsearch/node/MockNode.java | 4 - .../org/elasticsearch/node/package-info.java | 1 + .../java/org/xbib/elx/common}/AliasTest.java | 12 +- .../MockExtendedClientProviderTest.java | 16 + .../org/xbib/elx/common}/NetworkTest.java | 6 +- .../org/xbib/elx/common/NodeTestUtils.java | 213 ++ .../java/org/xbib/elx/common/SearchTest.java | 56 + .../java/org/xbib/elx/common}/SimpleTest.java | 12 +- .../org/xbib/elx/common/WildcardTest.java | 62 + .../org/xbib/elx/common/package-info.java | 1 + elx-common/src/test/resources/log4j2.xml | 13 + elx-http/build.gradle~ | 65 + elx-node/build.gradle | 3 + elx-node/build.gradle~ | 65 + .../org/xbib/elx/node/ExtendedNodeClient.java | 70 + .../elx/node/ExtendedNodeClientProvider.java | 10 + .../org.xbib.elx.api.ExtendedClientProvider | 1 + .../java/org/elasticsearch/node/MockNode.java | 34 + .../elx/node/ExtendeNodeDuplicateIDTest.java | 58 + .../ExtendedNodeClientSingleNodeTest.java | 39 + .../xbib/elx/node/ExtendedNodeClientTest.java | 177 +- .../node/ExtendedNodeClusterBlockTest.java | 20 +- .../elx/node/ExtendedNodeIndexAliasTest.java | 42 +- .../elx/node/ExtendedNodeReplicaTest.java | 32 +- .../ExtendedNodeUpdateReplicaLevelTest.java | 35 +- .../java/org/xbib/elx/node/NodeTestUtils.java | 201 + .../src/test}/resources/log4j2.xml | 2 +- elx-transport/build.gradle | 3 + elx-transport/build.gradle~ | 63 + .../transport/ExtendedTransportClient.java | 129 + .../ExtendedTransportClientProvider.java | 11 + .../xbib/elx}/transport/TransportClient.java | 49 +- .../org/xbib/elx/transport/package-info.java | 4 + .../org.xbib.elx.api.ExtendedClientProvider | 1 + .../java/org/elasticsearch/node/MockNode.java | 34 + .../org/elasticsearch/node/package-info.java | 1 + ...ExtendedTransportClientSingleNodeTest.java | 40 + .../ExtendedTransportClientTest.java | 122 +- .../ExtendedTransportDuplicateIDTest.java | 57 + .../ExtendedTransportIndexAliasTest.java | 65 + .../ExtendedTransportReplicaTest.java | 31 +- ...tendedTransportUpdateReplicaLevelTest.java | 33 +- .../xbib/elx/transport}/NodeTestUtils.java | 35 +- .../org/xbib/elx/transport/package-info.java | 1 + elx-transport/src/test/resources/log4j2.xml | 13 + gradle.properties | 20 +- gradle/ext.gradle | 8 - gradle/publish.gradle | 22 +- gradle/wrapper/gradle-wrapper.jar | Bin 54227 -> 55190 bytes gradle/wrapper/gradle-wrapper.properties | 4 +- gradlew | 8 +- gradlew.bat | 2 +- settings.gradle | 6 +- .../org/elasticsearch/node/package-info.java | 4 - .../org/xbib/elasticsearch/SearchTest.java | 70 - .../org/xbib/elasticsearch/WildcardTest.java | 69 - .../client/node/BulkNodeDuplicateIDTest.java | 62 - .../extras/client/node/package-info.java | 4 - .../extras/client/package-info.java | 4 - .../BulkTransportDuplicateIDTest.java | 60 - .../org/xbib/elasticsearch/package-info.java | 4 - .../java/suites/BulkNodeTestSuite.java | 23 - .../java/suites/BulkTransportTestSuite.java | 22 - .../java/suites/ListenerSuite.java | 23 - .../java/suites/MiscTestSuite.java | 21 - .../java/suites/TestListener.java | 44 - .../elasticsearch/extras/client/settings.json | 3 - .../extras/client/AbstractClient.java | 496 --- .../elasticsearch/extras/client/Clients.java | 105 - .../extras/client/Parameters.java | 24 - .../extras/client/node/BulkNodeClient.java | 513 --- .../extras/client/node/package-info.java | 4 - .../extras/client/package-info.java | 4 - .../client/transport/BulkTransportClient.java | 564 --- .../client/transport/MockTransportClient.java | 155 - .../extras/client/transport/package-info.java | 4 - 116 files changed, 14232 insertions(+), 2861 deletions(-) create mode 100644 config/pmd/category/java/bestpractices.xml create mode 100644 config/pmd/category/java/categories.properties create mode 100644 config/pmd/category/java/codestyle.xml create mode 100644 config/pmd/category/java/design.xml create mode 100644 config/pmd/category/java/documentation.xml create mode 100644 config/pmd/category/java/errorprone.xml create mode 100644 config/pmd/category/java/multithreading.xml create mode 100644 config/pmd/category/java/performance.xml create mode 100644 config/pmd/category/java/security.xml create mode 100644 elx-api/build.gradle create mode 100644 elx-api/build.gradle~ rename {src/main/java/org/xbib/elasticsearch/extras/client => elx-api/src/main/java/org/xbib/elx/api}/BulkControl.java (87%) rename {src/main/java/org/xbib/elasticsearch/extras/client => elx-api/src/main/java/org/xbib/elx/api}/BulkMetric.java (86%) rename src/main/java/org/xbib/elasticsearch/extras/client/ClientMethods.java => elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java (71%) create mode 100644 elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java rename {src/main/java/org/xbib/elasticsearch/extras/client => elx-api/src/main/java/org/xbib/elx/api}/IndexAliasAdder.java (80%) create mode 100644 elx-api/src/main/java/org/xbib/elx/api/package-info.java create mode 100644 elx-common/build.gradle create mode 100644 elx-common/build.gradle~ create mode 100644 elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java rename {src/main/java/org/xbib/elasticsearch/extras/client => elx-common/src/main/java/org/xbib/elx/common}/BulkProcessor.java (95%) create mode 100644 elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/Parameters.java rename {src/main/java/org/xbib/elasticsearch/extras/client => elx-common/src/main/java/org/xbib/elx/common}/SimpleBulkControl.java (95%) rename {src/main/java/org/xbib/elasticsearch/extras/client => elx-common/src/main/java/org/xbib/elx/common}/SimpleBulkMetric.java (55%) create mode 100644 elx-common/src/main/java/org/xbib/elx/common/io/ClasspathURLStreamHandler.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/io/ClasspathURLStreamHandlerFactory.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/io/package-info.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/management/IndexDefinition.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/management/IndexRetention.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/management/package-info.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/package-info.java rename {src/main/java/org/xbib/elasticsearch/extras/client => elx-common/src/main/java/org/xbib/elx/common/util}/NetworkUtils.java (92%) create mode 100644 elx-common/src/main/java/org/xbib/elx/common/util/package-info.java create mode 100644 elx-common/src/main/resources/META-INF/services/java.net.URLStreamHandlerFactory create mode 100644 elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider rename {src/integration-test => elx-common/src/test}/java/org/elasticsearch/node/MockNode.java (98%) create mode 100644 elx-common/src/test/java/org/elasticsearch/node/package-info.java rename {src/integration-test/java/org/xbib/elasticsearch => elx-common/src/test/java/org/xbib/elx/common}/AliasTest.java (92%) create mode 100644 elx-common/src/test/java/org/xbib/elx/common/MockExtendedClientProviderTest.java rename {src/integration-test/java/org/xbib/elasticsearch/extras/client => elx-common/src/test/java/org/xbib/elx/common}/NetworkTest.java (95%) create mode 100644 elx-common/src/test/java/org/xbib/elx/common/NodeTestUtils.java create mode 100644 elx-common/src/test/java/org/xbib/elx/common/SearchTest.java rename {src/integration-test/java/org/xbib/elasticsearch => elx-common/src/test/java/org/xbib/elx/common}/SimpleTest.java (91%) create mode 100644 elx-common/src/test/java/org/xbib/elx/common/WildcardTest.java create mode 100644 elx-common/src/test/java/org/xbib/elx/common/package-info.java create mode 100644 elx-common/src/test/resources/log4j2.xml create mode 100644 elx-http/build.gradle~ create mode 100644 elx-node/build.gradle create mode 100644 elx-node/build.gradle~ create mode 100644 elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java create mode 100644 elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java create mode 100644 elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider create mode 100644 elx-node/src/test/java/org/elasticsearch/node/MockNode.java create mode 100644 elx-node/src/test/java/org/xbib/elx/node/ExtendeNodeDuplicateIDTest.java create mode 100644 elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientSingleNodeTest.java rename src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClientTest.java => elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientTest.java (53%) rename src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClusterBlockTest.java => elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClusterBlockTest.java (76%) rename src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeIndexAliasTest.java => elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeIndexAliasTest.java (58%) rename src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeReplicaTest.java => elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeReplicaTest.java (78%) rename src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportUpdateReplicaLevelTest.java => elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeUpdateReplicaLevelTest.java (60%) create mode 100644 elx-node/src/test/java/org/xbib/elx/node/NodeTestUtils.java rename {src/integration-test => elx-node/src/test}/resources/log4j2.xml (80%) create mode 100644 elx-transport/build.gradle create mode 100644 elx-transport/build.gradle~ create mode 100644 elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java create mode 100644 elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java rename {src/main/java/org/xbib/elasticsearch/extras/client => elx-transport/src/main/java/org/xbib/elx}/transport/TransportClient.java (93%) create mode 100644 elx-transport/src/main/java/org/xbib/elx/transport/package-info.java create mode 100644 elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider create mode 100644 elx-transport/src/test/java/org/elasticsearch/node/MockNode.java create mode 100644 elx-transport/src/test/java/org/elasticsearch/node/package-info.java create mode 100644 elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientSingleNodeTest.java rename src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClientTest.java => elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientTest.java (53%) create mode 100644 elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportDuplicateIDTest.java create mode 100644 elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportIndexAliasTest.java rename src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportReplicaTest.java => elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportReplicaTest.java (78%) rename src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeUpdateReplicaLevelTest.java => elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportUpdateReplicaLevelTest.java (63%) rename {src/integration-test/java/org/xbib/elasticsearch => elx-transport/src/test/java/org/xbib/elx/transport}/NodeTestUtils.java (91%) create mode 100644 elx-transport/src/test/java/org/xbib/elx/transport/package-info.java create mode 100644 elx-transport/src/test/resources/log4j2.xml delete mode 100644 gradle/ext.gradle delete mode 100644 src/integration-test/java/org/elasticsearch/node/package-info.java delete mode 100644 src/integration-test/java/org/xbib/elasticsearch/SearchTest.java delete mode 100644 src/integration-test/java/org/xbib/elasticsearch/WildcardTest.java delete mode 100644 src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeDuplicateIDTest.java delete mode 100644 src/integration-test/java/org/xbib/elasticsearch/extras/client/node/package-info.java delete mode 100644 src/integration-test/java/org/xbib/elasticsearch/extras/client/package-info.java delete mode 100644 src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportDuplicateIDTest.java delete mode 100644 src/integration-test/java/org/xbib/elasticsearch/package-info.java delete mode 100644 src/integration-test/java/suites/BulkNodeTestSuite.java delete mode 100644 src/integration-test/java/suites/BulkTransportTestSuite.java delete mode 100644 src/integration-test/java/suites/ListenerSuite.java delete mode 100644 src/integration-test/java/suites/MiscTestSuite.java delete mode 100644 src/integration-test/java/suites/TestListener.java delete mode 100644 src/integration-test/resources/org/xbib/elasticsearch/extras/client/settings.json delete mode 100644 src/main/java/org/xbib/elasticsearch/extras/client/AbstractClient.java delete mode 100644 src/main/java/org/xbib/elasticsearch/extras/client/Clients.java delete mode 100644 src/main/java/org/xbib/elasticsearch/extras/client/Parameters.java delete mode 100644 src/main/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClient.java delete mode 100644 src/main/java/org/xbib/elasticsearch/extras/client/node/package-info.java delete mode 100644 src/main/java/org/xbib/elasticsearch/extras/client/package-info.java delete mode 100644 src/main/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClient.java delete mode 100644 src/main/java/org/xbib/elasticsearch/extras/client/transport/MockTransportClient.java delete mode 100644 src/main/java/org/xbib/elasticsearch/extras/client/transport/package-info.java diff --git a/.gitignore b/.gitignore index bf3e9b4..bdd8810 100644 --- a/.gitignore +++ b/.gitignore @@ -4,10 +4,10 @@ /.idea /target .DS_Store -*.iml /.settings /.classpath /.project /.gradle -/build -/plugins \ No newline at end of file +build +*.iml +*~ diff --git a/build.gradle b/build.gradle index 2f8478a..01e2104 100644 --- a/build.gradle +++ b/build.gradle @@ -1,9 +1,11 @@ - plugins { - id "org.sonarqube" version "2.2" + id "org.sonarqube" version "2.6.1" + id "io.codearte.nexus-staging" version "0.11.0" + id "com.github.spotbugs" version "1.6.9" + id "org.xbib.gradle.plugin.asciidoctor" version "1.5.6.0.1" } -printf "Host: %s\nOS: %s %s %s\nJVM: %s %s %s %s\nGroovy: %s\nGradle: %s\n" + +printf "Host: %s\nOS: %s %s %s\nJVM: %s %s %s %s\nGradle: %s Groovy: %s Java: %s\n" + "Build: group: ${project.group} name: ${project.name} version: ${project.version}\n", InetAddress.getLocalHost(), System.getProperty("os.name"), @@ -13,102 +15,143 @@ printf "Host: %s\nOS: %s %s %s\nJVM: %s %s %s %s\nGroovy: %s\nGradle: %s\n" + System.getProperty("java.vm.version"), System.getProperty("java.vm.vendor"), System.getProperty("java.vm.name"), + gradle.gradleVersion, GroovySystem.getVersion(), - gradle.gradleVersion + JavaVersion.current() -apply plugin: 'java' -apply plugin: 'maven' -apply plugin: 'signing' -apply plugin: 'findbugs' -apply plugin: 'pmd' -apply plugin: 'checkstyle' -apply plugin: "jacoco" +if (JavaVersion.current() < JavaVersion.VERSION_11) { + throw new GradleException("This build must be run with java 11 or higher") +} -apply from: 'gradle/ext.gradle' +subprojects { + apply plugin: 'java' + apply plugin: 'maven' + apply plugin: 'signing' + apply plugin: 'com.github.spotbugs' + apply plugin: 'pmd' + apply plugin: 'checkstyle' + apply plugin: 'org.xbib.gradle.plugin.asciidoctor' + + configurations { + asciidoclet + wagon + } -sourceSets { - integrationTest { - java { - srcDir file('src/integration-test/java') - compileClasspath += main.output - compileClasspath += test.output - } - resources { - srcDir file('src/integration-test/resources') + dependencies { + testCompile "junit:junit:${project.property('junit.version')}" + testCompile "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}" + testCompile "org.apache.logging.log4j:log4j-slf4j-impl:${project.property('log4j.version')}" + wagon "org.apache.maven.wagon:wagon-ssh:${project.property('wagon.version')}" + } + + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + + tasks.withType(JavaCompile) { + options.compilerArgs << "-Xlint:all" + if (!options.compilerArgs.contains("-processor")) { + options.compilerArgs << '-proc:none' } } -} -configurations { - wagon - integrationTestCompile.extendsFrom testCompile - integrationTestRuntime.extendsFrom testRuntime -} - -dependencies { - compile "org.xbib:metrics:1.0.0" - compile("org.elasticsearch:elasticsearch:2.2.1") { - exclude module: "securesm" + test { + jvmArgs =[ + '--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED', + '--add-opens=java.base/java.nio=ALL-UNNAMED' + ] + systemProperty 'jna.debug_load', 'true' + testLogging { + showStandardStreams = true + exceptionFormat = 'full' + } } - testCompile "net.java.dev.jna:jna:4.1.0" - testCompile "junit:junit:4.12" - testCompile "org.apache.logging.log4j:log4j-core:2.7" - testCompile "org.apache.logging.log4j:log4j-slf4j-impl:2.7" - wagon 'org.apache.maven.wagon:wagon-ssh-external:2.10' -} - -sourceCompatibility = JavaVersion.VERSION_1_8 -targetCompatibility = JavaVersion.VERSION_1_8 - -[compileJava, compileTestJava]*.options*.encoding = 'UTF-8' -tasks.withType(JavaCompile) { - options.compilerArgs << "-Xlint:all" -} - -task integrationTest(type: Test) { - include '**/MiscTestSuite.class' - include '**/BulkNodeTestSuite.class' - include '**/BulkTransportTestSuite.class' - testClassesDir = sourceSets.integrationTest.output.classesDir - classpath = configurations.integrationTestCompile - classpath += configurations.integrationTestRuntime - classpath += sourceSets.main.output - classpath += sourceSets.test.output - classpath += sourceSets.integrationTest.output - outputs.upToDateWhen { false } - systemProperty 'path.home', projectDir.absolutePath - testLogging.showStandardStreams = true -} - -integrationTest.mustRunAfter test -check.dependsOn integrationTest - -clean { - delete "plugins" - delete "logs" -} - -task javadocJar(type: Jar, dependsOn: classes) { - from javadoc - into "build/tmp" - classifier 'javadoc' -} - -task sourcesJar(type: Jar, dependsOn: classes) { - from sourceSets.main.allSource - into "build/tmp" - classifier 'sources' -} - -artifacts { - archives javadocJar, sourcesJar -} - -if (project.hasProperty('signing.keyId')) { - signing { - sign configurations.archives + + clean { + delete "plugins" + delete "logs" + delete "out" } -} -apply from: 'gradle/publish.gradle' -apply from: 'gradle/sonarqube.gradle' + /*javadoc { + options.docletpath = configurations.asciidoclet.files.asType(List) + options.doclet = 'org.asciidoctor.Asciidoclet' + options.overview = "src/docs/asciidoclet/overview.adoc" + options.addStringOption "-base-dir", "${projectDir}" + options.addStringOption "-attribute", + "name=${project.name},version=${project.version},title-link=https://github.com/xbib/${project.name}" + configure(options) { + noTimestamp = true + } + }*/ + + task javadocJar(type: Jar, dependsOn: javadoc) { + classifier 'javadoc' + } + + task sourcesJar(type: Jar, dependsOn: classes) { + from sourceSets.main.allSource + classifier 'sources' + } + + artifacts { + archives javadocJar, sourcesJar + } + + if (project.hasProperty('signing.keyId')) { + signing { + sign configurations.archives + } + } + + apply from: "${rootProject.projectDir}/gradle/publish.gradle" + + spotbugs { + effort = "max" + reportLevel = "low" + //includeFilter = file("findbugs-exclude.xml") + } + + tasks.withType(com.github.spotbugs.SpotBugsTask) { + ignoreFailures = true + reports { + xml.enabled = false + html.enabled = true + } + } + + tasks.withType(Pmd) { + ignoreFailures = true + reports { + xml.enabled = true + html.enabled = true + } + } + tasks.withType(Checkstyle) { + ignoreFailures = true + reports { + xml.enabled = true + html.enabled = true + } + } + + pmd { + toolVersion = '6.11.0' + ruleSets = ['category/java/bestpractices.xml'] + } + + checkstyle { + configFile = rootProject.file('config/checkstyle/checkstyle.xml') + ignoreFailures = true + showViolations = true + } + + sonarqube { + properties { + property "sonar.projectName", "${project.group} ${project.name}" + property "sonar.sourceEncoding", "UTF-8" + property "sonar.tests", "src/test/java" + property "sonar.scm.provider", "git" + property "sonar.junit.reportsPath", "build/test-results/test/" + } + } +} \ No newline at end of file diff --git a/config/pmd/category/java/bestpractices.xml b/config/pmd/category/java/bestpractices.xml new file mode 100644 index 0000000..816e8cc --- /dev/null +++ b/config/pmd/category/java/bestpractices.xml @@ -0,0 +1,1636 @@ + + + + + + Rules which enforce generally accepted best practices. + + + + + The abstract class does not contain any abstract methods. An abstract class suggests + an incomplete implementation, which is to be completed by subclasses implementing the + abstract methods. If the class is intended to be used as a base class only (not to be instantiated + directly) a protected constructor can be provided prevent direct instantiation. + + 3 + + + + + + + + + + + + + + + Instantiation by way of private constructors from outside of the constructor's class often causes the + generation of an accessor. A factory method, or non-privatization of the constructor can eliminate this + situation. The generated class file is actually an interface. It gives the accessing class the ability + to invoke a new hidden package scope constructor that takes the interface as a supplementary parameter. + This turns a private constructor effectively into one with package scope, and is challenging to discern. + + 3 + + + + + + + + When accessing a private field / method from another class, the Java compiler will generate a accessor methods + with package-private visibility. This adds overhead, and to the dex method count on Android. This situation can + be avoided by changing the visibility of the field / method from private to package-private. + + 3 + + + + + + + + Constructors and methods receiving arrays should clone objects and store the copy. + This prevents future changes from the user from affecting the original array. + + 3 + + + + + + + + Avoid printStackTrace(); use a logger call instead. + + 3 + + + + + + + + + + + + + + + Reassigning loop variables can lead to hard-to-find bugs. Prevent or limit how these variables can be changed. + + In foreach-loops, configured by the `foreachReassign` property: + - `deny`: Report any reassignment of the loop variable in the loop body. _This is the default._ + - `allow`: Don't check the loop variable. + - `firstOnly`: Report any reassignments of the loop variable, except as the first statement in the loop body. + _This is useful if some kind of normalization or clean-up of the value before using is permitted, but any other change of the variable is not._ + + In for-loops, configured by the `forReassign` property: + - `deny`: Report any reassignment of the control variable in the loop body. _This is the default._ + - `allow`: Don't check the control variable. + - `skip`: Report any reassignments of the control variable, except conditional increments/decrements (`++`, `--`, `+=`, `-=`). + _This prevents accidental reassignments or unconditional increments of the control variable._ + + 3 + + + + + + + + Reassigning values to incoming parameters is not recommended. Use temporary local variables instead. + + 2 + + + + + + + + StringBuffers/StringBuilders can grow considerably, and so may become a source of memory leaks + if held within objects with long lifetimes. + + 3 + + + + + + + + + + + + + + + Application with hard-coded IP addresses can become impossible to deploy in some cases. + Externalizing IP adresses is preferable. + + 3 + + + + + + + + Always check the return values of navigation methods (next, previous, first, last) of a ResultSet. + If the value return is 'false', it should be handled properly. + + 3 + + + + + + + + Avoid constants in interfaces. Interfaces should define types, constants are implementation details + better placed in classes or enums. See Effective Java, item 19. + + 3 + + + + + + + + + + + + + + + + By convention, the default label should be the last label in a switch statement. + + 3 + + + + + + + + + + + + + + + Reports loops that can be safely replaced with the foreach syntax. The rule considers loops over + lists, arrays and iterators. A loop is safe to replace if it only uses the index variable to + access an element of the list or array, only has one update statement, and loops through *every* + element of the list or array left to right. + + 3 + + l) { + for (int i = 0; i < l.size(); i++) { // pre Java 1.5 + System.out.println(l.get(i)); + } + + for (String s : l) { // post Java 1.5 + System.out.println(s); + } + } +} +]]> + + + + + + Having a lot of control variables in a 'for' loop makes it harder to see what range of values + the loop iterates over. By default this rule allows a regular 'for' loop with only one variable. + + 3 + + + + //ForInit/LocalVariableDeclaration[count(VariableDeclarator) > $maximumVariables] + + + + + + + + + + Whenever using a log level, one should check if the loglevel is actually enabled, or + otherwise skip the associate String creation and manipulation. + + 2 + + + + + + + + In JUnit 3, test suites are indicated by the suite() method. In JUnit 4, suites are indicated + through the @RunWith(Suite.class) annotation. + + 3 + + + + + + + + + + + + + + + In JUnit 3, the tearDown method was used to clean up all data entities required in running tests. + JUnit 4 skips the tearDown method and executes all methods annotated with @After after running each test. + JUnit 5 introduced @AfterEach and @AfterAll annotations to execute methods after each test or after all tests in the class, respectively. + + 3 + + + + + + + + + + + + + + + In JUnit 3, the setUp method was used to set up all data entities required in running tests. + JUnit 4 skips the setUp method and executes all methods annotated with @Before before all tests. + JUnit 5 introduced @BeforeEach and @BeforeAll annotations to execute methods before each test or before all tests in the class, respectively. + + 3 + + + + + + + + + + + + + + + In JUnit 3, the framework executed all methods which started with the word test as a unit test. + In JUnit 4, only methods annotated with the @Test annotation are executed. + In JUnit 5, one of the following annotations should be used for tests: @Test, @RepeatedTest, @TestFactory, @TestTemplate or @ParameterizedTest. + + 3 + + + + + + + + + + + + + + + + + JUnit assertions should include an informative message - i.e., use the three-argument version of + assertEquals(), not the two-argument version. + + 3 + + + + + + + + Unit tests should not contain too many asserts. Many asserts are indicative of a complex test, for which + it is harder to verify correctness. Consider breaking the test scenario into multiple, shorter test scenarios. + Customize the maximum number of assertions used by this Rule to suit your needs. + + This rule checks for JUnit4, JUnit5 and TestNG Tests, as well as methods starting with "test". + + 3 + + + + + $maximumAsserts] +]]> + + + + + + + + + + + JUnit tests should include at least one assertion. This makes the tests more robust, and using assert + with messages provide the developer a clearer idea of what the test does. + + 3 + + + + + + + + In JUnit4, use the @Test(expected) annotation to denote tests that should throw exceptions. + + 3 + + + + + + + + The use of implementation types (i.e., HashSet) as object references limits your ability to use alternate + implementations in the future as requirements change. Whenever available, referencing objects + by their interface types (i.e, Set) provides much more flexibility. + + 3 + + list = new ArrayList<>(); + + public HashSet getFoo() { + return new HashSet(); + } + + // preferred approach + private List list = new ArrayList<>(); + + public Set getFoo() { + return new HashSet(); + } +} +]]> + + + + + + Exposing internal arrays to the caller violates object encapsulation since elements can be + removed or replaced outside of the object that owns it. It is safer to return a copy of the array. + + 3 + + + + + + + + + Annotating overridden methods with @Override ensures at compile time that + the method really overrides one, which helps refactoring and clarifies intent. + + 3 + + + + + + + + Java allows the use of several variables declaration of the same type on one line. However, it + can lead to quite messy code. This rule looks for several declarations on the same line. + + 4 + + + + 1] + [$strictMode or count(distinct-values(VariableDeclarator/@BeginLine)) != count(VariableDeclarator)] +| +//FieldDeclaration + [count(VariableDeclarator) > 1] + [$strictMode or count(distinct-values(VariableDeclarator/@BeginLine)) != count(VariableDeclarator)] +]]> + + + + + + + + + + + + + Position literals first in comparisons, if the second argument is null then NullPointerExceptions + can be avoided, they will just return false. + + 3 + + + + + + + + + + + + + + + Position literals first in comparisons, if the second argument is null then NullPointerExceptions + can be avoided, they will just return false. + + 3 + + + + + + + + + + + + + + + Throwing a new exception from a catch block without passing the original exception into the + new exception will cause the original stack trace to be lost making it difficult to debug + effectively. + + 3 + + + + + + + + Consider replacing Enumeration usages with the newer java.util.Iterator + + 3 + + + + + + + + + + + + + + + Consider replacing Hashtable usage with the newer java.util.Map if thread safety is not required. + + 3 + + + //Type/ReferenceType/ClassOrInterfaceType[@Image='Hashtable'] + + + + + + + + + + Consider replacing Vector usages with the newer java.util.ArrayList if expensive thread-safe operations are not required. + + 3 + + + //Type/ReferenceType/ClassOrInterfaceType[@Image='Vector'] + + + + + + + + + + All switch statements should include a default option to catch any unspecified values. + + 3 + + + + + + + + + + + + + + References to System.(out|err).print are usually intended for debugging purposes and can remain in + the codebase even in production code. By using a logger one can enable/disable this behaviour at + will (and by priority) and avoid clogging the Standard out log. + + 2 + + + + + + + + + + + + + + + Avoid passing parameters to methods or constructors without actually referencing them in the method body. + + 3 + + + + + + + + Avoid unused import statements to prevent unwanted dependencies. + This rule will also find unused on demand imports, i.e. import com.foo.*. + + 4 + + + + + + + + Detects when a local variable is declared and/or assigned, but not used. + + 3 + + + + + + + + Detects when a private field is declared and/or assigned a value, but not used. + + 3 + + + + + + + + Unused Private Method detects when a private method is declared but is unused. + + 3 + + + + + + + + This rule detects JUnit assertions in object equality. These assertions should be made by more specific methods, like assertEquals. + + 3 + + + + + + + + + + + + + + + This rule detects JUnit assertions in object references equality. These assertions should be made by + more specific methods, like assertNull, assertNotNull. + + 3 + + + + + + + + + + + + + + + This rule detects JUnit assertions in object references equality. These assertions should be made + by more specific methods, like assertSame, assertNotSame. + + 3 + + + + + + + + + + + + + + + When asserting a value is the same as a literal or Boxed boolean, use assertTrue/assertFalse, instead of assertEquals. + + 3 + + + + + + + + + + + + + + + The isEmpty() method on java.util.Collection is provided to determine if a collection has any elements. + Comparing the value of size() to 0 does not convey intent as well as the isEmpty() method. + + 3 + + + + + + + + Java 7 introduced the try-with-resources statement. This statement ensures that each resource is closed at the end + of the statement. It avoids the need of explicitly closing the resources in a finally block. Additionally exceptions + are better handled: If an exception occurred both in the `try` block and `finally` block, then the exception from + the try block was suppressed. With the `try`-with-resources statement, the exception thrown from the try-block is + preserved. + + 3 + + + + + + + + + + + + + + + + + Java 5 introduced the varargs parameter declaration for methods and constructors. This syntactic + sugar provides flexibility for users of these methods and constructors, allowing them to avoid + having to deal with the creation of an array. + + 4 + + + + + + + + + + + + + + diff --git a/config/pmd/category/java/categories.properties b/config/pmd/category/java/categories.properties new file mode 100644 index 0000000..3189fd3 --- /dev/null +++ b/config/pmd/category/java/categories.properties @@ -0,0 +1,13 @@ +# +# BSD-style license; for more info see http://pmd.sourceforge.net/license.html +# + +rulesets.filenames=\ + category/java/bestpractices.xml,\ + category/java/codestyle.xml,\ + category/java/design.xml,\ + category/java/documentation.xml,\ + category/java/errorprone.xml,\ + category/java/multithreading.xml,\ + category/java/performance.xml,\ + category/java/security.xml diff --git a/config/pmd/category/java/codestyle.xml b/config/pmd/category/java/codestyle.xml new file mode 100644 index 0000000..ac2f0a0 --- /dev/null +++ b/config/pmd/category/java/codestyle.xml @@ -0,0 +1,2176 @@ + + + + + + Rules which enforce a specific coding style. + + + + + Abstract classes should be named 'AbstractXXX'. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by {% rule java/codestyle/ClassNamingConventions %}. + + 3 + + + + + + + + + + + + + + + + + + 3 + + + + + + + + Avoid using dollar signs in variable/method/class/interface names. + + 3 + + + + + + + Avoid using final local variables, turn them into fields. + 3 + + + + + + + + + + + + + + + Prefixing parameters by 'in' or 'out' pollutes the name of the parameters and reduces code readability. + To indicate whether or not a parameter will be modify in a method, its better to document method + behavior with Javadoc. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the more general rule {% rule java/codestyle/FormalParameterNamingConventions %}. + + 4 + + + + + + + + + + + + + + + + + + Do not use protected fields in final classes since they cannot be subclassed. + Clarify your intent by using private or package access modifiers instead. + + 3 + + + + + + + + + + + + + + + Do not use protected methods in most final classes since they cannot be subclassed. This should + only be allowed in final classes that extend other classes with protected methods (whose + visibility cannot be reduced). Clarify your intent by using private or package access modifiers instead. + + 3 + + + + + + + + + + + + + + + Unnecessary reliance on Java Native Interface (JNI) calls directly reduces application portability + and increases the maintenance burden. + + 2 + + + //Name[starts-with(@Image,'System.loadLibrary')] + + + + + + + + + + Methods that return boolean results should be named as predicate statements to denote this. + I.e, 'isReady()', 'hasValues()', 'canCommit()', 'willFail()', etc. Avoid the use of the 'get' + prefix for these methods. + + 4 + + + + + + + + + + + + + + + + It is a good practice to call super() in a constructor. If super() is not called but + another constructor (such as an overloaded constructor) is called, this rule will not report it. + + 3 + + + + 0 ] +/ClassOrInterfaceBody + /ClassOrInterfaceBodyDeclaration + /ConstructorDeclaration[ count (.//ExplicitConstructorInvocation)=0 ] +]]> + + + + + + + + + + + Configurable naming conventions for type declarations. This rule reports + type declarations which do not match the regex that applies to their + specific kind (e.g. enum or interface). Each regex can be configured through + properties. + + By default this rule uses the standard Java naming convention (Pascal case), + and reports utility class names not ending with 'Util'. + + 1 + + + + + + + + To avoid mistakes if we want that a Method, Constructor, Field or Nested class have a default access modifier + we must add a comment at the beginning of it's declaration. + By default the comment must be /* default */ or /* package */, if you want another, you have to provide a regular expression. + This rule ignores by default all cases that have a @VisibleForTesting annotation. Use the + property "ignoredAnnotations" to customize the recognized annotations. + + 3 + + + + + + + + Avoid negation within an "if" expression with an "else" clause. For example, rephrase: + `if (x != y) diff(); else same();` as: `if (x == y) same(); else diff();`. + + Most "if (x != y)" cases without an "else" are often return cases, so consistent use of this + rule makes the code easier to read. Also, this resolves trivial ordering problems, such + as "does the error case go first?" or "does the common case go first?". + + 3 + + + + + + + + Enforce a policy for braces on control statements. It is recommended to use braces on 'if ... else' + statements and loop statements, even if they are optional. This usually makes the code clearer, and + helps prepare the future when you need to add another statement. That said, this rule lets you control + which statements are required to have braces via properties. + + From 6.2.0 on, this rule supersedes WhileLoopMustUseBraces, ForLoopMustUseBraces, IfStmtMustUseBraces, + and IfElseStmtMustUseBraces. + + 3 + + + + + + + + + + + + + 1 + or (some $stmt (: in only the block statements until the next label :) + in following-sibling::BlockStatement except following-sibling::SwitchLabel[1]/following-sibling::BlockStatement + satisfies not($stmt/Statement/Block))] + ]]> + + + + + + + + + + Use explicit scoping instead of accidental usage of default package private level. + The rule allows methods and fields annotated with Guava's @VisibleForTesting. + + 3 + + + + + + + + + + + + Avoid importing anything from the package 'java.lang'. These classes are automatically imported (JLS 7.5.3). + + 4 + + + + + + + + Duplicate or overlapping import statements should be avoided. + + 4 + + + + + + + + Empty or auto-generated methods in an abstract class should be tagged as abstract. This helps to remove their inapproprate + usage by developers who should be implementing their own versions in the concrete subclasses. + + 1 + + + + + + + + + + + + + + No need to explicitly extend Object. + 4 + + + + + + + + + + + + + + + Fields should be declared at the top of the class, before any method declarations, constructors, initializers or inner classes. + + 3 + + + + + + + + + Configurable naming conventions for field declarations. This rule reports variable declarations + which do not match the regex that applies to their specific kind ---e.g. constants (static final), + enum constant, final field. Each regex can be configured through properties. + + By default this rule uses the standard Java naming convention (Camel case), and uses the ALL_UPPER + convention for constants and enum constants. + + 1 + + + + + + + + Some for loops can be simplified to while loops, this makes them more concise. + + 3 + + + + + + + + + + + + + + + Avoid using 'for' statements without using curly braces. If the code formatting or + indentation is lost then it becomes difficult to separate the code being controlled + from the rest. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the rule {% rule java/codestyle/ControlStatementBraces %}. + + 3 + + + //ForStatement[not(Statement/Block)] + + + + + + + + + + Configurable naming conventions for formal parameters of methods and lambdas. + This rule reports formal parameters which do not match the regex that applies to their + specific kind (e.g. lambda parameter, or final formal parameter). Each regex can be + configured through properties. + + By default this rule uses the standard Java naming convention (Camel case). + + 1 + + lambda1 = s_str -> { }; + + // lambda parameters with an explicit type can be configured separately + Consumer lambda1 = (String str) -> { }; + + } + + } + ]]> + + + + + + Names for references to generic values should be limited to a single uppercase letter. + + 4 + + + + 1 + or + string:upper-case(@Image) != @Image +] +]]> + + + + + extends BaseDao { + // This is ok... +} + +public interface GenericDao { + // Also this +} + +public interface GenericDao { + // 'e' should be an 'E' +} + +public interface GenericDao { + // 'EF' is not ok. +} +]]> + + + + + + + Identical `catch` branches use up vertical space and increase the complexity of code without + adding functionality. It's better style to collapse identical branches into a single multi-catch + branch. + + 3 + + + + + + + + Avoid using if..else statements without using surrounding braces. If the code formatting + or indentation is lost then it becomes difficult to separate the code being controlled + from the rest. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the rule {% rule java/codestyle/ControlStatementBraces %}. + + 3 + + + + + + + + + + + + + + + Avoid using if statements without using braces to surround the code block. If the code + formatting or indentation is lost then it becomes difficult to separate the code being + controlled from the rest. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the rule {% rule java/codestyle/ControlStatementBraces %}. + + 3 + + + + + + + + + + + + + + + This rule finds Linguistic Naming Antipatterns. It checks for fields, that are named, as if they should + be boolean but have a different type. It also checks for methods, that according to their name, should + return a boolean, but don't. Further, it checks, that getters return something and setters won't. + Finally, it checks that methods, that start with "to" - so called transform methods - actually return + something, since according to their name, they should convert or transform one object into another. + There is additionally an option, to check for methods that contain "To" in their name - which are + also transform methods. However, this is disabled by default, since this detection is prone to + false positives. + + For more information, see [Linguistic Antipatterns - What They Are and How + Developers Perceive Them](https://doi.org/10.1007/s10664-014-9350-8). + + 3 + + + + + + + + The Local Home interface of a Session EJB should be suffixed by 'LocalHome'. + + 4 + + + + + + + + + + + + + + + The Local Interface of a Session EJB should be suffixed by 'Local'. + + 4 + + + + + + + + + + + + + + + A local variable assigned only once can be declared final. + + 3 + + + + + + + + Configurable naming conventions for local variable declarations and other locally-scoped + variables. This rule reports variable declarations which do not match the regex that applies to their + specific kind (e.g. final variable, or catch-clause parameter). Each regex can be configured through + properties. + + By default this rule uses the standard Java naming convention (Camel case). + + 1 + + + + + + + + Fields, formal arguments, or local variable names that are too long can make the code difficult to follow. + + 3 + + + + + $minimum] +]]> + + + + + + + + + + + The EJB Specification states that any MessageDrivenBean or SessionBean should be suffixed by 'Bean'. + + 4 + + + + + + + + + + + + + + + A method argument that is never re-assigned within the method can be declared final. + + 3 + + + + + + + + Configurable naming conventions for method declarations. This rule reports + method declarations which do not match the regex that applies to their + specific kind (e.g. JUnit test or native method). Each regex can be + configured through properties. + + By default this rule uses the standard Java naming convention (Camel case). + + 1 + + + + + + + + Detects when a non-field has a name starting with 'm_'. This usually denotes a field and could be confusing. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the more general rule + {% rule java/codestyle/LocalVariableNamingConventions %}. + + 3 + + + + + + + + + + + + + + + Detects when a class or interface does not have a package definition. + + 3 + + + //ClassOrInterfaceDeclaration[count(preceding::PackageDeclaration) = 0] + + + + + + + + + + Since Java 1.7, numeric literals can use underscores to separate digits. This rule enforces that + numeric literals above a certain length use these underscores to increase readability. + + The rule only supports decimal (base 10) literals for now. The acceptable length under which literals + are not required to have underscores is configurable via a property. Even under that length, underscores + that are misplaced (not making groups of 3 digits) are reported. + + 3 + + + + + + + + + + + + + + + + + A method should have only one exit point, and that should be the last statement in the method. + + 3 + + 0) { + return "hey"; // first exit + } + return "hi"; // second exit + } +} +]]> + + + + + + Detects when a package definition contains uppercase characters. + + 3 + + + //PackageDeclaration/Name[lower-case(@Image)!=@Image] + + + + + + + + + + Checks for variables that are defined before they might be used. A reference is deemed to be premature if it is created right before a block of code that doesn't use it that also has the ability to return or throw an exception. + + 3 + + + + + + + + Remote Interface of a Session EJB should not have a suffix. + + 4 + + + + + + + + + + + + + + + A Remote Home interface type of a Session EJB should be suffixed by 'Home'. + + 4 + + + + + + + + + + + + + + + Short Classnames with fewer than e.g. five characters are not recommended. + + 4 + + + + + + + + + + + + + + + + Method names that are very short are not helpful to the reader. + + 3 + + + + + + + + + + + + + + + + Fields, local variables, or parameter names that are very short are not helpful to the reader. + + 3 + + + + + + + + + + + + + + + + + Field names using all uppercase characters - Sun's Java naming conventions indicating constants - should + be declared as final. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the more general rule {% rule java/codestyle/FieldNamingConventions %}. + + 3 + + + + + + + + + + + + + + + If you overuse the static import feature, it can make your program unreadable and + unmaintainable, polluting its namespace with all the static members you import. + Readers of your code (including you, a few months after you wrote it) will not know + which class a static member comes from (Sun 1.5 Language Guide). + + 3 + + + + + $maximumStaticImports] +]]> + + + + + + + + + + + + Avoid the use of value in annotations when it's the only element. + + 3 + + + + + + + + + This rule detects when a constructor is not necessary; i.e., when there is only one constructor and the + constructor is identical to the default constructor. The default constructor should has same access + modifier as the declaring class. In an enum type, the default constructor is implicitly private. + + 3 + + + + + + + + Import statements allow the use of non-fully qualified names. The use of a fully qualified name + which is covered by an import statement is redundant. Consider using the non-fully qualified name. + + 4 + + + + + + + + Avoid the creation of unnecessary local variables + + 3 + + + + + + + + Fields in interfaces and annotations are automatically `public static final`, and methods are `public abstract`. + Classes, interfaces or annotations nested in an interface or annotation are automatically `public static` + (all nested interfaces and annotations are automatically static). + Nested enums are automatically `static`. + For historical reasons, modifiers which are implied by the context are accepted by the compiler, but are superfluous. + + 3 + + + + + + + + Avoid the use of unnecessary return statements. + + 3 + + + + + + + + Use the diamond operator to let the type be inferred automatically. With the Diamond operator it is possible + to avoid duplication of the type parameters. + Instead, the compiler is now able to infer the parameter types for constructor calls, + which makes the code also more readable. + + 3 + + + + + + + + + strings = new ArrayList(); // unnecessary duplication of type parameters +List stringsWithDiamond = new ArrayList<>(); // using the diamond operator is more concise +]]> + + + + + Useless parentheses should be removed. + 4 + + + + 1] + /PrimaryPrefix/Expression + [not(./CastExpression)] + [not(./ConditionalExpression)] + [not(./AdditiveExpression)] + [not(./AssignmentOperator)] +| +//Expression[not(parent::PrimaryPrefix)]/PrimaryExpression[count(*)=1] + /PrimaryPrefix/Expression +| +//Expression/ConditionalAndExpression/PrimaryExpression/PrimaryPrefix/Expression[ + count(*)=1 and + count(./CastExpression)=0 and + count(./EqualityExpression/MultiplicativeExpression)=0 and + count(./ConditionalExpression)=0 and + count(./ConditionalOrExpression)=0] +| +//Expression/ConditionalOrExpression/PrimaryExpression/PrimaryPrefix/Expression[ + count(*)=1 and + not(./CastExpression) and + not(./ConditionalExpression) and + not(./EqualityExpression/MultiplicativeExpression)] +| +//Expression/ConditionalExpression/PrimaryExpression/PrimaryPrefix/Expression[ + count(*)=1 and + not(./CastExpression) and + not(./EqualityExpression)] +| +//Expression/AdditiveExpression[not(./PrimaryExpression/PrimaryPrefix/Literal[@StringLiteral='true'])] + /PrimaryExpression[1]/PrimaryPrefix/Expression[ + count(*)=1 and + not(./CastExpression) and + not(./AdditiveExpression[@Image = '-']) and + not(./ShiftExpression) and + not(./RelationalExpression) and + not(./InstanceOfExpression) and + not(./EqualityExpression) and + not(./AndExpression) and + not(./ExclusiveOrExpression) and + not(./InclusiveOrExpression) and + not(./ConditionalAndExpression) and + not(./ConditionalOrExpression) and + not(./ConditionalExpression)] +| +//Expression/EqualityExpression/PrimaryExpression/PrimaryPrefix/Expression[ + count(*)=1 and + not(./CastExpression) and + not(./AndExpression) and + not(./InclusiveOrExpression) and + not(./ExclusiveOrExpression) and + not(./ConditionalExpression) and + not(./ConditionalAndExpression) and + not(./ConditionalOrExpression) and + not(./EqualityExpression)] +]]> + + + + + + + + + + + Reports qualified this usages in the same class. + + 3 + + + + + + + + + + + + + + + A variable naming conventions rule - customize this to your liking. Currently, it + checks for final variables that should be fully capitalized and non-final variables + that should not include underscores. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the more general rules {% rule java/codestyle/FieldNamingConventions %}, + {% rule java/codestyle/FormalParameterNamingConventions %}, and + {% rule java/codestyle/LocalVariableNamingConventions %}. + + 1 + + + + + + + + Avoid using 'while' statements without using braces to surround the code block. If the code + formatting or indentation is lost then it becomes difficult to separate the code being + controlled from the rest. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the rule {% rule java/codestyle/ControlStatementBraces %}. + + 3 + + + //WhileStatement[not(Statement/Block)] + + + + + + + + diff --git a/config/pmd/category/java/design.xml b/config/pmd/category/java/design.xml new file mode 100644 index 0000000..ded3d80 --- /dev/null +++ b/config/pmd/category/java/design.xml @@ -0,0 +1,1657 @@ + + + + + + Rules that help you discover design issues. + + + + + If an abstract class does not provides any methods, it may be acting as a simple data container + that is not meant to be instantiated. In this case, it is probably better to use a private or + protected constructor in order to prevent instantiation than make the class misleadingly abstract. + + 1 + + + + + + + + + + + + + + + Avoid catching generic exceptions such as NullPointerException, RuntimeException, Exception in try-catch block + + 3 + + + + + + + + + + + + + + + Avoid creating deeply nested if-then statements since they are harder to read and error-prone to maintain. + + 3 + + y) { + if (y>z) { + if (z==x) { + // !! too deep + } + } + } + } +} +]]> + + + + + + Catch blocks that merely rethrow a caught exception only add to code size and runtime complexity. + + 3 + + + + + + + + + + + + + + + Catch blocks that merely rethrow a caught exception wrapped inside a new instance of the same type only add to + code size and runtime complexity. + + 3 + + + + + + + + + + + + + + + *Effective Java, 3rd Edition, Item 72: Favor the use of standard exceptions* +> +>Arguably, every erroneous method invocation boils down to an illegal argument or state, +but other exceptions are standardly used for certain kinds of illegal arguments and states. +If a caller passes null in some parameter for which null values are prohibited, convention dictates that +NullPointerException be thrown rather than IllegalArgumentException. + +To implement that, you are encouraged to use `java.util.Objects.requireNonNull()` +(introduced in Java 1.7). This method is designed primarily for doing parameter +validation in methods and constructors with multiple parameters. + +Your parameter validation could thus look like the following: +``` +public class Foo { + private String exampleValue; + + void setExampleValue(String exampleValue) { + // check, throw and assignment in a single standard call + this.exampleValue = Objects.requireNonNull(exampleValue, "exampleValue must not be null!"); + } + } +``` +]]> + + 1 + + + + + + + + + + + + + + + Avoid throwing certain exception types. Rather than throw a raw RuntimeException, Throwable, + Exception, or Error, use a subclassed exception or error instead. + + 1 + + + + + + + + + + + + + + + A class with only private constructors should be final, unless the private constructor + is invoked by a inner class. + + 1 + + + + = 1 ] +[count(./ClassOrInterfaceBody/ClassOrInterfaceBodyDeclaration/ConstructorDeclaration[(@Public = 'true') or (@Protected = 'true') or (@PackagePrivate = 'true')]) = 0 ] +[not(.//ClassOrInterfaceDeclaration)] +]]> + + + + + + + + + + + Sometimes two consecutive 'if' statements can be consolidated by separating their conditions with a boolean short-circuit operator. + + 3 + + + + + + + + + + + + + + + This rule counts the number of unique attributes, local variables, and return types within an object. + A number higher than the specified threshold can indicate a high degree of coupling. + + 3 + + + + + + + = 10. +Additionnally, classes with many methods of moderate complexity get reported as well once the total of their +methods' complexities reaches 80, even if none of the methods was directly reported. + +Reported methods should be broken down into several smaller methods. Reported classes should probably be broken down +into subcomponents.]]> + + 3 + + + + + + + + Data Classes are simple data holders, which reveal most of their state, and + without complex functionality. The lack of functionality may indicate that + their behaviour is defined elsewhere, which is a sign of poor data-behaviour + proximity. By directly exposing their internals, Data Classes break encapsulation, + and therefore reduce the system's maintainability and understandability. Moreover, + classes tend to strongly rely on their data representation, which makes for a brittle + design. + + Refactoring a Data Class should focus on restoring a good data-behaviour proximity. In + most cases, that means moving the operations defined on the data back into the class. + In some other cases it may make sense to remove entirely the class and move the data + into the former client classes. + + 3 + + + + + + + + Errors are system exceptions. Do not extend them. + + 3 + + + + + + + + + + + + + + + Using Exceptions as form of flow control is not recommended as they obscure true exceptions when debugging. + Either add the necessary validation or use an alternate control structure. + + 3 + + + + + + + + Excessive class file lengths are usually indications that the class may be burdened with excessive + responsibilities that could be provided by external classes or functions. In breaking these methods + apart the code becomes more manageable and ripe for reuse. + + 3 + + + + + + + + A high number of imports can indicate a high degree of coupling within an object. This rule + counts the number of unique imports and reports a violation if the count is above the + user-specified threshold. + + 3 + + + + + + + + When methods are excessively long this usually indicates that the method is doing more than its + name/signature might suggest. They also become challenging for others to digest since excessive + scrolling causes readers to lose focus. + Try to reduce the method length by creating helper methods and removing any copy/pasted code. + + 3 + + + + + + + + Methods with numerous parameters are a challenge to maintain, especially if most of them share the + same datatype. These situations usually denote the need for new objects to wrap the numerous parameters. + + 3 + + + + + + + + Classes with large numbers of public methods and attributes require disproportionate testing efforts + since combinational side effects grow rapidly and increase risk. Refactoring these classes into + smaller ones not only increases testability and reliability but also allows new variations to be + developed easily. + + 3 + + + + + + + + If a final field is assigned to a compile-time constant, it could be made static, thus saving overhead + in each object at runtime. + + 3 + + + + + + + + + + + + + + + The God Class rule detects the God Class design flaw using metrics. God classes do too many things, + are very big and overly complex. They should be split apart to be more object-oriented. + The rule uses the detection strategy described in "Object-Oriented Metrics in Practice". + The violations are reported against the entire class. + + See also the references: + + Michele Lanza and Radu Marinescu. Object-Oriented Metrics in Practice: + Using Software Metrics to Characterize, Evaluate, and Improve the Design + of Object-Oriented Systems. Springer, Berlin, 1 edition, October 2006. Page 80. + + 3 + + + + + Identifies private fields whose values never change once object initialization ends either in the declaration + of the field or by a constructor. This helps in converting existing classes to becoming immutable ones. + + 3 + + + + + + + + The Law of Demeter is a simple rule, that says "only talk to friends". It helps to reduce coupling between classes + or objects. + + See also the references: + + * Andrew Hunt, David Thomas, and Ward Cunningham. The Pragmatic Programmer. From Journeyman to Master. Addison-Wesley Longman, Amsterdam, October 1999.; + * K.J. Lieberherr and I.M. Holland. Assuring good style for object-oriented programs. Software, IEEE, 6(5):38–48, 1989.; + * <http://www.ccs.neu.edu/home/lieber/LoD.html> + * <http://en.wikipedia.org/wiki/Law_of_Demeter> + + 3 + + + + + + + + Use opposite operator instead of negating the whole expression with a logic complement operator. + + 3 + + + + + + + + + = + return false; + } + + return true; +} +]]> + + + + + + Avoid using classes from the configured package hierarchy outside of the package hierarchy, + except when using one of the configured allowed classes. + + 3 + + + + + + + + Complexity directly affects maintenance costs is determined by the number of decision points in a method + plus one for the method entry. The decision points include 'if', 'while', 'for', and 'case labels' calls. + Generally, numbers ranging from 1-4 denote low complexity, 5-7 denote moderate complexity, 8-10 denote + high complexity, and 11+ is very high complexity. Modified complexity treats switch statements as a single + decision point. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the rule {% rule java/design/CyclomaticComplexity %}. + + 3 + + + + + + + + This rule uses the NCSS (Non-Commenting Source Statements) algorithm to determine the number of lines + of code for a given constructor. NCSS ignores comments, and counts actual statements. Using this algorithm, + lines of code that are split are counted as one. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the rule {% rule java/design/NcssCount %}. + + 3 + + + + + + + + This rule uses the NCSS (Non-Commenting Source Statements) metric to determine the number of lines + of code in a class, method or constructor. NCSS ignores comments, blank lines, and only counts actual + statements. For more details on the calculation, see the documentation of + the [NCSS metric](/pmd_java_metrics_index.html#non-commenting-source-statements-ncss). + + 3 + + + + + + + + This rule uses the NCSS (Non-Commenting Source Statements) algorithm to determine the number of lines + of code for a given method. NCSS ignores comments, and counts actual statements. Using this algorithm, + lines of code that are split are counted as one. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the rule {% rule java/design/NcssCount %}. + + 3 + + + + + + + + This rule uses the NCSS (Non-Commenting Source Statements) algorithm to determine the number of lines + of code for a given type. NCSS ignores comments, and counts actual statements. Using this algorithm, + lines of code that are split are counted as one. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the rule {% rule java/design/NcssCount %}. + + 3 + + + + + + + + The NPath complexity of a method is the number of acyclic execution paths through that method. + While cyclomatic complexity counts the number of decision points in a method, NPath counts the number of + full paths from the beginning to the end of the block of the method. That metric grows exponentially, as + it multiplies the complexity of statements in the same block. For more details on the calculation, see the + documentation of the [NPath metric](/pmd_java_metrics_index.html#npath-complexity-npath). + + A threshold of 200 is generally considered the point where measures should be taken to reduce + complexity and increase readability. + + 3 + + + + + + + + A method/constructor shouldn't explicitly throw the generic java.lang.Exception, since it + is unclear which exceptions that can be thrown from the methods. It might be + difficult to document and understand such vague interfaces. Use either a class + derived from RuntimeException or a checked exception. + + 3 + + + + + + + + + + 3 + + + + + + + + + + + + + + + Avoid negation in an assertTrue or assertFalse test. + + For example, rephrase: + + assertTrue(!expr); + + as: + + assertFalse(expr); + + + 3 + + + + + + + + + + + + + + + Avoid unnecessary comparisons in boolean expressions, they serve no purpose and impacts readability. + + 3 + + + + + + + + + + + + + + + Avoid unnecessary if-then-else statements when returning a boolean. The result of + the conditional test can be returned instead. + + 3 + + + + + + + + No need to check for null before an instanceof; the instanceof keyword returns false when given a null argument. + + 3 + + + + + + + + + + + + + + + Fields whose scopes are limited to just single methods do not rely on the containing + object to provide them to other methods. They may be better implemented as local variables + within those methods. + + 3 + + + + + + + + Complexity directly affects maintenance costs is determined by the number of decision points in a method + plus one for the method entry. The decision points include 'if', 'while', 'for', and 'case labels' calls. + Generally, numbers ranging from 1-4 denote low complexity, 5-7 denote moderate complexity, 8-10 denote + high complexity, and 11+ is very high complexity. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the rule {% rule java/design/CyclomaticComplexity %}. + + 3 + + + + + + + + A high ratio of statements to labels in a switch statement implies that the switch statement + is overloaded. Consider moving the statements into new methods or creating subclasses based + on the switch variable. + + 3 + + + + + + + + Classes that have too many fields can become unwieldy and could be redesigned to have fewer fields, + possibly through grouping related fields in new objects. For example, a class with individual + city/state/zip fields could park them within a single Address field. + + 3 + + + + + + + + A class with too many methods is probably a good suspect for refactoring, in order to reduce its + complexity and find a way to have more fine grained objects. + + 3 + + + + + + $maxmethods + ] +]]> + + + + + + + + The overriding method merely calls the same method defined in a superclass. + + 3 + + + + + + + + When you write a public method, you should be thinking in terms of an API. If your method is public, it means other class + will use it, therefore, you want (or need) to offer a comprehensive and evolutive API. If you pass a lot of information + as a simple series of Strings, you may think of using an Object to represent all those information. You'll get a simpler + API (such as doWork(Workload workload), rather than a tedious series of Strings) and more importantly, if you need at some + point to pass extra data, you'll be able to do so by simply modifying or extending Workload without any modification to + your API. + + 3 + + + + 3 +] +]]> + + + + + + + + + + + For classes that only have static methods, consider making them utility classes. + Note that this doesn't apply to abstract classes, since their subclasses may + well include non-static methods. Also, if you want this class to be a utility class, + remember to add a private constructor to prevent instantiation. + (Note, that this use was known before PMD 5.1.0 as UseSingleton). + + 3 + + + + + + diff --git a/config/pmd/category/java/documentation.xml b/config/pmd/category/java/documentation.xml new file mode 100644 index 0000000..34b351a --- /dev/null +++ b/config/pmd/category/java/documentation.xml @@ -0,0 +1,144 @@ + + + + + + Rules that are related to code documentation. + + + + + A rule for the politically correct... we don't want to offend anyone. + + 3 + + + + + + + + Denotes whether comments are required (or unwanted) for specific language elements. + + 3 + + + + + + + + Determines whether the dimensions of non-header comments found are within the specified limits. + + 3 + + + + + + + + Uncommented Empty Constructor finds instances where a constructor does not + contain statements, but there is no comment. By explicitly commenting empty + constructors it is easier to distinguish between intentional (commented) + and unintentional empty constructors. + + 3 + + + + + + + + + + + + + + + + Uncommented Empty Method Body finds instances where a method body does not contain + statements, but there is no comment. By explicitly commenting empty method bodies + it is easier to distinguish between intentional (commented) and unintentional + empty methods. + + 3 + + + + + + + + + + + + + \ No newline at end of file diff --git a/config/pmd/category/java/errorprone.xml b/config/pmd/category/java/errorprone.xml new file mode 100644 index 0000000..cf289c3 --- /dev/null +++ b/config/pmd/category/java/errorprone.xml @@ -0,0 +1,3383 @@ + + + + + + Rules to detect constructs that are either broken, extremely confusing or prone to runtime errors. + + + + + Avoid assignments in operands; this can make code more complicated and harder to read. + + 3 + + + + + + + + Identifies a possible unsafe usage of a static field. + + 3 + + + + + + + + Methods such as getDeclaredConstructors(), getDeclaredConstructor(Class[]) and setAccessible(), + as the interface PrivilegedAction, allow for the runtime alteration of variable, class, or + method visibility, even if they are private. This violates the principle of encapsulation. + + 3 + + + + + + + + + + + + + + + Use of the term 'assert' will conflict with newer versions of Java since it is a reserved word. + + 2 + + + //VariableDeclaratorId[@Image='assert'] + + + + + + + + + + Using a branching statement as the last part of a loop may be a bug, and/or is confusing. + Ensure that the usage is not a bug, or consider using another approach. + + 2 + + 25) { + break; + } +} +]]> + + + + + + The method Object.finalize() is called by the garbage collector on an object when garbage collection determines + that there are no more references to the object. It should not be invoked by application logic. + + Note that Oracle has declared Object.finalize() as deprecated since JDK 9. + + 3 + + + + + + + + Code should never throw NullPointerExceptions under normal circumstances. A catch block may hide the + original error, causing other, more subtle problems later on. + + 3 + + + + + + + + + + + + + + + Catching Throwable errors is not recommended since its scope is very broad. It includes runtime issues such as + OutOfMemoryError that should be exposed and managed separately. + + 3 + + + + + + + + One might assume that the result of "new BigDecimal(0.1)" is exactly equal to 0.1, but it is actually + equal to .1000000000000000055511151231257827021181583404541015625. + This is because 0.1 cannot be represented exactly as a double (or as a binary fraction of any finite + length). Thus, the long value that is being passed in to the constructor is not exactly equal to 0.1, + appearances notwithstanding. + + The (String) constructor, on the other hand, is perfectly predictable: 'new BigDecimal("0.1")' is + exactly equal to 0.1, as one would expect. Therefore, it is generally recommended that the + (String) constructor be used in preference to this one. + + 3 + + + + + + + + + + + + + + + Code containing duplicate String literals can usually be improved by declaring the String as a constant field. + + 3 + + + + + + + + Use of the term 'enum' will conflict with newer versions of Java since it is a reserved word. + + 2 + + + //VariableDeclaratorId[@Image='enum'] + + + + + + + + + + It can be confusing to have a field name with the same name as a method. While this is permitted, + having information (field) and actions (method) is not clear naming. Developers versed in + Smalltalk often prefer this approach as the methods denote accessor methods. + + 3 + + + + + + + + It is somewhat confusing to have a field name matching the declaring class name. + This probably means that type and/or field names should be chosen more carefully. + + 3 + + + + + + + + Each caught exception type should be handled in its own catch clause. + + 3 + + + + + + + + + + + + + + + Avoid using hard-coded literals in conditional statements. By declaring them as static variables + or private members with descriptive names maintainability is enhanced. By default, the literals "-1" and "0" are ignored. + More exceptions can be defined with the property "ignoreMagicNumbers". + + 3 + + + + + + + + + + + = 0) { } // alternative approach + + if (aDouble > 0.0) {} // magic number 0.0 + if (aDouble >= Double.MIN_VALUE) {} // preferred approach +} +]]> + + + + + + Statements in a catch block that invoke accessors on the exception without using the information + only add to code size. Either remove the invocation, or use the return result. + + 2 + + + + + + + + + + + + + + + The use of multiple unary operators may be problematic, and/or confusing. + Ensure that the intended usage is not a bug, or consider simplifying the expression. + + 2 + + + + + + + + Integer literals should not start with zero since this denotes that the rest of literal will be + interpreted as an octal value. + + 3 + + + + + + + + Avoid equality comparisons with Double.NaN. Due to the implicit lack of representation + precision when comparing floating point numbers these are likely to cause logic errors. + + 3 + + + + + + + + + + + + + + + If a class is a bean, or is referenced by a bean directly or indirectly it needs to be serializable. + Member variables need to be marked as transient, static, or have accessor methods in the class. Marking + variables as transient is the safest and easiest modification. Accessor methods should follow the Java + naming conventions, i.e. for a variable named foo, getFoo() and setFoo() accessor methods should be provided. + + 3 + + + + + + + + The null check is broken since it will throw a NullPointerException itself. + It is likely that you used || instead of && or vice versa. + + 2 + + + + + + + Super should be called at the start of the method + 3 + + + + + + + + + + + + + + + Super should be called at the end of the method + + 3 + + + + + + + + + + + + + + + The skip() method may skip a smaller number of bytes than requested. Check the returned value to find out if it was the case or not. + + 3 + + + + + + + + When deriving an array of a specific class from your Collection, one should provide an array of + the same class as the parameter of the toArray() method. Doing otherwise you will will result + in a ClassCastException. + + 3 + + + + + + + + + + + + + + + The java Manual says "By convention, classes that implement this interface should override + Object.clone (which is protected) with a public method." + + 3 + + + + + + + + + + + + + + + The method clone() should only be implemented if the class implements the Cloneable interface with the exception of + a final method that only throws CloneNotSupportedException. + + The rule can also detect, if the class implements or extends a Cloneable class. + + 3 + + + + + + + + If a class implements cloneable the return type of the method clone() must be the class name. That way, the caller + of the clone method doesn't need to cast the returned clone to the correct type. + + Note: This is only possible with Java 1.5 or higher. + + 3 + + + + + + + + + + + + + + + The method clone() should throw a CloneNotSupportedException. + + 3 + + + + + + + + + + + + + + + Ensure that resources (like Connection, Statement, and ResultSet objects) are always closed after use. + + 3 + + + + + + + + Use equals() to compare object references; avoid comparing them with ==. + + 3 + + + + + + + + Calling overridable methods during construction poses a risk of invoking methods on an incompletely + constructed object and can be difficult to debug. + It may leave the sub-class unable to construct its superclass or forced to replicate the construction + process completely within itself, losing the ability to call super(). If the default constructor + contains a call to an overridable method, the subclass may be completely uninstantiable. Note that + this includes method calls throughout the control flow graph - i.e., if a constructor Foo() calls a + private method bar() that calls a public method buz(), this denotes a problem. + + 1 + + + + + + + The dataflow analysis tracks local definitions, undefinitions and references to variables on different paths on the data flow. + From those informations there can be found various problems. + + 1. UR - Anomaly: There is a reference to a variable that was not defined before. This is a bug and leads to an error. + 2. DU - Anomaly: A recently defined variable is undefined. These anomalies may appear in normal source text. + 3. DD - Anomaly: A recently defined variable is redefined. This is ominous but don't have to be a bug. + + 5 + + dd-anomaly + foo(buz); + buz = 2; +} // buz is undefined when leaving scope -> du-anomaly +]]> + + + + + + Calls to System.gc(), Runtime.getRuntime().gc(), and System.runFinalization() are not advised. Code should have the + same behavior whether the garbage collection is disabled using the option -Xdisableexplicitgc or not. + Moreover, "modern" jvms do a very good job handling garbage collections. If memory usage issues unrelated to memory + leaks develop within an application, it should be dealt with JVM options rather than within the code itself. + + 2 + + + + + + + + + + + + + + + Web applications should not call System.exit(), since only the web container or the + application server should stop the JVM. This rule also checks for the equivalent call Runtime.getRuntime().exit(). + + 3 + + + + + + + + + + + + + + + Extend Exception or RuntimeException instead of Throwable. + + 3 + + + + + + + + + + + + + + + Use Environment.getExternalStorageDirectory() instead of "/sdcard" + + 3 + + + //Literal[starts-with(@Image,'"/sdcard')] + + + + + + + + + + Throwing exceptions within a 'finally' block is confusing since they may mask other exceptions + or code defects. + Note: This is a PMD implementation of the Lint4j rule "A throw in a finally block" + + 4 + + + //FinallyStatement[descendant::ThrowStatement] + + + + + + + + + + Avoid importing anything from the 'sun.*' packages. These packages are not portable and are likely to change. + + 4 + + + + + + + + Don't use floating point for loop indices. If you must use floating point, use double + unless you're certain that float provides enough precision and you have a compelling + performance need (space or time). + + 3 + + + + + + + + + + + + + + + Empty Catch Block finds instances where an exception is caught, but nothing is done. + In most circumstances, this swallows an exception which should either be acted on + or reported. + + 3 + + + + + + + + + + + + + + + + + Empty finalize methods serve no purpose and should be removed. Note that Oracle has declared Object.finalize() as deprecated since JDK 9. + + 3 + + + + + + + + + + + + + + + Empty finally blocks serve no purpose and should be removed. + + 3 + + + + + + + + + + + + + + + Empty If Statement finds instances where a condition is checked but nothing is done about it. + + 3 + + + + + + + + + + + + + + + Empty initializers serve no purpose and should be removed. + + 3 + + + //Initializer/Block[count(*)=0] + + + + + + + + + + Empty block statements serve no purpose and should be removed. + + 3 + + + //BlockStatement/Statement/Block[count(*) = 0] + + + + + + + + + + An empty statement (or a semicolon by itself) that is not used as the sole body of a 'for' + or 'while' loop is probably a bug. It could also be a double semicolon, which has no purpose + and should be removed. + + 3 + + + + + + + + + + + + + + + Empty switch statements serve no purpose and should be removed. + + 3 + + + //SwitchStatement[count(*) = 1] + + + + + + + + + + Empty synchronized blocks serve no purpose and should be removed. + + 3 + + + //SynchronizedStatement/Block[1][count(*) = 0] + + + + + + + + + + Avoid empty try blocks - what's the point? + + 3 + + + + + + + + + + + + + + + Empty While Statement finds all instances where a while statement does nothing. + If it is a timing loop, then you should use Thread.sleep() for it; if it is + a while loop that does a lot in the exit expression, rewrite it to make it clearer. + + 3 + + + + + + + + + + + + + + + Tests for null should not use the equals() method. The '==' operator should be used instead. + + 1 + + + + + + + + + + + + + + + If the finalize() is implemented, its last action should be to call super.finalize. Note that Oracle has declared Object.finalize() as deprecated since JDK 9. + + 3 + + + + + + + + + + + + + + + + If the finalize() is implemented, it should do something besides just calling super.finalize(). Note that Oracle has declared Object.finalize() as deprecated since JDK 9. + + 3 + + + + + + + + + + + + + + + Methods named finalize() should not have parameters. It is confusing and most likely an attempt to + overload Object.finalize(). It will not be called by the VM. + + Note that Oracle has declared Object.finalize() as deprecated since JDK 9. + + 3 + + + + 0]] +]]> + + + + + + + + + + + When overriding the finalize(), the new method should be set as protected. If made public, + other classes may invoke it at inappropriate times. + + Note that Oracle has declared Object.finalize() as deprecated since JDK 9. + + 3 + + + + + + + + + + + + + + + Avoid idempotent operations - they have no effect. + + 3 + + + + + + + + There is no need to import a type that lives in the same package. + + 3 + + + + + + + + Avoid instantiating an object just to call getClass() on it; use the .class public member instead. + + 4 + + + + + + + + + + + + + + + Check for messages in slf4j loggers with non matching number of arguments and placeholders. + + 5 + + + + + + + + Avoid jumbled loop incrementers - its usually a mistake, and is confusing even if intentional. + + 3 + + + + + + + + + + + + + + Some JUnit framework methods are easy to misspell. + + 3 + + + + + + + + + + + + + + + The suite() method in a JUnit test needs to be both public and static. + + 3 + + + + + + + + + + + + + + + In most cases, the Logger reference can be declared as static and final. + + 2 + + + + + + + + + + + + + + + Non-constructor methods should not have the same name as the enclosing class. + + 3 + + + + + + + + The null check here is misplaced. If the variable is null a NullPointerException will be thrown. + Either the check is useless (the variable will never be "null") or it is incorrect. + + 3 + + + + + + + + + + + + + + + + + + Switch statements without break or return statements for each case option + may indicate problematic behaviour. Empty cases are ignored as these indicate an intentional fall-through. + + 3 + + + + + + + + + + + + + + + Serializable classes should provide a serialVersionUID field. + The serialVersionUID field is also needed for abstract base classes. Each individual class in the inheritance + chain needs an own serialVersionUID field. See also [Should an abstract class have a serialVersionUID](https://stackoverflow.com/questions/893259/should-an-abstract-class-have-a-serialversionuid). + + 3 + + + + + + + + + + + + + + + A class that has private constructors and does not have any static methods or fields cannot be used. + + 3 + + + + + + + + + + + + + + + Normally only one logger is used in each class. + + 2 + + + + + + + + A non-case label (e.g. a named break/continue label) was present in a switch statement. + This legal, but confusing. It is easy to mix up the case labels and the non-case labels. + + 3 + + + //SwitchStatement//BlockStatement/Statement/LabeledStatement + + + + + + + + + + A non-static initializer block will be called any time a constructor is invoked (just prior to + invoking the constructor). While this is a valid language construct, it is rarely used and is + confusing. + + 3 + + + + + + + + + + + + + + + Assigning a "null" to a variable (outside of its declaration) is usually bad form. Sometimes, this type + of assignment is an indication that the programmer doesn't completely understand what is going on in the code. + + NOTE: This sort of assignment may used in some cases to dereference objects and encourage garbage collection. + + 3 + + + + + + + + Override both public boolean Object.equals(Object other), and public int Object.hashCode(), or override neither. Even if you are inheriting a hashCode() from a parent class, consider implementing hashCode and explicitly delegating to your superclass. + + 3 + + + + + + + + Object clone() should be implemented with super.clone(). + + 2 + + + + 0 +] +]]> + + + + + + + + + + + A logger should normally be defined private static final and be associated with the correct class. + Private final Log log; is also allowed for rare cases where loggers need to be passed around, + with the restriction that the logger needs to be passed into the constructor. + + 3 + + + + + + + + + + + + + + + + For any method that returns an array, it is a better to return an empty array rather than a + null reference. This removes the need for null checking all results and avoids inadvertent + NullPointerExceptions. + + 1 + + + + + + + + + + + + + + + Avoid returning from a finally block, this can discard exceptions. + + 3 + + + + //FinallyStatement//ReturnStatement except //FinallyStatement//(MethodDeclaration|LambdaExpression)//ReturnStatement + + + + + + + + + + Be sure to specify a Locale when creating SimpleDateFormat instances to ensure that locale-appropriate + formatting is used. + + 3 + + + + + + + + + + + + + + + Some classes contain overloaded getInstance. The problem with overloaded getInstance methods + is that the instance created using the overloaded method is not cached and so, + for each call and new objects will be created for every invocation. + + 2 + + + + + + + + Some classes contain overloaded getInstance. The problem with overloaded getInstance methods + is that the instance created using the overloaded method is not cached and so, + for each call and new objects will be created for every invocation. + + 2 + + + + + + + + According to the J2EE specification, an EJB should not have any static fields + with write access. However, static read-only fields are allowed. This ensures proper + behavior especially when instances are distributed by the container on several JREs. + + 3 + + + + + + + + + + + + + + + Individual character values provided as initialization arguments will be converted into integers. + This can lead to internal buffer sizes that are larger than expected. Some examples: + + ``` + new StringBuffer() // 16 + new StringBuffer(6) // 6 + new StringBuffer("hello world") // 11 + 16 = 27 + new StringBuffer('A') // chr(A) = 65 + new StringBuffer("A") // 1 + 16 = 17 + + new StringBuilder() // 16 + new StringBuilder(6) // 6 + new StringBuilder("hello world") // 11 + 16 = 27 + new StringBuilder('C') // chr(C) = 67 + new StringBuilder("A") // 1 + 16 = 17 + ``` + + 4 + + + + + + + + + + + + + + + The method name and parameter number are suspiciously close to equals(Object), which can denote an + intention to override the equals(Object) method. + + 2 + + + + + + + + + + + + + + + The method name and return type are suspiciously close to hashCode(), which may denote an intention + to override the hashCode() method. + + 3 + + + + + + + + A suspicious octal escape sequence was found inside a String literal. + The Java language specification (section 3.10.6) says an octal + escape sequence inside a literal String shall consist of a backslash + followed by: + + OctalDigit | OctalDigit OctalDigit | ZeroToThree OctalDigit OctalDigit + + Any octal escape sequence followed by non-octal digits can be confusing, + e.g. "\038" is interpreted as the octal escape sequence "\03" followed by + the literal character "8". + + 3 + + + + + + + + Test classes end with the suffix Test. Having a non-test class with that name is not a good practice, + since most people will assume it is a test case. Test classes have test methods named testXXX. + + 3 + + + + + + + + Do not use "if" statements whose conditionals are always true or always false. + + 3 + + + + + + + + + + + + + + + A JUnit test assertion with a boolean literal is unnecessary since it always will evaluate to the same thing. + Consider using flow control (in case of assertTrue(false) or similar) or simply removing + statements like assertTrue(true) and assertFalse(false). If you just want a test to halt after finding + an error, use the fail() method and provide an indication message of why it did. + + 3 + + + + + + + + + + + + + + + Using equalsIgnoreCase() is faster than using toUpperCase/toLowerCase().equals() + + 3 + + + + + + + + Avoid the use temporary objects when converting primitives to Strings. Use the static conversion methods + on the wrapper classes instead. + + 3 + + + + + + + + After checking an object reference for null, you should invoke equals() on that object rather than passing it to another object's equals() method. + + 3 + + + + + + + + + + + + + + + To make sure the full stacktrace is printed out, use the logging statement with two arguments: a String and a Throwable. + + 3 + + + + + + + + + + + + + + + Using '==' or '!=' to compare strings only works if intern version is used on both sides. + Use the equals() method instead. + + 3 + + + + + + + + + + + + + + + An operation on an Immutable object (String, BigDecimal or BigInteger) won't change the object itself + since the result of the operation is a new object. Therefore, ignoring the operation result is an error. + + 3 + + + + + + + + When doing String.toLowerCase()/toUpperCase() conversions, use Locales to avoids problems with languages that + have unusual conventions, i.e. Turkish. + + 3 + + + + + + + + + + + + + + + In J2EE, the getClassLoader() method might not work as expected. Use + Thread.currentThread().getContextClassLoader() instead. + + 3 + + + //PrimarySuffix[@Image='getClassLoader'] + + + + + + + + diff --git a/config/pmd/category/java/multithreading.xml b/config/pmd/category/java/multithreading.xml new file mode 100644 index 0000000..d3e8327 --- /dev/null +++ b/config/pmd/category/java/multithreading.xml @@ -0,0 +1,393 @@ + + + + + + Rules that flag issues when dealing with multiple threads of execution. + + + + + Method-level synchronization can cause problems when new code is added to the method. + Block-level synchronization helps to ensure that only the code that needs synchronization + gets it. + + 3 + + + //MethodDeclaration[@Synchronized='true'] + + + + + + + + + + Avoid using java.lang.ThreadGroup; although it is intended to be used in a threaded environment + it contains methods that are not thread-safe. + + 3 + + + + + + + + + + + + + + + Use of the keyword 'volatile' is generally used to fine tune a Java application, and therefore, requires + a good expertise of the Java Memory Model. Moreover, its range of action is somewhat misknown. Therefore, + the volatile keyword should not be used for maintenance purpose and portability. + + 2 + + + //FieldDeclaration[contains(@Volatile,'true')] + + + + + + + + + + The J2EE specification explicitly forbids the use of threads. + + 3 + + + //ClassOrInterfaceType[@Image = 'Thread' or @Image = 'Runnable'] + + + + + + + + + + Explicitly calling Thread.run() method will execute in the caller's thread of control. Instead, call Thread.start() for the intended behavior. + + 4 + + + + + + + + + + + + + + + Partially created objects can be returned by the Double Checked Locking pattern when used in Java. + An optimizing JRE may assign a reference to the baz variable before it calls the constructor of the object the + reference points to. + + Note: With Java 5, you can make Double checked locking work, if you declare the variable to be `volatile`. + + For more details refer to: <http://www.javaworld.com/javaworld/jw-02-2001/jw-0209-double.html> + or <http://www.cs.umd.edu/~pugh/java/memoryModel/DoubleCheckedLocking.html> + + 1 + + + + + + + + Non-thread safe singletons can result in bad state changes. Eliminate + static singletons if possible by instantiating the object directly. Static + singletons are usually not needed as only a single instance exists anyway. + Other possible fixes are to synchronize the entire method or to use an + [initialize-on-demand holder class](https://en.wikipedia.org/wiki/Initialization-on-demand_holder_idiom). + + Refrain from using the double-checked locking pattern. The Java Memory Model doesn't + guarantee it to work unless the variable is declared as `volatile`, adding an uneeded + performance penalty. [Reference](http://www.cs.umd.edu/~pugh/java/memoryModel/DoubleCheckedLocking.html) + + See Effective Java, item 48. + + 3 + + + + + + + + SimpleDateFormat instances are not synchronized. Sun recommends using separate format instances + for each thread. If multiple threads must access a static formatter, the formatter must be + synchronized either on method or block level. + + This rule has been deprecated in favor of the rule {% rule UnsynchronizedStaticFormatter %}. + + 3 + + + + + + + + Instances of `java.text.Format` are generally not synchronized. + Sun recommends using separate format instances for each thread. + If multiple threads must access a static formatter, the formatter must be + synchronized either on method or block level. + + 3 + + + + + + + + Since Java5 brought a new implementation of the Map designed for multi-threaded access, you can + perform efficient map reads without blocking other threads. + + 3 + + + + + + + + + + + + + + + Thread.notify() awakens a thread monitoring the object. If more than one thread is monitoring, then only + one is chosen. The thread chosen is arbitrary; thus its usually safer to call notifyAll() instead. + + 3 + + + + + + + + + + + + + \ No newline at end of file diff --git a/config/pmd/category/java/performance.xml b/config/pmd/category/java/performance.xml new file mode 100644 index 0000000..1ce2d8d --- /dev/null +++ b/config/pmd/category/java/performance.xml @@ -0,0 +1,1006 @@ + + + + + + Rules that flag suboptimal code. + + + + + The conversion of literals to strings by concatenating them with empty strings is inefficient. + It is much better to use one of the type-specific toString() methods instead. + + 3 + + + + + + + + + + + + + + + Avoid concatenating characters as strings in StringBuffer/StringBuilder.append methods. + + 3 + + + + + + + + Instead of manually copying data between two arrays, use the efficient Arrays.copyOf or System.arraycopy method instead. + + 3 + + + + + + + + + + + + + + + The FileInputStream and FileOutputStream classes contains a finalizer method which will cause garbage + collection pauses. + See [JDK-8080225](https://bugs.openjdk.java.net/browse/JDK-8080225) for details. + + The FileReader and FileWriter constructors instantiate FileInputStream and FileOutputStream, + again causing garbage collection issues while finalizer methods are called. + + * Use `Files.newInputStream(Paths.get(fileName))` instead of `new FileInputStream(fileName)`. + * Use `Files.newOutputStream(Paths.get(fileName))` instead of `new FileOutputStream(fileName)`. + * Use `Files.newBufferedReader(Paths.get(fileName))` instead of `new FileReader(fileName)`. + * Use `Files.newBufferedWriter(Paths.get(fileName))` instead of `new FileWriter(fileName)`. + + Please note, that the `java.nio` API does not throw a `FileNotFoundException` anymore, instead + it throws a `NoSuchFileException`. If your code dealt explicitly with a `FileNotFoundException`, + then this needs to be adjusted. Both exceptions are subclasses of `IOException`, so catching + that one covers both. + + 1 + + + + + + + + + + + + + + + New objects created within loops should be checked to see if they can created outside them and reused. + + 3 + + + + + + + + Java uses the 'short' type to reduce memory usage, not to optimize calculation. In fact, the JVM does not have any + arithmetic capabilities for the short type: the JVM must convert the short into an int, do the proper calculation + and convert the int back to a short. Thus any storage gains found through use of the 'short' type may be offset by + adverse impacts on performance. + + 1 + + + + + + + + + + + + + + + Don't create instances of already existing BigInteger (BigInteger.ZERO, BigInteger.ONE) and + for Java 1.5 onwards, BigInteger.TEN and BigDecimal (BigDecimal.ZERO, BigDecimal.ONE, BigDecimal.TEN) + + 3 + + + + + + + + Avoid instantiating Boolean objects; you can reference Boolean.TRUE, Boolean.FALSE, or call Boolean.valueOf() instead. + Note that new Boolean() is deprecated since JDK 9 for that reason. + + 2 + + + + + + + + Calling new Byte() causes memory allocation that can be avoided by the static Byte.valueOf(). + It makes use of an internal cache that recycles earlier instances making it more memory efficient. + Note that new Byte() is deprecated since JDK 9 for that reason. + + 2 + + + + + + + + + + + + + + + Consecutive calls to StringBuffer/StringBuilder .append should be chained, reusing the target object. This can improve the performance + by producing a smaller bytecode, reducing overhead and improving inlining. A complete analysis can be found [here](https://github.com/pmd/pmd/issues/202#issuecomment-274349067) + + 3 + + + + + + + + Consecutively calling StringBuffer/StringBuilder.append(...) with literals should be avoided. + Since the literals are constants, they can already be combined into a single String literal and this String + can be appended in a single method call. + + 3 + + + + + + + + + + 3 + + 0) { + doSomething(); + } +} +]]> + + + + + + Avoid concatenating non-literals in a StringBuffer constructor or append() since intermediate buffers will + need to be be created and destroyed by the JVM. + + 3 + + + + + + + + Failing to pre-size a StringBuffer or StringBuilder properly could cause it to re-size many times + during runtime. This rule attempts to determine the total number the characters that are actually + passed into StringBuffer.append(), but represents a best guess "worst case" scenario. An empty + StringBuffer/StringBuilder constructor initializes the object to 16 characters. This default + is assumed if the length of the constructor can not be determined. + + 3 + + + + + + + + Calling new Integer() causes memory allocation that can be avoided by the static Integer.valueOf(). + It makes use of an internal cache that recycles earlier instances making it more memory efficient. + Note that new Integer() is deprecated since JDK 9 for that reason. + + 2 + + + + + + + + + + + + + + + Calling new Long() causes memory allocation that can be avoided by the static Long.valueOf(). + It makes use of an internal cache that recycles earlier instances making it more memory efficient. + Note that new Long() is deprecated since JDK 9 for that reason. + + 2 + + + + + + + + + + + + + + + Calls to a collection's `toArray(E[])` method should specify a target array of zero size. This allows the JVM + to optimize the memory allocation and copying as much as possible. + + Previous versions of this rule (pre PMD 6.0.0) suggested the opposite, but current JVM implementations + perform always better, when they have full control over the target array. And allocation an array via + reflection is nowadays as fast as the direct allocation. + + See also [Arrays of Wisdom of the Ancients](https://shipilev.net/blog/2016/arrays-wisdom-ancients/) + + Note: If you don't need an array of the correct type, then the simple `toArray()` method without an array + is faster, but returns only an array of type `Object[]`. + + 3 + + + + + + + + + foos = getFoos(); + +// much better; this one allows the jvm to allocate an array of the correct size and effectively skip +// the zeroing, since each array element will be overridden anyways +Foo[] fooArray = foos.toArray(new Foo[0]); + +// inefficient, the array needs to be zeroed out by the jvm before it is handed over to the toArray method +Foo[] fooArray = foos.toArray(new Foo[foos.size()]); +]]> + + + + + + Java will initialize fields with known default values so any explicit initialization of those same defaults + is redundant and results in a larger class file (approximately three additional bytecode instructions per field). + + 3 + + + + + + + + Since it passes in a literal of length 1, calls to (string).startsWith can be rewritten using (string).charAt(0) + at the expense of some readability. + + 3 + + + + + + + + + + + + + + + Calling new Short() causes memory allocation that can be avoided by the static Short.valueOf(). + It makes use of an internal cache that recycles earlier instances making it more memory efficient. + Note that new Short() is deprecated since JDK 9 for that reason. + + 2 + + + + + + + + + + + + + + + Avoid instantiating String objects; this is usually unnecessary since they are immutable and can be safely shared. + + 2 + + + + + + + + Avoid calling toString() on objects already known to be string instances; this is unnecessary. + + 3 + + + + + + + + Switch statements are intended to be used to support complex branching behaviour. Using a switch for only a few + cases is ill-advised, since switches are not as easy to understand as if-then statements. In these cases use the + if-then statement to increase code readability. + + 3 + + + + + + + + + + + + + + + + Most wrapper classes provide static conversion methods that avoid the need to create intermediate objects + just to create the primitive forms. Using these avoids the cost of creating objects that also need to be + garbage-collected later. + + 3 + + + + + + + + ArrayList is a much better Collection implementation than Vector if thread-safe operation is not required. + + 3 + + + + 0] + //AllocationExpression/ClassOrInterfaceType + [@Image='Vector' or @Image='java.util.Vector'] +]]> + + + + + + + + + + + (Arrays.asList(...)) if that is inconvenient for you (e.g. because of concurrent access). +]]> + + 3 + + + + + + + + + l= new ArrayList<>(100); + for (int i=0; i< 100; i++) { + l.add(ints[i]); + } + for (int i=0; i< 100; i++) { + l.add(a[i].toString()); // won't trigger the rule + } + } +} +]]> + + + + + + Use String.indexOf(char) when checking for the index of a single character; it executes faster. + + 3 + + + + + + + + No need to call String.valueOf to append to a string; just use the valueOf() argument directly. + + 3 + + + + + + + + The use of the '+=' operator for appending strings causes the JVM to create and use an internal StringBuffer. + If a non-trivial number of these concatenations are being used then the explicit use of a StringBuilder or + threadsafe StringBuffer is recommended to avoid this. + + 3 + + + + + + + + Use StringBuffer.length() to determine StringBuffer length rather than using StringBuffer.toString().equals("") + or StringBuffer.toString().length() == ... + + 3 + + + + + + + + + diff --git a/config/pmd/category/java/security.xml b/config/pmd/category/java/security.xml new file mode 100644 index 0000000..dbad352 --- /dev/null +++ b/config/pmd/category/java/security.xml @@ -0,0 +1,65 @@ + + + + + + Rules that flag potential security flaws. + + + + + Do not use hard coded values for cryptographic operations. Please store keys outside of source code. + + 3 + + + + + + + + Do not use hard coded initialization vector in cryptographic operations. Please use a randomly generated IV. + + 3 + + + + + + diff --git a/elx-api/build.gradle b/elx-api/build.gradle new file mode 100644 index 0000000..9e7b7be --- /dev/null +++ b/elx-api/build.gradle @@ -0,0 +1,19 @@ +dependencies { + compile "org.xbib:metrics:${project.property('xbib-metrics.version')}" + compile("org.elasticsearch:elasticsearch:${project.property('elasticsearch.version')}") { + // exclude ES jackson yaml, cbor, smile versions + exclude group: 'com.fasterxml.jackson.dataformat' + // dependencies that are not meant for client + exclude module: 'securesm' + // we use log4j2, not log4j + exclude group: 'log4j' + } + // override log4j2 of Elastic with ours + compile "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}" + // for Elasticsearch session, ES uses SMILE when encoding source for SearchRequest + compile "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${project.property('jackson-dataformat.version')}" + // CBOR ist default JSON content compression encoding in ES 2.2.1 + compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${project.property('jackson-dataformat.version')}" + // not used, but maybe in other projects + compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${project.property('jackson-dataformat.version')}" +} \ No newline at end of file diff --git a/elx-api/build.gradle~ b/elx-api/build.gradle~ new file mode 100644 index 0000000..02e43a4 --- /dev/null +++ b/elx-api/build.gradle~ @@ -0,0 +1,18 @@ + +dependencies { + compile("org.elasticsearch.client:transport:${rootProject.property('elasticsearch.version')}") { + exclude group: 'org.elasticsearch', module: 'securesm' + exclude group: 'org.elasticsearch.plugin', module: 'transport-netty3-client' + exclude group: 'org.elasticsearch.plugin', module: 'reindex-client' + exclude group: 'org.elasticsearch.plugin', module: 'percolator-client' + exclude group: 'org.elasticsearch.plugin', module: 'lang-mustache-client' + } + // we try to override the Elasticsearch netty by our netty version which might be more recent + compile "io.netty:netty-buffer:${rootProject.property('netty.version')}" + compile "io.netty:netty-codec-http:${rootProject.property('netty.version')}" + compile "io.netty:netty-handler:${rootProject.property('netty.version')}" +} + +jar { + baseName "${rootProject.name}-api" +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/BulkControl.java b/elx-api/src/main/java/org/xbib/elx/api/BulkControl.java similarity index 87% rename from src/main/java/org/xbib/elasticsearch/extras/client/BulkControl.java rename to elx-api/src/main/java/org/xbib/elx/api/BulkControl.java index 910f2f2..e0fcf84 100644 --- a/src/main/java/org/xbib/elasticsearch/extras/client/BulkControl.java +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkControl.java @@ -1,10 +1,8 @@ -package org.xbib.elasticsearch.extras.client; +package org.xbib.elx.api; import java.util.Map; import java.util.Set; -/** - */ public interface BulkControl { void startBulk(String indexName, long startRefreshInterval, long stopRefreshInterval); @@ -18,5 +16,4 @@ public interface BulkControl { Map getStartBulkRefreshIntervals(); Map getStopBulkRefreshIntervals(); - } diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/BulkMetric.java b/elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java similarity index 86% rename from src/main/java/org/xbib/elasticsearch/extras/client/BulkMetric.java rename to elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java index a45e9c2..3002b8c 100644 --- a/src/main/java/org/xbib/elasticsearch/extras/client/BulkMetric.java +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java @@ -1,11 +1,8 @@ -package org.xbib.elasticsearch.extras.client; +package org.xbib.elx.api; import org.xbib.metrics.Count; import org.xbib.metrics.Metered; -/** - * - */ public interface BulkMetric { Metered getTotalIngest(); @@ -27,5 +24,4 @@ public interface BulkMetric { void stop(); long elapsed(); - } diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/ClientMethods.java b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java similarity index 71% rename from src/main/java/org/xbib/elasticsearch/extras/client/ClientMethods.java rename to elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java index a683b63..4ba2496 100644 --- a/src/main/java/org/xbib/elasticsearch/extras/client/ClientMethods.java +++ b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java @@ -1,11 +1,11 @@ -package org.xbib.elasticsearch.extras.client; +package org.xbib.elx.api; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; - import java.io.IOException; import java.io.InputStream; import java.util.List; @@ -13,38 +13,53 @@ import java.util.Map; import java.util.concurrent.ExecutionException; /** - * Interface for providing convenient administrative methods for ingesting data into Elasticsearch. + * Interface for providing extended administrative methods for managing and ingesting Elasticsearch. */ -public interface ClientMethods extends Parameters { +public interface ExtendedClient { /** - * Initialize new ingest client, wrap an existing Elasticsearch client, and set up metrics. - * - * @param client the Elasticsearch client - * @param metric metric - * @param control control - * @return this ingest - * @throws IOException if client could not get created + * Set an Elasticsearch client to extend from it. May be null for TransportClient. + * @param client client + * @return an ELasticsearch client */ - ClientMethods init(ElasticsearchClient client, BulkMetric metric, BulkControl control) throws IOException; - - /** - * Initialize, create new ingest client, and set up metrics. - * - * @param settings settings - * @param metric metric - * @param control control - * @return this ingest - * @throws IOException if client could not get created - */ - ClientMethods init(Settings settings, BulkMetric metric, BulkControl control) throws IOException; + ExtendedClient setClient(ElasticsearchClient client); /** * Return Elasticsearch client. * * @return Elasticsearch client */ - ElasticsearchClient client(); + ElasticsearchClient getClient(); + + ExtendedClient setBulkMetric(BulkMetric bulkMetric); + + BulkMetric getBulkMetric(); + + ExtendedClient setBulkControl(BulkControl bulkControl); + + BulkControl getBulkControl(); + + /** + * Create new Elasticsearch client, wrap an existing Elasticsearch client. + * + * @param settings settings + * @return this client + * @throws IOException if init fails + */ + ExtendedClient init(Settings settings) throws IOException; + + /** + * Bulked index request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * + * @param index the index + * @param type the type + * @param id the id + * @param create true if document must be created + * @param source the source + * @return this + */ + ExtendedClient index(String index, String type, String id, boolean create, BytesReference source); /** * Index document. @@ -52,10 +67,20 @@ public interface ClientMethods extends Parameters { * @param index the index * @param type the type * @param id the id + * @param create true if document is to be created, false otherwise * @param source the source - * @return this + * @return this client methods */ - ClientMethods index(String index, String type, String id, String source); + ExtendedClient index(String index, String type, String id, boolean create, String source); + + /** + * Bulked index request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * + * @param indexRequest the index request to add + * @return this ingest + */ + ExtendedClient indexRequest(IndexRequest indexRequest); /** * Delete document. @@ -65,7 +90,29 @@ public interface ClientMethods extends Parameters { * @param id the id * @return this ingest */ - ClientMethods delete(String index, String type, String id); + ExtendedClient delete(String index, String type, String id); + + /** + * Bulked delete request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * + * @param deleteRequest the delete request to add + * @return this ingest + */ + ExtendedClient deleteRequest(DeleteRequest deleteRequest); + + /** + * Bulked update request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * Note that updates only work correctly when all operations between nodes are synchronized. + * + * @param index the index + * @param type the type + * @param id the id + * @param source the source + * @return this + */ + ExtendedClient update(String index, String type, String id, BytesReference source); /** * Update document. Use with precaution! Does not work in all cases. @@ -76,7 +123,17 @@ public interface ClientMethods extends Parameters { * @param source the source * @return this */ - ClientMethods update(String index, String type, String id, String source); + ExtendedClient update(String index, String type, String id, String source); + + /** + * Bulked update request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * Note that updates only work correctly when all operations between nodes are synchronized. + * + * @param updateRequest the update request to add + * @return this ingest + */ + ExtendedClient updateRequest(UpdateRequest updateRequest); /** * Set the maximum number of actions per request. @@ -84,7 +141,7 @@ public interface ClientMethods extends Parameters { * @param maxActionsPerRequest maximum number of actions per request * @return this ingest */ - ClientMethods maxActionsPerRequest(int maxActionsPerRequest); + ExtendedClient maxActionsPerRequest(int maxActionsPerRequest); /** * Set the maximum concurent requests. @@ -92,7 +149,7 @@ public interface ClientMethods extends Parameters { * @param maxConcurentRequests maximum number of concurrent ingest requests * @return this Ingest */ - ClientMethods maxConcurrentRequests(int maxConcurentRequests); + ExtendedClient maxConcurrentRequests(int maxConcurentRequests); /** * Set the maximum volume for request before flush. @@ -100,7 +157,7 @@ public interface ClientMethods extends Parameters { * @param maxVolume maximum volume * @return this ingest */ - ClientMethods maxVolumePerRequest(String maxVolume); + ExtendedClient maxVolumePerRequest(String maxVolume); /** * Set the flush interval for automatic flushing outstanding ingest requests. @@ -108,7 +165,7 @@ public interface ClientMethods extends Parameters { * @param flushInterval the flush interval, default is 30 seconds * @return this ingest */ - ClientMethods flushIngestInterval(String flushInterval); + ExtendedClient flushIngestInterval(String flushInterval); /** * Set mapping. @@ -141,7 +198,7 @@ public interface ClientMethods extends Parameters { * @param index index * @return this ingest */ - ClientMethods newIndex(String index); + ExtendedClient newIndex(String index); /** * Create a new index. @@ -153,7 +210,7 @@ public interface ClientMethods extends Parameters { * @return this ingest * @throws IOException if new index creation fails */ - ClientMethods newIndex(String index, String type, InputStream settings, InputStream mappings) throws IOException; + ExtendedClient newIndex(String index, String type, InputStream settings, InputStream mappings) throws IOException; /** * Create a new index. @@ -163,7 +220,7 @@ public interface ClientMethods extends Parameters { * @param mappings mappings * @return this ingest */ - ClientMethods newIndex(String index, Settings settings, Map mappings); + ExtendedClient newIndex(String index, Settings settings, Map mappings); /** * Create new mapping. @@ -173,7 +230,7 @@ public interface ClientMethods extends Parameters { * @param mapping mapping * @return this ingest */ - ClientMethods newMapping(String index, String type, Map mapping); + ExtendedClient newMapping(String index, String type, Map mapping); /** * Delete index. @@ -181,7 +238,7 @@ public interface ClientMethods extends Parameters { * @param index index * @return this ingest */ - ClientMethods deleteIndex(String index); + ExtendedClient deleteIndex(String index); /** * Start bulk mode. @@ -192,7 +249,8 @@ public interface ClientMethods extends Parameters { * @return this ingest * @throws IOException if bulk could not be started */ - ClientMethods startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds) throws IOException; + ExtendedClient startBulk(String index, long startRefreshIntervalSeconds, + long stopRefreshIntervalSeconds) throws IOException; /** * Stops bulk mode. @@ -201,42 +259,14 @@ public interface ClientMethods extends Parameters { * @return this Ingest * @throws IOException if bulk could not be stopped */ - ClientMethods stopBulk(String index) throws IOException; - - /** - * Bulked index request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when bulk limits are exceeded. - * - * @param indexRequest the index request to add - * @return this ingest - */ - ClientMethods bulkIndex(IndexRequest indexRequest); - - /** - * Bulked delete request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when bulk limits are exceeded. - * - * @param deleteRequest the delete request to add - * @return this ingest - */ - ClientMethods bulkDelete(DeleteRequest deleteRequest); - - /** - * Bulked update request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when bulk limits are exceeded. - * Note that updates only work correctly when all operations between nodes are synchronized! - * - * @param updateRequest the update request to add - * @return this ingest - */ - ClientMethods bulkUpdate(UpdateRequest updateRequest); + ExtendedClient stopBulk(String index) throws IOException; /** * Flush ingest, move all pending documents to the cluster. * * @return this */ - ClientMethods flushIngest(); + ExtendedClient flushIngest(); /** * Wait for all outstanding responses. @@ -246,7 +276,7 @@ public interface ClientMethods extends Parameters { * @throws InterruptedException if wait is interrupted * @throws ExecutionException if execution failed */ - ClientMethods waitForResponses(String maxWaitTime) throws InterruptedException, ExecutionException; + ExtendedClient waitForResponses(String maxWaitTime) throws InterruptedException, ExecutionException; /** * Refresh the index. @@ -363,11 +393,10 @@ public interface ClientMethods extends Parameters { Long mostRecentDocument(String index, String timestampfieldname) throws IOException; /** - * Get metric. - * - * @return metric + * Get cluster name. + * @return the cluster name */ - BulkMetric getMetric(); + String getClusterName(); /** * Returns true is a throwable exists. @@ -385,6 +414,7 @@ public interface ClientMethods extends Parameters { /** * Shutdown the ingesting. + * @throws IOException if shutdown fails */ - void shutdown(); + void shutdown() throws IOException; } diff --git a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java new file mode 100644 index 0000000..2a8904a --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java @@ -0,0 +1,7 @@ +package org.xbib.elx.api; + +@FunctionalInterface +public interface ExtendedClientProvider { + + C getExtendedClient(); +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/IndexAliasAdder.java b/elx-api/src/main/java/org/xbib/elx/api/IndexAliasAdder.java similarity index 80% rename from src/main/java/org/xbib/elasticsearch/extras/client/IndexAliasAdder.java rename to elx-api/src/main/java/org/xbib/elx/api/IndexAliasAdder.java index a659ab4..d92bca3 100644 --- a/src/main/java/org/xbib/elasticsearch/extras/client/IndexAliasAdder.java +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexAliasAdder.java @@ -1,10 +1,7 @@ -package org.xbib.elasticsearch.extras.client; +package org.xbib.elx.api; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; -/** - * - */ @FunctionalInterface public interface IndexAliasAdder { diff --git a/elx-api/src/main/java/org/xbib/elx/api/package-info.java b/elx-api/src/main/java/org/xbib/elx/api/package-info.java new file mode 100644 index 0000000..7991a43 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/package-info.java @@ -0,0 +1,4 @@ +/** + * The API of the Elasticsearch extensions. + */ +package org.xbib.elx.api; diff --git a/elx-common/build.gradle b/elx-common/build.gradle new file mode 100644 index 0000000..e794a8b --- /dev/null +++ b/elx-common/build.gradle @@ -0,0 +1,9 @@ +dependencies { + compile project(':elx-api') + compile "org.xbib:guice:${project.property('xbib-guice.version')}" + // add all dependencies to runtime source set, even that which are excluded by Elasticsearch jar, + // for metaprogramming. We are in Groovyland. + runtime "com.vividsolutions:jts:${project.property('jts.version')}" + runtime "com.github.spullara.mustache.java:compiler:${project.property('mustache.version')}" + runtime "net.java.dev.jna:jna:${project.property('jna.version')}" +} diff --git a/elx-common/build.gradle~ b/elx-common/build.gradle~ new file mode 100644 index 0000000..99099fb --- /dev/null +++ b/elx-common/build.gradle~ @@ -0,0 +1,65 @@ +buildscript { + repositories { + jcenter() + maven { + url 'http://xbib.org/repository' + } + } + dependencies { + classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.2.2.0" + } +} + +apply plugin: 'org.xbib.gradle.plugin.elasticsearch.build' + +configurations { + main + tests +} + +dependencies { + compile project(':api') + compile "org.xbib:metrics:${project.property('xbib-metrics.version')}" + compileOnly "org.apache.logging.log4j:log4j-api:${project.property('log4j.version')}" + testCompile "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" + testRuntime "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" +} + +jar { + baseName "${rootProject.name}-common" +} + +/* +task testJar(type: Jar, dependsOn: testClasses) { + baseName = "${project.archivesBaseName}-tests" + from sourceSets.test.output +} +*/ + +artifacts { + main jar + tests testJar + archives sourcesJar, javadocJar +} + +test { + enabled = false + jvmArgs "-javaagent:" + configurations.alpnagent.asPath + systemProperty 'path.home', project.buildDir.absolutePath + testLogging { + showStandardStreams = true + exceptionFormat = 'full' + } +} + +randomizedTest { + enabled = false +} + +esTest { + // test with the jars, not the classes, for security manager + // classpath = files(configurations.testRuntime) + configurations.main.artifacts.files + configurations.tests.artifacts.files + systemProperty 'tests.security.manager', 'true' +} +esTest.dependsOn jar, testJar + diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java new file mode 100644 index 0000000..69f2608 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java @@ -0,0 +1,1144 @@ +package org.xbib.elx.common; + +import com.carrotsearch.hppc.cursors.ObjectCursor; +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; +import org.elasticsearch.action.admin.indices.create.CreateIndexAction; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; +import org.elasticsearch.action.admin.indices.flush.FlushAction; +import org.elasticsearch.action.admin.indices.flush.FlushRequest; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequestBuilder; +import org.elasticsearch.action.admin.indices.get.GetIndexAction; +import org.elasticsearch.action.admin.indices.get.GetIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequestBuilder; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; +import org.elasticsearch.action.admin.indices.recovery.RecoveryAction; +import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest; +import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; +import org.xbib.elx.api.BulkControl; +import org.xbib.elx.api.BulkMetric; +import org.xbib.elx.api.ExtendedClient; +import org.xbib.elx.api.IndexAliasAdder; +import org.xbib.elx.common.management.IndexDefinition; +import org.xbib.elx.common.management.IndexRetention; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.io.StringWriter; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.existsQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; + +public abstract class AbstractExtendedClient implements ExtendedClient { + + private static final Logger logger = LogManager.getLogger(AbstractExtendedClient.class.getName()); + + private Map mappings; + + private ElasticsearchClient client; + + private BulkProcessor bulkProcessor; + + private BulkMetric bulkMetric; + + private BulkControl bulkControl; + + private Throwable throwable; + + private boolean closed; + + private int maxActionsPerRequest; + + private int maxConcurrentRequests; + + private String maxVolumePerRequest; + + private String flushIngestInterval; + + protected abstract ElasticsearchClient createClient(Settings settings) throws IOException; + + protected AbstractExtendedClient() { + maxActionsPerRequest = Parameters.DEFAULT_MAX_ACTIONS_PER_REQUEST.getNum(); + maxConcurrentRequests = Parameters.DEFAULT_MAX_CONCURRENT_REQUESTS.getNum(); + maxVolumePerRequest = Parameters.DEFAULT_MAX_VOLUME_PER_REQUEST.getString(); + flushIngestInterval = Parameters.DEFAULT_FLUSH_INTERVAL.getString(); + mappings = new HashMap<>(); + } + + @Override + public AbstractExtendedClient setClient(ElasticsearchClient client) { + this.client = client; + return this; + } + + @Override + public ElasticsearchClient getClient() { + return client; + } + + @Override + public AbstractExtendedClient setBulkMetric(BulkMetric metric) { + this.bulkMetric = metric; + return this; + } + + @Override + public BulkMetric getBulkMetric() { + return bulkMetric; + } + + @Override + public AbstractExtendedClient setBulkControl(BulkControl bulkControl) { + this.bulkControl = bulkControl; + return this; + } + + @Override + public BulkControl getBulkControl() { + return bulkControl; + } + + @Override + public AbstractExtendedClient init(Settings settings) throws IOException { + if (client == null) { + this.client = createClient(settings); + } + if (bulkMetric != null) { + bulkMetric.start(); + } + BulkProcessor.Listener listener = new BulkProcessor.Listener() { + + private final Logger logger = LogManager.getLogger("org.xbib.elx.BulkProcessor.Listener"); + + @Override + public void beforeBulk(long executionId, BulkRequest request) { + long l = 0; + if (bulkMetric != null) { + l = bulkMetric.getCurrentIngest().getCount(); + bulkMetric.getCurrentIngest().inc(); + int n = request.numberOfActions(); + bulkMetric.getSubmitted().inc(n); + bulkMetric.getCurrentIngestNumDocs().inc(n); + bulkMetric.getTotalIngestSizeInBytes().inc(request.estimatedSizeInBytes()); + } + logger.debug("before bulk [{}] [actions={}] [bytes={}] [concurrent requests={}]", + executionId, + request.numberOfActions(), + request.estimatedSizeInBytes(), + l); + } + + @Override + public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { + long l = 0; + if (bulkMetric != null) { + l = bulkMetric.getCurrentIngest().getCount(); + bulkMetric.getCurrentIngest().dec(); + bulkMetric.getSucceeded().inc(response.getItems().length); + } + int n = 0; + for (BulkItemResponse itemResponse : response.getItems()) { + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().dec(itemResponse.getIndex(), itemResponse.getType(), itemResponse.getId()); + } + if (itemResponse.isFailed()) { + n++; + if (bulkMetric != null) { + bulkMetric.getSucceeded().dec(1); + bulkMetric.getFailed().inc(1); + } + } + } + if (bulkMetric != null) { + logger.debug("after bulk [{}] [succeeded={}] [failed={}] [{}ms] {} concurrent requests", + executionId, + bulkMetric.getSucceeded().getCount(), + bulkMetric.getFailed().getCount(), + response.getTook().millis(), + l); + } + if (n > 0) { + logger.error("bulk [{}] failed with {} failed items, failure message = {}", + executionId, n, response.buildFailureMessage()); + } else { + if (bulkMetric != null) { + bulkMetric.getCurrentIngestNumDocs().dec(response.getItems().length); + } + } + } + + @Override + public void afterBulk(long executionId, BulkRequest request, Throwable failure) { + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().dec(); + } + throwable = failure; + closed = true; + logger.error("after bulk [" + executionId + "] error", failure); + } + }; + if (this.client != null) { + BulkProcessor.Builder builder = BulkProcessor.builder((Client)this.client, listener) + .setBulkActions(maxActionsPerRequest) + .setConcurrentRequests(maxConcurrentRequests) + .setFlushInterval(TimeValue.parseTimeValue(flushIngestInterval, null, "flushIngestInterval")); + if (maxVolumePerRequest != null) { + builder.setBulkSize(ByteSizeValue.parseBytesSizeValue(maxVolumePerRequest, "maxVolumePerRequest")); + } + this.bulkProcessor = builder.build(); + } + this.closed = false; + return this; + } + + @Override + public synchronized void shutdown() throws IOException { + ensureActive(); + if (bulkProcessor != null) { + logger.info("closing bulk processor..."); + bulkProcessor.close(); + } + if (bulkMetric != null) { + logger.info("stopping metric"); + bulkMetric.stop(); + } + if (bulkControl != null && bulkControl.indices() != null && !bulkControl.indices().isEmpty()) { + logger.info("stopping bulk mode for indices {}...", bulkControl.indices()); + for (String index : bulkControl.indices()) { + stopBulk(index); + } + } + } + + @Override + public ExtendedClient maxActionsPerRequest(int maxActionsPerRequest) { + this.maxActionsPerRequest = maxActionsPerRequest; + return this; + } + + @Override + public ExtendedClient maxConcurrentRequests(int maxConcurrentRequests) { + this.maxConcurrentRequests = maxConcurrentRequests; + return this; + } + + @Override + public ExtendedClient maxVolumePerRequest(String maxVolumePerRequest) { + this.maxVolumePerRequest = maxVolumePerRequest; + return this; + } + + @Override + public ExtendedClient flushIngestInterval(String flushIngestInterval) { + this.flushIngestInterval = flushIngestInterval; + return this; + } + + @Override + public ExtendedClient newIndex(String index) { + ensureActive(); + return newIndex(index, null, null); + } + + @Override + public ExtendedClient newIndex(String index, String type, InputStream settings, InputStream mappings) throws IOException { + mapping(type, mappings); + return newIndex(index, Settings.settingsBuilder().loadFromStream(".json", settings).build(), + this.mappings); + } + + @Override + public ExtendedClient newIndex(String index, Settings settings, Map mappings) { + ensureActive(); + if (index == null) { + logger.warn("no index name given to create index"); + return this; + } + CreateIndexRequestBuilder createIndexRequestBuilder = + new CreateIndexRequestBuilder(client, CreateIndexAction.INSTANCE).setIndex(index); + if (settings != null) { + logger.info("found settings {}", settings.getAsMap()); + createIndexRequestBuilder.setSettings(settings); + } + if (mappings != null) { + for (Map.Entry entry : mappings.entrySet()) { + String type = entry.getKey(); + String mapping = entry.getValue(); + logger.info("found mapping for {}", type); + createIndexRequestBuilder.addMapping(type, mapping, XContentType.JSON); + } + } + CreateIndexResponse createIndexResponse = createIndexRequestBuilder.execute().actionGet(); + logger.info("index {} created: {}", index, createIndexResponse); + return this; + } + + @Override + public ExtendedClient newMapping(String index, String type, Map mapping) { + PutMappingRequestBuilder putMappingRequestBuilder = + new PutMappingRequestBuilder(client, PutMappingAction.INSTANCE) + .setIndices(index) + .setType(type) + .setSource(mapping); + putMappingRequestBuilder.execute().actionGet(); + logger.info("mapping created for index {} and type {}", index, type); + return this; + } + + @Override + public ExtendedClient deleteIndex(String index) { + ensureActive(); + if (index == null) { + logger.warn("no index name given to delete index"); + return this; + } + DeleteIndexRequestBuilder deleteIndexRequestBuilder = + new DeleteIndexRequestBuilder(client, DeleteIndexAction.INSTANCE, index); + deleteIndexRequestBuilder.execute().actionGet(); + return this; + } + + @Override + public ExtendedClient startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds) + throws IOException { + ensureActive(); + if (bulkControl == null) { + return this; + } + if (!bulkControl.isBulk(index) && startRefreshIntervalSeconds > 0L && stopRefreshIntervalSeconds > 0L) { + bulkControl.startBulk(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds); + updateIndexSetting(index, "refresh_interval", startRefreshIntervalSeconds + "s"); + } + return this; + } + + @Override + public ExtendedClient stopBulk(String index) throws IOException { + ensureActive(); + if (bulkControl == null) { + return this; + } + if (bulkControl.isBulk(index)) { + long secs = bulkControl.getStopBulkRefreshIntervals().get(index); + if (secs > 0L) { + updateIndexSetting(index, "refresh_interval", secs + "s"); + } + bulkControl.finishBulk(index); + } + return this; + } + + @Override + public ExtendedClient flushIngest() { + ensureActive(); + logger.debug("flushing bulk processor"); + bulkProcessor.flush(); + return this; + } + + @Override + public ExtendedClient waitForResponses(String maxWaitTime) throws InterruptedException { + ensureActive(); + long millis = TimeValue.parseTimeValue(maxWaitTime, TimeValue.timeValueMinutes(1),"millis").getMillis(); + logger.debug("waiting for " + millis + " millis"); + while (!bulkProcessor.awaitClose(millis, TimeUnit.MILLISECONDS)) { + logger.warn("still waiting for responses"); + } + return this; + } + + @Override + public ExtendedClient index(String index, String type, String id, boolean create, BytesReference source) { + return indexRequest(new IndexRequest(index).type(type).id(id).create(create).source(source)); + } + + @Override + public ExtendedClient index(String index, String type, String id, boolean create, String source) { + return indexRequest(new IndexRequest(index).type(type).id(id).create(create).source(source.getBytes(StandardCharsets.UTF_8))); + } + + @Override + public ExtendedClient indexRequest(IndexRequest indexRequest) { + ensureActive(); + try { + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id()); + } + bulkProcessor.add(indexRequest); + } catch (Exception e) { + throwable = e; + closed = true; + logger.error("bulk add of index request failed: " + e.getMessage(), e); + } + return this; + } + + @Override + public ExtendedClient delete(String index, String type, String id) { + return deleteRequest(new DeleteRequest(index).type(type).id(id)); + } + + @Override + public ExtendedClient deleteRequest(DeleteRequest deleteRequest) { + ensureActive(); + try { + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().inc(deleteRequest.index(), deleteRequest.type(), deleteRequest.id()); + } + bulkProcessor.add(deleteRequest); + } catch (Exception e) { + throwable = e; + closed = true; + logger.error("bulk add of delete failed: " + e.getMessage(), e); + } + return this; + } + + @Override + public ExtendedClient update(String index, String type, String id, BytesReference source) { + return updateRequest(new UpdateRequest().index(index).type(type).id(id).upsert(source)); + } + + @Override + public ExtendedClient update(String index, String type, String id, String source) { + return updateRequest(new UpdateRequest().index(index).type(type).id(id).upsert(source.getBytes(StandardCharsets.UTF_8))); + } + + @Override + public ExtendedClient updateRequest(UpdateRequest updateRequest) { + ensureActive(); + try { + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().inc(updateRequest.index(), updateRequest.type(), updateRequest.id()); + } + bulkProcessor.add(updateRequest); + } catch (Exception e) { + throwable = e; + closed = true; + logger.error("bulk add of update request failed: " + e.getMessage(), e); + } + return this; + } + + @Override + public void mapping(String type, String mapping) { + mappings.put(type, mapping); + } + + @Override + public void mapping(String type, InputStream in) throws IOException { + if (type == null) { + return; + } + StringWriter sw = new StringWriter(); + Streams.copy(new InputStreamReader(in, StandardCharsets.UTF_8), sw); + mappings.put(type, sw.toString()); + } + + @Override + public int waitForRecovery(String index) throws IOException { + ensureActive(); + if (index == null) { + throw new IOException("unable to wait for recovery, no index no given"); + } + RecoveryResponse response = client.execute(RecoveryAction.INSTANCE, new RecoveryRequest(index)).actionGet(); + int shards = response.getTotalShards(); + client.execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest(index) + .waitForActiveShards(shards)).actionGet(); + return shards; + } + + @Override + public void waitForCluster(String statusString, String timeout) throws IOException { + ensureActive(); + ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString); + ClusterHealthResponse healthResponse = + client.execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest() + .waitForStatus(status).timeout(timeout)).actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + throw new IOException("cluster state is " + healthResponse.getStatus().name() + + " and not " + status.name() + + ", from here on, everything will fail!"); + } + } + + @Override + public String healthColor() { + ensureActive(); + try { + ClusterHealthResponse healthResponse = + client.execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().timeout(TimeValue.timeValueSeconds(30))).actionGet(); + ClusterHealthStatus status = healthResponse.getStatus(); + return status.name(); + } catch (ElasticsearchTimeoutException e) { + logger.warn(e.getMessage(), e); + return "TIMEOUT"; + } catch (NoNodeAvailableException e) { + logger.warn(e.getMessage(), e); + return "DISCONNECTED"; + } catch (Exception e) { + logger.warn(e.getMessage(), e); + return "[" + e.getMessage() + "]"; + } + } + + @Override + public int updateReplicaLevel(String index, int level) throws IOException { + waitForCluster("YELLOW", "30s"); + updateIndexSetting(index, "number_of_replicas", level); + return waitForRecovery(index); + } + + @Override + public void flushIndex(String index) { + ensureActive(); + if (index != null) { + client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet(); + } + } + + @Override + public void refreshIndex(String index) { + ensureActive(); + if (index != null) { + client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet(); + } + } + + @Override + public void putMapping(String index) { + ensureActive(); + if (mappings != null && !mappings.isEmpty()) { + for (Map.Entry me : mappings.entrySet()) { + client.execute(PutMappingAction.INSTANCE, + new PutMappingRequest(index).type(me.getKey()).source(me.getValue())).actionGet(); + } + } + } + + @Override + public String resolveAlias(String alias) { + ensureActive(); + GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client, GetAliasesAction.INSTANCE); + GetAliasesResponse getAliasesResponse = getAliasesRequestBuilder.setAliases(alias).execute().actionGet(); + if (!getAliasesResponse.getAliases().isEmpty()) { + return getAliasesResponse.getAliases().keys().iterator().next().value; + } + return alias; + } + + @Override + public String resolveMostRecentIndex(String alias) { + ensureActive(); + if (alias == null) { + return null; + } + GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client, GetAliasesAction.INSTANCE); + GetAliasesResponse getAliasesResponse = getAliasesRequestBuilder.setAliases(alias).execute().actionGet(); + Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); + Set indices = new TreeSet<>(Collections.reverseOrder()); + for (ObjectCursor indexName : getAliasesResponse.getAliases().keys()) { + Matcher m = pattern.matcher(indexName.value); + if (m.matches() && alias.equals(m.group(1))) { + indices.add(indexName.value); + } + } + return indices.isEmpty() ? alias : indices.iterator().next(); + } + + @Override + public Map getAliasFilters(String alias) { + GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client, GetAliasesAction.INSTANCE); + return getFilters(getAliasesRequestBuilder.setIndices(resolveAlias(alias)).execute().actionGet()); + } + + @Override + public void switchAliases(String index, String concreteIndex, List extraAliases) { + switchAliases(index, concreteIndex, extraAliases, null); + } + + @Override + public void switchAliases(String index, String concreteIndex, + List extraAliases, IndexAliasAdder adder) { + ensureActive(); + if (index.equals(concreteIndex)) { + return; + } + // two situations: 1. there is a new alias 2. there is already an old index with the alias + String oldIndex = resolveAlias(index); + final Map oldFilterMap = oldIndex.equals(index) ? null : getIndexFilters(oldIndex); + final List newAliases = new LinkedList<>(); + final List switchAliases = new LinkedList<>(); + IndicesAliasesRequestBuilder requestBuilder = new IndicesAliasesRequestBuilder(client, IndicesAliasesAction.INSTANCE); + if (oldFilterMap == null || !oldFilterMap.containsKey(index)) { + // never apply a filter for trunk index name + requestBuilder.addAlias(concreteIndex, index); + newAliases.add(index); + } + // switch existing aliases + if (oldFilterMap != null) { + for (Map.Entry entry : oldFilterMap.entrySet()) { + String alias = entry.getKey(); + String filter = entry.getValue(); + requestBuilder.removeAlias(oldIndex, alias); + if (filter != null) { + requestBuilder.addAlias(concreteIndex, alias, filter); + } else { + requestBuilder.addAlias(concreteIndex, alias); + } + switchAliases.add(alias); + } + } + // a list of aliases that should be added, check if new or old + if (extraAliases != null) { + for (String extraAlias : extraAliases) { + if (oldFilterMap == null || !oldFilterMap.containsKey(extraAlias)) { + // index alias adder only active on extra aliases, and if alias is new + if (adder != null) { + adder.addIndexAlias(requestBuilder, concreteIndex, extraAlias); + } else { + requestBuilder.addAlias(concreteIndex, extraAlias); + } + newAliases.add(extraAlias); + } else { + String filter = oldFilterMap.get(extraAlias); + requestBuilder.removeAlias(oldIndex, extraAlias); + if (filter != null) { + requestBuilder.addAlias(concreteIndex, extraAlias, filter); + } else { + requestBuilder.addAlias(concreteIndex, extraAlias); + } + switchAliases.add(extraAlias); + } + } + } + if (!newAliases.isEmpty() || !switchAliases.isEmpty()) { + logger.info("new aliases = {}, switch aliases = {}", newAliases, switchAliases); + requestBuilder.execute().actionGet(); + } + } + + @Override + public void performRetentionPolicy(String index, String concreteIndex, int timestampdiff, int mintokeep) { + if (timestampdiff == 0 && mintokeep == 0) { + return; + } + ensureActive(); + if (index.equals(concreteIndex)) { + return; + } + GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE); + GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet(); + Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); + Set indices = new TreeSet<>(); + logger.info("{} indices", getIndexResponse.getIndices().length); + for (String s : getIndexResponse.getIndices()) { + Matcher m = pattern.matcher(s); + if (m.matches() && index.equals(m.group(1)) && !s.equals(concreteIndex)) { + indices.add(s); + } + } + if (indices.isEmpty()) { + logger.info("no indices found, retention policy skipped"); + return; + } + if (mintokeep > 0 && indices.size() <= mintokeep) { + logger.info("{} indices found, not enough for retention policy ({}), skipped", + indices.size(), mintokeep); + return; + } else { + logger.info("candidates for deletion = {}", indices); + } + List indicesToDelete = new ArrayList<>(); + // our index + Matcher m1 = pattern.matcher(concreteIndex); + if (m1.matches()) { + Integer i1 = Integer.parseInt(m1.group(2)); + for (String s : indices) { + Matcher m2 = pattern.matcher(s); + if (m2.matches()) { + Integer i2 = Integer.parseInt(m2.group(2)); + int kept = indices.size() - indicesToDelete.size(); + if ((timestampdiff == 0 || (timestampdiff > 0 && i1 - i2 > timestampdiff)) && mintokeep <= kept) { + indicesToDelete.add(s); + } + } + } + } + logger.info("indices to delete = {}", indicesToDelete); + if (indicesToDelete.isEmpty()) { + logger.info("not enough indices found to delete, retention policy complete"); + return; + } + String[] s = indicesToDelete.toArray(new String[indicesToDelete.size()]); + DeleteIndexRequestBuilder requestBuilder = new DeleteIndexRequestBuilder(client, DeleteIndexAction.INSTANCE, s); + DeleteIndexResponse response = requestBuilder.execute().actionGet(); + if (!response.isAcknowledged()) { + logger.warn("retention delete index operation was not acknowledged"); + } + } + + @Override + public Long mostRecentDocument(String index, String timestampfieldname) { + ensureActive(); + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); + SortBuilder sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC); + SearchResponse searchResponse = searchRequestBuilder.setIndices(index) + .addField(timestampfieldname) + .setSize(1) + .addSort(sort) + .execute().actionGet(); + if (searchResponse.getHits().getHits().length == 1) { + SearchHit hit = searchResponse.getHits().getHits()[0]; + if (hit.getFields().get(timestampfieldname) != null) { + return hit.getFields().get(timestampfieldname).getValue(); + } else { + return 0L; + } + } + return null; + } + + @Override + public boolean hasThrowable() { + return throwable != null; + } + + @Override + public Throwable getThrowable() { + return throwable; + } + + private void updateIndexSetting(String index, String key, Object value) throws IOException { + ensureActive(); + if (index == null) { + throw new IOException("no index name given"); + } + if (key == null) { + throw new IOException("no key given"); + } + if (value == null) { + throw new IOException("no value given"); + } + Settings.Builder updateSettingsBuilder = Settings.settingsBuilder(); + updateSettingsBuilder.put(key, value.toString()); + UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index) + .settings(updateSettingsBuilder); + client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet(); + } + + private void ensureActive() { + if (this instanceof MockExtendedClient) { + return; + } + if (client == null) { + throw new IllegalStateException("no client"); + } + if (closed) { + throw new ElasticsearchException("client is closed"); + } + } + + public Map getIndexFilters(String index) { + GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client, GetAliasesAction.INSTANCE); + return getFilters(getAliasesRequestBuilder.setIndices(index).execute().actionGet()); + } + + private Map getFilters(GetAliasesResponse getAliasesResponse) { + Map result = new HashMap<>(); + for (ObjectObjectCursor> object : getAliasesResponse.getAliases()) { + List aliasMetaDataList = object.value; + for (AliasMetaData aliasMetaData : aliasMetaDataList) { + if (aliasMetaData.filteringRequired()) { + result.put(aliasMetaData.alias(), + new String(aliasMetaData.getFilter().uncompressed(), StandardCharsets.UTF_8) ); + } else { + result.put(aliasMetaData.alias(), null); + } + } + } + return result; + } + + @Override + public String getClusterName() { + ensureActive(); + try { + ClusterStateRequestBuilder clusterStateRequestBuilder = + new ClusterStateRequestBuilder(client, ClusterStateAction.INSTANCE).all(); + ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); + String name = clusterStateResponse.getClusterName().value(); + int nodeCount = clusterStateResponse.getState().getNodes().size(); + return name + " (" + nodeCount + " nodes connected)"; + } catch (ElasticsearchTimeoutException e) { + logger.warn(e.getMessage(), e); + return "TIMEOUT"; + } catch (NoNodeAvailableException e) { + logger.warn(e.getMessage(), e); + return "DISCONNECTED"; + } catch (Exception e) { + logger.warn(e.getMessage(), e); + return "[" + e.getMessage() + "]"; + } + } + + public IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) + throws MalformedURLException { + boolean isEnabled = settings.getAsBoolean("enabled", !(client instanceof MockExtendedClient)); + String indexName = settings.get("name", index); + String fullIndexName; + String dateTimePattern = settings.get("dateTimePattern"); + if (dateTimePattern != null) { + fullIndexName = resolveAlias(indexName + + DateTimeFormatter.ofPattern(dateTimePattern) + .withZone(ZoneId.systemDefault()) // not GMT + .format(LocalDate.now())); + logger.info("index name {} resolved to full index name = {}", indexName, fullIndexName); + } else { + fullIndexName = resolveMostRecentIndex(indexName); + logger.info("index name {} resolved to full index name = {}", indexName, fullIndexName); + } + IndexRetention indexRetention = new IndexRetention() + .setMinToKeep(settings.getAsInt("retention.mintokeep", 0)) + .setTimestampDiff(settings.getAsInt("retention.diff", 0)); + return new IndexDefinition() + .setIndex(indexName) + .setFullIndexName(fullIndexName) + .setType(settings.get("type")) + .setSettingsUrl(new URL(settings.get("settings"))) + .setMappingsUrl(new URL(settings.get("mapping"))) + .setDateTimePattern(dateTimePattern) + .setEnabled(isEnabled) + .setIgnoreErrors(settings.getAsBoolean("skiperrors", false)) + .setSwitchAliases(settings.getAsBoolean("aliases", true)) + .setReplicaLevel(settings.getAsInt("replica", 0)) + .setRetention(indexRetention); + } + + public void checkMapping(String index) { + ensureActive(); + GetMappingsRequestBuilder getMappingsRequestBuilder = new GetMappingsRequestBuilder(client, GetMappingsAction.INSTANCE) + .setIndices(index); + GetMappingsResponse getMappingsResponse = getMappingsRequestBuilder.execute().actionGet(); + ImmutableOpenMap> map = getMappingsResponse.getMappings(); + map.keys().forEach((Consumer>) stringObjectCursor -> { + ImmutableOpenMap mappings = map.get(stringObjectCursor.value); + for (ObjectObjectCursor cursor : mappings) { + String mappingName = cursor.key; + MappingMetaData mappingMetaData = cursor.value; + checkMapping(index, mappingName, mappingMetaData); + } + }); + } + + @SuppressWarnings("unchecked") + private void checkMapping(String index, String type, MappingMetaData mappingMetaData) { + try { + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); + SearchResponse searchResponse = searchRequestBuilder.setSize(0) + .setIndices(index) + .setTypes(type) + .setQuery(matchAllQuery()) + .execute() + .actionGet(); + long total = searchResponse.getHits().getTotalHits(); + if (total > 0L) { + Map fields = new TreeMap<>(); + Map root = mappingMetaData.getSourceAsMap(); + checkMapping(index, type, "", "", root, fields); + AtomicInteger empty = new AtomicInteger(); + Map map = sortByValue(fields); + map.forEach((key, value) -> { + logger.info("{} {} {}", + key, + value, + (double) value * 100 / total); + if (value == 0) { + empty.incrementAndGet(); + } + }); + logger.info("index={} type={} numfields={} fieldsnotused={}", + index, type, map.size(), empty.get()); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + @SuppressWarnings("unchecked") + private void checkMapping(String index, String type, + String pathDef, String fieldName, Map map, + Map fields) { + String path = pathDef; + if (!path.isEmpty() && !path.endsWith(".")) { + path = path + "."; + } + if (!"properties".equals(fieldName)) { + path = path + fieldName; + } + if (map.containsKey("index")) { + String mode = (String) map.get("index"); + if ("no".equals(mode)) { + return; + } + } + for (Map.Entry entry : map.entrySet()) { + String key = entry.getKey(); + Object o = entry.getValue(); + if (o instanceof Map) { + Map child = (Map) o; + o = map.get("type"); + String fieldType = o instanceof String ? o.toString() : null; + // do not recurse into our custom field mapper + if (!"standardnumber".equals(fieldType) && !"ref".equals(fieldType)) { + checkMapping(index, type, path, key, child, fields); + } + } else if ("type".equals(key)) { + QueryBuilder filterBuilder = existsQuery(path); + QueryBuilder queryBuilder = constantScoreQuery(filterBuilder); + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); + SearchResponse searchResponse = searchRequestBuilder.setSize(0) + .setIndices(index) + .setTypes(type) + .setQuery(queryBuilder) + .execute() + .actionGet(); + fields.put(path, searchResponse.getHits().totalHits()); + } + } + } + + private static > Map sortByValue(Map map) { + Map result = new LinkedHashMap<>(); + map.entrySet().stream().sorted(Comparator.comparing(Map.Entry::getValue)) + .forEachOrdered(e -> result.put(e.getKey(), e.getValue())); + return result; + } + + @SuppressWarnings("unchecked") + public void createIndex(IndexDefinition indexDefinition) + throws IOException { + ensureActive(); + waitForCluster("YELLOW", "30s"); + URL indexSettings = indexDefinition.getSettingsUrl(); + if (indexSettings == null) { + throw new IllegalArgumentException("no settings defined for index " + indexDefinition.getIndex()); + } + URL indexMappings = indexDefinition.getMappingsUrl(); + if (indexMappings == null) { + throw new IllegalArgumentException("no mappings defined for index " + indexDefinition.getIndex()); + } + try (InputStream indexSettingsInput = indexSettings.openStream(); + InputStream indexMappingsInput = indexMappings.openStream()) { + // multiple type? + if (indexDefinition.getType() == null) { + Map mapping = new HashMap<>(); + // get type names from input stream + Reader reader = new InputStreamReader(indexMappingsInput, StandardCharsets.UTF_8); + Map map = JsonXContent.jsonXContent.createParser(reader).mapOrdered(); + for (Map.Entry entry : map.entrySet()) { + mapping.put(entry.getKey(), JsonXContent.contentBuilder().map((Map) entry.getValue()).string()); + } + Settings settings = Settings.settingsBuilder() + .loadFromStream("", indexSettingsInput) + .build(); + newIndex(indexDefinition.getFullIndexName(), settings, mapping); + } else { + newIndex(indexDefinition.getFullIndexName(), + indexDefinition.getType(), indexSettingsInput, indexMappingsInput); + } + } catch (IOException e) { + if (indexDefinition.ignoreErrors()) { + logger.warn(e.getMessage(), e); + logger.warn("warning while creating index '{}' with settings at {} and mappings at {}", + indexDefinition.getFullIndexName(), indexSettings, indexMappings); + } else { + logger.error("error while creating index '{}' with settings at {} and mappings at {}", + indexDefinition.getFullIndexName(), indexSettings, indexMappings); + throw new IOException(e); + } + } + } + + public void startBulk(Map defs) throws IOException { + ensureActive(); + for (Map.Entry entry : defs.entrySet()) { + IndexDefinition def = entry.getValue(); + startBulk(def.getFullIndexName(), -1, 1); + } + } + + public void stopBulk(Map defs) throws IOException { + ensureActive(); + if (defs == null) { + return; + } + try { + logger.info("flush bulk"); + flushIngest(); + logger.info("waiting for all bulk responses from cluster"); + waitForResponses("120s"); + logger.info("all bulk responses received"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.error(e.getMessage(), e); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } finally { + logger.info("updating cluster settings of {}", defs.keySet()); + for (Map.Entry entry : defs.entrySet()) { + IndexDefinition def = entry.getValue(); + stopBulk(def.getFullIndexName()); + } + } + } + + public void forceMerge(Map defs) { + for (Map.Entry entry : defs.entrySet()) { + if (entry.getValue().hasForceMerge()) { + logger.info("force merge of {}", entry.getKey()); + try { + ForceMergeRequestBuilder forceMergeRequestBuilder = + new ForceMergeRequestBuilder(client, ForceMergeAction.INSTANCE); + forceMergeRequestBuilder.setIndices(entry.getValue().getFullIndexName()); + forceMergeRequestBuilder.execute().get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.error(e.getMessage(), e); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + } + } + + public void switchIndex(IndexDefinition indexDefinition, List extraAliases) { + if (extraAliases == null) { + return; + } + if (indexDefinition.isSwitchAliases()) { + // filter out null/empty values + List validAliases = extraAliases.stream() + .filter(a -> a != null && !a.isEmpty()) + .collect(Collectors.toList()); + try { + switchAliases(indexDefinition.getIndex(), + indexDefinition.getFullIndexName(), validAliases); + } catch (Exception e) { + logger.warn("switching index failed: " + e.getMessage(), e); + } + } + } + + public void switchIndex(IndexDefinition indexDefinition, + List extraAliases, IndexAliasAdder indexAliasAdder) { + if (extraAliases == null) { + return; + } + if (indexDefinition.isSwitchAliases()) { + // filter out null/empty values + List validAliases = extraAliases.stream() + .filter(a -> a != null && !a.isEmpty()) + .collect(Collectors.toList()); + try { + switchAliases(indexDefinition.getIndex(), + indexDefinition.getFullIndexName(), validAliases, indexAliasAdder); + } catch (Exception e) { + logger.warn("switching index failed: " + e.getMessage(), e); + } + } + } + + public void replica(IndexDefinition indexDefinition) { + if (indexDefinition.getReplicaLevel() > 0) { + try { + updateReplicaLevel(indexDefinition.getFullIndexName(), indexDefinition.getReplicaLevel()); + } catch (Exception e) { + logger.warn("setting replica failed: " + e.getMessage(), e); + } + } + } +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/BulkProcessor.java b/elx-common/src/main/java/org/xbib/elx/common/BulkProcessor.java similarity index 95% rename from src/main/java/org/xbib/elasticsearch/extras/client/BulkProcessor.java rename to elx-common/src/main/java/org/xbib/elx/common/BulkProcessor.java index b32637e..10f6e62 100644 --- a/src/main/java/org/xbib/elasticsearch/extras/client/BulkProcessor.java +++ b/elx-common/src/main/java/org/xbib/elx/common/BulkProcessor.java @@ -1,4 +1,4 @@ -package org.xbib.elasticsearch.extras.client; +package org.xbib.elx.common; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; @@ -248,11 +248,17 @@ public class BulkProcessor implements Closeable { public static class Builder { private final Client client; + private final Listener listener; + private String name; + private int concurrentRequests = 1; + private int bulkActions = 1000; + private ByteSizeValue bulkSize = new ByteSizeValue(5, ByteSizeUnit.MB); + private TimeValue flushInterval = null; /** @@ -281,7 +287,7 @@ public class BulkProcessor implements Closeable { /** * Sets the number of concurrent requests allowed to be executed. A value of 0 means that only a single * request will be allowed to be executed. A value of 1 means 1 concurrent request is allowed to be executed - * while accumulating new bulk requests. Defaults to 1. + * while accumulating new bulk requests. Defaults to {@code 1}. * * @param concurrentRequests maximum number of concurrent requests * @return this builder @@ -293,9 +299,9 @@ public class BulkProcessor implements Closeable { /** * Sets when to flush a new bulk request based on the number of actions currently added. Defaults to - * 1000. Can be set to -1 to disable it. + * {@code 1000}. Can be set to {@code -1} to disable it. * - * @param bulkActions mbulk actions + * @param bulkActions bulk actions * @return this builder */ public Builder setBulkActions(int bulkActions) { @@ -305,7 +311,7 @@ public class BulkProcessor implements Closeable { /** * Sets when to flush a new bulk request based on the size of actions currently added. Defaults to - * 5mb. Can be set to -1 to disable it. + * {@code 5mb}. Can be set to {@code -1} to disable it. * * @param bulkSize bulk size * @return this builder @@ -318,7 +324,7 @@ public class BulkProcessor implements Closeable { /** * Sets a flush interval flushing *any* bulk actions pending if the interval passes. Defaults to not set. * Note, both {@link #setBulkActions(int)} and {@link #setBulkSize(org.elasticsearch.common.unit.ByteSizeValue)} - * can be set to -1 with the flush interval set allowing for complete async processing of bulk actions. + * can be set to {@code -1} with the flush interval set allowing for complete async processing of bulk actions. * * @param flushInterval flush interval * @return this builder @@ -365,8 +371,10 @@ public class BulkProcessor implements Closeable { } - private class SyncBulkRequestHandler implements BulkRequestHandler { + private static class SyncBulkRequestHandler implements BulkRequestHandler { + private final Client client; + private final BulkProcessor.Listener listener; SyncBulkRequestHandler(Client client, BulkProcessor.Listener listener) { @@ -390,15 +398,19 @@ public class BulkProcessor implements Closeable { } @Override - public boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException { + public boolean awaitClose(long timeout, TimeUnit unit) { return true; } } - private class AsyncBulkRequestHandler implements BulkRequestHandler { + private static class AsyncBulkRequestHandler implements BulkRequestHandler { + private final Client client; + private final BulkProcessor.Listener listener; + private final Semaphore semaphore; + private final int concurrentRequests; private AsyncBulkRequestHandler(Client client, BulkProcessor.Listener listener, int concurrentRequests) { @@ -450,8 +462,8 @@ public class BulkProcessor implements Closeable { @Override public boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException { - if (semaphore.tryAcquire(this.concurrentRequests, timeout, unit)) { - semaphore.release(this.concurrentRequests); + if (semaphore.tryAcquire(concurrentRequests, timeout, unit)) { + semaphore.release(concurrentRequests); return true; } return false; diff --git a/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java b/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java new file mode 100644 index 0000000..cc15697 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java @@ -0,0 +1,124 @@ +package org.xbib.elx.common; + +import org.elasticsearch.Version; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.xbib.elx.api.BulkControl; +import org.xbib.elx.api.BulkMetric; +import org.xbib.elx.api.ExtendedClient; +import org.xbib.elx.api.ExtendedClientProvider; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.ServiceLoader; + +@SuppressWarnings("rawtypes") +public class ClientBuilder { + + private final ElasticsearchClient client; + + private final Settings.Builder settingsBuilder; + + private Map, ExtendedClientProvider> providerMap; + + private Class provider; + + private BulkMetric metric; + + private BulkControl control; + + public ClientBuilder() { + this(null); + } + + public ClientBuilder(ElasticsearchClient client) { + this(client, Thread.currentThread().getContextClassLoader()); + } + + public ClientBuilder(ElasticsearchClient client, ClassLoader classLoader) { + this.client = client; + this.settingsBuilder = Settings.builder(); + settingsBuilder.put("node.name", "elx-client-" + Version.CURRENT); + this.providerMap = new HashMap<>(); + ServiceLoader serviceLoader = ServiceLoader.load(ExtendedClientProvider.class, + classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader()); + for (ExtendedClientProvider provider : serviceLoader) { + providerMap.put(provider.getClass(), provider); + } + this.metric = new SimpleBulkMetric(); + this.control = new SimpleBulkControl(); + } + + public static ClientBuilder builder() { + return new ClientBuilder(); + } + + public static ClientBuilder builder(ElasticsearchClient client) { + return new ClientBuilder(client); + } + + public ClientBuilder provider(Class provider) { + this.provider = provider; + return this; + } + + public ClientBuilder put(String key, String value) { + settingsBuilder.put(key, value); + return this; + } + + public ClientBuilder put(String key, Integer value) { + settingsBuilder.put(key, value); + return this; + } + + public ClientBuilder put(String key, Long value) { + settingsBuilder.put(key, value); + return this; + } + + public ClientBuilder put(String key, Double value) { + settingsBuilder.put(key, value); + return this; + } + + public ClientBuilder put(String key, ByteSizeValue value) { + settingsBuilder.put(key, value); + return this; + } + + public ClientBuilder put(String key, TimeValue value) { + settingsBuilder.put(key, value); + return this; + } + + public ClientBuilder put(Settings settings) { + settingsBuilder.put(settings); + return this; + } + + public ClientBuilder setMetric(BulkMetric metric) { + this.metric = metric; + return this; + } + + public ClientBuilder setControl(BulkControl control) { + this.control = control; + return this; + } + + @SuppressWarnings("unchecked") + public C build() throws IOException { + if (provider == null) { + throw new IllegalArgumentException("no provider"); + } + return (C) providerMap.get(provider).getExtendedClient() + .setClient(client) + .setBulkMetric(metric) + .setBulkControl(control) + .init(settingsBuilder.build()); + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java new file mode 100644 index 0000000..dc01807 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java @@ -0,0 +1,146 @@ +package org.xbib.elx.common; + +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; + +import java.util.Map; + +/** + * Mock client, it does not perform actions on a cluster. Useful for testing or dry runs. + */ +public class MockExtendedClient extends AbstractExtendedClient { + + @Override + public ElasticsearchClient getClient() { + return null; + } + + @Override + public MockExtendedClient init(Settings settings) { + return this; + } + + @Override + protected ElasticsearchClient createClient(Settings settings) { + return null; + } + + @Override + public MockExtendedClient maxActionsPerRequest(int maxActions) { + return this; + } + + @Override + public MockExtendedClient maxConcurrentRequests(int maxConcurrentRequests) { + return this; + } + + @Override + public MockExtendedClient maxVolumePerRequest(String maxVolumePerRequest) { + return this; + } + + @Override + public MockExtendedClient flushIngestInterval(String interval) { + return this; + } + + @Override + public MockExtendedClient index(String index, String type, String id, boolean create, String source) { + return this; + } + + @Override + public MockExtendedClient delete(String index, String type, String id) { + return this; + } + + @Override + public MockExtendedClient update(String index, String type, String id, String source) { + return this; + } + + @Override + public MockExtendedClient indexRequest(IndexRequest indexRequest) { + return this; + } + + @Override + public MockExtendedClient deleteRequest(DeleteRequest deleteRequest) { + return this; + } + + @Override + public MockExtendedClient updateRequest(UpdateRequest updateRequest) { + return this; + } + + @Override + public MockExtendedClient flushIngest() { + return this; + } + + @Override + public MockExtendedClient waitForResponses(String timeValue) { + return this; + } + + @Override + public MockExtendedClient startBulk(String index, long startRefreshInterval, long stopRefreshIterval) { + return this; + } + + @Override + public MockExtendedClient stopBulk(String index) { + return this; + } + + @Override + public MockExtendedClient deleteIndex(String index) { + return this; + } + + @Override + public MockExtendedClient newIndex(String index) { + return this; + } + + @Override + public MockExtendedClient newMapping(String index, String type, Map mapping) { + return this; + } + + @Override + public void putMapping(String index) { + } + + @Override + public void refreshIndex(String index) { + } + + @Override + public void flushIndex(String index) { + } + + @Override + public void waitForCluster(String healthColor, String timeValue) { + } + + @Override + public int waitForRecovery(String index) { + return -1; + } + + @Override + public int updateReplicaLevel(String index, int level) { + return -1; + } + + @Override + public void shutdown() { + // nothing to do + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java new file mode 100644 index 0000000..87e65cc --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java @@ -0,0 +1,10 @@ +package org.xbib.elx.common; + +import org.xbib.elx.api.ExtendedClientProvider; + +public class MockExtendedClientProvider implements ExtendedClientProvider { + @Override + public MockExtendedClient getExtendedClient() { + return new MockExtendedClient(); + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/Parameters.java b/elx-common/src/main/java/org/xbib/elx/common/Parameters.java new file mode 100644 index 0000000..017c780 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/Parameters.java @@ -0,0 +1,40 @@ +package org.xbib.elx.common; + +public enum Parameters { + + DEFAULT_MAX_ACTIONS_PER_REQUEST(1000), + + DEFAULT_MAX_CONCURRENT_REQUESTS(Runtime.getRuntime().availableProcessors()), + + DEFAULT_MAX_VOLUME_PER_REQUEST("10mb"), + + DEFAULT_FLUSH_INTERVAL("30s"), + + MAX_ACTIONS_PER_REQUEST ("max_actions_per_request"), + + MAX_CONCURRENT_REQUESTS("max_concurrent_requests"), + + MAX_VOLUME_PER_REQUEST("max_volume_per_request"), + + FLUSH_INTERVAL("flush_interval"); + + int num; + + String string; + + Parameters(int num) { + this.num = num; + } + + Parameters(String string) { + this.string = string; + } + + int getNum() { + return num; + } + + String getString() { + return string; + } +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkControl.java b/elx-common/src/main/java/org/xbib/elx/common/SimpleBulkControl.java similarity index 95% rename from src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkControl.java rename to elx-common/src/main/java/org/xbib/elx/common/SimpleBulkControl.java index b9a92d6..624cec5 100644 --- a/src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkControl.java +++ b/elx-common/src/main/java/org/xbib/elx/common/SimpleBulkControl.java @@ -1,4 +1,6 @@ -package org.xbib.elasticsearch.extras.client; +package org.xbib.elx.common; + +import org.xbib.elx.api.BulkControl; import java.util.HashMap; import java.util.HashSet; diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkMetric.java b/elx-common/src/main/java/org/xbib/elx/common/SimpleBulkMetric.java similarity index 55% rename from src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkMetric.java rename to elx-common/src/main/java/org/xbib/elx/common/SimpleBulkMetric.java index e836816..1a9dd35 100644 --- a/src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkMetric.java +++ b/elx-common/src/main/java/org/xbib/elx/common/SimpleBulkMetric.java @@ -1,32 +1,48 @@ -package org.xbib.elasticsearch.extras.client; +package org.xbib.elx.common; +import org.xbib.elx.api.BulkMetric; import org.xbib.metrics.Count; import org.xbib.metrics.CountMetric; import org.xbib.metrics.Meter; import org.xbib.metrics.Metered; -/** - * - */ + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + public class SimpleBulkMetric implements BulkMetric { - private final Meter totalIngest = new Meter(); + private final Meter totalIngest; - private final Count totalIngestSizeInBytes = new CountMetric(); + private final Count totalIngestSizeInBytes; - private final Count currentIngest = new CountMetric(); + private final Count currentIngest; - private final Count currentIngestNumDocs = new CountMetric(); + private final Count currentIngestNumDocs; - private final Count submitted = new CountMetric(); + private final Count submitted; - private final Count succeeded = new CountMetric(); + private final Count succeeded; - private final Count failed = new CountMetric(); + private final Count failed; private Long started; private Long stopped; + public SimpleBulkMetric() { + this(Executors.newSingleThreadScheduledExecutor()); + } + + public SimpleBulkMetric(ScheduledExecutorService executorService) { + totalIngest = new Meter(executorService); + totalIngestSizeInBytes = new CountMetric(); + currentIngest = new CountMetric(); + currentIngestNumDocs = new CountMetric(); + submitted = new CountMetric(); + succeeded = new CountMetric(); + failed = new CountMetric(); + } + @Override public Metered getTotalIngest() { return totalIngest; @@ -65,7 +81,7 @@ public class SimpleBulkMetric implements BulkMetric { @Override public void start() { this.started = System.nanoTime(); - totalIngest.spawn(5L); + totalIngest.start(5L); } @Override diff --git a/elx-common/src/main/java/org/xbib/elx/common/io/ClasspathURLStreamHandler.java b/elx-common/src/main/java/org/xbib/elx/common/io/ClasspathURLStreamHandler.java new file mode 100644 index 0000000..e7d8727 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/io/ClasspathURLStreamHandler.java @@ -0,0 +1,25 @@ +package org.xbib.elx.common.io; + +import java.io.IOException; +import java.net.URL; +import java.net.URLConnection; +import java.net.URLStreamHandler; + +public class ClasspathURLStreamHandler extends URLStreamHandler { + + private final ClassLoader classLoader; + + public ClasspathURLStreamHandler() { + this.classLoader = getClass().getClassLoader(); + } + + public ClasspathURLStreamHandler(ClassLoader classLoader) { + this.classLoader = classLoader; + } + + @Override + protected URLConnection openConnection(URL u) throws IOException { + final URL resourceUrl = classLoader.getResource(u.getPath()); + return resourceUrl != null ? resourceUrl.openConnection() : null; + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/io/ClasspathURLStreamHandlerFactory.java b/elx-common/src/main/java/org/xbib/elx/common/io/ClasspathURLStreamHandlerFactory.java new file mode 100644 index 0000000..00c7c83 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/io/ClasspathURLStreamHandlerFactory.java @@ -0,0 +1,12 @@ +package org.xbib.elx.common.io; + +import java.net.URLStreamHandler; +import java.net.URLStreamHandlerFactory; + +public class ClasspathURLStreamHandlerFactory implements URLStreamHandlerFactory { + + @Override + public URLStreamHandler createURLStreamHandler(String protocol) { + return "classpath".equals(protocol) ? new ClasspathURLStreamHandler() : null; + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/io/package-info.java b/elx-common/src/main/java/org/xbib/elx/common/io/package-info.java new file mode 100644 index 0000000..62a8d78 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/io/package-info.java @@ -0,0 +1 @@ +package org.xbib.elx.common.io; \ No newline at end of file diff --git a/elx-common/src/main/java/org/xbib/elx/common/management/IndexDefinition.java b/elx-common/src/main/java/org/xbib/elx/common/management/IndexDefinition.java new file mode 100644 index 0000000..37dcc45 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/management/IndexDefinition.java @@ -0,0 +1,139 @@ +package org.xbib.elx.common.management; + +import java.net.URL; + +public class IndexDefinition { + + private String index; + + private String type; + + private String fullIndexName; + + private String dateTimePattern; + + private URL settingsUrl; + + private URL mappingsUrl; + + private boolean enabled; + + private boolean ignoreErrors; + + private boolean switchAliases; + + private boolean hasForceMerge; + + private int replicaLevel; + + private IndexRetention indexRetention; + + public IndexDefinition setIndex(String index) { + this.index = index; + return this; + } + + public String getIndex() { + return index; + } + + public IndexDefinition setFullIndexName(String fullIndexName) { + this.fullIndexName = fullIndexName; + return this; + } + + public String getFullIndexName() { + return fullIndexName; + } + + public IndexDefinition setType(String type) { + this.type = type; + return this; + } + + public String getType() { + return type; + } + + public IndexDefinition setSettingsUrl(URL settingsUrl) { + this.settingsUrl = settingsUrl; + return this; + } + + public URL getSettingsUrl() { + return settingsUrl; + } + + public IndexDefinition setMappingsUrl(URL mappingsUrl) { + this.mappingsUrl = mappingsUrl; + return this; + } + + public URL getMappingsUrl() { + return mappingsUrl; + } + + public IndexDefinition setDateTimePattern(String timeWindow) { + this.dateTimePattern = timeWindow; + return this; + } + + public String getDateTimePattern() { + return dateTimePattern; + } + + public IndexDefinition setEnabled(boolean enabled) { + this.enabled = enabled; + return this; + } + + public boolean isEnabled() { + return enabled; + } + + public IndexDefinition setIgnoreErrors(boolean ignoreErrors) { + this.ignoreErrors = ignoreErrors; + return this; + } + + public boolean ignoreErrors() { + return ignoreErrors; + } + + public IndexDefinition setSwitchAliases(boolean switchAliases) { + this.switchAliases = switchAliases; + return this; + } + + public boolean isSwitchAliases() { + return switchAliases; + } + + public IndexDefinition setForceMerge(boolean hasForceMerge) { + this.hasForceMerge = hasForceMerge; + return this; + } + + public boolean hasForceMerge() { + return hasForceMerge; + } + + public IndexDefinition setReplicaLevel(int replicaLevel) { + this.replicaLevel = replicaLevel; + return this; + } + + public int getReplicaLevel() { + return replicaLevel; + } + + public IndexDefinition setRetention(IndexRetention indexRetention) { + this.indexRetention = indexRetention; + return this; + } + + public IndexRetention getRetention() { + return indexRetention; + } + +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/management/IndexRetention.java b/elx-common/src/main/java/org/xbib/elx/common/management/IndexRetention.java new file mode 100644 index 0000000..8024ef4 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/management/IndexRetention.java @@ -0,0 +1,27 @@ +package org.xbib.elx.common.management; + +public class IndexRetention { + + private int timestampDiff; + + private int minToKeep; + + public IndexRetention setTimestampDiff(int timestampDiff) { + this.timestampDiff = timestampDiff; + return this; + } + + public int getTimestampDiff() { + return timestampDiff; + } + + public IndexRetention setMinToKeep(int minToKeep) { + this.minToKeep = minToKeep; + return this; + } + + public int getMinToKeep() { + return minToKeep; + } + +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/management/package-info.java b/elx-common/src/main/java/org/xbib/elx/common/management/package-info.java new file mode 100644 index 0000000..0d98623 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/management/package-info.java @@ -0,0 +1 @@ +package org.xbib.elx.common.management; \ No newline at end of file diff --git a/elx-common/src/main/java/org/xbib/elx/common/package-info.java b/elx-common/src/main/java/org/xbib/elx/common/package-info.java new file mode 100644 index 0000000..4971f08 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/package-info.java @@ -0,0 +1,4 @@ +/** + * Common classes for Elasticsearch client extensions. + */ +package org.xbib.elx.common; diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/NetworkUtils.java b/elx-common/src/main/java/org/xbib/elx/common/util/NetworkUtils.java similarity index 92% rename from src/main/java/org/xbib/elasticsearch/extras/client/NetworkUtils.java rename to elx-common/src/main/java/org/xbib/elx/common/util/NetworkUtils.java index 9c5ffc2..11dd014 100644 --- a/src/main/java/org/xbib/elasticsearch/extras/client/NetworkUtils.java +++ b/elx-common/src/main/java/org/xbib/elx/common/util/NetworkUtils.java @@ -1,7 +1,7 @@ -package org.xbib.elasticsearch.extras.client; +package org.xbib.elx.common.util; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.io.IOException; import java.net.Inet4Address; @@ -11,6 +11,7 @@ import java.net.NetworkInterface; import java.net.SocketException; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.Enumeration; import java.util.List; import java.util.Locale; @@ -20,7 +21,7 @@ import java.util.Locale; */ public class NetworkUtils { - private static final ESLogger logger = ESLoggerFactory.getLogger(NetworkUtils.class.getName()); + private static final Logger logger = LogManager.getLogger(NetworkUtils.class.getName()); private static final String IPV4_SETTING = "java.net.preferIPv4Stack"; @@ -102,10 +103,8 @@ public class NetworkUtils { NetworkInterface networkInterface = interfaces.nextElement(); allInterfaces.add(networkInterface); Enumeration subInterfaces = networkInterface.getSubInterfaces(); - if (subInterfaces.hasMoreElements()) { - while (subInterfaces.hasMoreElements()) { - allInterfaces.add(subInterfaces.nextElement()); - } + while (subInterfaces.hasMoreElements()) { + allInterfaces.add(subInterfaces.nextElement()); } } sortInterfaces(allInterfaces); @@ -223,10 +222,8 @@ public class NetworkUtils { NetworkInterface networkInterface = interfaces.nextElement(); networkInterfaces.add(networkInterface); Enumeration subInterfaces = networkInterface.getSubInterfaces(); - if (subInterfaces.hasMoreElements()) { - while (subInterfaces.hasMoreElements()) { - networkInterfaces.add(subInterfaces.nextElement()); - } + while (subInterfaces.hasMoreElements()) { + networkInterfaces.add(subInterfaces.nextElement()); } } sortInterfaces(networkInterfaces); @@ -234,7 +231,7 @@ public class NetworkUtils { } private static void sortInterfaces(List interfaces) { - Collections.sort(interfaces, (o1, o2) -> Integer.compare(o1.getIndex(), o2.getIndex())); + Collections.sort(interfaces, Comparator.comparingInt(NetworkInterface::getIndex)); } private static void sortAddresses(List addressList) { diff --git a/elx-common/src/main/java/org/xbib/elx/common/util/package-info.java b/elx-common/src/main/java/org/xbib/elx/common/util/package-info.java new file mode 100644 index 0000000..3c41bfe --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/util/package-info.java @@ -0,0 +1 @@ +package org.xbib.elx.common.util; \ No newline at end of file diff --git a/elx-common/src/main/resources/META-INF/services/java.net.URLStreamHandlerFactory b/elx-common/src/main/resources/META-INF/services/java.net.URLStreamHandlerFactory new file mode 100644 index 0000000..bb6d620 --- /dev/null +++ b/elx-common/src/main/resources/META-INF/services/java.net.URLStreamHandlerFactory @@ -0,0 +1 @@ +org.xbib.elx.common.io.ClasspathURLStreamHandlerFactory \ No newline at end of file diff --git a/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider b/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider new file mode 100644 index 0000000..9729b83 --- /dev/null +++ b/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider @@ -0,0 +1 @@ +org.xbib.elx.common.MockExtendedClientProvider \ No newline at end of file diff --git a/src/integration-test/java/org/elasticsearch/node/MockNode.java b/elx-common/src/test/java/org/elasticsearch/node/MockNode.java similarity index 98% rename from src/integration-test/java/org/elasticsearch/node/MockNode.java rename to elx-common/src/test/java/org/elasticsearch/node/MockNode.java index b0c02eb..aad8b8b 100644 --- a/src/integration-test/java/org/elasticsearch/node/MockNode.java +++ b/elx-common/src/test/java/org/elasticsearch/node/MockNode.java @@ -8,9 +8,6 @@ import org.elasticsearch.plugins.Plugin; import java.util.ArrayList; import java.util.Collection; -/** - * - */ public class MockNode extends Node { public MockNode() { @@ -34,5 +31,4 @@ public class MockNode extends Node { list.add(classpathPlugin); return list; } - } diff --git a/elx-common/src/test/java/org/elasticsearch/node/package-info.java b/elx-common/src/test/java/org/elasticsearch/node/package-info.java new file mode 100644 index 0000000..8ffed8c --- /dev/null +++ b/elx-common/src/test/java/org/elasticsearch/node/package-info.java @@ -0,0 +1 @@ +package org.elasticsearch.node; \ No newline at end of file diff --git a/src/integration-test/java/org/xbib/elasticsearch/AliasTest.java b/elx-common/src/test/java/org/xbib/elx/common/AliasTest.java similarity index 92% rename from src/integration-test/java/org/xbib/elasticsearch/AliasTest.java rename to elx-common/src/test/java/org/xbib/elx/common/AliasTest.java index 970268e..e9106d0 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/AliasTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/AliasTest.java @@ -1,9 +1,11 @@ -package org.xbib.elasticsearch; +package org.xbib.elx.common; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import com.carrotsearch.hppc.cursors.ObjectCursor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; @@ -12,8 +14,6 @@ import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.cluster.metadata.AliasAction; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.junit.Test; import java.io.IOException; @@ -29,7 +29,7 @@ import java.util.regex.Pattern; */ public class AliasTest extends NodeTestUtils { - private static final ESLogger logger = ESLoggerFactory.getLogger(AliasTest.class.getName()); + private static final Logger logger = LogManager.getLogger(AliasTest.class.getName()); @Test public void testAlias() throws IOException { @@ -53,7 +53,7 @@ public class AliasTest extends NodeTestUtils { } @Test - public void testMostRecentIndex() throws IOException { + public void testMostRecentIndex() { String alias = "test"; CreateIndexRequest indexRequest = new CreateIndexRequest("test20160101"); client("1").admin().indices().create(indexRequest).actionGet(); @@ -86,7 +86,7 @@ public class AliasTest extends NodeTestUtils { assertEquals("test20160103", it.next()); assertEquals("test20160102", it.next()); assertEquals("test20160101", it.next()); - logger.info("result={}", result); + logger.info("success: result={}", result); } } diff --git a/elx-common/src/test/java/org/xbib/elx/common/MockExtendedClientProviderTest.java b/elx-common/src/test/java/org/xbib/elx/common/MockExtendedClientProviderTest.java new file mode 100644 index 0000000..8474c1c --- /dev/null +++ b/elx-common/src/test/java/org/xbib/elx/common/MockExtendedClientProviderTest.java @@ -0,0 +1,16 @@ +package org.xbib.elx.common; + +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertNotNull; + +public class MockExtendedClientProviderTest { + + @Test + public void testMockExtendedProvider() throws IOException { + MockExtendedClient client = ClientBuilder.builder().provider(MockExtendedClientProvider.class).build(); + assertNotNull(client); + } +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/NetworkTest.java b/elx-common/src/test/java/org/xbib/elx/common/NetworkTest.java similarity index 95% rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/NetworkTest.java rename to elx-common/src/test/java/org/xbib/elx/common/NetworkTest.java index b9e7a87..248b906 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/NetworkTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/NetworkTest.java @@ -1,4 +1,4 @@ -package org.xbib.elasticsearch.extras.client; +package org.xbib.elx.common; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -9,15 +9,13 @@ import java.net.NetworkInterface; import java.util.Collections; import java.util.Enumeration; -/** - * - */ public class NetworkTest { private static final Logger logger = LogManager.getLogger(NetworkTest.class); @Test public void testNetwork() throws Exception { + // walk very slowly over all interfaces Enumeration nets = NetworkInterface.getNetworkInterfaces(); for (NetworkInterface netint : Collections.list(nets)) { System.out.println("checking network interface = " + netint.getName()); diff --git a/elx-common/src/test/java/org/xbib/elx/common/NodeTestUtils.java b/elx-common/src/test/java/org/xbib/elx/common/NodeTestUtils.java new file mode 100644 index 0000000..86e30c6 --- /dev/null +++ b/elx-common/src/test/java/org/xbib/elx/common/NodeTestUtils.java @@ -0,0 +1,213 @@ +package org.xbib.elx.common; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.client.support.AbstractClient; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.node.MockNode; +import org.elasticsearch.node.Node; +import org.junit.After; +import org.junit.Before; +import org.xbib.elx.common.util.NetworkUtils; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.elasticsearch.common.settings.Settings.settingsBuilder; + +public class NodeTestUtils { + + private static final Logger logger = LogManager.getLogger("test"); + + private static Random random = new Random(); + + private static char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); + + private Map nodes = new HashMap<>(); + + private Map clients = new HashMap<>(); + + private AtomicInteger counter = new AtomicInteger(); + + private String cluster; + + private String host; + + private int port; + + private static void deleteFiles() throws IOException { + Path directory = Paths.get(getHome() + "/data"); + Files.walkFileTree(directory, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + + }); + + } + + @Before + public void startNodes() { + try { + logger.info("starting"); + setClusterName(); + startNode("1"); + findNodeAddress(); + try { + ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN) + .timeout(TimeValue.timeValueSeconds(30))).actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + throw new IOException("cluster state is " + healthResponse.getStatus().name() + + ", from here on, everything will fail!"); + } + } catch (ElasticsearchTimeoutException e) { + throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); + } + } catch (Throwable t) { + logger.error("startNodes failed", t); + } + } + + @After + public void stopNodes() { + try { + closeNodes(); + } catch (Exception e) { + logger.error("can not close nodes", e); + } finally { + try { + deleteFiles(); + logger.info("data files wiped"); + Thread.sleep(2000L); // let OS commit changes + } catch (IOException e) { + logger.error(e.getMessage(), e); + } catch (InterruptedException e) { + // ignore + } + } + } + + protected void setClusterName() { + this.cluster = "test-helper-cluster-" + + NetworkUtils.getLocalAddress().getHostName() + + "-" + System.getProperty("user.name") + + "-" + counter.incrementAndGet(); + } + + protected String getClusterName() { + return cluster; + } + + protected Settings getSettings() { + return settingsBuilder() + .put("host", host) + .put("port", port) + .put("cluster.name", cluster) + .put("path.home", getHome()) + .build(); + } + + protected Settings getNodeSettings() { + return settingsBuilder() + .put("cluster.name", cluster) + .put("cluster.routing.schedule", "50ms") + .put("cluster.routing.allocation.disk.threshold_enabled", false) + .put("discovery.zen.multicast.enabled", true) + .put("discovery.zen.multicast.ping_timeout", "5s") + .put("http.enabled", true) + .put("threadpool.bulk.size", Runtime.getRuntime().availableProcessors()) + .put("threadpool.bulk.queue_size", 16 * Runtime.getRuntime().availableProcessors()) // default is 50, too low + .put("index.number_of_replicas", 0) + .put("path.home", getHome()) + .build(); + } + + protected static String getHome() { + return System.getProperty("path.home", System.getProperty("user.dir")); + } + + public void startNode(String id) { + buildNode(id).start(); + } + + public AbstractClient client(String id) { + return clients.get(id); + } + + private void closeNodes() { + logger.info("closing all clients"); + for (AbstractClient client : clients.values()) { + client.close(); + } + clients.clear(); + logger.info("closing all nodes"); + for (Node node : nodes.values()) { + if (node != null) { + node.close(); + } + } + nodes.clear(); + logger.info("all nodes closed"); + } + + protected void findNodeAddress() { + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); + NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet(); + Object obj = response.iterator().next().getTransport().getAddress() + .publishAddress(); + if (obj instanceof InetSocketTransportAddress) { + InetSocketTransportAddress address = (InetSocketTransportAddress) obj; + host = address.address().getHostName(); + port = address.address().getPort(); + } + } + + private Node buildNode(String id) { + Settings nodeSettings = settingsBuilder() + .put(getNodeSettings()) + .put("name", id) + .build(); + Node node = new MockNode(nodeSettings); + AbstractClient client = (AbstractClient) node.client(); + nodes.put(id, node); + clients.put(id, client); + logger.info("clients={}", clients); + return node; + } + + protected String randomString(int len) { + final char[] buf = new char[len]; + final int n = numbersAndLetters.length - 1; + for (int i = 0; i < buf.length; i++) { + buf[i] = numbersAndLetters[random.nextInt(n)]; + } + return new String(buf); + } +} diff --git a/elx-common/src/test/java/org/xbib/elx/common/SearchTest.java b/elx-common/src/test/java/org/xbib/elx/common/SearchTest.java new file mode 100644 index 0000000..63892d0 --- /dev/null +++ b/elx-common/src/test/java/org/xbib/elx/common/SearchTest.java @@ -0,0 +1,56 @@ +package org.xbib.elx.common; + +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.bulk.BulkAction; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.sort.SortOrder; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +public class SearchTest extends NodeTestUtils { + + @Test + public void testSearch() throws Exception { + Client client = client("1"); + BulkRequestBuilder builder = new BulkRequestBuilder(client, BulkAction.INSTANCE); + for (int i = 0; i < 1000; i++) { + IndexRequest indexRequest = new IndexRequest("pages", "row") + .source(XContentFactory.jsonBuilder() + .startObject() + .field("user1", "joerg") + .field("user2", "joerg") + .field("user3", "joerg") + .field("user4", "joerg") + .field("user5", "joerg") + .field("user6", "joerg") + .field("user7", "joerg") + .field("user8", "joerg") + .field("user9", "joerg") + .field("rowcount", i) + .field("rs", 1234)); + builder.add(indexRequest); + } + client.bulk(builder.request()).actionGet(); + client.admin().indices().refresh(new RefreshRequest()).actionGet(); + + for (int i = 0; i < 100; i++) { + QueryBuilder queryStringBuilder = QueryBuilders.queryStringQuery("rs:" + 1234); + SearchRequestBuilder requestBuilder = client.prepareSearch() + .setIndices("pages") + .setTypes("row") + .setQuery(queryStringBuilder) + .addSort("rowcount", SortOrder.DESC) + .setFrom(i * 10).setSize(10); + SearchResponse searchResponse = requestBuilder.execute().actionGet(); + assertTrue(searchResponse.getHits().getTotalHits() > 0); + } + } +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/SimpleTest.java b/elx-common/src/test/java/org/xbib/elx/common/SimpleTest.java similarity index 91% rename from src/integration-test/java/org/xbib/elasticsearch/SimpleTest.java rename to elx-common/src/test/java/org/xbib/elx/common/SimpleTest.java index 0af13df..75cdc29 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/SimpleTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/SimpleTest.java @@ -1,4 +1,4 @@ -package org.xbib.elasticsearch; +package org.xbib.elx.common; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; @@ -10,16 +10,14 @@ import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; import org.elasticsearch.action.index.IndexAction; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexNotFoundException; import org.junit.Test; -/** - * - */ public class SimpleTest extends NodeTestUtils { protected Settings getNodeSettings() { return settingsBuilder() - .put("path.home", System.getProperty("path.home")) + .put(super.getNodeSettings()) .put("index.analysis.analyzer.default.filter.0", "lowercase") .put("index.analysis.analyzer.default.filter.1", "trim") .put("index.analysis.analyzer.default.tokenizer", "keyword") @@ -32,8 +30,8 @@ public class SimpleTest extends NodeTestUtils { DeleteIndexRequestBuilder deleteIndexRequestBuilder = new DeleteIndexRequestBuilder(client("1"), DeleteIndexAction.INSTANCE, "test"); deleteIndexRequestBuilder.execute().actionGet(); - } catch (Exception e) { - // ignore + } catch (IndexNotFoundException e) { + // ignore if index not found } IndexRequestBuilder indexRequestBuilder = new IndexRequestBuilder(client("1"), IndexAction.INSTANCE); indexRequestBuilder diff --git a/elx-common/src/test/java/org/xbib/elx/common/WildcardTest.java b/elx-common/src/test/java/org/xbib/elx/common/WildcardTest.java new file mode 100644 index 0000000..783b440 --- /dev/null +++ b/elx-common/src/test/java/org/xbib/elx/common/WildcardTest.java @@ -0,0 +1,62 @@ +package org.xbib.elx.common; + +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.junit.Test; + +import java.io.IOException; + +public class WildcardTest extends NodeTestUtils { + + protected Settings getNodeSettings() { + return Settings.settingsBuilder() + .put(super.getNodeSettings()) + .put("cluster.routing.allocation.disk.threshold_enabled", false) + .put("discovery.zen.multicast.enabled", false) + .put("http.enabled", false) + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .build(); + } + + @Test + public void testWildcard() throws Exception { + index(client("1"), "1", "010"); + index(client("1"), "2", "0*0"); + // exact + validateCount(client("1"), QueryBuilders.queryStringQuery("010").defaultField("field"), 1); + validateCount(client("1"), QueryBuilders.queryStringQuery("0\\*0").defaultField("field"), 1); + // pattern + validateCount(client("1"), QueryBuilders.queryStringQuery("0*0").defaultField("field"), 1); // 2? + validateCount(client("1"), QueryBuilders.queryStringQuery("0?0").defaultField("field"), 1); // 2? + validateCount(client("1"), QueryBuilders.queryStringQuery("0**0").defaultField("field"), 1); // 2? + validateCount(client("1"), QueryBuilders.queryStringQuery("0??0").defaultField("field"), 0); + validateCount(client("1"), QueryBuilders.queryStringQuery("*10").defaultField("field"), 1); + validateCount(client("1"), QueryBuilders.queryStringQuery("*1*").defaultField("field"), 1); + validateCount(client("1"), QueryBuilders.queryStringQuery("*\\*0").defaultField("field"), 0); // 1? + validateCount(client("1"), QueryBuilders.queryStringQuery("*\\**").defaultField("field"), 0); // 1? + } + + private void index(Client client, String id, String fieldValue) throws IOException { + client.index(new IndexRequest("index", "type", id) + .source(XContentFactory.jsonBuilder().startObject().field("field", fieldValue).endObject()) + .refresh(true)).actionGet(); + } + + private long count(Client client, QueryBuilder queryBuilder) { + return client.prepareSearch("index").setTypes("type") + .setQuery(queryBuilder) + .execute().actionGet().getHits().getTotalHits(); + } + + private void validateCount(Client client, QueryBuilder queryBuilder, long expectedHits) { + final long actualHits = count(client, queryBuilder); + if (actualHits != expectedHits) { + throw new RuntimeException("actualHits=" + actualHits + ", expectedHits=" + expectedHits); + } + } +} diff --git a/elx-common/src/test/java/org/xbib/elx/common/package-info.java b/elx-common/src/test/java/org/xbib/elx/common/package-info.java new file mode 100644 index 0000000..9a9e4ce --- /dev/null +++ b/elx-common/src/test/java/org/xbib/elx/common/package-info.java @@ -0,0 +1 @@ +package org.xbib.elx.common; \ No newline at end of file diff --git a/elx-common/src/test/resources/log4j2.xml b/elx-common/src/test/resources/log4j2.xml new file mode 100644 index 0000000..6c323f8 --- /dev/null +++ b/elx-common/src/test/resources/log4j2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/elx-http/build.gradle~ b/elx-http/build.gradle~ new file mode 100644 index 0000000..da70162 --- /dev/null +++ b/elx-http/build.gradle~ @@ -0,0 +1,65 @@ +buildscript { + repositories { + jcenter() + maven { + url 'http://xbib.org/repository' + } + } + dependencies { + classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.2.2.0" + } +} + +apply plugin: 'org.xbib.gradle.plugin.elasticsearch.build' + +configurations { + main + tests +} + +dependencies { + compile project(':common') + compile "org.xbib:netty-http-client:${project.property('xbib-netty-http-client.version')}" + testCompile "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" + testRuntime "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" +} + +jar { + baseName "${rootProject.name}-common" +} + +/* +task testJar(type: Jar, dependsOn: testClasses) { + baseName = "${project.archivesBaseName}-tests" + from sourceSets.test.output +} +*/ + +artifacts { + main jar + tests testJar + archives sourcesJar, javadocJar +} + +test { + enabled = true + include '**/SimpleTest.*' + testLogging { + showStandardStreams = true + exceptionFormat = 'full' + } +} + +randomizedTest { + enabled = false +} + +esTest { + enabled = true + // test with the jars, not the classes, for security manager + // classpath = files(configurations.testRuntime) + configurations.main.artifacts.files + configurations.tests.artifacts.files + systemProperty 'tests.security.manager', 'true' + // maybe we like some extra security policy for our code + systemProperty 'tests.security.policy', '/extra-security.policy' +} +esTest.dependsOn jar, testJar diff --git a/elx-node/build.gradle b/elx-node/build.gradle new file mode 100644 index 0000000..bc5e01e --- /dev/null +++ b/elx-node/build.gradle @@ -0,0 +1,3 @@ +dependencies { + compile project(':elx-common') +} \ No newline at end of file diff --git a/elx-node/build.gradle~ b/elx-node/build.gradle~ new file mode 100644 index 0000000..0da2929 --- /dev/null +++ b/elx-node/build.gradle~ @@ -0,0 +1,65 @@ +buildscript { + repositories { + jcenter() + maven { + url 'http://xbib.org/repository' + } + } + dependencies { + classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.2.3.4" + } +} + +apply plugin: 'org.xbib.gradle.plugin.elasticsearch.build' + +configurations { + main + tests +} + +dependencies { + compile project(':common') + testCompile "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" + testRuntime "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" +} + +jar { + baseName "${rootProject.name}-node" +} + +/* +task testJar(type: Jar, dependsOn: testClasses) { + baseName = "${project.archivesBaseName}-tests" + from sourceSets.test.output +} +*/ + +artifacts { + main jar + tests testJar + archives sourcesJar, javadocJar +} + +test { + enabled = false + jvmArgs "-javaagent:" + configurations.alpnagent.asPath + systemProperty 'path.home', projectDir.absolutePath + testLogging { + showStandardStreams = true + exceptionFormat = 'full' + } +} + +randomizedTest { + enabled = false +} + + +esTest { + // test with the jars, not the classes, for security manager + // classpath = files(configurations.testRuntime) + configurations.main.artifacts.files + configurations.tests.artifacts.files + systemProperty 'tests.security.manager', 'true' + // maybe we like some extra security policy for our code + systemProperty 'tests.security.policy', '/extra-security.policy' +} +esTest.dependsOn jar, testJar diff --git a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java b/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java new file mode 100644 index 0000000..493a596 --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java @@ -0,0 +1,70 @@ +package org.xbib.elx.node; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.node.Node; +import org.elasticsearch.plugins.Plugin; +import org.xbib.elx.common.AbstractExtendedClient; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; + +public class ExtendedNodeClient extends AbstractExtendedClient { + + private static final Logger logger = LogManager.getLogger(ExtendedNodeClient.class.getName()); + + private Node node; + + @Override + protected ElasticsearchClient createClient(Settings settings) throws IOException { + if (settings != null) { + String version = System.getProperty("os.name") + + " " + System.getProperty("java.vm.name") + + " " + System.getProperty("java.vm.vendor") + + " " + System.getProperty("java.runtime.version") + + " " + System.getProperty("java.vm.version"); + Settings effectiveSettings = Settings.builder().put(settings) + .put("node.client", true) + .put("node.master", false) + .put("node.data", false) + .build(); + logger.info("creating node client on {} with effective settings {}", + version, effectiveSettings.toString()); + Collection> plugins = Collections.emptyList(); + this.node = new BulkNode(new Environment(effectiveSettings), plugins); + try { + node.start(); + } catch (Exception e) { + throw new IOException(e); + } + return node.client(); + } + return null; + } + + + @Override + public void shutdown() throws IOException { + super.shutdown(); + try { + if (node != null) { + logger.debug("closing node..."); + node.close(); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + private static class BulkNode extends Node { + + BulkNode(Environment env, Collection> classpathPlugins) { + super(env, Version.CURRENT, classpathPlugins); + } + } +} diff --git a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java b/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java new file mode 100644 index 0000000..46a4e9a --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java @@ -0,0 +1,10 @@ +package org.xbib.elx.node; + +import org.xbib.elx.api.ExtendedClientProvider; + +public class ExtendedNodeClientProvider implements ExtendedClientProvider { + @Override + public ExtendedNodeClient getExtendedClient() { + return new ExtendedNodeClient(); + } +} diff --git a/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider new file mode 100644 index 0000000..372aaad --- /dev/null +++ b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider @@ -0,0 +1 @@ +org.xbib.elx.node.ExtendedNodeClientProvider \ No newline at end of file diff --git a/elx-node/src/test/java/org/elasticsearch/node/MockNode.java b/elx-node/src/test/java/org/elasticsearch/node/MockNode.java new file mode 100644 index 0000000..aad8b8b --- /dev/null +++ b/elx-node/src/test/java/org/elasticsearch/node/MockNode.java @@ -0,0 +1,34 @@ +package org.elasticsearch.node; + +import org.elasticsearch.Version; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.internal.InternalSettingsPreparer; +import org.elasticsearch.plugins.Plugin; + +import java.util.ArrayList; +import java.util.Collection; + +public class MockNode extends Node { + + public MockNode() { + super(Settings.EMPTY); + } + + public MockNode(Settings settings) { + super(settings); + } + + public MockNode(Settings settings, Collection> classpathPlugins) { + super(InternalSettingsPreparer.prepareEnvironment(settings, null), Version.CURRENT, classpathPlugins); + } + + public MockNode(Settings settings, Class classpathPlugin) { + this(settings, list(classpathPlugin)); + } + + private static Collection> list(Class classpathPlugin) { + Collection> list = new ArrayList<>(); + list.add(classpathPlugin); + return list; + } +} diff --git a/elx-node/src/test/java/org/xbib/elx/node/ExtendeNodeDuplicateIDTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendeNodeDuplicateIDTest.java new file mode 100644 index 0000000..97cb185 --- /dev/null +++ b/elx-node/src/test/java/org/xbib/elx/node/ExtendeNodeDuplicateIDTest.java @@ -0,0 +1,58 @@ +package org.xbib.elx.node; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.junit.Ignore; +import org.junit.Test; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.Parameters; + +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.junit.Assert.*; + +@Ignore +public class ExtendeNodeDuplicateIDTest extends NodeTestUtils { + + private static final Logger logger = LogManager.getLogger(ExtendeNodeDuplicateIDTest.class.getSimpleName()); + + private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; + + private static final Long ACTIONS = 12345L; + + @Test + public void testDuplicateDocIDs() throws Exception { + long numactions = ACTIONS; + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .build(); + try { + client.newIndex("test"); + for (int i = 0; i < ACTIONS; i++) { + client.index("test", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flushIngest(); + client.waitForResponses("30s"); + client.refreshIndex("test"); + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) + .setIndices("test") + .setTypes("test") + .setQuery(matchAllQuery()); + long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); + logger.info("hits = {}", hits); + assertTrue(hits < ACTIONS); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + client.shutdown(); + assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + } + } +} diff --git a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientSingleNodeTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientSingleNodeTest.java new file mode 100644 index 0000000..0d7335d --- /dev/null +++ b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientSingleNodeTest.java @@ -0,0 +1,39 @@ +package org.xbib.elx.node; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.junit.Test; +import org.xbib.elx.common.ClientBuilder; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +public class ExtendedNodeClientSingleNodeTest extends NodeTestUtils { + + private static final Logger logger = LogManager.getLogger(ExtendedNodeClientSingleNodeTest.class.getSimpleName()); + + @Test + public void testSingleDocNodeClient() throws Exception { + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .build(); + try { + client.newIndex("test"); + client.index("test", "test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + client.flushIngest(); + client.waitForResponses("30s"); + } catch (InterruptedException e) { + // ignore + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + client.shutdown(); + } + } +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClientTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientTest.java similarity index 53% rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClientTest.java rename to elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientTest.java index 77b004f..957972d 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClientTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientTest.java @@ -1,43 +1,37 @@ -package org.xbib.elasticsearch.extras.client.node; +package org.xbib.elx.node; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.junit.Before; import org.junit.Test; -import org.xbib.elasticsearch.NodeTestUtils; -import org.xbib.elasticsearch.extras.client.Clients; -import org.xbib.elasticsearch.extras.client.SimpleBulkControl; -import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.Parameters; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -/** - * - */ -public class BulkNodeClientTest extends NodeTestUtils { +public class ExtendedNodeClientTest extends NodeTestUtils { - private static final ESLogger logger = ESLoggerFactory.getLogger(BulkNodeClientTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(ExtendedNodeClientTest.class.getSimpleName()); - private static final Long MAX_ACTIONS = 1000L; + private static final Long ACTIONS = 25000L; - private static final Long NUM_ACTIONS = 1234L; + private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; @Before public void startNodes() { @@ -49,13 +43,38 @@ public class BulkNodeClientTest extends NodeTestUtils { } } + @Test + public void testSingleDocNodeClient() throws Exception { + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) + .build(); + try { + client.newIndex("test"); + client.index("test", "test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + client.flushIngest(); + client.waitForResponses("30s"); + } catch (InterruptedException e) { + // ignore + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + client.shutdown(); + } + } + @Test public void testNewIndexNodeClient() throws Exception { - final BulkNodeClient client = Clients.builder() - .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5)) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkNodeClient(client("1")); + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) + .build(); client.newIndex("test"); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); @@ -66,11 +85,10 @@ public class BulkNodeClientTest extends NodeTestUtils { @Test public void testMappingNodeClient() throws Exception { - final BulkNodeClient client = Clients.builder() - .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5)) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkNodeClient(client("1")); + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) + .build(); XContentBuilder builder = jsonBuilder() .startObject() .startObject("test") @@ -85,7 +103,7 @@ public class BulkNodeClientTest extends NodeTestUtils { client.newIndex("test"); GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test"); GetMappingsResponse getMappingsResponse = - client.client().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); + client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); logger.info("mappings={}", getMappingsResponse.getMappings()); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); @@ -94,59 +112,34 @@ public class BulkNodeClientTest extends NodeTestUtils { client.shutdown(); } - @Test - public void testSingleDocNodeClient() { - final BulkNodeClient client = Clients.builder() - .put(Clients.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS) - .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(30)) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkNodeClient(client("1")); - try { - client.newIndex("test"); - client.index("test", "test", "1", "{ \"name\" : \"Hello World\"}"); // single doc ingest - client.flushIngest(); - client.waitForResponses("30s"); - } catch (InterruptedException e) { - // ignore - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } catch (ExecutionException e) { - logger.error(e.getMessage(), e); - } finally { - assertEquals(1, client.getMetric().getSucceeded().getCount()); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); - } - assertFalse(client.hasThrowable()); - client.shutdown(); - } - } - @Test public void testRandomDocsNodeClient() throws Exception { - long numactions = NUM_ACTIONS; - final BulkNodeClient client = Clients.builder() - .put(Clients.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS) - .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60)) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkNodeClient(client("1")); + long numactions = ACTIONS; + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); try { client.newIndex("test"); - for (int i = 0; i < NUM_ACTIONS; i++) { - client.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + for (int i = 0; i < ACTIONS; i++) { + client.index("test", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.waitForResponses("30s"); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - assertEquals(numactions, client.getMetric().getSucceeded().getCount()); + assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); } assertFalse(client.hasThrowable()); + client.refreshIndex("test"); + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) + .setQuery(QueryBuilders.matchAllQuery()).setSize(0); + assertEquals(numactions, + searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); client.shutdown(); } } @@ -154,15 +147,15 @@ public class BulkNodeClientTest extends NodeTestUtils { @Test public void testThreadedRandomDocsNodeClient() throws Exception { int maxthreads = Runtime.getRuntime().availableProcessors(); - Long maxactions = MAX_ACTIONS; - final Long maxloop = NUM_ACTIONS; - logger.info("NodeClient max={} maxactions={} maxloop={}", maxthreads, maxactions, maxloop); - final BulkNodeClient client = Clients.builder() - .put(Clients.MAX_ACTIONS_PER_REQUEST, maxactions) - .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60))// disable auto flush for this test - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkNodeClient(client("1")); + Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; + final Long actions = ACTIONS; + logger.info("NodeClient max={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); try { client.newIndex("test") .startBulk("test", -1, 1000); @@ -170,39 +163,39 @@ public class BulkNodeClientTest extends NodeTestUtils { EsExecutors.daemonThreadFactory("bulk-nodeclient-test")); final CountDownLatch latch = new CountDownLatch(maxthreads); for (int i = 0; i < maxthreads; i++) { - pool.execute(new Runnable() { - public void run() { - for (int i = 0; i < maxloop; i++) { - client.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - latch.countDown(); + pool.execute(() -> { + for (int i1 = 0; i1 < actions; i1++) { + client.index("test", "test", null, false,"{ \"name\" : \"" + randomString(32) + "\"}"); } + latch.countDown(); }); } - logger.info("waiting for max 30 seconds..."); - latch.await(30, TimeUnit.SECONDS); - logger.info("flush..."); - client.flushIngest(); - client.waitForResponses("30s"); - logger.info("got all responses, thread pool shutdown..."); - pool.shutdown(); - logger.info("pool is shut down"); + logger.info("waiting for latch..."); + if (latch.await(5, TimeUnit.MINUTES)) { + logger.info("last flush..."); + client.flushIngest(); + client.waitForResponses("60s"); + logger.info("got all responses, pool shutdown..."); + pool.shutdown(); + logger.info("pool is shut down"); + } else { + logger.warn("latch timeout"); + } } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { client.stopBulk("test"); - assertEquals(maxthreads * maxloop, client.getMetric().getSucceeded().getCount()); + assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount()); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); } assertFalse(client.hasThrowable()); client.refreshIndex("test"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE) + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) .setQuery(QueryBuilders.matchAllQuery()).setSize(0); - assertEquals(maxthreads * maxloop, + assertEquals(maxthreads * actions, searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); client.shutdown(); } } - } diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClusterBlockTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClusterBlockTest.java similarity index 76% rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClusterBlockTest.java rename to elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClusterBlockTest.java index 09c628d..b38555a 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClusterBlockTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClusterBlockTest.java @@ -1,24 +1,22 @@ -package org.xbib.elasticsearch.extras.client.node; +package org.xbib.elx.node; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; -import org.xbib.elasticsearch.NodeTestUtils; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -/** - * - */ -public class BulkNodeClusterBlockTest extends NodeTestUtils { +@Ignore +public class ExtendedNodeClusterBlockTest extends NodeTestUtils { - private static final ESLogger logger = ESLoggerFactory.getLogger("test"); + private static final Logger logger = LogManager.getLogger("test"); @Before public void startNodes() { @@ -33,6 +31,7 @@ public class BulkNodeClusterBlockTest extends NodeTestUtils { } } + @Override protected Settings getNodeSettings() { return Settings.settingsBuilder() .put(super.getNodeSettings()) @@ -44,8 +43,7 @@ public class BulkNodeClusterBlockTest extends NodeTestUtils { public void testClusterBlock() throws Exception { BulkRequestBuilder brb = client("1").prepareBulk(); XContentBuilder builder = jsonBuilder().startObject().field("field1", "value1").endObject(); - String jsonString = builder.string(); - IndexRequestBuilder irb = client("1").prepareIndex("test", "test", "1").setSource(jsonString); + IndexRequestBuilder irb = client("1").prepareIndex("test", "test", "1").setSource(builder); brb.add(irb); brb.execute().actionGet(); } diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeIndexAliasTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeIndexAliasTest.java similarity index 58% rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeIndexAliasTest.java rename to elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeIndexAliasTest.java index eb5256c..1503fee 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeIndexAliasTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeIndexAliasTest.java @@ -1,17 +1,12 @@ -package org.xbib.elasticsearch.extras.client.node; +package org.xbib.elx.node; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilders; +import org.junit.Ignore; import org.junit.Test; -import org.xbib.elasticsearch.NodeTestUtils; -import org.xbib.elasticsearch.extras.client.Clients; -import org.xbib.elasticsearch.extras.client.IndexAliasAdder; -import org.xbib.elasticsearch.extras.client.SimpleBulkControl; -import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; +import org.xbib.elx.common.ClientBuilder; import java.util.Arrays; import java.util.List; @@ -19,23 +14,20 @@ import java.util.Map; import static org.junit.Assert.assertFalse; -/** - * - */ -public class BulkNodeIndexAliasTest extends NodeTestUtils { +@Ignore +public class ExtendedNodeIndexAliasTest extends NodeTestUtils { - private static final ESLogger logger = ESLoggerFactory.getLogger(BulkNodeIndexAliasTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(ExtendedNodeIndexAliasTest.class.getSimpleName()); @Test public void testIndexAlias() throws Exception { - final BulkNodeClient client = Clients.builder() - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkNodeClient(client("1")); + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .build(); try { client.newIndex("test1234"); for (int i = 0; i < 1; i++) { - client.index("test1234", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test1234", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.refreshIndex("test1234"); @@ -45,18 +37,14 @@ public class BulkNodeIndexAliasTest extends NodeTestUtils { client.newIndex("test5678"); for (int i = 0; i < 1; i++) { - client.index("test5678", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test5678", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.refreshIndex("test5678"); simpleAliases = Arrays.asList("d", "e", "f"); - client.switchAliases("test", "test5678", simpleAliases, new IndexAliasAdder() { - @Override - public void addIndexAlias(IndicesAliasesRequestBuilder builder, String index, String alias) { - builder.addAlias(index, alias, QueryBuilders.termQuery("my_key", alias)); - } - }); + client.switchAliases("test", "test5678", simpleAliases, (builder, index, alias) -> + builder.addAlias(index, alias, QueryBuilders.termQuery("my_key", alias))); Map aliases = client.getIndexFilters("test5678"); logger.info("aliases of index test5678 = {}", aliases); diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeReplicaTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeReplicaTest.java similarity index 78% rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeReplicaTest.java rename to elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeReplicaTest.java index b4fec6b..89de2df 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeReplicaTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeReplicaTest.java @@ -1,5 +1,7 @@ -package org.xbib.elasticsearch.extras.client.node; +package org.xbib.elx.node; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.admin.indices.stats.IndexShardStats; import org.elasticsearch.action.admin.indices.stats.IndexStats; @@ -9,15 +11,11 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.indexing.IndexingStats; +import org.junit.Ignore; import org.junit.Test; -import org.xbib.elasticsearch.NodeTestUtils; -import org.xbib.elasticsearch.extras.client.Clients; -import org.xbib.elasticsearch.extras.client.SimpleBulkControl; -import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; +import org.xbib.elx.common.ClientBuilder; import java.util.Map; @@ -25,9 +23,10 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -public class BulkNodeReplicaTest extends NodeTestUtils { +@Ignore +public class ExtendedNodeReplicaTest extends NodeTestUtils { - private final static ESLogger logger = ESLoggerFactory.getLogger(BulkNodeReplicaTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(ExtendedNodeReplicaTest.class.getSimpleName()); @Test public void testReplicaLevel() throws Exception { @@ -47,20 +46,19 @@ public class BulkNodeReplicaTest extends NodeTestUtils { .put("index.number_of_replicas", 1) .build(); - final BulkNodeClient client = Clients.builder() - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkNodeClient(client("1")); + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .build(); try { client.newIndex("test1", settingsTest1, null) .newIndex("test2", settingsTest2, null); client.waitForCluster("GREEN", "30s"); for (int i = 0; i < 1234; i++) { - client.index("test1", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test1", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } for (int i = 0; i < 1234; i++) { - client.index("test2", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test2", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.waitForResponses("30s"); @@ -70,13 +68,13 @@ public class BulkNodeReplicaTest extends NodeTestUtils { logger.info("refreshing"); client.refreshIndex("test1"); client.refreshIndex("test2"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE) + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) .setIndices("test1", "test2") .setQuery(matchAllQuery()); long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); logger.info("query total hits={}", hits); assertEquals(2468, hits); - IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.client(), IndicesStatsAction.INSTANCE) + IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.getClient(), IndicesStatsAction.INSTANCE) .all(); IndicesStatsResponse response = indicesStatsRequestBuilder.execute().actionGet(); for (Map.Entry m : response.getIndices().entrySet()) { diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportUpdateReplicaLevelTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeUpdateReplicaLevelTest.java similarity index 60% rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportUpdateReplicaLevelTest.java rename to elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeUpdateReplicaLevelTest.java index 1f56df8..a1b29a4 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportUpdateReplicaLevelTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeUpdateReplicaLevelTest.java @@ -1,54 +1,47 @@ -package org.xbib.elasticsearch.extras.client.transport; +package org.xbib.elx.node; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; +import org.junit.Ignore; import org.junit.Test; -import org.xbib.elasticsearch.NodeTestUtils; -import org.xbib.elasticsearch.extras.client.Clients; -import org.xbib.elasticsearch.extras.client.SimpleBulkControl; -import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; +import org.xbib.elx.common.ClientBuilder; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -/** - * - */ -public class BulkTransportUpdateReplicaLevelTest extends NodeTestUtils { +@Ignore +public class ExtendedNodeUpdateReplicaLevelTest extends NodeTestUtils { - private static final ESLogger logger = - ESLoggerFactory.getLogger(BulkTransportUpdateReplicaLevelTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(ExtendedNodeUpdateReplicaLevelTest.class.getSimpleName()); @Test public void testUpdateReplicaLevel() throws Exception { - int numberOfShards = 2; + long numberOfShards = 2; int replicaLevel = 3; // we need 3 nodes for replica level 3 startNode("2"); startNode("3"); - int shardsAfterReplica; + long shardsAfterReplica; Settings settings = Settings.settingsBuilder() .put("index.number_of_shards", numberOfShards) .put("index.number_of_replicas", 0) .build(); - final BulkTransportClient client = Clients.builder() - .put(getSettings()) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkTransportClient(); + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .build(); try { client.newIndex("replicatest", settings, null); client.waitForCluster("GREEN", "30s"); for (int i = 0; i < 12345; i++) { - client.index("replicatest", "replicatest", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("replicatest", "replicatest", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.waitForResponses("30s"); diff --git a/elx-node/src/test/java/org/xbib/elx/node/NodeTestUtils.java b/elx-node/src/test/java/org/xbib/elx/node/NodeTestUtils.java new file mode 100644 index 0000000..9a4750e --- /dev/null +++ b/elx-node/src/test/java/org/xbib/elx/node/NodeTestUtils.java @@ -0,0 +1,201 @@ +package org.xbib.elx.node; + +import static org.elasticsearch.common.settings.Settings.settingsBuilder; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.client.support.AbstractClient; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.node.MockNode; +import org.elasticsearch.node.Node; +import org.junit.After; +import org.junit.Before; +import org.xbib.elx.common.util.NetworkUtils; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +public class NodeTestUtils { + + private static final Logger logger = LogManager.getLogger("test"); + + private static Random random = new Random(); + + private static char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); + + private Map nodes = new HashMap<>(); + + private Map clients = new HashMap<>(); + + private AtomicInteger counter = new AtomicInteger(); + + private String cluster; + + private String host; + + private int port; + + private static void deleteFiles() throws IOException { + Path directory = Paths.get(getHome() + "/data"); + Files.walkFileTree(directory, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + + }); + + } + + @Before + public void startNodes() { + try { + logger.info("starting"); + setClusterName(); + startNode("1"); + findNodeAddress(); + try { + ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN) + .timeout(TimeValue.timeValueSeconds(30))).actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + throw new IOException("cluster state is " + healthResponse.getStatus().name() + + ", from here on, everything will fail!"); + } + } catch (ElasticsearchTimeoutException e) { + throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); + } + } catch (Throwable t) { + logger.error("startNodes failed", t); + } + } + + @After + public void stopNodes() { + try { + closeNodes(); + } catch (Exception e) { + logger.error("can not close nodes", e); + } finally { + try { + deleteFiles(); + logger.info("data files wiped"); + Thread.sleep(2000L); // let OS commit changes + } catch (IOException e) { + logger.error(e.getMessage(), e); + } catch (InterruptedException e) { + // ignore + } + } + } + + protected void setClusterName() { + this.cluster = "test-helper-cluster-" + + NetworkUtils.getLocalAddress().getHostName() + + "-" + System.getProperty("user.name") + + "-" + counter.incrementAndGet(); + } + + protected Settings getNodeSettings() { + return settingsBuilder() + .put("cluster.name", cluster) + .put("cluster.routing.schedule", "50ms") + .put("cluster.routing.allocation.disk.threshold_enabled", false) + .put("discovery.zen.multicast.enabled", true) + .put("discovery.zen.multicast.ping_timeout", "5s") + .put("http.enabled", true) + .put("threadpool.bulk.size", Runtime.getRuntime().availableProcessors()) + .put("threadpool.bulk.queue_size", 16 * Runtime.getRuntime().availableProcessors()) // default is 50, too low + .put("index.number_of_replicas", 0) + .put("path.home", getHome()) + .build(); + } + + protected static String getHome() { + return System.getProperty("path.home", System.getProperty("user.dir")); + } + + public void startNode(String id) { + buildNode(id).start(); + } + + public AbstractClient client(String id) { + return clients.get(id); + } + + private void closeNodes() { + logger.info("closing all clients"); + for (AbstractClient client : clients.values()) { + client.close(); + } + clients.clear(); + logger.info("closing all nodes"); + for (Node node : nodes.values()) { + if (node != null) { + node.close(); + } + } + nodes.clear(); + logger.info("all nodes closed"); + } + + protected void findNodeAddress() { + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); + NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet(); + Object obj = response.iterator().next().getTransport().getAddress() + .publishAddress(); + if (obj instanceof InetSocketTransportAddress) { + InetSocketTransportAddress address = (InetSocketTransportAddress) obj; + host = address.address().getHostName(); + port = address.address().getPort(); + } + } + + private Node buildNode(String id) { + Settings nodeSettings = settingsBuilder() + .put(getNodeSettings()) + .put("name", id) + .build(); + logger.info("settings={}", nodeSettings.getAsMap()); + Node node = new MockNode(nodeSettings); + AbstractClient client = (AbstractClient) node.client(); + nodes.put(id, node); + clients.put(id, client); + logger.info("clients={}", clients); + return node; + } + + protected String randomString(int len) { + final char[] buf = new char[len]; + final int n = numbersAndLetters.length - 1; + for (int i = 0; i < buf.length; i++) { + buf[i] = numbersAndLetters[random.nextInt(n)]; + } + return new String(buf); + } +} diff --git a/src/integration-test/resources/log4j2.xml b/elx-node/src/test/resources/log4j2.xml similarity index 80% rename from src/integration-test/resources/log4j2.xml rename to elx-node/src/test/resources/log4j2.xml index b175dfc..1258d7f 100644 --- a/src/integration-test/resources/log4j2.xml +++ b/elx-node/src/test/resources/log4j2.xml @@ -2,7 +2,7 @@ - + diff --git a/elx-transport/build.gradle b/elx-transport/build.gradle new file mode 100644 index 0000000..bc5e01e --- /dev/null +++ b/elx-transport/build.gradle @@ -0,0 +1,3 @@ +dependencies { + compile project(':elx-common') +} \ No newline at end of file diff --git a/elx-transport/build.gradle~ b/elx-transport/build.gradle~ new file mode 100644 index 0000000..b47f835 --- /dev/null +++ b/elx-transport/build.gradle~ @@ -0,0 +1,63 @@ +buildscript { + repositories { + jcenter() + maven { + url 'http://xbib.org/repository' + } + } + dependencies { + classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.2.2.0" + } +} + +apply plugin: 'org.xbib.gradle.plugin.elasticsearch.build' + +configurations { + main + tests +} + +dependencies { + compile project(':common') + testCompile "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" + testRuntime "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" +} + +jar { + baseName "${rootProject.name}-transport" +} + +task testJar(type: Jar, dependsOn: testClasses) { + baseName = "${project.archivesBaseName}-tests" + from sourceSets.test.output +} + +artifacts { + main jar + tests testJar + archives sourcesJar, javadocJar +} + +esTest { + enabled = true + // test with the jars, not the classes, for security manager + classpath = files(configurations.testRuntime) + configurations.main.artifacts.files + configurations.tests.artifacts.files + systemProperty 'tests.security.manager', 'true' + // maybe we like some extra security policy for our code + systemProperty 'tests.security.policy', '/extra-security.policy' +} +esTest.dependsOn jar, testJar + +randomizedTest { + enabled = false +} + +test { + enabled = false + jvmArgs "-javaagent:" + configurations.alpnagent.asPath + systemProperty 'path.home', projectDir.absolutePath + testLogging { + showStandardStreams = true + exceptionFormat = 'full' + } +} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java new file mode 100644 index 0000000..f66ac58 --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java @@ -0,0 +1,129 @@ +package org.xbib.elx.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.xbib.elx.common.AbstractExtendedClient; +import org.xbib.elx.common.util.NetworkUtils; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Transport client with additional methods using the BulkProcessor. + */ +public class ExtendedTransportClient extends AbstractExtendedClient { + + private static final Logger logger = LogManager.getLogger(ExtendedTransportClient.class.getName()); + + @Override + protected ElasticsearchClient createClient(Settings settings) { + if (settings != null) { + String systemIdentifier = System.getProperty("os.name") + + " " + System.getProperty("java.vm.name") + + " " + System.getProperty("java.vm.vendor") + + " " + System.getProperty("java.vm.version") + + " Elasticsearch " + Version.CURRENT.toString(); + logger.info("creating transport client on {} with effective settings {}", + systemIdentifier, settings.getAsMap()); + TransportClient.Builder builder = TransportClient.builder() + .settings(Settings.builder() + .put("cluster.name", settings.get("cluster.name")) + .put("processors", settings.getAsInt("processors", Runtime.getRuntime().availableProcessors())) + .put("client.transport.ignore_cluster_name", true) + .build()); + return builder.build(); + } + return null; + } + + @Override + public ExtendedTransportClient init(Settings settings) throws IOException { + super.init(settings); + // additional auto-connect + try { + Collection addrs = findAddresses(settings); + if (!connect(addrs, settings.getAsBoolean("autodiscover", false))) { + throw new NoNodeAvailableException("no cluster nodes available, check settings " + + settings.toString()); + } + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + return this; + } + + @Override + public synchronized void shutdown() throws IOException { + super.shutdown(); + logger.info("shutting down..."); + if (getClient() != null) { + TransportClient client = (TransportClient) getClient(); + client.close(); + client.threadPool().shutdown(); + } + logger.info("shutting down completed"); + } + + private Collection findAddresses(Settings settings) throws IOException { + final int defaultPort = settings.getAsInt("port", 9300); + Collection addresses = new ArrayList<>(); + for (String hostname : settings.getAsArray("host")) { + String[] splitHost = hostname.split(":", 2); + if (splitHost.length == 2) { + try { + String host = splitHost[0]; + InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null); + int port = Integer.parseInt(splitHost[1]); + InetSocketTransportAddress address = new InetSocketTransportAddress(inetAddress, port); + addresses.add(address); + } catch (NumberFormatException e) { + logger.warn(e.getMessage(), e); + } + } + if (splitHost.length == 1) { + String host = splitHost[0]; + InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null); + InetSocketTransportAddress address = new InetSocketTransportAddress(inetAddress, defaultPort); + addresses.add(address); + } + } + return addresses; + } + + private boolean connect(Collection addresses, boolean autodiscover) { + if (getClient() == null) { + throw new IllegalStateException("no client present"); + } + logger.debug("trying to connect to {}", addresses); + TransportClient transportClient = (TransportClient) getClient(); + transportClient.addTransportAddresses(addresses); + List nodes = transportClient.connectedNodes(); + logger.info("connected to nodes = {}", nodes); + if (nodes != null && !nodes.isEmpty()) { + if (autodiscover) { + logger.debug("trying to auto-discover all nodes..."); + ClusterStateRequestBuilder clusterStateRequestBuilder = + new ClusterStateRequestBuilder(getClient(), ClusterStateAction.INSTANCE); + ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); + DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes(); + transportClient.addDiscoveryNodes(discoveryNodes); + logger.info("after auto-discovery: connected to {}", transportClient.connectedNodes()); + } + return true; + } + return false; + } +} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java b/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java new file mode 100644 index 0000000..669d21a --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.transport; + +import org.xbib.elx.api.ExtendedClientProvider; + +public class ExtendedTransportClientProvider implements ExtendedClientProvider { + + @Override + public ExtendedTransportClient getExtendedClient() { + return new ExtendedTransportClient(); + } +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/transport/TransportClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportClient.java similarity index 93% rename from src/main/java/org/xbib/elasticsearch/extras/client/transport/TransportClient.java rename to elx-transport/src/main/java/org/xbib/elx/transport/TransportClient.java index 3912ce7..827f657 100644 --- a/src/main/java/org/xbib/elasticsearch/extras/client/transport/TransportClient.java +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportClient.java @@ -1,9 +1,8 @@ -package org.xbib.elasticsearch.extras.client.transport; +package org.xbib.elx.transport; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; -import com.google.common.collect.ImmutableMap; import org.elasticsearch.Version; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionListener; @@ -25,7 +24,6 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterNameModule; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Injector; @@ -54,6 +52,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -278,16 +277,11 @@ public class TransportClient extends AbstractClient { } } try { + FutureTransportResponseHandler responseHandler = new LivenessResponseHandler(); LivenessResponse livenessResponse = transportService.submitRequest(listedNode, TransportLivenessAction.NAME, headers.applyTo(new LivenessRequest()), TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STATE) - .withTimeout(pingTimeout).build(), - new FutureTransportResponseHandler() { - @Override - public LivenessResponse newInstance() { - return new LivenessResponse(); - } - }).txGet(); + .withTimeout(pingTimeout).build(),responseHandler).txGet(); if (!clusterName.equals(livenessResponse.getClusterName())) { logger.warn("node {} not part of the cluster {}, ignoring...", listedNode, clusterName); newFilteredNodes.add(listedNode); @@ -347,12 +341,10 @@ public class TransportClient extends AbstractClient { } } - /** - * - */ public static class Builder { private Settings settings = Settings.EMPTY; + private List> pluginClasses = new ArrayList<>(); public Builder settings(Settings.Builder settings) { @@ -395,12 +387,7 @@ public class TransportClient extends AbstractClient { modules.add(new ClusterNameModule(this.settings)); modules.add(new ThreadPoolModule(threadPool)); modules.add(new TransportModule(this.settings)); - modules.add(new SearchModule() { - @Override - protected void configure() { - // noop - } - }); + modules.add(new TransportSearchModule()); modules.add(new ActionModule(true)); modules.add(new ClientTransportModule()); modules.add(new CircuitBreakerModule(this.settings)); @@ -419,27 +406,39 @@ public class TransportClient extends AbstractClient { } /** - * The {@link ProxyActionMap} must be declared public. + * The {@link ProxyActionMap} must be declared public for injection. */ @SuppressWarnings({"unchecked", "rawtypes"}) public static class ProxyActionMap { - private final ImmutableMap proxies; + private final Map proxies; @Inject public ProxyActionMap(Settings settings, TransportService transportService, Map actions) { - MapBuilder actionsBuilder = new MapBuilder<>(); + this.proxies = new LinkedHashMap<>(); for (GenericAction action : actions.values()) { if (action instanceof Action) { - actionsBuilder.put((Action) action, new TransportActionNodeProxy(settings, action, transportService)); + this.proxies.put((Action) action, new TransportActionNodeProxy(settings, action, transportService)); } } - this.proxies = actionsBuilder.immutableMap(); } - public ImmutableMap getProxies() { + public Map getProxies() { return proxies; } } + private static class LivenessResponseHandler extends FutureTransportResponseHandler { + @Override + public LivenessResponse newInstance() { + return new LivenessResponse(); + } + } + + private static class TransportSearchModule extends SearchModule { + @Override + protected void configure() { + // noop + } + } } diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/package-info.java b/elx-transport/src/main/java/org/xbib/elx/transport/package-info.java new file mode 100644 index 0000000..3697854 --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/package-info.java @@ -0,0 +1,4 @@ +/** + * Classes for Elasticsearch transport client extensions. + */ +package org.xbib.elx.transport; diff --git a/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider new file mode 100644 index 0000000..640e2f9 --- /dev/null +++ b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider @@ -0,0 +1 @@ +org.xbib.elx.transport.ExtendedTransportClientProvider \ No newline at end of file diff --git a/elx-transport/src/test/java/org/elasticsearch/node/MockNode.java b/elx-transport/src/test/java/org/elasticsearch/node/MockNode.java new file mode 100644 index 0000000..aad8b8b --- /dev/null +++ b/elx-transport/src/test/java/org/elasticsearch/node/MockNode.java @@ -0,0 +1,34 @@ +package org.elasticsearch.node; + +import org.elasticsearch.Version; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.internal.InternalSettingsPreparer; +import org.elasticsearch.plugins.Plugin; + +import java.util.ArrayList; +import java.util.Collection; + +public class MockNode extends Node { + + public MockNode() { + super(Settings.EMPTY); + } + + public MockNode(Settings settings) { + super(settings); + } + + public MockNode(Settings settings, Collection> classpathPlugins) { + super(InternalSettingsPreparer.prepareEnvironment(settings, null), Version.CURRENT, classpathPlugins); + } + + public MockNode(Settings settings, Class classpathPlugin) { + this(settings, list(classpathPlugin)); + } + + private static Collection> list(Class classpathPlugin) { + Collection> list = new ArrayList<>(); + list.add(classpathPlugin); + return list; + } +} diff --git a/elx-transport/src/test/java/org/elasticsearch/node/package-info.java b/elx-transport/src/test/java/org/elasticsearch/node/package-info.java new file mode 100644 index 0000000..8ffed8c --- /dev/null +++ b/elx-transport/src/test/java/org/elasticsearch/node/package-info.java @@ -0,0 +1 @@ +package org.elasticsearch.node; \ No newline at end of file diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientSingleNodeTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientSingleNodeTest.java new file mode 100644 index 0000000..cb9dba9 --- /dev/null +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientSingleNodeTest.java @@ -0,0 +1,40 @@ +package org.xbib.elx.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.junit.Test; +import org.xbib.elx.common.ClientBuilder; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +public class ExtendedTransportClientSingleNodeTest extends NodeTestUtils { + + private static final Logger logger = LogManager.getLogger(ExtendedTransportClientSingleNodeTest.class.getSimpleName()); + + @Test + public void testSingleDocNodeClient() throws Exception { + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) + .put(getSettings()) + .build(); + try { + client.newIndex("test"); + client.index("test", "test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + client.flushIngest(); + client.waitForResponses("30s"); + } catch (InterruptedException e) { + // ignore + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + client.shutdown(); + } + } +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClientTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientTest.java similarity index 53% rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClientTest.java rename to elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientTest.java index c7c82e0..cfca3da 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClientTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientTest.java @@ -1,40 +1,34 @@ -package org.xbib.elasticsearch.extras.client.transport; +package org.xbib.elx.transport; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.query.QueryBuilders; import org.junit.Before; import org.junit.Test; -import org.xbib.elasticsearch.NodeTestUtils; -import org.xbib.elasticsearch.extras.client.Clients; -import org.xbib.elasticsearch.extras.client.SimpleBulkControl; -import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.Parameters; -import java.io.IOException; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -/** - * - */ -public class BulkTransportClientTest extends NodeTestUtils { - private static final ESLogger logger = ESLoggerFactory.getLogger(BulkTransportClientTest.class.getSimpleName()); +public class ExtendedTransportClientTest extends NodeTestUtils { - private static final Long MAX_ACTIONS = 1000L; + private static final Logger logger = LogManager.getLogger(ExtendedTransportClientTest.class.getSimpleName()); - private static final Long NUM_ACTIONS = 1234L; + private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; + + private static final Long ACTIONS = 1234L; @Before public void startNodes() { @@ -47,13 +41,12 @@ public class BulkTransportClientTest extends NodeTestUtils { } @Test - public void testBulkClient() throws IOException { - final BulkTransportClient client = Clients.builder() + public void testBulkClient() throws Exception { + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) .put(getSettings()) - .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60)) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkTransportClient(); + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); client.newIndex("test"); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); @@ -75,27 +68,24 @@ public class BulkTransportClientTest extends NodeTestUtils { } @Test - public void testSingleDocBulkClient() throws IOException { - final BulkTransportClient client = Clients.builder() + public void testSingleDocBulkClient() throws Exception { + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) .put(getSettings()) - .put(Clients.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS) - .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60)) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkTransportClient(); + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); try { client.newIndex("test"); - client.index("test", "test", "1", "{ \"name\" : \"Hello World\"}"); // single doc ingest + client.index("test", "test", "1", true, "{ \"name\" : \"Hello World\"}"); client.flushIngest(); client.waitForResponses("30s"); } catch (InterruptedException e) { // ignore - } catch (ExecutionException e) { - logger.error(e.getMessage(), e); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - assertEquals(1, client.getMetric().getSucceeded().getCount()); + assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); } @@ -105,30 +95,27 @@ public class BulkTransportClientTest extends NodeTestUtils { } @Test - public void testRandomDocsBulkClient() throws IOException { - long numactions = NUM_ACTIONS; - final BulkTransportClient client = Clients.builder() + public void testRandomDocsBulkClient() throws Exception { + long numactions = ACTIONS; + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) .put(getSettings()) - .put(Clients.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS) - .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60)) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkTransportClient(); + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); try { client.newIndex("test"); - for (int i = 0; i < NUM_ACTIONS; i++) { - client.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + for (int i = 0; i < ACTIONS; i++) { + client.index("test", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.waitForResponses("30s"); } catch (InterruptedException e) { // ignore - } catch (ExecutionException e) { - logger.error(e.getMessage(), e); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - assertEquals(numactions, client.getMetric().getSucceeded().getCount()); + assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); } @@ -140,54 +127,54 @@ public class BulkTransportClientTest extends NodeTestUtils { @Test public void testThreadedRandomDocsBulkClient() throws Exception { int maxthreads = Runtime.getRuntime().availableProcessors(); - long maxactions = MAX_ACTIONS; - final long maxloop = NUM_ACTIONS; + long maxactions = MAX_ACTIONS_PER_REQUEST; + final long maxloop = ACTIONS; Settings settingsForIndex = Settings.settingsBuilder() .put("index.number_of_shards", 2) .put("index.number_of_replicas", 1) .build(); - final BulkTransportClient client = Clients.builder() + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) .put(getSettings()) - .put(Clients.MAX_ACTIONS_PER_REQUEST, maxactions) - .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60)) // = disable autoflush for this test - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkTransportClient(); + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxactions) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); try { client.newIndex("test", settingsForIndex, null) .startBulk("test", -1, 1000); - ThreadPoolExecutor pool = - EsExecutors.newFixed("bulkclient-test", maxthreads, 30, EsExecutors.daemonThreadFactory("bulkclient-test")); + ThreadPoolExecutor pool = EsExecutors.newFixed("bulkclient-test", maxthreads, 30, + EsExecutors.daemonThreadFactory("bulkclient-test")); final CountDownLatch latch = new CountDownLatch(maxthreads); for (int i = 0; i < maxthreads; i++) { pool.execute(() -> { for (int i1 = 0; i1 < maxloop; i1++) { - client.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } latch.countDown(); }); } - logger.info("waiting for max 30 seconds..."); - latch.await(30, TimeUnit.SECONDS); - logger.info("client flush ..."); - client.flushIngest(); - client.waitForResponses("30s"); - logger.info("thread pool to be shut down ..."); - pool.shutdown(); - logger.info("poot shut down"); + logger.info("waiting for latch..."); + if (latch.await(60, TimeUnit.SECONDS)) { + logger.info("flush ..."); + client.flushIngest(); + client.waitForResponses("30s"); + logger.info("pool to be shut down ..."); + pool.shutdown(); + logger.info("poot shut down"); + } } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { client.stopBulk("test"); - assertEquals(maxthreads * maxloop, client.getMetric().getSucceeded().getCount()); + assertEquals(maxthreads * maxloop, client.getBulkMetric().getSucceeded().getCount()); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); } assertFalse(client.hasThrowable()); client.refreshIndex("test"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE) + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) // to avoid NPE at org.elasticsearch.action.search.SearchRequest.writeTo(SearchRequest.java:580) .setIndices("_all") .setQuery(QueryBuilders.matchAllQuery()) @@ -197,5 +184,4 @@ public class BulkTransportClientTest extends NodeTestUtils { client.shutdown(); } } - } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportDuplicateIDTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportDuplicateIDTest.java new file mode 100644 index 0000000..1fa73c9 --- /dev/null +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportDuplicateIDTest.java @@ -0,0 +1,57 @@ +package org.xbib.elx.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.junit.Test; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.Parameters; + +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.junit.Assert.*; + +public class ExtendedTransportDuplicateIDTest extends NodeTestUtils { + + private final static Logger logger = LogManager.getLogger(ExtendedTransportDuplicateIDTest.class.getSimpleName()); + + private final static Long MAX_ACTIONS_PER_REQUEST = 1000L; + + private final static Long ACTIONS = 12345L; + + @Test + public void testDuplicateDocIDs() throws Exception { + long numactions = ACTIONS; + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) + .put(getSettings()) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .build(); + try { + client.newIndex("test"); + for (int i = 0; i < ACTIONS; i++) { + client.index("test", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flushIngest(); + client.waitForResponses("30s"); + client.refreshIndex("test"); + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) + .setIndices("test") + .setTypes("test") + .setQuery(matchAllQuery()); + long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); + logger.info("hits = {}", hits); + assertTrue(hits < ACTIONS); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + client.shutdown(); + assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + } + } +} diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportIndexAliasTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportIndexAliasTest.java new file mode 100644 index 0000000..a2dfb4c --- /dev/null +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportIndexAliasTest.java @@ -0,0 +1,65 @@ +package org.xbib.elx.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.index.query.QueryBuilders; +import org.junit.Ignore; +import org.junit.Test; +import org.xbib.elx.common.ClientBuilder; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertFalse; + +@Ignore +public class ExtendedTransportIndexAliasTest extends NodeTestUtils { + + private static final Logger logger = LogManager.getLogger(ExtendedTransportIndexAliasTest.class.getSimpleName()); + + @Test + public void testIndexAlias() throws Exception { + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) + .build(); + try { + client.newIndex("test1234"); + for (int i = 0; i < 1; i++) { + client.index("test1234", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flushIngest(); + client.refreshIndex("test1234"); + + List simpleAliases = Arrays.asList("a", "b", "c"); + client.switchAliases("test", "test1234", simpleAliases); + + client.newIndex("test5678"); + for (int i = 0; i < 1; i++) { + client.index("test5678", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flushIngest(); + client.refreshIndex("test5678"); + + simpleAliases = Arrays.asList("d", "e", "f"); + client.switchAliases("test", "test5678", simpleAliases, (builder, index, alias) -> + builder.addAlias(index, alias, QueryBuilders.termQuery("my_key", alias))); + Map aliases = client.getIndexFilters("test5678"); + logger.info("aliases of index test5678 = {}", aliases); + + aliases = client.getAliasFilters("test"); + logger.info("aliases of alias test = {}", aliases); + + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + client.waitForResponses("30s"); + client.shutdown(); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + } + } +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportReplicaTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportReplicaTest.java similarity index 78% rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportReplicaTest.java rename to elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportReplicaTest.java index bc8f449..6b6d6d4 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportReplicaTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportReplicaTest.java @@ -1,5 +1,7 @@ -package org.xbib.elasticsearch.extras.client.transport; +package org.xbib.elx.transport; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.admin.indices.stats.IndexShardStats; import org.elasticsearch.action.admin.indices.stats.IndexStats; @@ -9,15 +11,10 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.indexing.IndexingStats; import org.junit.Test; -import org.xbib.elasticsearch.NodeTestUtils; -import org.xbib.elasticsearch.extras.client.Clients; -import org.xbib.elasticsearch.extras.client.SimpleBulkControl; -import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; +import org.xbib.elx.common.ClientBuilder; import java.util.Map; @@ -28,9 +25,9 @@ import static org.junit.Assert.assertFalse; /** * */ -public class BulkTransportReplicaTest extends NodeTestUtils { +public class ExtendedTransportReplicaTest extends NodeTestUtils { - private static final ESLogger logger = ESLoggerFactory.getLogger(BulkTransportReplicaTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(ExtendedTransportReplicaTest.class.getSimpleName()); @Test public void testReplicaLevel() throws Exception { @@ -50,20 +47,20 @@ public class BulkTransportReplicaTest extends NodeTestUtils { .put("index.number_of_replicas", 1) .build(); - final BulkTransportClient client = Clients.builder() + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) .put(getSettings()) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkTransportClient(); + .build(); + try { client.newIndex("test1", settingsTest1, null) .newIndex("test2", settingsTest2, null); client.waitForCluster("GREEN", "30s"); for (int i = 0; i < 1234; i++) { - client.index("test1", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test1", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } for (int i = 0; i < 1234; i++) { - client.index("test2", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test2", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.waitForResponses("30s"); @@ -73,13 +70,13 @@ public class BulkTransportReplicaTest extends NodeTestUtils { logger.info("refreshing"); client.refreshIndex("test1"); client.refreshIndex("test2"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE) + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) .setIndices("test1", "test2") .setQuery(matchAllQuery()); long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); logger.info("query total hits={}", hits); assertEquals(2468, hits); - IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.client(), + IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.getClient(), IndicesStatsAction.INSTANCE).all(); IndicesStatsResponse response = indicesStatsRequestBuilder.execute().actionGet(); for (Map.Entry m : response.getIndices().entrySet()) { diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeUpdateReplicaLevelTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportUpdateReplicaLevelTest.java similarity index 63% rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeUpdateReplicaLevelTest.java rename to elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportUpdateReplicaLevelTest.java index 5dc9202..17d71d6 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeUpdateReplicaLevelTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportUpdateReplicaLevelTest.java @@ -1,29 +1,23 @@ -package org.xbib.elasticsearch.extras.client.node; +package org.xbib.elx.transport; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; import org.junit.Test; -import org.xbib.elasticsearch.NodeTestUtils; -import org.xbib.elasticsearch.extras.client.Clients; -import org.xbib.elasticsearch.extras.client.SimpleBulkControl; -import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; +import org.xbib.elx.common.ClientBuilder; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -/** - * - */ -public class BulkNodeUpdateReplicaLevelTest extends NodeTestUtils { +public class ExtendedTransportUpdateReplicaLevelTest extends NodeTestUtils { - private static final ESLogger logger = ESLoggerFactory.getLogger(BulkNodeUpdateReplicaLevelTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(ExtendedTransportUpdateReplicaLevelTest.class.getSimpleName()); @Test public void testUpdateReplicaLevel() throws Exception { - int numberOfShards = 2; + long numberOfShards = 2; int replicaLevel = 3; // we need 3 nodes for replica level 3 @@ -32,21 +26,21 @@ public class BulkNodeUpdateReplicaLevelTest extends NodeTestUtils { int shardsAfterReplica; + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) + .put(getSettings()) + .build(); + Settings settings = Settings.settingsBuilder() .put("index.number_of_shards", numberOfShards) .put("index.number_of_replicas", 0) .build(); - final BulkNodeClient client = Clients.builder() - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkNodeClient(client("1")); - try { client.newIndex("replicatest", settings, null); client.waitForCluster("GREEN", "30s"); for (int i = 0; i < 12345; i++) { - client.index("replicatest", "replicatest", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("replicatest", "replicatest", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.waitForResponses("30s"); @@ -62,5 +56,4 @@ public class BulkNodeUpdateReplicaLevelTest extends NodeTestUtils { assertFalse(client.hasThrowable()); } } - } diff --git a/src/integration-test/java/org/xbib/elasticsearch/NodeTestUtils.java b/elx-transport/src/test/java/org/xbib/elx/transport/NodeTestUtils.java similarity index 91% rename from src/integration-test/java/org/xbib/elasticsearch/NodeTestUtils.java rename to elx-transport/src/test/java/org/xbib/elx/transport/NodeTestUtils.java index d098332..736f87a 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/NodeTestUtils.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/NodeTestUtils.java @@ -1,7 +1,7 @@ -package org.xbib.elasticsearch; - -import static org.elasticsearch.common.settings.Settings.settingsBuilder; +package org.xbib.elx.transport; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; @@ -10,8 +10,6 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.unit.TimeValue; @@ -19,22 +17,25 @@ import org.elasticsearch.node.MockNode; import org.elasticsearch.node.Node; import org.junit.After; import org.junit.Before; -import org.xbib.elasticsearch.extras.client.NetworkUtils; +import org.xbib.elx.common.util.NetworkUtils; import java.io.IOException; -import java.nio.file.*; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.util.HashMap; import java.util.Map; import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; -/** - * - */ +import static org.elasticsearch.common.settings.Settings.settingsBuilder; + public class NodeTestUtils { - private static final ESLogger logger = ESLoggerFactory.getLogger("test"); + private static final Logger logger = LogManager.getLogger("test"); private static Random random = new Random(); @@ -53,7 +54,7 @@ public class NodeTestUtils { private int port; private static void deleteFiles() throws IOException { - Path directory = Paths.get(System.getProperty("path.home") + "/data"); + Path directory = Paths.get(getHome() + "/data"); Files.walkFileTree(directory, new SimpleFileVisitor() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { @@ -148,11 +149,11 @@ public class NodeTestUtils { .build(); } - protected String getHome() { - return System.getProperty("path.home"); + protected static String getHome() { + return System.getProperty("path.home", System.getProperty("user.dir")); } - public void startNode(String id) throws IOException { + public void startNode(String id) { buildNode(id).start(); } @@ -160,7 +161,7 @@ public class NodeTestUtils { return clients.get(id); } - private void closeNodes() throws IOException { + private void closeNodes() { logger.info("closing all clients"); for (AbstractClient client : clients.values()) { client.close(); @@ -188,7 +189,7 @@ public class NodeTestUtils { } } - private Node buildNode(String id) throws IOException { + private Node buildNode(String id) { Settings nodeSettings = settingsBuilder() .put(getNodeSettings()) .put("name", id) diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/package-info.java b/elx-transport/src/test/java/org/xbib/elx/transport/package-info.java new file mode 100644 index 0000000..7abcc5a --- /dev/null +++ b/elx-transport/src/test/java/org/xbib/elx/transport/package-info.java @@ -0,0 +1 @@ +package org.xbib.elx.transport; \ No newline at end of file diff --git a/elx-transport/src/test/resources/log4j2.xml b/elx-transport/src/test/resources/log4j2.xml new file mode 100644 index 0000000..6c323f8 --- /dev/null +++ b/elx-transport/src/test/resources/log4j2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index 3f32f32..b6d2ad5 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,19 @@ group = org.xbib -name = elasticsearch-extras-client -version = 2.2.1.2 +name = elx +version = 2.2.1.3 + +xbib-metrics.version = 1.1.0 +xbib-guice.version = 4.0.4 + +elasticsearch.version = 2.2.1 +jna.version = 4.5.2 +log4j.version = 2.11.1 +mustache.version = 0.9.5 +jts.version = 1.13 +jackson-dataformat.version = 2.8.11 + +junit.version = 4.12 +wagon.version = 3.0.0 +asciidoclet.version = 1.5.4 + +org.gradle.warning.mode = all diff --git a/gradle/ext.gradle b/gradle/ext.gradle deleted file mode 100644 index 7bb7c73..0000000 --- a/gradle/ext.gradle +++ /dev/null @@ -1,8 +0,0 @@ -ext { - user = 'xbib' - name = 'elasticsearch-extras-client' - description = 'Some extras implemented for using Elasticsearch clients (node and transport)' - scmUrl = 'https://github.com/' + user + '/' + name - scmConnection = 'scm:git:git://github.com/' + user + '/' + name + '.git' - scmDeveloperConnection = 'scm:git:git://github.com/' + user + '/' + name + '.git' -} diff --git a/gradle/publish.gradle b/gradle/publish.gradle index 0337849..8675487 100644 --- a/gradle/publish.gradle +++ b/gradle/publish.gradle @@ -1,12 +1,19 @@ +ext { + description = 'Extensions for Elasticsearch clients (node and transport)' + scmUrl = 'https://github.com/jprante/elx' + scmConnection = 'scm:git:git://github.com/jprante/elx.git' + scmDeveloperConnection = 'scm:git:git://github.com/jprante/elx.git' +} -task xbibUpload(type: Upload) { +task xbibUpload(type: Upload, dependsOn: build) { + group = 'publish' configuration = configurations.archives uploadDescriptor = true repositories { if (project.hasProperty("xbibUsername")) { mavenDeployer { configuration = configurations.wagon - repository(url: 'scpexe://xbib.org/repository') { + repository(url: uri(project.property('xbibUrl'))) { authentication(userName: xbibUsername, privateKey: xbibPrivateKey) } } @@ -14,7 +21,8 @@ task xbibUpload(type: Upload) { } } -task sonaTypeUpload(type: Upload) { +task sonaTypeUpload(type: Upload, dependsOn: build) { + group = 'publish' configuration = configurations.archives uploadDescriptor = true repositories { @@ -34,7 +42,7 @@ task sonaTypeUpload(type: Upload) { name project.name description description packaging 'jar' - inceptionYear '2012' + inceptionYear '2019' url scmUrl organization { name 'xbib' @@ -42,7 +50,7 @@ task sonaTypeUpload(type: Upload) { } developers { developer { - id user + id 'xbib' name 'Jörg Prante' email 'joergprante@gmail.com' url 'https://github.com/jprante' @@ -64,3 +72,7 @@ task sonaTypeUpload(type: Upload) { } } } + +nexusStaging { + packageGroup = "org.xbib" +} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 51288f9c2f05faf8d42e1a751a387ca7923882c3..87b738cbd051603d91cc39de6cb000dd98fe6b02 100644 GIT binary patch delta 50055 zcmZ5`1B@oX((UYyZQHhO+qP}v+p%riw#^;ewr#$B_kYRD{gbYAx|2$}lTMvF=k!7; za7ryOoPsnk2ow+y1OyNekRVVz94YMooo(y+cm@9L@W%_t;!p*llcJ#%q1`#%IsF3q zUkMciApc36CjX!LjUMs;t}E8eR)Il*fX*R-fM^pLpwSW^N3a0VYTD|^>L}ka=_Up^ zpvD4$5rOPTbm7g*YIRLg-2}gAf7ys^!2KnRk~5qCJ0yLB?)?Sc^IR;iTj>kgRI|Rc z@*Vjd$!GbJ{#O7gpps@eo%!Q(iUb|ANU~F4z9!1>Zq3Rvo%`Xv zK{CN&WcuQ`;2LAUn=WZnwQwMAa#fB|*X1$b@4^dix6Uy@?mfV^6E#)O%w=%AqvOEI zJd02VQ9Hyqym%7#D<_A|wK8qGBgPPQ)HmF)lpd1||K)xzn@DtZLo8kJPFSyAI%(qC z%s52u=dt2i;n{P;=?&O@evvvksv4aG@}MVpc@&)AE@O`K1mg4&B<-+s-23(H&9!t&`bnO*ye%sGp8+?4iMW zRZUbZzM|1cBRER0`Mz@V?)HgXtIV2ISc@^JQPnA>xl%3%np1JMMvBs~avvh5f+KVV z^VxO~IdI*hV4x2d(_J-<;%}YGDn;e10FC(WSXmjsK~dEa{)N)?JUvOjAW3sHq2sbU zQb&cSPk3DWRwhilOzDo0&**d;MPIu|xT*3LBUfSg_hVv-?p}a6YVN*8oP9Rhx#xlg zTDxz_zOzsMzO;|1#aEG7V3C-zaBCRS_)mO9?H%lrH}j>=l2bI&4h6#Cuwxb;=8=$UtL_oqu2=A-pO zxI-{g9QLmpZ8MwHYBTzAN!Yqb!=0B*G&{O?MNRrzK|0}7PKp!g6%w@*`$ygWz0!jy z3{R7mM{8Q=ifgDg3v)vMBm|MIF)5h1u=OOMmP>~m%I&B!#sQ^kG-KdhDc&czGtqWL zRDU0Ik4tua2k*>sqsJ%GdM2Mp@@@#YRX$0iPE;(LS()}zVZZyY@XPX3& zN8KVUMJuoow+wbFcGxnFewN51uw!YCAyC&7GOcgB6H(yKUCa{;7v2>Ck!7bA$Om^ z?JuYGLGk))z!72aagIOQPiMf8Mqto(Kr0V9{u7nI-|OCpJqi&XVU$7%ghZ$pd%g)e zei$@sNE+pcrMmQ%$sdhR{s48JOSyD_-#d9)k64m_)ftLx0wZpYws7SG{_iOu;p_#P z;n;MFK`#C@Q`!8>e4I*X=8XJw3Re83+7JMHmmXh=oX@Nj>vAfNgLf7$2bo}X8i`l7 zw`(l2zo(BUd0Be$!_%Ly)5O1Pg&Vq(_jQ)6HoG8Q4w=C6n^&q%e*PP#AI-(0s4yQJ z8AQWE)*gZRYObG0p%c+7BrS@99fSLp{%nr*nd{&a@ZW{hLh;+T;9td^{};snDVXx3 z2#HyA(12ztH~IkR6|n9A{#_v*kAT&XRiuOj6>oNQyguP@%Kc@`V_5i}7rmQN%Z9K6;fQ^ z6SQhUbNv`+Y+4@lyo9!uZD?Y53&lyAvT5Iz63izAF1Ois8X)NT=zc+dvm81|7*_r) z-~S5;Mvtz`v3~;r$^3%>mj8Et$i%B*3_!B7t~|0J3U4E=jc$vEx~=G;u)t=4TcrNn zVp<#)QlgcF0@C2ZtlT>5gdB6W)_bhXtcZS)#XkJ4IOa`K04$5>i=p}Pw(~TvyWR21 z%?x41v{#=|4HA$R76IOiuYkO2PP z;W(O!41+ceNaMSX?oQ%G>>gyrp~^9;*p@b^sdjvsbFtNe&5#7|t>zZHFpW^vnqngd z{gHZwqp()30EXcRm8ZNK4_YKa@jEl+vfbdzq4>k*;>O>kA>|#j*djL)s@P+}OlBI7 z56Lf|!0<--fqQug(VZ~R-(v%nq=1FyW_6P^Zt$^PoAH*4h^N`d96mYJD0p;BnS>F} z91JpSEp}+xg`15O1-9#U3W7Z8&rFVEraqELHZ|RhYhdNW{Ah8Q;Ga&JkjwCTgT{?{qg2CzNl{6mbGpF>eat@=s3T8G-!@@;e-U>1m6%J7O#ng@PJxxI z#T@Ap?{8O3!gdnjR}Ja6k6~<#^0gzuZ%B2M1cLN`{w`r$O7)YfTzVM}`QRGI`}N*4dCgVg-Gt5i2~+bZrM>8dm%6u`;cu!0_+QV6 zwS_0M4-N<@3l|88@c%s{2mqb2jiIx1wkD*H&Zy=OVE($NbZKGlslBH`?owmfk~;=> zA{<$|$7CX*D*15SZJ$R{etx}D_vlJ{LrO|m3bvRusE}o9u1H`*fO5C#X*_zJ4CGwE z-$^;|h(cjn;Tp|C>xI8D)UCK$;zFOr{y5YAxbug(@olqJ?;Yk(HsF4lVEJGr;I!Yo zagk; z$9PR7NI8%XgxH%2@W9hoS zstCGBEX7=5)<)?{#uC*;OX*7X<4nNv7)Vg#PMU_}Qq31s0AV%khLNN?N^4k6^YM3r z5R*VkY}t0vE9`u-hQCSLvh7nzQzbTxM{de38S{B;t7Z~~gtEUYspB0sr`xinI4K3o zW@PnpIOCqlvV+jEo4bps%djsww->xCPBv%rzb(}5;nzB5HqftH+OpFuBE9Tr80+N~ z7Sd&kc)0gq0c&?n#!`)WEw#Nb5Mix25A3YNp-vNV4RLaAL2^qfZ$32?N%duq02le4i z7c?@kSI^^M!z##6mvhkMmuFe5#~N}~yD1@;J&>c;*q1lslHsHqaHIP%DF&;?#!{>$ z(v;MV0Fy~c&w3F#HQO^SvcCL@!s(`r!~OTJ$(o>(Hb`!{qj@`@{+cZvwM;qAYkx@c z)N+_l2`?WVQ<>p(xbs8_`6q6v7p!$Td(SO7qk&8lng`D1bQx;G$rIy4BEa?(ocdc^ z#Ci=S;>5ZHN1>D|l|s6VcYrFWv)975%X;Ag0M|QW0hN(XUo-7WYEm9+-bkX7&^E+| z-6WKdyWD;p&mC+l&EVj-zPgj-o9W{2%uvl3is5rE7hz@5n zHDeA*Nz^K(`Ady(xT5Z_R`#U7RZc)k{Cgg2qSuh4nKJXK){(C|<~(wiwriaC+A(I+ zfCsF#d~GQCZsL=62~pj->q}SoH9MY?HZ?RyR3yrow$d=hJmQJz^8!>Xrp_mj!UpNO zC{f;wn2j7}joYE>BVZ+~mS-Iy;tb9yS$a}kEo7=SIrB;bQQGAq2sT_V%|K~HGQ zC7mHs=$+Wn(;ms0OuK9~gM1eDLpf=;1LA;X(p}4kedkaolUPD`%?9;#+BI@_ZKv4F z)?2!+!2$ULbDDPbYhr<;iy~q4dS*2pS(-tkNs1h$4}1mLP||Ft*@4hoLuh(KKn$Hi zM5{%vwvz;$wv$NAy^huZkaWjk8tG{9MjUd=qeLyKuha(l5s5L9vcq0UDX;ltoS$?7 z6M=?a+3mK_H{`$@Eph6lN|A+OrBcd-$``yA6G-H@EZMiTRQArVt73?e<|Ut;H=NFwnjU}9sQu+3Eot-B_%_kB1< zVOr^74aImDL4&4b&9WiVt<^!!IIZqFFs}LfwwhT1HQu9aNofAqSPdj)fLcx}bjnch zn?mh!i;_)Ov+!UNDWqXJ*hzn=KyThOZF)+mQ@cu+o3o*)Et(Hlf@ZLjQv~4{!b(C? zUBp))aHw|LieImU1Zm|T!QFKRO#&OjKmcqGLB97 zGYkitEC1mMDV|1~sk%Kuv3VG{ENL}$kQcRWB4x6SYEfX>ixJ#f%`Pdhvg6XhD#wUK zxmptiwn1H}g|)7FBf)ATA%ew5@@nD7mik=lzK%zThEjP?7aZChkPa(NxC*Cs{Oq%J zbQ=<;dmUB!b`-m0BqM=RkB;u_E*StL)A3h&O?stcKO%OTgy+Lau}nRMw?pc|KEz#E zs?v-SA=WaRy>O0PlY4mAM;kl+TwGOgu6jOR6OdtsK_{+$YWS9ykZjsSJXM+R^K6 za$0sO9&HSjDLWZ*<+ZEUC})=3>{kn4-IF8iQ0j4sB8EenUfhGeG#Za4O@B+wF$c z>KT*mE!WWm$eg`zr%+=5ek{k8r-e|x_NuBVKa$;Aa2BubX!Qynlv^uT6m56e7|_1& zu^CD2lk;HIzGjmBsgO8o*|`6qNJh^yN&rA?s+&!vi&{sxYd$ta8NpiGp9}RQT7>z3RbJA3u7J7oS)!`C{8q4(sl(4(dp;9ONKFlQadCX@wbc2#5 z-Kk(AJL#+jJ)QdZ`TH{%am;*!0-=T()0V_hL?zweU~`v^AW$o zCcH42t@2II5q#d>X)mpkOwl&Rp+B+3T8u!oWE$+NbtQL?&=B{)+b{56s9FjZcuT1S zUsDTx!jZKyi|bmSjb8R{vCufQJQq9)y1<>T;A%C`}kRhN+9Od3nq);3TdX7w3#`U`eJzWENTV+OW> z)ztnDx1RWL#0jG-ZmS7`Clob*h-|(#bDJ$( zW~i{Xnd8SqrInhRZvD$*RmJkox#|*ioomH8?@WIA9da3r?L^*P5*e3wg(T#rfP`;` z8dz;r#T!Y+ZwtuGWa>&L3%|YN&Zi7O3XIm^H#WoIH_W=0U0!_B75#3@bI1KdNCx~$ z_h*4z$IO9i$EB-}Dxi_jmwo|ib>hwUur2g@X9lD_b@5|13V1afZrY`RH;fKP3_oUe zPn>aR4kM=iJ%%~EBYm3}Mw&i{V03{a9KHY_pY&vu@sGM=M1%YRG)Ex)9!(D*3j7u| zs}Qa9&UJpfPZN=cz%MJ2YSqY4q_i(qDO0N}!2LiRV=>zUEtc64R;p*9oji>WzF`>< z1yD_TXZz&v7995h;rGuKVRh|_dZ|+vw$>YzpHs_~it>@FPnD*8ZlF{pT=GD=5iTxk zny$Hx3@^1v)F*$bEbI6iZ4CpcVy_$QV+PA%z4Zg0!SvV`==DcNz(3kFiKZJS&A7lm zZIhG*j>LpiWVe+43xlEy>|)`a(;Eqx{ttUpas9SU^=P>%7Xi znbt37=nVejc6TH9eRaH(qq*tYtkC+adshe%&Sqd{zW9NNPhuK=Z;%s!*qau(@)a%H zxVo`$kL5!jvYtsn9q21RI4|kLp4i6eGK2FBKeN9*>x+qFL4x9bn<+?M>5LcKu$oqc zx8SlQ>|z2{8Sac&u+2Nv?U}W$-xvP{inA7rj$yzB>ArWu!q%f}>^6*DW)`J}616Hr z)^pRSKu~JYabN%d3zcIDg)?^`acR0=I33^Bkas?9!*c)Nz^j4D4og_Qwnm(O zF`3=ChW{AwlY53yXA6F3;*bp4R-XkVQ-d##_s=IA<|ZB3y-XQeaJWeQTGiGoi6D#I zB?(-Ym&QM0zbF**0&7|`iKsP9Xq~U>9-2s`sq{lzB#}N-L|XtD(~un1Ha_-YIeq?~ zXieB`WwgRC#(qwCmwq)Bb4cKup2ss@tlbxVeuqTDHxDs7Q(1-|O6=N|0DkrJ68gau?+sOMp6yTcm|9hR<5I@J_KiK*!tZ z(&>8);gs3W?8*S7+4O6eI;Zt$O(v;)m3mTt`6P7o#+p8&7+$9r$gC;O`h`xL>KT^Z zx%w7pgCxkKF%@Io%d%MeesI1a9XomX#S5tzZR2<3V!4Il&93JtuZQ57Y+OUHogctQ zm&p^bXxX0wG5`3^8RGMQpE4|vaN$5Sng;>T88%XK>EHt_%#52vYK?7rpc++gd7eIj z7Hg9UQ(V$I#5H`iHGRih3jcK+Euau%x%2V$7-90(8$uv|W-K-ag5Fb~U%1uImMiD%JSfsGW=I0#Xt4@vxz zFk)uoZ|z9Fyx0hqbvN|TPR^{Ak2{dMfxf_#eEGDY3OXm!gIOu<^eI_1$kCuXnz2LO zh=6L2+-K{ZTz!Z^1y!zn-3(^Kj<0fM@oJ>05s>nFojSc1GQ)fqA|%>_EYWIH%?7j| zl!4RCiOenz9p;!SX#$to7#tyVxsO@vRsWA)|9v?^P9KOCFd(2H7$6|#|8RWVVFEz1 z`i3*MI_3|q=0EE0>SI}kMa5m83?>y}BZRXecO;!P5~H0O=R%vkie2kAY=t_y zshlCCP0`vit6<=f4-dn9B~wY_5%O3*$uNiw1Nar|yBCa!!O+`C&DldL!~4AE-1D&2 zyYq7OF8}lOg%5<&U+cmXVXl>Tpb2Q6wDOP>N6}w&kQ2w5DH}{(yzUBtp7PMMl@-lW zmKLNHZI*yV&t$A^f#%{eI9qg+95iCK!Id1&5s?(fi|KH5O}aZW^N`Xn#`OCwpI>2& zlIlvcCk1m}dNp(hl5&R%tG{&1>i>NAH@_a5`0Z{FTG?D&?0yEyB}Si#J`@hy;1JX;Nq1wKyXTQEvxR5`oM!Ys7IKK` z&F3Xbz-5e9sm^xO>9E|&QqF7PIEyx!x7A>sxx~EOIUC~%oyO{KOy{P|x==nAC@ zuRNEZ+gOvWX??G^?qZ^Is!6~4_+fIx5iRqOVuwj*u&DnQVWDt$FzA9~kuD?*LxgBD zMf)~4nOPw^6;q=S>&JI2|L>`By|SXhdf)l`)AULms~4Hl)0BZ%;TAyXvQ74yi@2-S z)&}c>HW3CC{GqHJ&0nW>#gx#StEaw4mmkLDAj%yY8o9J10hz%SRv+O(c2*y00rNML zwbn550gLercbVRE#h0Nz7dvaj7~cY>5I7g-V&sSUTrw$gnhiA#t8tgc2wYNUPdT_X zJ5fH}<|2FMzGMCQky(@by%DsLf~!= z5`!xEsXpk}mOkLj&Kr@XhRAv5TcR3F56B(F(=QQS!O5q!U{T;5RjBvd27mS23jdq( zqYsbdzINjWnI*;QZZ#7BL1DWkx)R(Kd2}AjcP3pn)%k^CFbu!~x-NY+K}L_nh9z)q zaEPCQ0a7U_UKchEDA9I;KmNWrm|s*bh6k28OPAd)Y!_u*8l1VzdUu+d|4-5bfh;@} zbVHgR6K(EaNSWGZ9WI;irW>%FLyBkSF4JAE8mrfK|5it>L3)@^Bz4VMZo%5f7GLJy zpwFAgPSGEo&B_4c&JDOISUnkfk87o}1dVJ+L?5m(*R-ll<@S)<%?=0*Shk%gHA%p-M$vyHbKPh=v1`uu)4V%XtPq)J5L#nWLM+LXgK(3 zRh`cBiG`A00*4>B7{y$<>1G@^)`HH_#d#llPMP!VPA9-hsmn)5m2~1;xT+21#AWhi zSwjzZVz0VeL00VZaZznIJB$4KeAO6iNJ$EH&p&-A=By2B9Y;)C4C)KQ46Y-OkN6K+ zQJf$J zpwBZHi(RI*#hHvld4HS#vT}FVl&JC?mYdF~@%h>_8q)ciAFeL-XkDE`v!ocBjC`+{sfr))_^W0VkvHXSosbM z4+=y5CnI8Ks756UT9rk&Gkoi&%CAi2v4*`nlr1?N>)Gx4;CmO09d1iVi=ebr;uH#H zzEXf3x67EIxHnVUW+t&Mn|YZLt2O_o`h0)_B-$ZtKf61F(TVyE)RUPRHz`ND1-;Uu&>v#U}m17 zWt?ul{7UjQ~Bn^ck zMM=^$1iC&CEm~k8K%|o-P{bojrU@EYPdu7mpuZFeQYPVhDcoCL!F3TNO#pvrX!d+~ zImvmk#sA~?{hJ{W6jU)v(u2%crypb=2?uP|P*+;F6iTBWG`m zeU`1`w>8G-Lenlc%#O?dkF3jm-P}e`*+gUIwBk(Y8(2h>H^A6R+r_iF6w`#-tHnOI zwHjZq>6|)1mt?c@N>^PNWcJ#tVjxkjP@j>`(5wS=elF9Gm|h;24BEB#wllnklxw^- zXcHf&m4K%aqRFk$Xw~)WZRE0IqU_P&*e9JB#0IFrJbMgQf~hcD&r#cUnT9f=B_sT( zyyiw39OW%ZibkZ=Amy(Y@R9T?Jpwtwo=ODfUTNp-bIT*#)0FRku!MK6VxP;c{;7+M zA~k^ZN)LD@>WjbY2B;6O627&qc2~H!0XslT9jrp+sJZxzO{s0YSM4UV{Dv5@JKY@N zX71&snDKT(Y3q(BX;6+9a`A@edbAgE!OT0|NgoW`CTuyggJJX7R-!UE*s6>!%4B>? zw7<5UjcE}FhRlms<-_FC72#xa{C%M*7$X3CK$rc*5mwQEe!(ck7(m2cWcGqg{jscm z!m(|C+EJiK7+0*kRhQ)o-Gm6==O!wsq6VXcRKzJV=Mm`$rHnDaDO3+6l|dr&fdwDb z{w~lS2m?#6($7dEUD{`N_(kd-;wDSVS$aClMVVD_0Zuf4)oECrx^H3da`qp?5&i%F z_2?tC*<=6rJ%b1p2#D%GZCI`@SYq55A|P7>Tn9xR>nE0Gk}2v>q-``2BNa3n&qTR_ z3<3fJHH9dgWWfQ`!1*=Y^fZ*-M6=G5RF_rF%3+7z+R@?S1?h_SO}AI~vC|8Hd7{De z_viIXV(UZCOZU%8?@JCmuhS9MpK7JzD00fxNOEcghgggpqL9*YkBk5$LAw40Qvi?f zNVgu(LZmt5KqD#-_{`J3r(Ra-oeyXv$N^E808NiVf22ch6f{%{*C9H!2c#aAfj(JC zYUCa&6y3fbX6@s?s$Q3tf6&U9M|A+JM|OZ!m);0AN)<8?`kff2{r&zF)NhZ-0Px$q zSFcKpZL*M7?>jj9?eaX78>9z!YCv1i1k`rm3zY}{b}d18Qj{bnzCs#UDc)m9eoGrb67jxppVvtoAUX??v9I z4J{4IW=*s^%wiB~$4xZ@010AzdFe>wxG{Beb^JJU?OJ?hSUODuUOEaSVy;HZYcsrgE615-6#7gc)Ur7$)K}L-O(5g zjAp@@{OIv5S$kE!Cbdx~{qv$RzL3;7mTCFW+_+VKR2zmOmgbT}01QmT@$Eyj;JofJ zUsOw}jUYTJ4Y0SBjyw26C&<=T*&vru9L!u(ho-$fA9e{1+%Z^*YndT6M-omc^^^LQ zqflo>t$Z8IWdGl6_CcDIE)ymqGObwe-dycuiqjQ1MmOzg#j+-sh^ouhmiY_d&5Ydn zrap25%YB=K)UyXI08iTtT0W`Tcn0B)#7=4G2=hjSF^o=uCai<{uzc9c(-A^cal;CZ zKt@z|!zf?lnk4l*Paps3sag)QF`tSIKQim$!4^uvKG{N;`Uv`cX{Mg4ed<>#&uIw? zzC?(Nlpxz5aoO@$te%>E?AMYg`|?+|o=+4a0 ztutgZ-Q>q=*ghKhfW8dJXrBIVD$?R5Pl4q0^b~WqOy9TENUiE8Sg-0GH{#~bK}ZDu zE|~R?+;MO_Kut(-!ZePs97LqW!zTt<%3_(! zXp*;D9kQ0o5NjeEdN>Y3Td}rU8MbCWIW&9;SN0kMD3Qjh&cLxw@M)M9?pNH5v&hvA`zV;p?dFEB2vy|~cpW0z&l{8BP1 z5_Iptb8ZxHk#8;5jA-G8)nls|&YX%DQHj-bE>G!4HQ)GqD=}bFQ%!B589Wl!qid6! z^Uyp3c=DK7H(TNQxDK;3ZGApksP4Y)OI$dq5AuDmXwamvfsl3ebcT%F8VchU8n8lx z?L!|6&4XW_^i`S%JAM)PEGu>DGi2DjQHi`aO>}#y>a+WDe2;U~ zojMC?0n%)(&N~yfvpk?0uXaagMwIG!yTDKGDU!mCAuLQxMfCs>)2Y->?UV zo0c}2HQ;2YrJ5bfbeug-ch8NRoXrpUO0YiFhmpkqnB~58{XIojU>n{1w9k?wFs&j_ z=VH!S){Rz7do5VAHr_KVj}BnUFyoWXh+K+EFmHu!wN_8E@wv0>z z#ET4$*Hhb3xTd&QlQt#;Z46Qj*`Iji97T0FQ z50G{Q0Y2O}JL98L6oax`1fn z3tz`;c|_0OrgAd)wKr87;{n|jsB)spqpoBm}SHvx8X!0@;a62$2Cp}FrwYHr^V$A z?lb7Fr3-$S{6~NnieNR1Fhi4+8S`LOSLIuqHpFOM^9&RqSM}Z3#3=WDlhN9Uv(-$a zcEnD0in|WE;*xbtTl59Om@&hSS$3GT_ej|8>>F$QTS>!5i|SbMYp9{^G{Iu*ds#M7 zJLFEc&Xf~YQ-+>N+(_;KtK4{f1G^}0uyzuLFRl?5kF2EiCw?^OB0JSNm8;wuDm&ca z8%k|oURtTgD9^eLqN)tWZu62t@m3K)6GZ`_`^NPu^(tyi&%eZe)Tl^MstD~tfA;G* zDvd6YIHZe`qw<8YPj-Y*p2k++7^F;P7H8plHs{@7*7u6*y!17-k!aEDS0O3Lt0kKt zhso;oJxVF414b!MIn)@^&AL=Z zvE&AjZEDEB;s2WuyzgTiC-OoG0{Sm%s9-<@=e)PZS1GQ_7T#>|e1Pz!E>+A3Nc8Vd zW*%GtKpJ^B`4@x)ui-UGi#&ufC7D_JoXQ0zp}Hu3lnXZ5l8DnWK?bgoqKal(*}RW5 zH&4Su1>4AAbNA+~weXO}wn`~jITHO;gONfQ%N7uWvE7GMrW?w1`w0y1%&{5rxWD@a zaSxHS9B5lJ#Q*e%d_;(VHvo<)uHW`0 zi9c9s3qtYiREmlB>~aY>tt554tX2Y9em+Iw)|pwx-$^d+4PbblTQwAVh*EMt z<%&NAWnoa`wcogYe80~hrZ4T-GHq?X40j&9d2T*@dv@HPn*qJIM1N#GtTC~L*~Co~ z6Q_!7WY%yZ4Vg;-C>o4qX=(_!CfD{@M`%uIeK^%x-)-)OtA5THJgyq{ zhMiV;H{y};t1X8qJ1Of1!6Jo8vvKz+y{0cYIo($~^$*$BgbKB1U16dl9t}95aaa@!BO(9azJq9Ys zJDbMd*K@rox_|#MXb*3m+yrNRo{6Tg!+PA)laWTAGVf`FZA#lK-@w7^3W;2^Gd>h< z0jY2?%2E(}T3J*J#5^$r3%olN!rdNMP;??BJ0WkPChg^j$BLkAB;yvfYrm zs}TvLYOyGv}L8gJ!$+b`sy#P!YHfo+$#tnu}WFWCZ|C&A;YIiT#)exQGDDzJ2 z!r)D0gsS?$)_7Y;)t$q}G5)SmV`X5|zeLeI71K#vVz0jyJgovi7i#|XHpaUW3vY46 zptiv7Nwy?UeW-O6#2A0FEN2*^w~BJN;f$Id7xTx>4(*Z@+V*^l`=DlTIRFs7u>%UN zTDZDuuBFgJ#P*f`>M=dX4a!Gz8A*<-7}$UB)LkT@^UGFf%5FW2T=|g}Qx?jc2MZpM zG&0+K=z{i5AqlJ;0TH~rrl#!(iY@lEu-bp}{7`AMNYJ?J{7%tA(vg~1!?8(*bNk%c zQSG6`v-s2Us3Y5B(|IYKBnqG_OAj1@Y!8C-d$SAan+j`b83#u*=&jy-NPqI~Y4V#H zUA}q4dY~5!>*ZaQzS8Ez5=TaPW+z*j5hAhA5LIrlM$0$h;fZu$!b=w=!Hk_I%HRP% z;*?^`Pqj^Y?*n_Rw>`n4HNfWsdL=#!uTAqQZxiR2wI?R}3Tvek;00vyM3x|>=n7ok zaa`lawM|PNMykvitk#U`dL(RwwwA4qqR~+L_)Vz{XL2T zh=-=IJum!gJxt&zGqxDeVzc}ax{V~FZf`VK%|oMAzjJ3mGyxsiG_r}A;$ssuoSc$s zd=heeB5FCL%6g>g+XUj@f>Z9ZlY@G<^|+R3>f53s;ZR1PA!eIU4m zV$JwbToG)d1OUJYVi00s6bh$62ZVTWo&x_edVUZ?-R{pj+&t7`3h%zIhfsg^jZtyH zAk@fD@U}4&*A7`-Ck0l@wAKp~={20vZ%TgAzE3K{gY=N}V$k2Fi>k~v{D%_&jF;E$ z7)Sv9Aoli*Vw6(mr_K=Ie>rC=We_v~S`;847wSYoermwd5#~(C z%N4mTUua(uB|i__lO_o;zwe1(y4_QqXYTnYx*s;t!Qm0-aK6W6H$2-|h>I{OrzkAN z%nSdja;)d-M!In~VstPaP^30pcU658E|Xb%~5iJ17CMsU5E@o^%t(QhL|NR5fk~)na?7 zc3NTG3~n7P)WzkfQ05T9%(^<{yiZW<6kX%PmAXDDE>(^TOm<0T)tPz+u28k=9)Vps zS0!3@P76|e;(O6{73k;GZ`O3%B={#%%;I}fc2kw9F7%Ewni^Z3&Jwq`j^^e{EVkMjolZakW$87KZKfLD;z0FLv=|G5o$e${7Zp+r zrq-%PE$Bf%7;$yASCXZXF3GyqHrCRoV0RaBMRV)L+&z~w0=4e9gg6Jr+^38( zlWbMBT+{^JRmPkeB%1J)U_KA+Lcr3QbDc%N_8IhuvA^wmgn&ZrJY(tfuT~+ z73;Y_KIB|iV%k6{nMsc3^|A~Q*IPYhHJDzm1ZE7h;}rd17m$H1AX*2CQxGBpk#C3$eEj*5w@Fe4B2<2Xp^NGU2zky6@UZyQ@La%AvZb!h39w8KUrhw#74k7?JD;=p3+p1goM2m&>5 zt*10XYVD!Z_HtgU!7E2#7gyRK#SpBlsk;<3f3M|{quZ8aIt_hhmJ%BqE5|+-KM3`S zDOICWG2sW>>XmibEJ~m9fgQC|K7wjhcB;YeCoh)S`Q(}*)_90MR_nN3;Um0&@-CM` z^$SLQmRY;C`D=I)!V?N>&=El3k?SwzUk70CLh9c0_+5d=beL@$^TSa3Ax9fD&&WBN zOC=vIBSUr%&>>7dGXA?wQezDWkCksxZD=ftak!B~g?w8>?+o3Ix-Xa=J2;M@v~7Z1 z_GV>|(v`PI1WOneu-0Nc74S=UyQsn@A;rcWiVKh&m<1(EUMwhK!yteGhl2h^0=bkv zYoCf8q2O!%LTf*2tPvOhiCqzagF?$#lZopUQf-QjM1Okw(`&L&D1AWwtvhND65%$w<&6}y zn2aeXW}u4MGtH@>GaIr_OP#Wb^56s$D@hsbb?t6i}QPH2CcTK zT;ktqCbAwC*F+K)aR^#YN1?MN`Gn<7;{`4;g0oUqd5h#6h-&9|^qJIQbKKE=WY_}D zZ08SmO@dAuK0|{YF4fMw&X=f{Is0j^Y`vmwK_Dmqzi<0=|k@ zR*u3M(tC;&`U1PER~(+wnNtrGcCJD>Wn@q3eCrugvKvCf^^5BFFa7F!R-fvDmA|7; zR6lX{3UA^*!1H%=!5MtoVOJ_|Vm**VMbZ$hb~>7x5mo@vBm9S{+pEVp7I9F}vGLa1 zYO0-))=D4nACNgx=e4}sY^t4M_sXuw8^t%h?{b!x@~+HU)i?7;FekyEan`#!M2NHU zw5g^e7iBnmFW%kkbo|V-C-KKLb{xho3Vv8xglDTgI+;kKd@+)YWKV0PwIyqjE?S5~ zy>8@ij#a=>c6P>!YKT$+;74`pBwza|_u;Fc6kKI{@P)yaiwZfs+jx~|AJ&%@PGx%n zzQToitK?p|Oa92tB6wrcifWe`xQ=~(e`-iL5^UjBejNTJG!tkl%2`M&wN-Yb#5>cK z_mZ_qp|i-=UcAqOdICNZ)RX0qb)5w``Z_zkeiM*$W*aM3H+y<7w4mFN3R*W0(l?HD zfUrR=TFH(O=u|*N40Di5sz(d8j3fvH#MmlL^U@+hOjvu&T5y+kIzg2%F2Z2%C3q&f zrLs2_dlZ^DH%qOW3!}M38U2Cuq#bNY5qGDSA-Wp(E7^M}`nl5Q4?NXo<^7=1tN1HI zgDU{gR23!GOb3}x#V=)F_KCq?bB}!gOkyKe3!Shvjv z985$ZY6E>TqyC0|i=g1Cb!7gknGE=+PNlYg4lwcxi#3m^_{9K7gW#keK7sO;j?%~J zZ=1(%3WWTBBL7+fPtYXmCs-=?bj>ThngWsmQ?+HE6(F5m`SFDL0R9=`2N*r@GgI0m48mitJb%)eWy z@*QU{e#_%kB_5u6iI^+q$$l~v+b@Zeb}St^f!uq@kP(yp*kM1^w+>vqH_|`6a!UBk zB=BR&O1ZEq^Ry4-x6&d?jJu490C?&|nF4KYzR&W^Vn;DR6yNZF$52(^c#San%#*YX z{jLYajD6cS)(o(G2-o@RQ~VK~C1{gbnx5qGCFW?XcV1yUs!rDnW_8sTJfVZm)jaf8 z!E_+`R|GnNc{kiRRZ$Q07z6^CPPfVi{#U4=fO0?`Jks>E!!|w7$WbKjD&PUShmDTD z3e^41t=CLKG8hipNe(c;ehC`v^u^?50K-~K5Q-U}{Kd}L4%&=ae=vktwq(J~Nb4Ow zeF(%|pJ#RYxp*Y|cdmGxWy@3~!H2?ML$++IPjg|-=otC|hb3rrBtK(n%+HQY>@bEI zZIvztN*gAg*Oaoz)ZdmLNev^@rd!6eo6UAu^N1b1 zv|I9{!$T&_zBosxFW4S2&kbB6NDF*05!JBf6ickZLaziBI@D(9ui@J83Jjfr-@^U2 zs&2Asgkn-LKM>C_K9EPc*cJeT4rNvboK;vtwt%_@NG@=(_yzoi4=`L40ce3tNkjhv zwnrQ_=WCzvGky<6QWu)PK0KPDf(gcIn=ORG0UIhv`zg zF=}6sM58&ryJ^!Q@}xiFJipVsCMBv?BhbvINJ=-uLH`<-bgUYpaMZHE5Y>OW@(U(ToNzv zl2w532Lt-9tJ}6D;RDAeKAM7BG~FuY6xOcx)|Kav3ef>hE}h`ot+}x9aD5FxoJ%MK zR@5#{wBSZXiVx&*`Q;m!MmMC6rYpkfBsA(BgR?S44!3@c3@EkZ88x^*dcjl^!mG%#B9UDyCHnehoz^|6`NF$;Co0zwIxpcoW|{geg)!GkR6h&hb2uO z7<$GMe1FFlSTiIMFPd0WaD$j`KY*NRKcJfOM&`3N`{Dck%PT1J>yQj|rg(UZr2%iM zETkaR0z^+t9k9y#Y&Tx+FlHpWxg+n@A%$u|!dw^Hi1rBzn34s|Y z&<@g-GA}{}e9|bxr>@mm{Hi0RzQDRWCK!3D)d8on$r4)=>SwE=Ey7=v9cM#y*cER# zP-V#y?ia;Qz7@6PR@DV+?eN)rxCcFZGBQ@odGL~03g}5CV?lkH9Rl0<{c)=JSXVg^ zZ#~nIOqT40v&CZR&%5F$y+cll7fOpRB)7fv@_)EG$L`F!rd@YO9ox2j#kQS}ZQHzJ z+w9o3la6iMw$tIxe!ji;7;F83Icm(RI_FtO*-Em@miZ+qXL^ok<*lH%I_lsb=W&}a znV;QrIO#YsY{*ln7<<)sSY&*#VIL`qO`a7evb&?6W^@~bM!G$A?I^ZW=`9!n7V87V z3O2!-fzw6tjaV})ASjkB$r%>p|8gn$vxTY@#Qo$-?#IQ zz9Yz(}G)7+iH%RAG!sUW*)^x+6#!tG;%!R3=y^>jX)vd4aJ2@ZYh;hT%MnRPhC@+8nHdm z9)AR))WCi)p(rC6fs=W0lD58bKxt#`=y^e`5p2XTi6S!9a$N%MrBqABT;~nNwjtQB zzfc!Oi9e8pb+g^({s&}R1EQxaBuCrA6PSqNL(L0PhO1r|2EQv?SR*N^CmfqGE4?+4 zSU=G+EEC(Y9cA}!hbWVJ@8C&HbX?&c0jl;iC&V#Ah0KJ0S=i?One3&DgkX0y>n)T&7KspMbbPeYx9JV;HoUk?RjNA@X5@Y zg56>(>N2O+{M)U8Wpmx-riN@^GYDa0 zZPylLkH_?Ag{jCECyVEID+Ed zYrK*u)TU|bl7J;zqPj#OR%A$LdT$i5QJgA_Vx^+wZ<3hoC(P7h}jGtvx|3<0~ zj2QZ9xc`$;afl;QNC-ArZ01^~1~o+|D2_{)Vpb2NxV7;rKd@H1^;Y}mM^ay(P-N5>tCtwVwb}-VuJ~b}% zPYLJDpIQ!t^+Tg2f>U`!L$JQ`11y}SiCp$mI%U6LDsI|B{!CZIpCO5tPS|-f>pcGe zh@%NPY{>uqnOM`?y2t_45;cB@GZwxX!r)+U^JV^SN`=imW$e{ihaB5woLuU^BM090 zUqZG*8PNe9lvqdeaJkapZdm0l<4VxhKUN@99E4d5a-k8Dn6kHkA$MDO%0#Eko(M@> zzp4=8+WEnX5aN>gM8sYpANa&*zrYswLiYFsAMX*ASFLhWPo3F5fvIj<2yv5{qr@&mM(eD?e{2_>uquH;F;Hi(|SfWrM$>%k%P}|1z10{y*D@(64 zoN9P+a=zh#y&6#CoTVkrynd&tc*YD09rg7qhzXUE=1_#%E#L)-(U#Mm^AGbl(JcqS zVlhKMerS)Ol%e17*nw`yTaLC`0*tmq!_P$52AtGPin@dh?Zmi_4h8S+3Z>Ah7Ll3D)U6%M^46a|VFso=wkoDsyfE zs?wgV13(oBnCFXNri=_7^;T6ZlZqJ;LTvur{F`_xKJfJ)wehBh>&fohz{v8=@gex% z1E4(vJkZK+P9Jsn3n5j-)+lOMV1?X?8bgP@Gm?pkR9fODp;o`%ZF(%Rx}mWWHCk^_ zK2OYS4~E~5V9foG5>^bW^!)s>Dosg5!6-0VaZhdukQ}vLyEFvY&QsC3ceEnMh(Iyk^ttAf!>J%S8%c*gj zk=4e9!*HC}7vmm27DqfhaNUXGkY_*s|*JCO3wEGZgd^4i(e2fQn_VJvp_ksNkY-QhKKYf^`e1N&!hKcYUS}v3nUBb(Mf!@$9-`nA>GtK{&54dUD!>rPdH> zybwt^SOWAFnF9d=Dv!(_lA|Q*jd}*f91oeC{xVj|)JqLQY_J$qwRRHnp?17c+U>P3 zZkwS1##36pW1W6NY!?o~OzQO+RVI>Xfqau#&JivO?W;5DRAo(`n6F$NO37aCG!f)qZM-LXfa zCi;!4Qu0tZ+JX0^j~+nhEo6pbpodtSUsMD-)GR1cK_UO~HOk(TI;%KQ1$n|t5h!z{ z^_Mkg<{l~?iin9wWP6D($0#YhPLfCq5H*n$pNA`D7S%Ycevd`Ih?i&Q+9x3?W^suM zbDspQSYD|5OxYy4l9cdB<=7Mh#}G`;pbyLQ4oef z=B^Idk~Y%)=e}PcbhB<52UPRv6DpvX60NL%ks!PKD0D7m=^gK|to@f#yEWoWab-R$ z2Q>E16YXJec^5s{eAOqM&In(Ky5}qXeMkWHxOVYmF?VWy9<+mRY$K^|UHTgzVLC$z zhyqI*6kSP{fRdLSP9eDrNk+T6Ml`-$&N-oyf*c@kLH#I5oR*=rHJQ}>ZyM28YbE~Q zW|EG4TeAz$jqGZ_DkChgW>ul(fCI8b0{8?=F8~z_0#*u}O>g}s0eo>t& zT!Q=d*i^pRQ7F$;RU@4hPURB||0^9tsKCE#1`=eEF82N(1Is(!PS}3HZ6*r#|QCZ>z5wi57(QCwc|>h%DC3MjBSzsGC%O7R`$@p zjRsnYv@i@n3nLsgG+rZVYz_S37(DVYh)z~TtJPjDhq)y@Eff!A zkHQ$Sa5xjHxD+2M1Cwj*(=2ExH)>);xwDm-{VK`_qMI#KH2-4X2=fkD~TZnFO3 zh$_gKT@)m5rO(LybVfe3-jdsNNM~==LNgU_xua=udt|-1-R6jI4->%<21(eiw1Kxq zQV(&z+ThHx$ZLCqUC^y8QV-e?jDa_I!1Yc?z?UKBHq8E;*iIw5ps z0I-`8IT4&^&W3@`*bkJ320;m?U0f@X9Zv&N*;QgceJhFVV0t4Ar_EGWK)_CArtv1f zPOTFa0=3KdQlx0VoX%4ABa>&7plqgq`2JfSconZ1$J>4#vYqiclU;Gk@`U#4lBk>W zjg8{3EoA%f)iqrjXzUCN$Wp@2^3*6S9T2Rkxk`%WwP29V+OA}3p#7n=+C+Z9>I+b- z;hFy3I?kiX`k3Lg$VDnDk79HAN73$XGhD&%Ij5n*bbwS$0H1uBab>d2{Hm16)2Hmm z&RGh{;;OTYw!aWT$CG;Ynl>Q9*IS4TBLMO~rrVXI0OLc*q9r=IPu9qCo~bec=gIF- zOxnOXiN)wdEdZn6cN=BgUMbS)R2V2kxU186-;}?#SbR%7E-;9IDq`-ICFz7R2Xyy4 z!yqQ)JnH=?w5kg*P8{s1sT4a13Q^=K-Rdux_Xy>>J8%U zn07!z#d0*O+4fSW-rjRh|J!)~ZEqmB=Y_yNj+<3AV5@cpfjaVwltDpNwfBa*{mD06 z62%F+gD;D3CJ_fy^X>|9J#Bba&mDIN+x7e-y9cRni(OzH2OFH?3n))>w7Q!|NX3$5!%G3QGw_3(s^(m4v=WRuzb!9bMCsE z!XmIdtIBsrn#0OwRueu7v2hBNF6(Sx7mXdBk9H_JepAus(OJ4L0)6VN$HVKa?{RFA zimBTLzpdBNFVCtH28Pk@qV^d9{w?8K-Te3wz->2DMSm&oISMFd$>$Pgs4d-4le^3a<*zB0EhLV%p61m3;CTPZn$>@kfa|0}H7QH&||x#y`!lYGE3@sa3s!rDw*F<#9>WJjP5xseO6PGBPK} z923V3AiQh@?kY+VS>Bo41u7m;|Megg44Fi8R&nqyz6uxjT_Nf5A@ny(Yi*!!2{J1f zviW>*;Fm$SEM!rSjkc``iB7 z)$~{0;R%&#`IqeS9aZr=Z=+i5Cg!k1Vkc}LC&uq2%BoloFP_+-$;JL;t>Y8v;* zJN4xK2JEBA=qqi$%fRN zds~on!*~(+SPzm)9t@su$BVOWE*8{=F*BFC;2v2JN=<fS`=udr{-9)-l7WU*y1pJ?HA+h!u`UgF3SvM9ZH`%BL*`YCI% z_E^$2c@66N%-4U2SwXbPTkYe0o>C(hq* z$YumWVJrD8!IBYbMgW@s(Tk%u5tomRt*>hgF&9#Y(EdV(t1wInD*!8oyV-!81O#qB zCEbN(USna7Qz!Q+zyNzH&5vTCfpQ10Mb!ex%VU!EDr0LT+Je!T+(@6O0Y%qUx*C#& z5v@yMVhsLTLd%UMx3*@fiD0*;NnWtN`=Yuu%YI}Xi84IXkeDIaGKm~R>QXXCW6%7X za=Eq-MNBjH0kxQOSR%Bf33mTIWYp$lEkvLe76$;}EZHHJsjW>(;Ag|>N}mD57^DLq zJeD5m$mcGMZH3vh9b-(@mL7tH7xHAv=+pm%+|7ZVMTXI-lCPH2vc|2pG%WdCb>WE@ z7DUZgO&IBhj-lvstc(PvFs*L17LYAr?H(z{Db$wxjk)-@x5l%uQkVgN52xDv_n!W( z$qgu;{)I6V&fW?kQDylH3NE}jF;XteHNsb53}OI~CKk6VlOV7!54do{qW43tMF%}46A1xu9pzI?E1t8HT>Bm^*$ zF@NiLmp?Npk-A0J#>Ujdu2xB1zv+XdkGPCz+(>1AF&IwfZ+wTT)7%S%e))?$sFvxM zL(|!QG<4o|`IxtTbUV_^`Dk&<(W&8LB|tlA7F_eTJSWnc%ys?|7rV z0LcE{L*D7Z?X>0jMi^n3v6>&tkEjLQ!O|JT1QgUf;}2ZZ7iQKku|fcm@_)npc*spP z!h4;OM-3d*yFrF}Z;s43^tzFGJlg$0>hA97xS3Vp?t(E%PyYaXfO;(4VyuL4?Ilai zcmZ*;0UZjdZnFXpc}M*oEX<-W|_6NQ|d~vrKVn#r1psGK2t4xus{s7Lykwax0 zZ|w0^ZV$`p$o@+h!wIn92}ScYfN$9)U<)tnLEqwSR&HCzeMFocLgR&Ss@({&A5)OG zg@m0ywjgPEcNhwHNRp+^+Lh^yBX7gGhCFL)dpyoT5BDpYw5f^*Ft)*Yh(MWSpz; z@7#_7$Wem6fM}pNl)KC>PdYfW{0I+OLuqdU#55-6uNVS%xdIdpo77%~%P`L$)G@wT0@r>$sOFP{^&^cjUWPf1F|hy{vFTmd&R;i)P@i zMyX^xVMunv9MEke)Y=711#P^xFT%=!Al=kd0rp2zu`Q%Dm z1%pSrgeezt88eCoKlrHfwc96Z`gnJ-XH8S5M&-Hre6gx)In%BJ!dO48H%$oGzWe&q zt_ibniuob=AB3^NiXTrpNIH=OPFw+%^4C8V!r?&@Yqpq+k00bWH3@xRYPhfF#9}@{ z=&zMGoV)jZ7(zK9V#ng#0TntEfCu);oWB#+UsMAV*6*l|jB-H^cV3hPqFO;e8$H4d z3isn+O(PjY_yuz95w$~cPO!v)Ed%7}p5<_9`~qLDe6;jn7<>Lsw>Fp*y7=?D-_{m%aCOJl^LN)@*Dxn0iDlnOePYC{#8O74{zuKDkdnp3icgDv3HuF&? zGV0?cQV+m?GccZkD92b|f9U=l?5i_FSi>R>`Ndhof(%|r%VfV zV%{pyCz@JBKKAl(CqTG9jwIOGl`QsU3_`elN6qn0Q024yxgKu!BbB=Za%zQX~$Ljtt6Q3U7Zu>Bmdl(*%WV$|_K{d6n(|~>$=3)=u zKcn)%k=9au?(VUJ8Y+o>jathw5TomcAw_oi*Y9!tyXbv*7bk5CsI9>*WvI5UuHs;Il zJ;g%cNPO!NlAu@=!Sv09p31uZ%Cjkudu{1+410VVnt z%?75YN}FjG@mc$xV_SCWEE!8lK~=MkH|s>M*-Ogc?PB`=asb&3x(lXyPdx+2-z^Zp zCdDPz4Wr`538G16_iOz6&1&sCs?%K%a%$M@oW`bwa9z6Nt%_A6V>qvxphq;#*ACl$ zA}C~?if2;M<jb-vk*SOiuRqbBa z7`O44kf>DSPMIr$Wt$q$LY(X*a9tsu*DvGnN*IJVD)9nwJ9>KlLRuhcvbqiO& zVfQjg*J|3+pXl&IetY9==MCuq>pgPgnxZAU)Gd)Ub+U=cZbmSvP&9hjdiWC9EHYUy z!+bN!%}A1Ep(=n$+{qE&;{~*r*sM9+mH22i{pTvUf2~w-MOvVvU3dcusH^wiNJ%fL z7UAh?Dqeo-D#i~>qVXF`n&3iB{!zFkIeKA2Iqw=c2ohsZtY<9Sqkta*TA=Zpussry zq!v=1#02FqJsz8Y{d&QWc_BkN^KK{esGX-Zp6ip|$R{rG#a6dOow3b2Yq2sd_ zS=%^W?v-VGaRG|;n58_7t@pK~Z^uSJ3)f|Q5=|5Ug__L&>TZa4EYDO!XPWm+H|UAI z!B6{wUvl%lsWfh1ud>f~k2mN=GPYQ0hPD|+rh9Ts-b=b7c!oE`x@MQ=q0KP94bYqy=$;grCkgMZatGJ1?0v&<+KQE_Z^mL!4>Cqy&P*2%|SHt-uOR8~($!4DFZN zA)COdjv=cN$R2#Jpruu%FVGxO7XToVL~&%y9K|o5WyGU)Dxt_e=yM4by`2jA$rT$l zs0;jsfpVWUp^a(M72`nfiE8JtaKe`#(yGyyyg*hoRpbvWGQ`pynukr9&NqOFo?3-I zu9VEGat2PRa+21}QmeiO@8FC-)@Ie^0DLDH%DXZ&DFvb~9lWb-|$83oQ` zH9XPZB@ttVxh>K0pw~!2)iS3Yr5SymErgA8eY~R?#O3_0l0-i-b9PWG!8_Ef%pFRM z$dKbB^d3ErAA5psr7YxJg?Y;875gl`Kncl!Pi5&S=pHOF*zQd+mUTBg8TuAWBwgsS zJ@C0!-8Or$d2Hg04DD4x96E{Fs;MF%Q^N&69U6Gj%A7+^tX1y_zaBaY`us4YWmX&L z%grEgy5A-X)|g_vh;dk*zi{xosz`yBP|_elMc#>IQuaW5iG(rE^hJ!Jx5v)MM zL-_YJ(ca65gj3ipQ>UWHakFUZXw2^crhX+O;C%EmI7oIDb4XiDqEq2LklVaVBh-P)i}t*h7(gGE2ktc&0{<8j~FGl;>S zj7YX*%znljLznA`st={>(>Y|*d9nfoEhbtrL`XW=PuS{re=L7SH|y@E4)fyMQlHHo zoDuKR1$^y(f&Aa5Ao0sk>wmjP5}}M~zR}@=KNdImt9icL1ZD6%mIdc}GDx|rAVFQh28~s#R;R0aac0L2k>;S<|Eb|s#oI*7L0R2kxS0s0UBc1k0CdT)H&ZIt-4IN zxDHCyN*4RB8|#(rcQNujWkY16Xt2^c@!0g0Q{#=2Z&{&IhLmX=jTF*vy&sixuaxQ~ zcZwGbR+)@-QRri;Ri6MOTo?=!+E_`KpKF!yu$I!xOFI(%+=jUlU|^;DccK@#U4a>) z>pXS_yOh-Th+9M_g^d-ptVi3yptFs0vSQS_=!_RTo|j9*t)3BG>KJh)+~k6(st6;j z76#?gYyK*1uxhuF8=GG7+Dx#(WU#{~{MvrbtT@mejtnoe(Jvj+2S*-6#90$=;02|) z%M4I-*@g9LrYMok3z>oRb;JWM#%*3a0ua3|m{M&q5Jp7P zy$P^g`Ijrr?l(=neCMk6*Nd-P!PFKXIeJ87i!o*m&f^Q8r>$e|H`lxKZF%6&8@6sm z#;9i#?_7Pzd+*i){hw}Vw0rmz{09|p&~q;my~869`cS}e-n2nz0@aS2a1a8%OQGgp zf$o39r(bHxjk3?b!TG{N7_hxb7U^5Kf&9oiMsesDoxL-uOms$Voh=p|2#u4(4nkO3 z%Mn%?Mvm=6Nu4(HCavjp*+Tg{=Ku2JXCh4vedix>WuH~?I*d8~ShT`CZT&GlY+D6i zU!y4ACB*_lK#S>|*cTn7HxDvmhMT%J`p_vZR0t9c7&9qi;_$;E}2d* z;x(AoX|+_*H7K}3-u*scSwWdmWgRBd-azV2PzT6#tN$syTvIfYREv*qtW?waYhMB$ z&6v)gmt4tJaFISZkf16hCO?bXmLMyrZvj15?a&G=MjId66C7pIRuh13x-ZrSzi1ZR zi0~7nVkI6;+-C{MJ38Q?T|_w$YcMZIlcF(;fz=)H%;=WJ?u&^E1v9u6U@y-#;mOZ6 zoeU`QATHhRZp7ghLFT3Se8U4p&Y;QU4!I&LsG<#d2coHUBuHh$Fo;(gU)C4kJj>uW ztD6DAdHQY2+|fv2KT0mk-Qo7B$rT89R z8UE^^j#yDP=q>K-uF4Op!OHFPujms_M|evjlTB4Y8e}QjX!Atc{71GSBkJPqoPn8P z*23f<Dta;XO(Q-PkEiJDR;bPv9)|8!e zVY}lOv@Syo`AyNR^?*&GeJPXR3DrfAZ4#_fSjD0Zi(W5n)BDUFc_hL1p*up#dOn-2J(&8@gu$M zCOisX;~6ze@W4~E6q6>sEcs_idjs1I_vH-!c@ZL#m{dhri&{h2ib|nWoon^+vVcg6 zEe>^p>BC?UA$I^dRIyYJIba##8+8DDE&ULY$h z)#DBryTL=ZaPZ&d^lm9cqFT@r+_omfeZdEHAbyKw+!=mFz$6y9N5|Tfl?fM@dygwAlPjL;XcSNFpig4`yFB#Y97|`A$s02;=~Fla=(6r z-7==gxP{t1wM+Mka!MlYa_uzbvbTY#82>ro7q!|G7q@HFWAuWs)=u?9F<_T>cSh^u z-q}4t6$AW7k9|HfUL^mpKKL$$8JJd?KUec>F?ae z^i2k}b2MW}lNL-+0UrCiNhq=}g#k4}`<9G36&{kq8;Af;1jgfw+X^m4p* zlOK5-KF9F`ut-ztj|bUcxM0A69B0A&d4fC({p9s2)D;muc~l-0UE#$>Mx{SYL;CGQ zzhZ+Pn}$83SeTr_0E!J)wnv^58SLDL5`dytJ_Cb>Cq4lKB)-xa<0v+?Cd8ngFeIO= zIQHW$)Q+I{$7vwV%PabH6r9oByUb)xqx@%lq}4y@XBtkvn5v4H^vbOKv*T#U?wrA> z#XzA2O!4cA*1oZR(W6ksuI?ez9Xv9mC4(}dI6J#6&3njQ%XI+2J?frmxKb+D z7LvKu7fXg@E2Yk@QucDATqYSS$Ff`s%nlx}nz@IS4}ANcoaeiP@g~O58QtIkCDd${ zgR<*7Z(#rTU>m-)o?c(!#vds}! z<=1td7tnXbJ+@$E8E74K*Q3eq=aBTA?Haou&m3+uF7{IP2{Lr8Y(c&ZS@q!e%Edun zhOMUiNx#T0TzC${p*pEWPI%~1J}r!kkKm^uZApU-$(OV2)hKu<4dQ2&!u2r+9ClWv z-LHbv(ShAF)(MXMl0{!b4_WeJtJXscq)C>?Q`K&9YcK7t*aS+bHhL;yWKwx~px42$~0g>IqY28<0Hz zMT=%g*+%PTvxbzxb$jcDzrKnNR@8V}oPQ0FY6#*1%o~X?LEZ*N=&Cnt^zjwsbN-h?|g`_|upAcxt zS98cbubG!?sxkh%Gw)8R3P~|01sUYc-36~O8(z(wLn*So4ptQFoT;DD?geEA_1f&* z<~F7ntLP6qOHM@oyuYKK1Lx_vcCea!GLBno3ImSirg>+CCovw$?U(=IjOP@srp09; zy=Hlbn3X~<^-3%bsn2SkMN_R14K%RXC^RY)+SZ^YK*#W87#{q3w>jXMsbB}xn=-GV z9hR9->4u5VV0J85+0(xP9*EJQ^B{No|+?g9{MR6e(ahIfv-O( zVEQ(=^{ptN=B6otyYFGjtf;i_cJ+`(Gvia#YCGytmIB&ro{cV)5$60S*DDaFSWXq} zcshrdh^WvjViQe9{g-~33A3kIhh#->7C_}mdz5Te4y~!ggjHBr=T=ikaM!=xX0}yB zxLM;H&xEB(Pct&ULQeKDIQ(^P2(zhQb3=90;IzR4+fGbL*}3agXQ{h5D%3skQ^E9FnxEw9&O(0uUt zy;c~anDw@ZJW$?~A=TF|(1M3k(jrOAPKoI32Ln{3PLZ$7Acl8zY!D!>1vmW6w!s+m z@!S+-QD9OBqWf4}hVjEF2kCtzBoL_*^`_!C_Fx*0X#|Z!m&VI$U_qQ?JvG}imu}Kc zrp`K9+;XcMrnq(-JI61M@$f>4Xg>=_{k|zzeg+EM(yN3FjF5|za@>%W)qS+%QlleG zaRbvJZXt@KFtMdeFmIqM#Hs7pvExV+$*wzZRI?|h@BoJbh7m3b{SY4d)T~c{t7EQF zMBDj0oRrc;Y^?8BP z5mo|Xarro$qSp|@k7)dKg7TlYF>SV+M^cQ^=bQgpf-G(J=#?tQN(#mn(f3{b|cBF7O%Ay8RkldG9Q=)h^#9A!$hz5dV zU+@KD+-&nT4k3{K1(BN?X&rZ}fIdq};z%By8M; zXv)*!pBJZRPkAc9#Z$xZc&TGU#mA#)$|U~IvnEXYBLWjnFP`(*AwW zOpYuO`BH7NVS3l)lu;UmQ%$^3N~{#qbW7S>iGr`jfE;y2K?=6V!VOD{f%uCH{-yGXbs?hcsVh0)I!BxCvn+$wrQ7-cnR<*Mt$5Zz95I+-4MJFTX# z-rr9z+x?hJOP%o(!K0Ieik6U&Ai@)1xN>-K*uY5m356Soi@*^=2F6G%*hdocO*{+% zYMwW#Mv-EzHH4(gAIN$X<&q}_q+kM;mQ8%fDo)R`tkG4R!Wu(absEGl$+Pj7-OO^bm=$Q0Ai%)v|y z)i(t0u^!Mt-4l-jt}#u4POChBFPxl3?>0?SJNm1}`bcz-`sQRDtC^2e1zodxfctcv zn~PYWKEHBu1m~$_$GxxCm0o2B-)gKHb$Y!}UCB@qAX45oOh|{BS)3nZmG2GIq;~cc zsSbeL7eXXrao~Y7#6`hP6qMP;63*1J#Dvhn^T?|?9ZK;P;cdjHfZn67h8i|4jnjdX zApqTW%*yQ;195^5)2w)>+{i$aj>)^0;Z>EDi5Z+ou`Ula;*yKw^O;w;>`z;?-u(B4J8*EpT>)!acw41-_Hu;g0;o33fZb>`j)vAwCXBKNO=2qN3#*7c#2UXwNP zQEO1%A}L&uCbJmPYukU#@;W2cu3@ih!jmWPR!k)zR`@c z>ni;+?*-|dzB^8YY<2))CzN?gciM3-0n{ktVrpXQ)6~?>_U(STh2V#&!@6ILBB(`_ zOCn2kfjMEgB&c$&v&?y#!N~7U&+JIcQ6#p0h0#RT3UZIdrYc~>xw?Ft^~z$APY!s< z0-iX>r%p(Go%{1Gv#{f61u{eVRVRWxeKEsscBpgZp>9l<-N|hNY%aaF5@FdOT#AmN~eSUh&jBcoHRy{4P3aCkL#_mzYQrjmVA^cEoFG^0l)gmD_j)GA;CzlZhY zKAK-8WVRWH_o5rx!55Z*i}e*~o@P$)Y1MAyaC(2ZcovAKcgO73aw)9q#|-_TU3>~x zBrfoXDycK`J14#F*Go z9QYqJKWk^4ZGJitDM%hm&PjCpNfUW8Rjw*5?5Pz__>||VuAt9Ngcn*ij))~@sRir7 zdpwVd56Y(x^44yGVpWSW%QSIN!GaGL7t&zDGh_2Xx#|{Wg2=Q}WDoJ(aS7kA7QQ|M z9O91xKn~^~ewiDB4B7qPZnAKAc<1N0NSF!p;|I_G!uYBh4|uM$E{6KKgkDJ!InbIG z)LaP9`qR62BSAlt6g)%A!CsT1k-TeE$N|lolVn_0H!pUkMZ#>2h-$v@i)+5+WV>6K zo0o=D-!M>y9Pm-F$@_MiWyhb*%LV*=JBRt<{j%&6;Xpgk6}-uhMkgjkj5ljhV)T*lfdzWV5+z;!FpY`qmUz zlxs+0*o-sMw8@Ppu|L9Z5A@b(q7mtY^1r&e{E{bbLvagC2?iZ^uJM1yMQD~(F(2fh z&eV=V1V*V_t+y!RVI3MQvmjuXz-n2>vXq_;vyf5ixj8Af^e_? z%D`NrW%Y&y7-Su_j>mDO6z?5+L9^X(4r4$|tr&-So|f(KbsLYki?y5CW#vrxShZW|dF7v0@o+3V04gVJ@73q<3Qouwh?{H)U~}E_GVy+*bnZE7UvL zfa|tvW^qWcLtX~8qk^6g2&Her>?&JArxpw=U6$CM*-*(&Qj-}{^~UWtoxnyVuOjZb zLmi~+Yj(@+dB?jq@pWoAz3V#U=fd!bOJub%ibIMD%-g=`2uA5cMv_E z_znLr3eC40d|vqoUba}iWht$4 zmJm0lcUz>rWM@2fye`qU#~H5F3@?nFGeCqjQZc0;Q%fj8M=VCC=*VF(uV)7txL#0W z1d=GCHBoGSB#f9DMk2{4F8K$wxXeYC``6<~Aa3XY5+=02NR|y=i-3B#*wxAK)8jct z%xV@2$Cc0%h?8ZKJhwmR5gz!U^-qi}2ItvZ2PlG>Rv}Ax#TM@rS|;pavRPTP_}pR- zu~uks!uC^@FkQ~b^`OX^m$HL?0yP$2y3zl>Cp1!5Xv%(Pylb$;Ydd-1|Mn!iz11nS zD2<0k5QnLl3!=L-6$h2d%496W76v`%;hpWcrcT&~0|&qUMlz(m=zBjAVBPLdT|oqj zTI~rRPi=ggb-m=w_<>(HNPo%MU}_bq^e1RJH(1Acaiqz+tP4!PwZ$Ex&Jn@{nbRRRZ)_c*Je=M5GTS!JqatYWB zm**Lpb37Nog<3}OotBMz+?8n>W$iU61OfJ+V1Z=m*wxopuK~9y5SLNc<4-z$$>Eh} zkK4OexZ|=*`LfE`Y&K$hNs4S2qVJb7U4#zxW`b^;&;Ai(W8P{3GfWYwraul>mTGrw zim4j|{jWw!#w0SaEL%(?a8d0?d$F~Do!dhdZI|K77>+@_y`BKSR-G=w3HugABOoIS zHi>15uZ=0ppMdWI((^ftKE0Vt494%Uvg(+k)3z(`!47CZCk^!>3O-J6M^5Bn?MvQD zx6V3+16I3)2k=ynYtkPwxfAhG!4`F<&M8J7CXZn0mu)R0lX#gvPiYP5X=T=k4`?1- zqW;~zc?3*WbY(IcleX9ZhZ5%MyQw`aTo>dH8XBxQ?mR> z)!jANzx|#tO7cr7{a;5)38A=E3qRXe{L{hg*{tAO7^k1HXa`=_)uZP;n(q+XM;j z?(Xgm0fM``1rP4dpo8n+?iSqL-Q6Wv2m}uS0)+gN?7O>v_Um)*xpQWouBqF%x~_E9 zQ_V4{KgdJTL1?Jt5P(?r`0J#P77@5PX;2CP6>*LTQ+O%3)&V535jcr>Sksh2;NaT| zGy=AXO?LQjah)*Q5HE&M93kQXPOntuTtrjefQe91#QozK?;ax5kGNf3WP6e#FVv(z zD~WnXzG4`?TS=qtqS(`XwNB*hw)okTXu;~UQ}R7G$d;b7gVF}YIJvHlE>8ucN7c4$ZWFe@3D|P3>2trDI*eICz-bopP6|47Fr-c9}T@a zEx2=b4ysJFz6V@x5ASD_Z`XWjdzltSO!sMY_^g!Hmlhi~j*A~Qkjr}Zd6>V^<_+1L z*~qy5WI9zQ!A`Ai=kgEP0S&3C^h_cZ=A3&Gj=mj?@*UH3GoaOSwzUI;-J1Zd8~ZV? zj_SAXbvTKFdpDd6G|NLE}_&^rMPSRCN z-_IUYUEOq_?yPpn1LCf#;}TFBa+lrV5f95xn23V5+bnvkq zEN4`SUUYy?;yKfHY6;2y-GFqs#a5&uN9H_B5!4q80wF?kyCGtZM>K5fo)*QE&~@;Q z-ZRqLN77cB6N77krUey8UOH#>M(ja1ENS6#`pSZ0%UvvtkT!fFG|g+ZZMK=)EI5Dmcy-i9|Rn*pSvVq z%C=!H(Lw#N^nv#%L%^5vD`M1Z)m&4x6vubVtkdD`wHV?bt{;Rhng}up zQ@E5mK(ek(oT}E(avj}aU+z0>j<(fJE+Ml4iGp#sr+qHmIE!|j4nIavPDg|hS5oj# zyoLCGs`2BCgG+Vj2`H0{A3O}GGg{Rd+jcD!jjrUG|0|KaB z(T3jH41TgMT{x}9ZE5O#oj+Hb!}cSSL6SP2FPOo5qTQIl?IO)G=c$ArE~pcZWndc{ zTS&-oh9bAb?M7mQSc=^=FxBZRRn)*cg5kYrI3%SabBvO`R9Tt56vkH$-AGT_Ft4Sk3!*D^UPXRo1v7-R!>Zw~gr3Or*kVkZow2hKJ&I-t~y} z=dWtqTf$cf&sv^R}ASzn@-1F(1f1o3CdADbMn5nXL0W(A70oXCe-b6n&YW zZr@D1{tW&G+tJe|OkBoi>|sOTWb&<;@#eaMogBV$R6}rZ4_W{0>v-exL;bra?7MSl z0?Uk3{sx*_%1JN4tLEI5W%PK!xs;Eg;vNm|8ZKXsM}MZ9%E~9y2_!>2e`e0MjAbo! zr|7=2Htvlrfv^tF-RD(Iby11L@rCV#Qrn0Iv9}vupR#_U;Y|oMgB(zt;TeZ4CPtO% zEA#eLulYxYaBnq!!qN<-@zK15$?xwzi2H>Bq8Mv>2M$54^oX9S+0o8`NFmSOyb5hU zQja~&XS^Q)I8~)n9Z|#>9>$Uxp|CIU8y^x3Kb9iU>>Pwo7S_oSB{;dt+bZUlniK3l zcO`TmW!6m>O;Y8=(dD>%grPbhZw4O1fc?maLW- zTZB!%CAGT=^$a_~7w{H~8LFxre2>)xfP5yM<}*SJ?fK7oE7ed>VF~!|>=wM%_1A;k z*#}~Pz4D?8m;k|(aC-$CpCKY%;q5w919BpHIs8mG9&M%xTfa8$%2ehBj-z1Vb*$s< z;)6Iw*br_2glO{hR?E~Y`PY+47ulQOjH>$cyx!gbsPy5aIA|Avq%m4q20C{=X$0Qo z^_Zk6=j4NQi@9v(S(mp&t$0A=hZ}hoL-iVf&lV?sOqqCk!+Y0NZbM*uCMDr8@X&pe z$zemQ{*)`rV97R>4XSsXGJ!hEWfddUGp``&95zAUEJ8C!;E1yrNn39kS-oeUkS8a* zhCge|=wLz3#|!W4j$ZP`nCqlF^E_O_WCOZZ^PFIGO|FJq$GE6Nda*2Po3v^}v!@>* z3)h~7U|3T*X;aN1hkU|9;>h!mW-y-;g>0phtmDnomw*X$#<(@q6altf*lm|f;!h@l zO_Tx(B#{Ekxzp9d?4wtuwUPIC^iTTe-&NGoI0g@$4e)L?%cyy$M^iVF<|%nNEAd|g z6SUGcU%O>iMaaLtFcFFaNCLmVq(+!NQvI+<7xsq2r}1eGF+If_geSx|U(_7g?1IMmJ|ZjGU{-pjqFw%&u(@Hh$|J;1K!m^T zSFR4!mhJxe4I<=Y=l+PCPn=OYx@wxcl)@3&=qp2eu)4Sz3DW z8;h9mkG|kaCNQN$2jK;If-VXz1>+uZ_y)+T*v6=;$p>G4o-d{dJ_SQRogT#WvGtXR zskh;XlO$PiS+9|-%@mkt5@UPth}tbUb|uvu0FCF9FEdIqgW!j(`Ka;Ozriq?LZ3cq z4qz|PVv*%M!2bDCLIYY}q!Mh)Lm zCwgp2>;f_vNd`y4QRu~?niZ5VB=L_w8T{V-GQNdTv=N+fE1=p z^RZMSRn{e#TJEXf`8ovUcBko$Q_?96i6vi~_Vx^t&nN~@`3avbCR@4$kx;+(UzvQr zMvyRHu25}|Jz2Bn{!r>Q@&WkMnnzL6x4{sNPRuuUo%_HPAA3_HeyqlTS$+ z(rjTDr}Eta%}CLx;;4LkaSfeDq!-W1V(Y)9J3e8nO)AObeafVcbz42LsVFCpOa?Bd zFV4lQ6F9un>F)yS>!{R~o5D{bxdvrabzLdcTf(n0wpIqDo7Sc2S8iLT3jj!t1G>wD z^vdY@t?xc5uc-sa0TtE3ok(W=uDMzEv(8g4s2$nx1b(LkwaX-3*U+_BQj^b<=9(Q9 z9ZMB^TWkue)6>&)%p}4i-N~4p-;6IVooyR;N<3U@?ATb63T8{BNEgsvANfI5Bxifw zgrXOa6F>~&H^_y;EoLszdI0*}?y$-IU2yl5Uv%emqZ*ksc!H@WO(XnlqUQ1SW}(h= zmf+BjVUV7tqRr(sH1ZyT$HFCg=!W}mh=t}Qa_Lz1F^2J3C=b4#<=4hCoaUj^REp@W z^S<3B?}oKd*W(b+7~P?44}U0itR&rZWaQQy?<++!TO{rnme?+=^_5AvLl7^D$6_#o zTlZL??k#ghM;}&Ek9(oqT*-8Hy*9+m}+khY6z9R-oCR3ZUQ5NL&KC&^sQwoOA z9DtOgz>#En~La)rhF^m`)N znV+H8ZMr%r-ZDFDeGfB}hk0!Tz7 zibr!$IpTbPRPvMLM=uJFN1SP!{uk@0gVA8URkmj~c>ic9m=b%W_rO|1-_(RK(_-OX zi}_sf4ytL9lx~&tmL30(TqUj?wSL0$Wy>e33~u!npF4`=qgdX&f){ZyM@}k9GN%Hl zb6UyxQBenx``kOxJ^@V^j?eFJ=r7v|=YQB5jK@e)UV2jzXoIo3hMA+6i)fqNgEQ}t2AMu;OEHa&k$h}bTP zi@Z~@(--gDAw??!HzQ*m@fQ08fbd=KFq6`9z+Cs`kRpjYDY9Io%GX~?qVGsaJ$-~y zPkG-taJC3d2BuY3Np+foPHlm!7J`2uHFU_=;KC+==Wd*&9r2?lRG{vuVY%D}E z5x#Y!^gEXNgNbVBnU%vsYPgOIvz#p3Q|RY37=KYo1{~IjZ%3pN^+6Z>&76ygUQtwfMbZo~q$FezBO240rfQ?u7o+@X)Ik zxUb5OL@W0+MS0?b8l1L#2^$znr`3$kl!b3U^S#$Ea@$JJY(tZOQZ$d(=$VS{Lf19| z&@N0sY=&+>nqP$j^+P;;Wy*iVXLU9$7MzUNPS_XUOu2p>VKjX`3?Ud=baMZOPSj{# z_g?PKYv6v-0NvF0hA#A!3S@lrb2Z#I>S{}Y4B9VBC(_;WC&>;mEQzHnChZCBCq9-~ zB)DtHD{m{E33q(P_L_Z*8tgPy!Q%})v6skY!oic>tBEnT;Ny7#qhkw4m!J9$s54j( z{^8!DLlm|d-%=XWtp2+KUN~|u`2p;(T0mmvf&h)5O5g>pmph-M3Fl-z=(noq(M{5m zszZ5X1hxt>a5QwUf9ltyxp;NkUZ51k?_)l&Qb9)2hy-31JA3G(*@47CGZ#}ceqX1j zE`B_I{XrYZFLWX)s*p3(6vq`M&RoJ)bU?s_+$g|>6-I~1o;F~=Cs?~z1N23LCt1O3 zm;`J-Q??3StX4Jix&7FHlu(|F*zPNvF8tDY4fZ8H+qK(jI#w!-V#}s@Iv)O37G*sr zp^guw#}Nua)7|bQ9paAo%U+YQcrlt8b?_3Wh`XKxQKLm;1?sl^aJE{^hnlnXcC*b4 zV%;-8DrK%S!xJ$-DUGPVI?2Qobu9vpiUMk{1diI%S=|_#0L-86!nM1|PpsQ!6QV*s z)dOLXSZ6gnJp(M(36B=5&@9M5;kj5vnteWKIe)8KBP1%nm-5WtIbZMO&%6e2f85Rp zWZMF=QHmoOycv0e6Cy6e#TJOf&YFV6WRhrl2-qqUtkHHChl;N=|0pnmhsST+pAG~+jwmQuLnCNK{a*aUvBA4YNrfD z8E3;bx%Ej8HcEYyS%F9MQEGyy4fjl&q_1u zdQ%uV{{aAE?^&;*!>PWW^A`P-b>DOEZrSVS`xCv7Ld)&eOe&lO7ug$Zyc$#EM~Oi1 zcEy#Yy8MEn1cO#XL2zQ=O_<^9=KXVhBnj@#=3GEa+#S=;_BB^JPJ^!m1MAp5%m;iG zKv<8ygmr!t``PO7o^u|?o%#AJ$flGghVQ>kk8{ehgw?&6!K!o{iLOkcdSdB2x}%6hBk~H6 z;kyIeNWWRv5XOqtRvvN|I!_r9^bOadWb(8^&_s?#Z#%9lX-LlQH8L6dftXp)>%eP> z4qo&70(iHGU_bgyzvFX%MyMfI>&x70YQO2yq@dmD?L8J4lpI9OWa~9mw+=Fsgw|R{ zX5y`rcp5tGLB|wEYd9zlQZ>h0=>2M`^J5l(=vJa5%>+#Rvh*(f^oQ~swKG#IKFQlz zpu>iFEvHVtXIY48piu<^OIV3@0tJYPGAq3s0w9etidHR~afq;RNJtFfU=qp%x)}-= zl_(IZ(-aCX>n~M4KG7ri@uRxbUE)04`p?kzaRJAlZGTFFG{VL8MnKMq+zmoF$%d@f zrBH2$d42v0^F6_^=py0ai8&*R4Q9!Vm(s5W4;5v+I^s*|^CRjn|Fl&T#Q;rcu&uU% zS7JGSufkf+gD;}$yN(Mg;Dil&EJWA9%_V+pbY48d+)tOBWD<1EM$+i`QH>Qe{Rm zbX~X%ZknPP=(N$7#crCbxTc?PCOueH%&e-0#yoQhF8Q}jIM7{+1*zP>X?TaLWJa(; zrBPxN`^R*>!B`>e=De2W>q#&xBc)f46P(5r-$7(0!!H=oJ3@-9!x+QL3Zp5=e$aNy zo3i03JKqO5TY^Il$rHkZy3n3{c7)_F=@aP32>Bc8_8_#2xT~P{q)64;B_o96pyH@fGjoE z8Y>M%%ZI>mIhD9b20P3o6m9&>4$X1D@bBLUru7sW6W?HDu62fSqhZz@C@J{iW@XcE zb!x0enf25%ZpzUYg*Ft^Un(Q!HFB4lO=G30`81^YG=QtMIqPzdhz=;WWQ|>A*;l3W zXxkl7@@z6AvmX8T674jtgS-v46K#?U~oqr^!60+PP8>A?QT$U*6pnKP)H1M zH0b0pM{=-}bu!waUX8;peo3)pmHuokdC!M&B(-(SMY(vPdPdgu3~ECVdC5M5oyOMh ztvKz$VLbz%A*UQIXd4pA*#Muu za`xzlNRR6eH6>mtDKrZqdx5q&SXBP#lQEW~cjPUo?%(=HXEj{k8xA}1&+!8pgJ#%c z9t6HJgmGOnaFJRPO2U38k7~jMk$mc%#Lq5eVP)6N&YMQ632**St$K64tR)-_2naiR zaP-yRnI)ZpQ~(fUIO2wc~{e*tK7_pa9C&o zyFPLv|18^6{{Db0vRZI}NbLQfHfRbC5Yw*J8)aW6S`Ee0j((t)9>p799x5{H z^7^r_-pjrGfGwFDXIS_|Z9I)viCgG3)V-Z7R=AQf!vlpvA z&AAf4mJdKvPLmIvU3{2Zl^H3g9DSMWC&UU``&do`a$l0Erzty4XH@6Ps2t}a{%X+? zr^m0TIx|=NcJ!T(n4J4kMG65pe8*tah5E`q<1h`uN0MP#F2%TVnT1o+wS%Sc5Q#gp zdOX{&DwUu&yJFmcJG-o<>f5AW$vtE*lSVJDO#?tjniFOcgXl=+)C+o|$`ZeJT&XjmrDgRH=OIVL2u>ZNT|6@3q%v7N*bA7Z?R`0 z4a=dc)-mq$0uaMc@ z9-h;3c7)poC#KnQ24Q$Vvh>1!$ySudt(;HplMj4-M$ltKrV*&DaFG`nfi^R*Jc z2dk>;ua^kR;ukHH69yX=kJ}Wzv}8?zl? zA+iUiK!&bfCP80q!u6IZx+PN%aIv)3ZI*?vUj_#g_Mwou|LW0%sW?Col>1gXBB{(T ztfQr|qN-Is7TPM+j=;{*rUg2>np+Wt`mh!0xtO2jU4$&7M|1l4+&V?Nev*{t>{qoY zR7b%KJoZt&I^69lb??{C z1_g%3q3FdUoFM6paPei*ZKy+_+rw%d;Ps}TZ3!KaIwSG7lXAw5b3~L{$_j%42)HO1 z=(Et-G@N2UP7W&^Bv>`Bvw8CmQAm2pe4-?F$KJ0Ol#6RVM8C49Mc*5UuzYj8W?i)$ zL+v_}t2yC5JjL?`E{PMyW}+HlIHjk3bDE@Zh`Qg2m-ky{;#-9uMyB_Y843K;nOX+( z7oky-+Q=DBU1T0Qd^z;{Rfk+Q04Ym4#iK6M$58)^xQ{ITOLn3iQa4RZAxdv}cBhee zpFNLqId8l72Fk`IGoiWHrMfO+fOt-batqMR(ZE5=Xlw`v_H#%zqrx76b2n)H?rM-Y z){buU^-_lUWj|+RR)yMNYjtO}BJoV97$I}B){=+)Y4B`U5G*tcoqDb=;LIUWkEm4D zSg=GrqXJf6Z_3MWBRhi=W>Fp}6{Kb~&);c@;xwa~ZG{x>s-l~S#jYHA#Y&dtpY=s? z{z3=QmNIPY;I(X}nhVlals?i{AR+QrmDFzk`P=d!<{9`A!M0c$W?)m4%}3uhWfdW; zGnAzMCiY^*jrI_?a$u_gU^-zn*4DkOO^|!tK}N>|4Lrl%^wDo$pfxLCnQ&*?G zuQ)0emt)2qT$aweCU`Mi7YA90Hre)4WjY7AMeRu&>sH6vux;lK8L!K)u(uwMC(%QD zZ=SA(8x7-&#lcAhV2DEQ3IVX0w;HeplKUPSN%XMZTf5{1@gBhfxAKN_A#Ei{RQ5h) zkmQk!M+gwULjGQ0U^fj33)+Snt-eYa z&k}yUJH+!{WtWAGS(cHInU9~8Go29;(&&_lt+F%YyCGh!eiTHVg z2^EGS7OcL0bdgP$*o{;YCo_+t5PA%ClqV2_c$kyHN(27^r)KrA3@F49Z*#+$f}+}A zYAVMRCp92E%sekcffuDRUlV5%`e{glu zgrZ2voc&A-WSO9g#>>&@1P+jbux5Al?{Dko5iXJgI7ihi^ba@9G8B#T7Zl&Vb=Ua_ zt(jl>WwcW?tG&S=&thT@wXuLSt4zNW8)YTfLW%C^t26#5>REB&G=bo2DOc@*$C`eh zNzD3y7-o8RMmGPc0 zRw>f3tx%VXuVUP4TToZwG=a-Qc^-+vsBjVpc(yKPCy*sE36jjA?r=EqNN6W;5J~ZPUFg#G zBon`}=Hn>lN_7L9gS497a_+U55R5+i7ga$rYkc``nt?KW8+J}V^G4n^g|CKGyi`ZA za-hHxI{1BBg{Pa{BDcx!oTmb{>p3OanD`^VwZEwdtkpSq1|Zdp#zuZlQL!pV=JVk{dfgjJ%4m{yaMNsR4mMP&M|* zpY9{#0Or-LX_HB7SB+FyaPL66o})$IW^{e$QGH-*T{-T7M^QfMK^FFx&=G$Z(lV%S zJ{w*o3X4AZeULAE{R-~Yw|y)O10_JPERL9FpMG5(F38r8#{!)`3cDI-4)Xb^eQaMi zMZpSW6E*BsDf#7Epcsqr>QT)NlsB{_Oh@sJOYKrDK(zELosw@56IJCm`bMZ8k*TqQ z&JlWD5i4nV!LFmRnq?-r>4L7x22yG)4LzNeXctL?ooIw*<*IM6&)wx)u>*iWQ5zP` ze#Q44mGk-_?c_28tsi8c78qXCJg`?!Gbr4pi*qq_m%Z$B&jergMVLxil4@!idr%j{ z7v#}TRuZS}wmj`DEm)Om*Bm7yqRMDh`|pGlOfVEAF;DAI7@b8nzrnqOYB_+s&Bxb% zN4aQ%X=cw!GJ+!%i+H`d7G(@Lpy-mHxUWObD}ouFXP`8h@M;>Tw@!QKCPOc9q+pju z6Ka9!AwyfZ5XRCDi94wnF>W+;r1}!@TJVVfc5JduV`qr>P{M#7(8qm)Ujq_7@Zkp% zy1JyNGTed74xJtiqB@CgcCz7v6E=v1V^mkNrKXX5Q##h%2t2cbVXpybZjol}0dM1k z-ZxL{eRJIvveJmsravKf@%w!BXtQ3nHpx$VdZ>GLF!rH=KEt2rd?Y@@8S{`*C^(Oe zw`}6+fu4JM*lgUbwLD;C{vca7FxZT4HKAKiC9dY7IJtkINfUMl67Xw^Em|25Tuhkpbq(CkM@(| ze#Nx8@n9QY{Bc*%aZjRE>1~eUG;P5g|Mb^~Re7XaI$?F44t!Wk5^30O1TBX zi?jIdK=nQT8I@a`&r4U?z~gmh@VD!GXHxN>bOXNmLUO z=hjp&q27$E_Ky$%93HLGz;@K=&(Q%)rI)EoJ`BvBOBl~Y;ox}{C+*jgEn#|Z%YB=Vo{FP zp=O_!KN>Q((Mb3^8=|R&ly|z-U31~wIGxSk$FakXy>mx5J{dI6QCXi33lMPCQrOIX z=PHS?Ztrmlu}|g==~_RKX~K`+o#N~I7^#6|G%=?3IeF9zo^4ScpSrP*)HXd&?2}dK zNizuFh1c~E5HPY$KCe=x8?olnFmDFFMW`P2M;evWu$mx`C;mu>q-8 zPB4I8@H%q~xOzn0MP9qF=$+%`S(dU!N$<;lPCel~Djl&f%K_`D)|q#J8?%GThRo{1 z4W|1EPGjoeE?%D`x^gBR6qkwZG>Axx49P@TF*kAo+{$;2sLX(HuX(+4bkP#Zt{HER z-=%Iv^XdqAmtTMN0_h$%H%J|2Rgc%IXFfBtPw(tq@>xxORvv%w^-;H2MaJ_cYF6^- zE-|f1G3hnc3-aoMeEbN5%)RnpAN?5)2SWn&z?|ZfszTG1o5F>&z8P?n#7?d@i*36i>wS0G znV1pGpv_kxf;+CGYSEVDQ(W)hL%S2iU?mpcT!84#ey*eXyeAxM?-BD4<)KE;NUPpK zjGY$wYu2WpKd^E&>j<8u7rax9iv3r`>r-Mr0K%E3^yGT(nFLc4zNX#^ju!&*x^Et! zz2e?A0WJ9X zzcjzu{*tU56kuo1(lsV4?6@cfjcP&|%>l#8JpOIv>>I5qgJbizLjQP)#Y9Ed>Tu8D zSip~R8?EJ=SD^$9Ol$JJ0yLVXZ-&fHjw)o{&H8F_C);}W0wsm7$Vc`mcW6i(RelN! z-^1JHE6)Q(_g!iXV@25~MXcfgVu#<8ynXrH5zOXIw;7CT;_7Z>wi&|JS|`Ylwc1mx zDsGs)!>va@wz9-gmRklYiNsVG_xWN;Y5}-7smgF3t}h=@k1cB~Yf+Ti`R?bdTKlt` z1lK3+j%UnYl%LZOU6t+z2Xemm8hWOho=uHC5U}-=mVX z&tlpO%I;jd_j0@O*yosuw*v4#F|GkZzEG|kkH2J{)!rOlH3>Q+pVV3ah)=XI>5Z8BpyB|%)P zwZ>&7@U^J#kBrbWD^x{RX{^u{C1SQFv#40F7ueaX^FqNVU?rKj==AjO3GP0O%3HC^5xIup~Ss{;%W~c(d1&-w*QQV>L(zVLMr;nb4jB-;U(S)zaZ zw2X3wR%hi*!H302h-*FX1(NRT?dSjP!;6T0zWh!Csq1hGUP{cbGkHOKal6s;D+%kD!0vQD!pbSE4aA;vm9AE>|L#EVwgcA?=_slG0im|`H~%s%WuU$-_1A0)*HMZ zn1XxkUc6}L)No4Fl%|V_!8-Sw*P2rW(+#nM6!BBZc-8q*>k*pNW}^Ij!;W>cC^uWa z-L^WYvL;tnaU0pr%R6S`xVK^`3kpjGx7jnbs%@;J*IE}lb7pY^aHw|Qd77;#C=*Od zwQObB?~EW|dB(a#LwJ0yNPe5W z+$d>=_^4acZ*lBRd@dg<`o+}u=gm*if#qyxJ%QQ}mXO~0i>qTX;B4v`GyM5Gd+w{J zutfrt0|fYOvXeF=fR1^z?dQ>#Ojx4t1UvCi1Sorf!LU5Jak{BJ6mTn7ul!U7^}X~5 zJ+Fn4cSljNhAxiTU7LkPxSr8O)vnE21}s-JpWhBDUvt91ZeR0ZePN(7TV^Gd*8Ty+ z=wRqmKW3>5<<1@e5d_3(_Bgbp<0ANtrvr^YrV-%^6O^)515A;Qxt(ZzlFZ?+W|I{&LYijAuddak`Aub9B}R58$Hs;6zODj_zrCE~041?0 zuFh&wy{sK!5%9ffO|_(4uaU$%B6~CZp!s7nfikN2v7)Uk?MS}#pvaFMGQvwN;XGcK zir7SL;enQbics9C>}spGx78%xpJ`J_Q&C_5UTTERM`4bP%jTMnT?D{4m>TIvy12QKh|1j_k_@FaG}CH zI+Aw)VX=^41cPe~6=W9bEa;LSgdvFw;`#{mex&Og{#-qM6y!Be7`zb;9u72)u zELxhGy#_y~e;0efTnocL1I7w_VWaMG4_yh!670oEJfoTxy^4Q_MXpEOXXcenhxwLx z5dHO(6MGyj1U>OE`@&b#34+VGPkbeq$@k=SN;A(4Nena^FPDA7ZXc{Q=7*L6b3cqR zPE-WPg|lh9l6127)*-7|H*g*?H;#Y6OMTcGnaKcr4@CP?f`|1^Cj#(h)E%Bu+_S}F z+=-~{(IDLa-57uSU}smjm#9B%j`?$cMPE56r6BO5#*M#>49jl+wj+5`%tc4ow(Iz3 z67Mf?T2;?QAHD#TcLpR@CuP@Izm(vbQlS;#I@3^EWEj%)G zYl{ISOJ%RJ9MG0uU+-jp?hISY&d+iaIvZ@}s71FSo7iVRvB_#II5Xxvz^Bd*F589T zIYSl{TLaw({lLq02Ho>qz7k{ZztiGUv#Ll0=#^sHCn*q5^2it$mpHB@K!tKUIhB(AdARYx3tZr1n${qf`{7cYdbE=VLGPHehtQD3 zXfHbrFWS@1I7R%C-X!D5G&;X9AdOA?mkVH%3GMKS$dvDkN^9785S zfc|;mpgNSG!EKdt2M*8$SJ;kEM3*dTa5wM}6E4bVV~c-1Y6iaJN=e~$?Qx=qna{v< zHO(75^{w!B!mNc%D_DannGaeX+Gcn~H!(;oZ4vyu)xMQFnK;Ag$p>|5(zlKen`HE8 z2bE)qEljhrd~ipucVnuhGO|c)q}5qe0I_$Mt}!e8Pf@;iCdM{bsUH}OR5jg70Lbv9 z%gBY9?VrC!UO&we9>wre-wDy4A-Ww)_J|wp%O4!Mta!Mhh@PgdnXJ!?oft`qN$RM; ztEwQRnvb>rH^0$)7shP^ys#<>E+NSDzjY-m7gKXP(9|p?AXeQ_dFc(7@OJHmbdbe& z6=+lJPz%+{@`qOx5!NzV?7AWSt)`x8uRG66&oTDOuI%=sQPazzWtXe~$=6EyCc0GD z1&bM*zO%x|Ti|+vy8wvv0T5RMU1hoqW$-%$;fTWIFkgc zZ(LnW20w%B6oR~$D?(w|wI)oxWT)IPF}3M+TI_~^r%|Ijh@w5o(DW07Yp76oe|jr< zglvJiAkcxY>b*<%JA@zDt_DViO}5;Yn$$~j+n6nqfb*o+_gWk9&M1IsswGN}&2+yXDjwIuW%n)7yT2t29VAf`ydhX2_%Hzy@Nvepq-y zxUZ+91VBt{+h2TQ-kfx-W6rf&@!07l0yTh(RfU7=PkqQ2RiOR!U2pLA?e?KqseDEJ zo5k4{hUF`~uLL+$1@k`%FX@w0z9|}!qi!n4121A2Js>YrZA$hgRGD~;uiY! zUSkJT<1TuvD&{Dv8?V;)MUwZ)DQur2 z0wm8G^ZXOFz8eQKywg~e7Gxe9rM`W7fNCRpcOI&l63JA{%z zQ9wH2^hie}{DvmjcpvB@;WZ2KLv?kr~ak2h3@Zc@xr~zn+k7C^$UhOI}=I4KsNenzj z#QnfQorvI|PJ!Q30Tk!10GWx`xaIEj3hozgEy66HFvu%Q|CLJPQkLT{3IQ<)kk5ef zUmvCiIFWySV_z>K{01yQ6ofp;P5SGbUkyCTem5}oe+>=dTr3M@I*2L!UksR+;Qzz) zfZ-Wt5a~HMO?=J{8G7YxdsKE*r(t+2F!m; z1n-N3h+l!&5wStHBQ#{cL|Ci7_!okISO9O7;O*~L9Ax|YH&A($i0l_|^ys335PT#G z;G^LB8wK7M2Pwh+4+sxBAEx>Z=57(n>I3urW)nj|kpBVhg#RBH4)kpl<2TqlNTIL* z44MFMzo--cZpA?zcz=UYR9qabTr6B&6>MDHEF3`dQ&?obv?AyCgKGgtn};HSX-WR> z2;LV5RpS5dOFYc<8<=nw%H$1J9|%_e&jG0?`wh%9Bmg~+A^isTk=yO@f=7-58v=si z52@~y{{z1UC5#dO20!lOeX0b5Ho+UkAK+W2-(Uj-RM6AtUy_~P?0qu>KW)LI{TIB$ z{2MGnfy(sX)2va>1Yiaiqi|z{fB^sHKdm?@i2Hw?U_laNe~soJm0|yZ{u+@p+27#& zQ9QC=L-$9q&Odty~*PyJt8ys-lHc;SPz7XO+z|FR83 zKnVV!ivN2sP#DBEjr7}b{2~SgnJ(Z!vN(bYT!=s^3zWZA{Dar>4*^3?zk$DKF2O>6 zga2i9{O`C^x%>w6zefUH&i)1dgVyU0$?v^Ddy_oBCH_O61vc|P_1E+UOQZz(&i-|v ze-QutA??}kH&DR}19U#|*I)lZukr`@67U;*Q9}rd_@9mZORxX|A@v7XF7!7zVUqo~ zuK%XTLqLEX=0B}C=n426STRHQ8~8_3;6JcmR;L&D8+i7C5L7+)m#+V0U4h-lpHg)b ze}n5>UxR$xa6pO+e|7jHyX6n*M#=wlfbK>E@%`@r|Hu{iqk}`*|2p9QBmJLHbgH&i@Tf`Ws&ybXW`qQ-CyP{&GtHzGV8Z{fe}Lu8ZjY&rJXC&FR0A<$FQ> zvjSwl{KtPU+y8YN+qP}YS?nqwDEv z{JKwi1Nc$}IHIx~I0P&R2sAWEwXdg45+Vh{e@<>4k8lwX5D?8IVFd!y-$QM;(y*Zzav1$p+P`W(NahZnE_FLXlp2c>yqew zbl`tuH?l;5fQu%xcy%VDwPG|GWyX-@E7r*37HTHIKz5z zCl0MN34NFg52JKHs|OJEw!?OUJGf@eALBD;-^am;abXPgn$(VVLm+==B;1`D_Ium+ z^YJEf+anr-@)n%qBRX1%81ha(MS8qTV$KJ?gS8Z#u1F2PVb2kr9-x`{C!Z?a-<=~1 zqK#Q?1kE+QomyOtV6BhD^n8#u^r4SsYbI8wzAMaE1?5jF+`I$3R-9K+@(eXGeGsz*FJRjDWZjXi6*3rVUa za+Ly>hm?=hP=W$o0pD}ef_xi_90WY|$snj#nG1ZJi(@!yTnaX|u54H-PXy}lJJRFgswy2&o7hYq z-bA)#d%$GM4-)~UsG2IadZHtDr#|K8t7~u)dga0j_9;7T9SJ@SEqW4^Prw67Da_tP z6j~EnM|u*GDpQ6AtA%GEJl@$Re7{3GFDaiW8@G@TZuENbKy46m!qT;8o6mpq(0ZldMawrdN@WyAAb#4 zkA8MYQjFLhIW!NGbZB;G|77n|sQwe3?Zb4>T)S_fU-GT*;$4~A@q{x|#|$GgUTzJ^ z&kK+l6X$ZGcZ|10AbkwSSeaElw=c`CGDc8z7K1(DgYApCZ5p~&+v=LSP`ShH(HunS z(H&HIqV>+4AsEg4NwgnPN_9!I!q`?x(;1Hk3rqYra3s5$T(z2`ZhxHq9+oYqclxPS z)=9COy=0$?Q&MJ7Nk`sCovHMklwEIlE)($iNE?pu1!wt{jH4hJkorllr0w6{35CqmsZhBl*%8+`D^4{w~@tc*_VC1D8}M_kW6YRFEQCumoIB zF=4jG3Rc-|ROY{WoLgoq8`j?ArsvP-UFazJXK&Z@pKa{ye~N3Q32^jFun=BKfMG)N zXr_g?Q_IYkaF^(Q2MlcST=zUX(aZ!I#4a;2tO^oNnXTMn%5kjlv*c>1|GP*H1qq=8 zUFg|@de6$*Gy>$7CF~HN$Iz>ibm>jbZKKz`>y{$Lo!M7rJX>e?ZYf1rYAq`iTYv+f5+E9v6U`Fmw>ZN0L!0A z8(}!Yfe`WD=t^FDvX8_FLkahc2tzUVj)=a9vtdGCfdv6VUoz6|LK}huLI%r=O?D=d zoq#i{3DDwh6(zbt3(&bOYKv0=6qC zk*M@t^1V-J!T1Fvd#vQq069+ zo$;J_0@B)w(NL1?fPtdNLaUKKzvrhkzYC-S4C^7-ELE~r6KP-;w*~v0S(&z^<~@7D zibd;m$;z&t*=<`?Z`;eRz{_qgdgdAb>PF7x^$db&ughv)D^T?%M@Z#dLXrNMM8gq0Zu5pJ3)1 zy-=J;g25+5?dReFD1- zl$ZEr7#ggAeD3P}LO1(|O~@-83a#5qCGyjzjPF;2Q2qfoD8P;)frUF+woGG|zm4Zg zyDCD^04{ZRTq&g}ihX%koWBWL@lb7>V|hJBQhB}XYnACwWykEaB7pB)UiO9v8y->n zqZPnmM&QbY`HGubWVqC=Z;6NK=&>!LfNqx3E-kSpCO- zS+=*3Xw$R_4}gN0?50_%3kEYdK1Eb6l-!LJl9W8BqJ=ffL0K8h5rq5x9PYQveql=B zfww5Ea@xv=@?ELM`U-Y94KE=X20R!60HFU{3Y}hQyH)=ynzsH4=|6FDwPLbzH8HohU`oj+gHFktL`q?2 zKm*+O)WCv^C_fo?uyV@sUAPP8KoJc@pz@{RKu=_Ry+8Wz5P@Kx`JMSfLkAYqQE(_h zr<_xsYR#rnZWb~zvnLhf;l7pjXmIx6-%ujS00dzO{N(}z{%HANxfFk(WV8+vWyu2 zR@5oQDsHs!G*v#B^pptCp>-T}#@A!9p8}lTHBk!^4pZGtU<-y}Nqn?M&;9<|z*Yph zvHBa1I$#(@9>%bc->IS-p&m0$-C?jiX(aVwj;>jJOQP?^9_q39tbVU3=!UNUj&^>ArbqYain5*~i#3x+!cU&``O-GfT%^d?@qN<`sKdQ(e z_2#hRD8?HbQaL4txy;g86LKU%C`l8t)6Wf6=kxLPHIcKakE=Z0H(LR*S(7K(CQKK0 z5@KI)Zx%b876kU;+XyL@pA~YRB*0&j+s)6e8;xvk-M?7IDmklcmKK_h0@n_&tp}Z7 zADgo6EiP#QmaS_uqNA%4Uha1cy@)+g&Ik4Z4H%FNGp+8G#NFk5W3i0ll8@|{F$A7f zBh2=IvPmL+$VH8oT4Mm``^i;r2)CZOVcq&DU3ZH}6#mDe7(-7hzrXjAGk&kJd^Y5; z3&zvCY^RV4ky}H#_n+qhY8GqCG#qw&m3gTN7cRn2DDkrBLS6mgt zS|fhKN#b!+Ng@J;-0aWk&)jFI-weYu^fOjS&^l-O%`wlUeGLEwphdZXNUY>|7D>XN z?~n~^b$~SLWC&bYbg$!YO1c0egt*@@2qoU?!%!|Nqjb2GcVY-@cdg+S%rzu2Hik#N z=^S6sUhMo(d0O2#qu!(RCkT4O2@WVC?5l7o)7yy!i~cgj>Tx1u;ESQv_}K1y$w*AqYVJ7`R3oIKg-gX^TbDecWHM> zGe`7PqzmGVE!P~eFMCjs`e&vlh9?G3Z25PWSH^mAFJ;?nFD#FF8j{HlCnxE!1NFRk zscFuY9c@TDL#%Re9pS{LzVl2H9$0?B5e;ikY4-Hjbv!h74BGissW^Z9eZuLdGVgu0 z&(oeIz<&n>$5pqyL%<~E?{_pPyDH;qm#WTQG75yItc^jMlM{hNlDB-%Mwz!;G^^Hq zc$-hHBAa75r5oGFx8m;mU69N)l zQ~m5?@QHvIy}1~(2kHouHTf45@hr2#M55lvM8&1=vTw{|+%duT0|UTi^qov>=`tfR zz2QleAuypb(NG`gKlx=TeV)cL}8m(){lV#zy&~v5XOB~{i z;hun^`rk$ixQN%J?W$r>TDlk!Kq+qb2yS(k*6`VwgU{~qm-DgLbKgZ}pt3pExR&b9 z!HqP60pGK;FY-aAIdurPtfUf&!o1k}P$SkR29di!>sBVIKrKdKfuSYiruPM98)IU+ zTH_mt0Wk+!=>v=^a(Ujccd}||wtp@d>m)!wj2o#mvavTBjUzYLM*J{@=w} zLqm2YR<)(pTryxw{@+kQ`%MuX(KCPLvi6`x!|(TVPTGJ^wUg1q>f zI2UjNTBE}V2~;>Z*m9XJ*0H52>p(#3?r&M89Xq6a6MTCY?qH1uQ+7krdnTO&FPZ(n z6!~b9luP&K7$FVB+1)YidG7qjb-ZO~z;i!i#&9ok29e$oDQzml1@jxNPrqGj(GT{^ zXB^*vbRKOBk2AmeYjc8g^3|X4wSK3M(}n5;Mv|HYzb60wAH)OYKWL)?Uh#$bUuJBB z0|8bR{kk0eyTZ6)J3d0 zjtsA$byg=x7;S=A8aFW&7V85G zzkAD)K+m7E!Q0hewtH+|v+~Ps)J7tO%vI!QPklqGMCRz%SvgL!N=g>xu6ffs>Zo0iftoJ@NzA^Le_S6=m8B31&a&Cvm{9c;~{#RyPyswQ16o z+CN0XU>K8zmFDkH_bgLn>}z;d7yL^!on$5;qU{=N?JqNaYP0f1c`UsHA@bfNPJgy@ zV*v?@rxwGDm0P}kdS1$cv-lG#d!`GGn!dFn<4!#i8-fofJ$$X`5|a z4I>P)+T)b5=hDS8Qf%1}PaB38+>ph+0f4}cYkD0|5ntZ?Q8(WFaX9)c2G2n4@tEjF z7^ZV?oC=Sg;&d&SY@hxjGouNWIv!x!fB;U1ITrl*b6gLVf)Q>>d31$JW<3S4^Ae|e zC>A(kapujy?k+j3L(l+32zO_$a6N{aoL;9|QB{|$!^ zv8(XWpywGFxjFBJ3^MzUDnoUi>sI?M*g6;Pu_{hB0b|?>v8ItoofdL%@Jl~nm+}6t zkD7xTnHT2#E=f9O2m!;UYJ9U>@~<>Hgo!KFBEy!Wo+3;{F~7iYt3K2kfFokOqxWLE zrOM15joj zHuM9HOVM5xqR{Ke6+NoYN$^@O2ow<9&ks{Fzs=vlU#*(Mw?lWn@?Ekfs(t}Y839VydkN}lU)1*=@FgeW3LH3 zfmoI(kgVHNb4fA!k}VlWR!~wlG}U9IuDT;+bnf=IdJ)lyp(-Z&hf<|uU#w(s1Ik2z zlxt}03OXEIU%q9~pHf2NKgdRi&1VS^j4LS>;eP1{&~gDtu9HaaKPX?nq5ik=vL&`n zYz6(t-cuUnWdJ)X?lmrL?VEb7K9P2pn}>RGBB&4~tzno%{?zixB#PY|2q_9Ne`N=i z?}~&5h&1@w!WS!%{nxYHJ{jHCr~0t~AU&fsJk!Q?&`3%OPt*8w0XZYQT_SEm(1@KdPjhxvgzB)N(@Q8Yh!1T40TMa6Vz zn0;xpMRZFm3*9GHD66OVS>|KzRoynpfEuW`p7j2g;#a<^$K%OOH2=6B2M(1{OOfNm zC3!CL27t2@M1mh0*;rrz-xB;S^SMdH+Y69zHKr>gz8NLN^{Y7z4OXLN@`0VR_OamL zUPw*Yn%@~6S4_CdoPHD02^Hx*_wwW2~Y9KH&pb=65=)X5)qR=B1>g@ z*IZBoFHgwHxezFY?v5b^r+s1Vk7D1cd#+bpim23~=p;DuONm zo;qq_ba2+^cB zR3)lxuA7x5;QQkRTo@<8fjp_@fY{1}AvlL~wJB-95^QR$*TM|0!$CZ$ zqtW32QS2*==;MP&jNNG63=p|uuEtEi@<+MnhJXXU8GvI+{%M)G#B{6GsdlZ=!4kv|^672N z%inqzS@Z^om`Hptv4z#n<6{x?&BVTIJibQk*2QNfpXsHRkryytpIoD&)_pgrQRT)n z^=1uuOQIu#paoVl=auClXL;(bzU#d5Q0JHKEaWx~%bv9MUo3 z4}bG(+$@QAT~*=-)9B-6_{nmzt`1q+O~gW*F^uUAKAUV_B+>)GGbXx^C{zY0);nb} z&qtj8*J;dsr}vWoIg9`d2nh3korX!12%x8?r-A;>DZ@>IKiSJJtykI>s;9~%P0>n= zPL2Ua0bOQzL6S=u5$<+4Ap`34{Gn@vZ*>vkggDDRyPU!NJ2v1|s%lS>^k{X`y!mC* zF2L`)t-oze@%!U5K?t-b(kb9aU`G2LPlS z>`8$2P$`xQ52JJw^{4`oI=zf{Ey|R41qz*Z+F|QX!HA{?@^2q@U4 zTC-BoN8Rkpp705?v(5CNet0~VC7p_WQhDa4Su%jRi7!=s1;H@ux!cZ?7XU}P1#N?2 z$9hQpZt2%x4oUPGW?yC0$;Va5#gJ=lLA< zl?Bh_oEqrI^@r%RaSgr!j$|~Ko7taJ z%+^n;DOkoWhdaXQXSO>`-Rvk{SUOEEa-LlZ)a%_EoVaR?Qr^fUsWw5XQJPmt1IZxU zqv7!(bueCwYSzaPnIuml3k!c5U=RV(FbWRYFbaUd(M)$`KO5Bj0dY z+P7Nn4996Rz4^dagh_wcnJb&>C~mH{U*-6JKVN-k-uwn;?0RP(4QpE_HYhZmZ^T&K zC65=@ZPFH!x5=yT0ZgDfvWOc=N9mw~>5wLqE6{N}l!W<+LkAOYI3{cUh?+eM1-tQ1wuDc%Q z;K~g7l=2QIBpMPE`}0MT$|3n886o9ry}#B0<+06x(f>I?=vxw@yoR_S{#4a|g0G8R z%n{L^6woS!Gg|}%$pc$R0?)caOO69`0VE12gw zdOb3*Iu{t=0bpsqe)C}a!ehOkNhL->x$p>mu!Os?FOne0F*#MYcll#H-*@F_@sXM5C;?3KEwf;XFfqo4i9d>}$oI$mpB43c90W95I?aA$w(Q zin*h0NclSSL}%Qw(yTa#&`F(Bw6(^&JRMr;58Ba(u@mjNtuf0ntI#VBdi`rd{6UQ> zUL+0u9*zDtc;(e#zb*gkj5(415%B*8FC<13z^5O&7EYkj3+t3+5VEebp7ny{Iz0kh zkupp$ER2I#hA^HegqcS(pZyO?w#zH}jOyp`j&|D`%`)3w^%$?JWD0bL%eu{vn7={& zA6jpfw_GhbJND_&Cq6H^JbV5bul{>{dptg$dkuX*jPbtP;q~r1;fYcY#8|Oe7y287^l36f+{~Gsv1cV0+(F#~^ln+I3(*R?W+O}(Jz3AMd6$fcQ zhtn$>yZat$tD>E{W;?Cs@Oa7#2mtn>k~QKCx4cw!rlV})N=00!zN7XFY_RQF0+%>t z$>$bSLq3%{X)jf)%w|iHf8+Da(s06Vm@^%?N3XCka!C;mzgsnKHmV!z4N_4m09SbJ z489@hX77F&jHMZ>`kA4gH6MmfzG^WluY_umh+UAD0T#2a(+iW+spkY7<)w%_JTy}* z!!70pXp$Bgu*&mN_z@`fCHkB+U3O73PxipR68AAtk1{{K%bX`G*~gUx^wQr1nh8S= zMWiiGR#N^cznlgBtr7Grm19*i0lef}lnFiOc-!sJ(U)OZOYL*nBlK{JKOJeT@>v|3 zBfWoR=bl#5K?|%sW319|zrL#m+~VoV|G;uIZ!qR_W^_s?x6~Qd>I(aC45=l`=e0PY zSZrpgIl~SdEm;eQ!N@*L9DI*H&5AD|F#U#*{?%yaI69SMBKxGf+7XEh2k;x1*mBG| z7Bv4@V$>U5bLUTJxWpcC%==SjH13IWGmJ>U=h7elYMc=-;vdqNEq4A!NvN*9C0p_b zV~Q~|KR(7=WW3|fLDMD1Pj}qQTp$w5!yjvJ={^7_khnSHA7B<*yk}*k8EH?2pIf^( z^ZA4Jm8x?2T>6M`cyEOp2&e~Qy_Llq9W;k)xqspIFStOq@K$$Ln<$QR^i&^o$L~cK z>;*3y3fOO!Gbu{bQm#JFF74QP8w}&|1||quzm>!b9n4MdcRq${&_*#vH4b`Jy-|J_ z0~@^M#x^`|C zt&_vgi;&LmKI3(+k!}<&H#xMwJV&__u)o>S%#I!{vkbiHKAmsUKkafH*j+uW@T}tc zN@V76UgD&BRR(|5yt*CjQ73kLHT5c?+Uq> zF2?Nj;t$L8)?zY}I`lq)<*nBCB$xndE#TBY(WZMf>?@B>*7W>Q#;r14k*M9%mw1(^ zEfS7m_HXh<>@YxN_+=D?*&AwYjl*Qq4YU=jdPWqo#CAH`eCvn_z!bNtSaEH_$zkQa>=FYxfh7{--| z9oYfV6(OSPazi1m2A3K=S7dd#2rz8$iPyNlp{*J6{<=Rd$KTu4K zk7pgD^}VgQHsdf1za7Iqpr%lf9sBi?R5UIc{=+D#kt@Bb`1S$yDSoAJ`qu$B%2_|s zc*r}%E>Ta;2vjEQ9YAArj{WVE>3>t;t*>sz?Z5hcKW_M#dVp69ytz*bJJZ2z|8A6QVhFl9hZ1tNRt{ZZo4V z9V;hY&G%yyuplJLr`3$=>CP^wKiVnIbdj%} zU0eP>1x#K)l`U$?=t!_2fa5)}V${;>$jIC!IN1VJ16Xl&u*fNv@i1qKHb3+jvDUIY z6b#m)P-U}pG>kMD;`Z#s7k1}m!AxnEXvtyIvzK%ZgUcKpS?OGM*{_oo02ve-Y>Vx- z8`Ies`%L#GK_v@q%LdPV+4CADiPBr?mgtrS(Uz2$?0GZ1_m!H#{7-Etn8d#rnX)CK z%W70~ge#_cOLfr$R@v>_EvP)&)Mc}5<55_yY_X>m^^Ig!nmS#Rian*ld@ud^$ZxVA zGctQkP9We{V6^EQT59v00Y2nB>zSR`3QRtKW-RCAr(LEV^>o^8?8;S=3sqK+XMA z1(J;z?b9%k7gSb2YS3UirI3z#DAAa9eFO@Yz7HQ1v=t-O#b^*5V2{G=E9uf<>(U#} z9XQzasEuw96pPNU$T#M&P#Tna(-xj%e4qR29AL{uCX|c#|Cz^H&Jk=`8QP}8M{I-&c=+@4`v`5MyTwQHL5rJ*wy zuuu6inr3>6SG*bnS|WP~Ng3NCt=L@8w`IlI_+1rCSgkKD97$LOHE|Q@-Ux;6<^$20 zzlHqbv#q!qyCc8aeFDM^g_YPf_n=~0;wbcMWy*EQ)yR3~)gf+Cp0G0Wb{RKn*VkS0 z!z>Xh46*_=+A>9kNlp-;=9b`E-$E#!5F8C?k@`li%yYV! z(F3A>Ddx?#4Kiaoc%KuS)f_`jqzrm;Q;peB<$Xu`*Jywv3FJ_ znbwh4d!@fd1TB>FEd8iWLG~@roujf~UuCr?2kWnWfZ5Rw&!gF0k|)b6wp9~L91ZrQ z>5~zDESC@kC{40(zWX9`m9J?AM zww++sN?Q*yhEkSC$Vw?4O(#Uf6ZP^nqK-i@rF1YA2@sxsE2q2DLY5W^Ru^fEp2fL` zIcn4%%x}b2rCsL(B7KS(kniN(KNnFMh}_B(<0D4`P$%Cg&v(7Z=)c3hrSxyP;Gw;* zAm&Q=!WDHRet&|3xgw`L9ScR7=np_^q6|8KO+^lu^q)y;$K)4KvK$jnn&G;Lq7D$q zisjlTkVXNY^d!mcsLuXsh|W0VkJyoWRHwsbyz<`f5=Tp?E}4h*5x1pkPM8I~OewKs zX2D+rYD@u;2W3@OZhdNpLEdLpvGNF!RNg;mv0(`+_xH&lhQ5u6qKBCNem|l|uojsx zAt^DdN*vAQ4Z$btZR!pZiA>yHvUzasDLCC1DP9M|o8!_9)C?-y6~aTUSaG zlA=nA=txPPQY3xjEld8z#g%_T_WwXJn3S>~r~q9$Po|%6 zYvK8WM9w6P{37;Z_AoLs)x9}zNc%-2{~#6ri2&iWf5R&fhFEN90NYg5znv>%DxiwO zq`R$dK@9ovo|AkNtu=r-lKWGBnI1nvM#V!RJ!kKyDlWsEGv;b!C?9j>FuQ zQ{e%Wf_@ZAWOv6w7Vxhi3Y{l~7nQ{RK@<@;jNrjQ5b%GD|Ndh@nGbWf z=0BQ{v=y89e0b#hKBSc@|UnQ}p?FgvaEs6R0Znr!IkF~0iXUD$>oCKcY{RVEr|?Y|GS`4iLkGVBoMwaJ!-k4s zL9gAafD}c}U+v}Dm6{E-;4tYh& z*ZEaM>Z9rm2IpPzCg%)E%D4{`;w!!IbcpH&K|0{!W2XL}>K?xS{Kx5IdTn7bSos~) zisY-5X5psV_+kLI)Q$5{RUn#{$H)_J>L8s%#U|pjC9Z%&xcOW@FRl>l$hzvcJ)*cg zE)ITqlBljITby4f(}4&sl8SUmq?9N><{`{6Odjj-*xxUw4=eo=u#Q=}DBJqzCZ@Mn zK?QxmQaC?Z707)yofibAX%{g7MUBDTI5A;m~WcGbO z6)CJnpNb(7HwvC@Rj3IkdG6kwz!f5t1`*{{CoZ)4qtrw;RV@M{w6EYRJ zj-QXTFS=E>OPb#fi?pRo7G5QJ|u6%fpwZ#$&( z6^5UmKFszb;%7W0c*Y$g!YyYZusQg3UkChl)Xmk~aPg}Q|9euH>RWdh4tLken4=!s zdhiqBALdRSCrTcU@bDiQkh*8#@!uYxx|0Df7^*nF%c8V>uZ^F>dE@UZJc!}#GbfH_ zf-3rkm-6r*9niYx`tglv1XxN+j_FSk?PK)nt3SY740W13kl^|1#qXntaGE;cbA794 z2NJn|l?0RcD~-Q;d@Fz*uKM0ncxO(zLT*1#6FwZdzRP9!>aa5 z-fPHfX#t4RpzH%xZT z#J;iF)nkPyp0y`22I zx99kEqO5%Hsz4)$gFzIK|oCsZB;>V zvQT=w@yqdiu3VjcXZ_zl4HuA}Q?b`rS=%IA1s;*7V*py-wftFSJ+J%Nr}DZ;ESt8& z#X5zYqzO0kWWiLwj`+?$={YiT)gBt^$Bo+L+XfQY$t;cC(#LG+ zP6-(%o%MZOwzsv$Q`P2cZmHgbE)I{%k*Isxji|}7VV9-43sYNlPoSwCkp1wR-|y_;);k^EJ`Dm0YX?) z6Ibakwx%gG^@C(!O*QJ_ck(HF~*@wI?y)=%qSwXY5@00MYM)-ryOrO!qexncLP`_1@|mc z%5DXhl=Z5p9EE#Sml#S%j#0*@Ep)$9g$7JN7$05A?wq;zmsCn$`6i_+EN!*fl_Xbt z7;#J{qTYY7f_5yEJ<6W6JPN#=o7!<{Lur}ab%-5o=WWr=%NPy`yi6r z77*Q;OOP^5fKTl-KcaU>1txM%IEOA~s=Z&75-}DI9F{5u5(24wpGKk5 zX15%dCl_Vs6q~%I=r#zS^Uh4K|E-<+ig$7IroCXNrTEC8#l&0MuYyjTlx0yY`0JRE zj_oI3;By5SRo&GZ;coW6yGvRaR9feY11v^H@a`gMB##*qVp!D3M_@1!C-6!(8Jg>>)Eps-U0+SF4)rNHvt|`X5%v0x=T7Zcb$Nw*MKw>U2bi)~0aIQ% zQ_AbwD4b(l3cj=T$v&1W)8zXe{y+()(mxwDZ-UnnE%8Lp4_&!pc;&_0IaU^tggMx% zMZR2i>MRXrWj*Ox!>_W}C)QQPGJAO(VH9riX_f2ms)MX@HL4`K8v3&up6uf>7?&l3 zIy3EMva96|hOeAVo9#1O72nsDfb48j)lzBKG}aMS)l+sup1v1b4tuIFEh-CB!Am`? z8hsT4sKE;)ec%g<}R4t9d-PSq_0=Z3<;g&zcFRf{FcaTk07h!5*73P^M z6iM$?b&dLq=xYoyP8qT-=-I!lXs>;%CHO|)UI<=5G*Z*Ptv$9LC>a(t0sVVb>#B!V z=_s-GMjC8QE>Y3hmX&SaFl5Qbu^X=^8=#tZJBAgXm3v>DXi3$x(G*p%$(!kR3V9AM zF7+3nAr2LOUL$t;0MmBw*5ojC(Z0_5fHEvEnm~)q7_D7|mAt*mT8OEc%FXgX4aVOZ z4Y$eh?^$~=?p^wpRS)I1fRjF*DFtcc7Mfq>aXh7!$7>*UI(lLnO5_EK+!ysvpDLosvOi9!;eBhwqkT5Mg;VdqS+3iLhl zSu{)ydF(9wezgu+WLlQjoiDUp440gJ?^13y4ziPJIkU8%>hYL=AL}|EGu2!RFlwSi8O?JI-!*&}F_I$0~<_`8B6*Tw7;g{rR&~=~cx7 z@~-AU9MwvB@77uLS=?nd?E3e*5}{Uw1LRdCM1ybw00PeXwtL73lpx{YgRtehi*k}RspDFzH!|t5S@!yYbb#Uh3L$x4Go3 zer;w?;p_V7_5`x$D2mmeWYk$p!`O)eo(bp%P#L|Ho?u!|j^cOHXY4Q;8{;Qij+RctNr7WI51*>J<$&-P^;{%ctz#rL9H(H>JX&uH|qoQEkF+x z(1GL;j)XCfcVp*|83A`jbOVp{`#A|@ZDNdrJA2NhW`j#NgYBv>(_kJ3JCJk_unD<0b1f^=RJyQC1M^#?fZXn=Fzi;1Ecl*PPu`VG~i&v8+WGqN1Ay<-p&!!QU$W@El#^n;N@P2Q2(v z!{*IQZ_dq;j~tgh{;KotF<;G~QIlNEx|6e+H|DO2<)es$D>zw4gWjVd8oa}KhYq{7 zT}bo8Zj6gE4@wXwm0_YvE|O+WS|%+LSG|=c*AvadZ>=Ap*#0w;+!(+x<4bS^*bmJj zM)_fuy6#}LKT(6vfUDju3r%&;sTNivlhAKqKrtN~5Ya6Q5pl0L3zq5(0vOXtboFe4 zS5czOk@#?t5o8kAkrNxSk?$$pQuMPZcDuw-FExoYb+09N97YLj zZ7|Xx;-VdP+S)a1>e|}cS6|v*di%<5|4Gl^E-y<~8AdS5e*f29C!Wi{xwrmChVP## zbI1!|-^{{)u2LkE#+VfH*@OAn-vNu24?K**7W{?>FocI4`3TBEaOU>K>|vn!BPnGx zuy^Q;i1T&)8rg+IOJM9-6;oe0uyaq3wIFjRi9;2T*0^&pcjx?CzOtn}Vh28;*)u?J z{YeYZYM|?TZDP46?u%Xml|5*0nKbS5`tH zeEPs^vkLVFwp-|I7qoikE##DXN@-IK)|1si?O!zHz-sgF^L;x=+nofg6OwYxox?%Q zpB^zm#-wst69!?2?RsY^Kv28OuY%#g)j!)b51z*^TzJJRWr4dibCJ+!8r+zLirT0O z`~`k*tTystXrP-Daf{llELC@mVSeDtWNyH1D!31cg(>haojy^)hQtoKQ7SEJe+)F1 zDut#@$5p9TaZx2^NIYDvH3!?>&XS@k4`6t?Mh+)uo!v%LpCb?#P(E}b*JhxrARgnf z(yCjIa;ZDZUN1-}r>#4^Su2q|e{4%>M83OHmLtEE1GLmq$fyb<^Hpf5dVV^yRA>>1 zO#623;lhlvGbz_`M9aZd1rC)-wRB$+o3;v&rk-{(Xcq`T#gv~15G0uot(P@$lKL7* zMdVA@u9#*s=*8gy!p4A%AUu1pWR}`=S&WQ-AbJILQqW=y+OsC~h>$Mj(X!(*cavk= zFHa;9LJotyBQQ5=v%>u=czZQFK<8$hkLmlO#EAn6hnG3zoS0EDmvriJ?OpPntt%GN z*g2*m$I~--6uAyLOwn84A@uFlE%CA!KbP5rK%^G0{M=;2~h}N9D^x&obS2RqNe3+0!3cr3d0u- zI&k!0kqDB2jQmD(iwf>u>fsWa)s$9Yt+iroi_F2b{HrVU%2Rv2Fl)Qph*u6dvq^9@ zDX{V?vr=7)2Cz6BgKiBhIHy&Gk`|#YO}s-FW9x6~KV#KgKX^ws`W)ub2+9|Lozr-n z@|9})wFBOzg@Ah ztg3mrYOOKGoCD80GntU?K6%YoLPRIE8ROR_)!WAbalEMBkDDIDQ-Log5~phBa;0#< zieX$jzHm+O)MbCH-ABpQVO-DcK2Kc3(w<28$YAVMhor9FTTu~P0#H=p_wYlUsi zZYvU_PcSTyux#Np$U<#?W~j}OUK}A&Oy*Qkhmc!85Pt5=Y(atqY!o>tI9IiGbjDj^RlW4EK8(@9OFO5aa_QM7iGaHk`|6??`S<# zA!-nT*xvonWx7yn?E|%QHwWA3P`5|B=?Ds#x*{}j6~1_#Mn`oodrDS*p_<` zVDdJME*wC@r(L3oti%q^7{@QBU;3Jvla~m{S;s<2C^?7{KuGN7M3#&nVJtI>*el0n zb0M{bUB32c#-i$sl%|cW5AU4zsPV>m!DgXwZ?nHDaR!vM&(bqCiX3XeE#0kH+=&bT zt-dCjPFFvWWLmoRzsiR%m$5R665B2om_l`q2@Ahkhi%;XO0DO1Y54@Q#=nqN9L_?k zf8^4a@{#Ac^lc>OHE0^kWrLQMb_b=5Zo$awUX<=FWViG6>7;_j?-X5JT^b7|Xt|B8 zjqNM4rf>%;%G?*3WQvYY+>^FSbSqc@0s&N?kv`RPia)gb@*IDu6P&kfQOGG`($h_o zo|hzQF%59k^!y{$W$`)6R@I&JhkH`ZF)uVgkgu!s4$iA!Om`2vju&6ItQ973B}MrQ z8h}b=WEpB^qpheKMD%C3rmQ2SJGyi#I~KKqKxmrUD`AZNo|bBR>|dD|_qHEkVNZvd z!mDI#d&joi3;kO0)uu}<-BYOnJ6OUfzN6#?rX+>qT|bA z0i*w8xk)$NCJF!G)1VfF3J?K!X0+z7{!O_BRT~@JG}0r z+OSSh7Y)gB6uzB`&4A6q!a?DyB2huI1QFFf7HRtW`>$CFq3PU%#>jjYZ?#sqGk8&3 z8>s=tff=j5jTI&eFa21*`jiP-HHm|rGE{R0a;It7V-r;X> z=2he%w?dP4)g?tN)Z!}GTI#PPd^8rm{5wJ)rQKmAA1xd}DnzX^Ldhi8{K6ZlYdMd& zZG~50&fixFcWnRZlM;Wxfj~Q=)Suj*W6F!EVQM?k9y)E$V;3i}eF`C-@njnLSMx1m zrEK|E3jT_lXQi($-#Ry|$|8zj4QCigsbz<}>|F!81x$8JVI*(Yg-97Z-O7s%1NcZ%HN_=Sl}kzhADQCGw*&vEvJSrN z$iU0La+v~xR*UDvZOOkggAPz=7@6s1z%uFk>EtS7$w}Ur4K6rmM z*|v!Z;el=UDe{w+@LIOhkDHbz6^YVXTf`rCBy{{1SPUgT5`P0LGA^(MEsjo4bRNpq zI`b%p&k|=Y#w~FGxhtzih87F6&cR_igHz4_H8H+HbMdtRFrF5#$ZB-BdQ6p*$}&FF zLLg(SuS0hsiMusak@>+8A#v@8cde@b+P#r$8SV7IK3abo;LZU2#(O(1K-mcswC2#~ z5b^YT51N&YWo9T=k(X#F0W-Tj3+S?8JG8V3*zsY+%n9C7skVd5u)4`8No=iqlOg+( zDT~&_em6(}?(tIx5A)BJ*R=PfvHkNiwoTi?d51`R^(|pa;#rK#Iy=4x=QZA%d-Y|Kkxr+nw%gn;8lNw980b&WM6gR9^ ziKhZ4UwqybqWbCYbHd3Rb(eT4_K8_yUBp(^xp!E=dVlort#L^gPiqRI!1^+tu_2w- zU6hD$n_d8<2F4W6=fJbJo;*<2S(6%1umas41#8j5A$ZSJia@8iCewOsKRQ10u+!L3 zAtqY0&bykU!i|J(Eoh9VRnonSmY#8lfeI|Dsc}WzbA;k1!gKy&;cngnYuv*dDNe>^ zdoC)VByfU}T{51Y8J#Vjcd0~DxtLjYjAVSC!x30q;ortzYm3Q`5MoQc)+~_p{Z8>r z3YK7ZqiOrYyj=g8_a$dM^33#$_PW4*pJnQNJ@aNLQJqngR(OKKwx)r#mNItH2MQ); zpalH+_sWIi6F6q%2Iz?sEyveS7^I&Hc{5@M}-VVn**__e|Ld46X>?zb?^lo#80l zZZkFc#N7{tQOu|(a>VK3;yqG_l2mcXiaPEXUdJln=U;@`La=9KiSgGs)iCo%ow$qu zsZtH4pthznQ5cl2W^*Yg<04NPBLhzv;z@?xVOn?(K0#gKcyULNVB)=w#)CjIY7xb` zMIZK9F)3k9Te+^j(3XyejAct9U-p9Azh!Kw5#Ib;lTeidlvo55vy1k`L`T)ZamNK) zk?K`uoyvm`?yWhOIX0Lastapob;J_^Ql_)Zr(}PjW(?XepC0wBIX8v=2{)Y1+{3pa zHyq8tzJYIfJ;+o(860Y>1>BbxAh74r`^KF^m*C~?tP!R!sW6JqYw}2SBEy_e0<3M~ z*iG>#OQDb8XCi;PQP>lFefmZNO+mX%cQ0oB4D;eXrwSS^)O)WmH|6p{^A`n2Cl^qNgXlb|zqok|DDbCuoMJ4$8 z!EJ^JRHrAkD>R7Kncp-S5&fsix>dQ#x+_;zsch0a5e+6g*)3*fZZ|c{ko!=Qp(Fhc zF(v^qPEce&?e~{f=r$1GV%u!k7r~p#;8qecK1STFPY2A8$%1Vz!DT9ry%!Mu$!0Vx z(X1dRh$&Xaev_zHft_#MbC-s+VY(?BaZ_qt=3!E{+`#e=a@LMMs65;Oed*1=>z9TP zcUh9)HgY3TNb|)MSp0Sk&eJ4K#*WP2bCZ`NG^!#^#HD|NJDMi|oX{lr8-?6YA8v7S zMKE4ru{s|ryRXWqw$q!_=bL2eK4X+hsn=Nqi@naw$mh)L{*%`(>oxqHYfg_XiINYz zmV>-+VuCZPgK%MCr41GN7Khj=^>JEZC!36G^K#33lxqKT=<{8Umzq51Sgfp%W~^7Z zYVK=zE?g*C%zRb=gNowrE64+;JqsSl0*`>@^ZXg1g*kSv;LY zwg#@VnuBctGn#CTjt5NCP4+f}DLpqXmU zz}87(MhDgnI7os!fFFCr*|>~T>sbZfDd7@HYu#$YFm=mI) z=b;5?Lz#3+b^EUDBjUx+@Txh2Pkc*h9vEXf(iZ5h9*xCSd@WSF5^_lVzQE0wUA=C@ zF>;pjU?bthc56-4kIaJKApWOVXQn`jFMq-$z?l0mNf-{Y$7@YI1b$jC$RrO0+Bug5 z`361;Kx!d>p67sg8O?QY`mW0eJf%B*Shp4Yo)w-k<7Q)+mpx1VgeAb2)OO;v$Uhge zj=R=-(5CX;5;WsO=TZ?GZ#V@pnm`adC7yW$89c=oJVhw&|AN%OFQ|THrtpsRdXq~d zVL@NNgm}PSap7)W-8N_{eCfpGz5=VviZY}JltN!X$7TQ!%JD{OA#lDRSLa)s;PwH) z5aj>1eEIyuM@8r@!jr?P7Lcr^I$h^p#;DrkPc{J;(?G)b`5c^Wq3`I8-|)uY;R;9H zf2Tg-^1occAqmnE>p(4VYyZ&r8Q`yZu3LEyKLL>qK1{ptO@^xNlyv*H5gQO6x!ouM z#;9xaUPGi1b7bt=CK(Q-^8hl8N+;V2Z1M#8R>f(ZU<2Qh-*M1!SUkom)g)xDbLGsv zgi(XvP};+EPgxs#-Ms=+duvKaH>-ys?-t}E-k4Qh(JHz3{14qWP8|?(aZ}&3+lnSS z2GUB-FzP%tFp@dMJ2mNSqPNH0l>5FkrB-U$l zX^k?j^G@-NNt+x{r_vc$PS>4tozE}b?F}?iA-6UBJEvREU7z09?_I+Q-|jDrKp6vQ zsN8n{!YAFrv~;LaQYbkJ^2PllZXCdB05|(dJ{^)*I#qkDEyFRMpKewD%VnjpYN&3V zJRNd_*f*kJwhHo>zT1@s_%Wq7`|_;->0VA$K+JwW z6`y}h`3nYL^&YaDP%k;Em*gNDPubr5OC)|y5=+kg{ylUT0e?3ILQZ^$ZhU0+-Trv| ztm?;0xqpp>=2i0e@@I?imrs9;ZTSmYSJe);kC!SEKyL8)7Nqs#?_kW0Au4~b9O{Sk zAdnB;--e2srp}7}8$E#7_U>jUU=ugQ^BKu3dsu`u7SOm)L2MsyT+xXgT)fw0OAvnInrnQ z`=U%8<6C%kE}I@Kf(2goU(hsV+)8HkwAymf@w!RFb-rf~TLt7`W4Mu}m|K`pm^`%j za_Kyc`J$3SpoxX%8kh2WXffxbiM=tbtsbHVvvwJnG3Q~3<`;a}A(4_qCVQZm68=*6 zP;YW2G{jx>)KnpY>nxb=0E(j)RO1gGku?WVhm%m9v@t^|bk3)uDl-}$kQ5iL4)C%I z8)jMV2*ab3Ql%%N8VaY?kirT(Lfn!_@inHf?j@M-9xIyDr)Wh)!?jE*ZzGaz2;hn{ z6EVXlS#TBS0S>I13E`UQm_s7f3@ywgXD~DW4i%1`K>Z+ycdmCi06ER4l8XM`i=njU zYnBWqx^mkbDzg!ZBAe)AV`N8#Tc4oa6^OC~7A135Bt{kG9)^dOp1sPysR6e&haQr} z=)!`!I6h(nk>M(u^hrL|@kwWN*#zMGnsoG8EzZ2BbrX(fV~O4%>v&*Rt%!4$$UPOf ze>)~{!*OA)|W3o)k{J!|31BRc^XWUrd*Ws`99WHc(?fr_g5Za5I1Y)5HqV< zi&QT^>j%s?n%dEBm#lhWkmd?(4Q9;d6IC} zJcj%}xkV8u8&k~I;7FN+SMF&23`Diry$us=k(p;R(T5rg4s z`C(XfiLx2{fUdc1=GVCtS+&`gINn4HP}DkUWONB{Z!z;?vg8O5>sKZNuv>E_JJuxu5_0DJjjdo`Nvfw=+6I!Q@b zbAu)nbl7!vcf+~Xu?DY3QLTxYr&L00(u4fK;tEh@fH=7*MbTPCp`}5UT1nVrzRC{M zot>o&g+6-5UyF@Ukr>gaxW6p+cwaK%rSl@Z)rRg*6!ei=8EIsjRtPfh14+vyQPzJ& z3dq<(qdKDft@_hjK>tx2By`~an}a&0C5!iHutKOyb$4K}H}qf5b!{`Rtg6V7+oxsI z_NITv0U#Xd&2npt9aBhk(%sK7@-C+1$P7mXIukC|dmT#Kv2dl!-WgO;Q?v&SjU=lY z$>H8N4LQ<`5H>{MY0j4FAG!<{&Er+ z+QQ;886*ZpHQw$<(A~hjXkpna&QP=ZBNycU38AJ2p$c_6J8ue)V=?S0p7H&Y?Qm2{P^JgROT| zmAl*GWTYRxGHzh0#M6ck(U`-CId!0rD0Zu@veD6hz*@IB7L&ZEHp#WFx94D+tM*TBKz|ZSYFP8`qzr>5ZwonO8S}55*rzy!JJ8CX5%(LumqxCTh3whHe}23cAsb zKzyK#fgm`+HM=&y0Fo{E zQ>zsl_!Fxs_5cb~U0G;Cdmf{V$zJ}l8q!3O@8G9 z_DTF58Y)|j`bp?4R01seQIZiLq>5|(E%gpbRAdNPf)@J)zd@IzclIbNNlTnDD|6~R z5rDOpXD83i&1pB4W3X8Cz_992v1_qqR^&>w17sd1(J%tD`;1%Q*6kqD0k=d}!{l-#Zeo`^ynUwo7@z9?P$I9SAa}p>!8Z0)^D8aQKFIN*b%YGK)qpVIR5t5+a8wHKPUCyI(yV`*8?sz&@&q$V>imZ~QLJ z5cniH*}_6!NCio^7w1F;0j$Xf!M7QeST7)jpxWdk)pkd(ik97xUO;y&Ojan=TRMGZ zPmWhlWnNGGT0Se1LQ9-bXI2g!z##b zA|0!ls*qdWDrqa?yIMCEjYy4wJ_F$pxqK~PnVjJn!m4*Kc~c*CScP4%v;VwFl1(<; z;`DvqDPNquu?VXTY!jYgnT_Xq+x8<0GCbE9&VGK({b-qjWfM-qt;LKcBl#68QY#0y2II%59BXDlkVec zKo+Uu6{+EtX;V=T)~PyO{_?i^YghVxSH|^{*w_?#amibP>&8(QgZDRHeX59Wz+`m< zCX~E4LfoR18>efX)3Vxcb%cN8d%HaUUXoC=zcZgC@a2en%Sj((T{Kbtc^?_G!~SThl$?QNqTWW0FeiG4RLOjO{91IUsgeAe|3>{~yUj&@zurt)J0w`5*p= z_NQ%(lW;SN0*F%7l*1B4`4WxOUV2ohq-bik&;-?7)wooi9~dtWQ-UObk~0c|s-|l< zQn&h0?If|Vm>cGM5M$--SD+pVXdi8zbnS9HnwYre=l20t>nrz{pBQz^-=WNw2&IJ7 z(-GBr3HvQtOR(GO9}Cp^+e2Pd!Az!43Q7t#LG$hD4IoIRyEa;FTl6p|C_A0(GN0oV zY8OhDL-5oc5u_+fN2a zsnC}B0PUikQ`*)ghj=V4?dJ<_WA?J?&KBM+Ylx;8QRE-A>Cw&B8kr1t8mktrMF*M1 znnwnHgliGZl<~?U41!vG?)4I(2Z(|s9SU`ksNjz#aK8XDH&=C~ZULyM&%aGlPqN=d z=NqXuj3Kq>E8HSn7iy9pU@F{V5Mb|JaUh&;0NvtmH>oL|(SNThcW<%!^^!a>mkF`1 zBMH&Ey-ATeZARVb39{^CPg=YsgUu$Z8ECe`>Un|e1+w=;Fs_C@7lvpy-{d`TKSRUK z<534O%=;2H#Q-QjLsJBVX&l1w2`IDT%yS?x=z6-Eu@s|v)EvJ&;%FGW>IRQlXNh_Wxvg`3}DjGk1p>3PG7s85ULC zOSsxDnfNezL!0$h(VU3!HVD55!8o&1Gq-YmfStL4`9A0KQtSPFdRiCg=!POhDs@zk zEdnPJ_R5}rc#?LWR9nJ4C;xOA|cDU>)1ooIl3Q$af z`=~#wLQ$+v1juiOk3CJv*g{yQ0?^hfaEUPj$HKI05QX0a36imuu=Tm1tp(ttV+1i0 z_k{@J^1jlPf(lF{j-=wsaPjei$r2575QQWH7Lu`zup|nq{wF`mMzHm=>C4^%W57mS z3mcy>|NoLef`h3hAo+)~@lE|^b^ak7B7Po>pU1%2+{%d7!PwB)+|G&4&cW8s*ulx% z*pYRF8@`VL7HFnZRj_>TBN0}8MNMr7SoR`RTt;(|@;Kzz*2~zECpNoeuMxIZv>x@U zMWnF81@Dr$9^z=5KpP7CVFT@hRhF^dc7n>zuSRf)S%%e!xAZM9@%5$mRc8Fr36J1! zZTS~=89eH%yEW1B7D&MV5JCTW*MwP9YQ+C6#AZz{;(vTE^9K!LPO#4*NhnW41}I+2 zBL67<%p)`>HffKgP^9}SuhAM)?vaNMVIdR$LWTwZhPzZtwsOtb96Tv_0sTr6A($P2 zxXFujEv6P)2Yqoo-r{z=;ys@F?+);*bZ?cQIyaHbusBXoJ=l>_X)ZOSNS_c6&e*(s zj;7d7HCubmPPZQannTFt?rdXquZwn{A`Q~@(pPZ%FC4cqh1X$QUkxV@ zBqPG)!ppUPjnk4{cDvytYPx8;#revv+72TaR}i z?T*I7nPE?=38~O%eGD!KjJTt?Xc|}poia1GN<&O)Sme#3oa=R2;TNuKJ>_9IX~x>{ z==0RMo5VZ^$}`xZ?>B4!w`V>c(?ljev&Z-!PMd-Z+*5d=tB_u9?hRv%bH~=A=Z7)( zcG?Z_*-O`4a#Zu;vZLw!V4lww*v};lnZzLp9a-75rNXhOnGEbAvy6O>D)oB|`v$Mj zKu%Xkm)u-K=;YjI&Op$4#*tEralOpDS9nfY} z6rTPLOtFv9`j3QIa6qDIBM#vwYQi^L@RXGo_bv;8Wu5+@@byBJ8^bF~el}W_1HCz( z;5B$m4lp^lo^bf2*bZF0SYKY+VXNT)eA`aCe?}HDIr9ty?xa9}WV+vjO`dN{T%YyE zslMo5Tbj9Ps!Rpr9ODF)=0IUvJjUh~uht(dcJrXPxBv0G|5&ar0^eN0kL6NA{J)kv zHG>4GO0t%l|H0b6|6!k>fMZjHPQ&;Mz$E77hce20L5_+bJB4*Hx?18cRl5hjP&hd1JXU?`DY=9yV+CQ9 zI~*CBQlc^q)4`cf!zgdXo*P@g+aNkz10kU0aLr`Xd4gQ4(aO?!+cb#Kar`!Ohpqp> zss3F$iLmob6O>-O|IMYrB%-9_DBiebZ@E?TurJQTNE$*@Hi8gvY^)bCYzsX6T-bKx z-mK>AUSXVIfftDrfk{S4ws%f>ZjjJjvx@x-U2uJbfl1qaaNA-2P^W(bGH{<&#RR~H zsi4JTt8hj}7#(dp$+}W>SNIS4G#aDE2^qO9GI_PdLNk zAlpO=9uyStxhr80NH@Y66XAsk$ID_FgBuX{5vy`fD7?hpTJr#+k|rz9r$Dusz*5%B z@U_2x6*)zd1RdPBVB}qkvRcMlc>NLGF0pV3hB8v8PA*>}w6u1RN8`i0y^4#8%$bei zt=$B{>qWuc!)651BNBVlAl@x z$o1ni>;4}|?mxCnA?T*ARIF z4LGszFXTS~ao<7-;oLuU%3?}!TIxr0@13789S>E9MF@?GeT5TFZv(=nZ{%;nYd&Hs zMyvHw#R*|k*^Akmwyxd0cldyxAmf{IYmLLrKx)Vp%_i-v$GvsZ18WlWQ0z8AY6o%; zwJ`voy>*isT8G*QrZN;1<%^y=EAx+KA?zeB%y*+1${n^7qu7q>{asFKzl4GyB`PuG ziDXo9aFVQ^Jj3cE)>&~x6>W1+%wno0G@PizqFiENiS)G8jay~|w@7nTjHAgO(NFF! zB!6jOEWw<03PFmu_PZ8~b-^KxECb7#7}FlWBtJ9euIbtC+ChH!}*+pMAx?yEaUW}BGoVQ(b56BEn3>xl4^{UDOq0U0)s4}p% zI6@#rV~zYWk(GfwZO+q={wf77(NvGdnFEiLY^*!#RClOixjngjVgKQ_r#@uJsx z8Fo{X!=RptrjYHll2bu06u8se4^06mIIXQ=&67r2GViaw0;_8-f#AT?{8Uwi6sO13 z=ppIi($^5|GhzRCGeEhZTROS&x24}Qz22zJkH%k4Z2r4HMQVVu!r-{+zAe{NZsaWX zrhJFMYOWD&H(x`?2nCG$r(~q=7tjg)6e=caMg@JfmmT>Iji)Ew`~D&zH*V%$%wLCo((m?{yD9Sr5qg8oiMT|6VAQn~+SbFTZ_dl**^V-M@ zGe`LMR`#X_UY^c=*GF@zD3cPQLE{j@JH}QOBKbyYoHF5XmPLu!$ra~zz%8|xDN}Ppbv$u7BKI>DEN^#4BJ|Z+jm4T z@VYAeMroUgz&onXPzw$_QdJ=j05KpB~N8>04uVsz24#)&kBP2BU9M?A37z`Dyy;$Tpu8 zrBdx)5WQ+RyYvC5UDzEX9b~&gaNGmM-(%oJGN2dQ*K3}Oj(k@B;cL94S)Uy{+#kf} z9UP@?OrnkA-o#72OJm&$jHZns7Ya?;%%@@X5Wl+(dEc`ue2&d%KVv%dJQ|9L#Y$U}=#bFSdL`@ER+svH^i>-Y) z;yN#(3-B1uw^bzBc82KVL(HCgr@%nqqsrM8)O@Mmf0?^g&Yd!*ze!}aX-{C@4eNQt zn26pNud#;NPD#9JJNGcOya>yqt@5j)>5~m)UQp(ToKDd(hciy-#R%kX8`5d@q?!sc zhB398jGH_ZFB0awQqR>Y$$M7^rin%h~B3i2rMT6;+(hf1no#c9M(2MU-wrs_NEu#-dn%Bi6NSeJ9Qzd)O#rkF`z2v(Ycx z=#piM>t#iTFiYhCrA%v=G&Vri1kYyd_}FIDHOOHfsBJ1I4&HpUV6+mAIa;}l#9dBk z1Mu}fg+mlVr0e8jhM%8EAO-NTkL4pUz*jrxXGw5J(7@q`ovGf?AJ`e0%bXF&?Qy*x z;rPNGkc3V@YcWgE&#>9fQ4(k7HCvIQq&bhsL37`~OQg z|7&i##xcb|`Q=H1pBooI1IFdBE!0wY`Y#ITpI+nvmr`1DV2MXNIv(E;Bt@+@JHC5< ze3Py7PdDI^-yx8~B7rAXHqATMRjIWjGV7AsSC*6H=D6$5#MkjZ%DO_Bs^@1tn~RBr zCV@qDNu`xEs1ZVod&llchytOm8V!?4PSLo8r*T3O+F|=eyZOZIh2P>5XPh8jrYB(A zz>!|y7@j$AQ6F%Rv}s=dC+Ezx(wZ*$5eY5^0RrOqubWh*U;q@frLcb-r9r*HMMkDE zea|(LQk2pLMVExgOaR3nI*#XsO(OVm#6-n}ob)?wrrMXN-qjFDc@PCNUpu%00NiCQU+(O&; zvLb1NCL4$H830T3>-|E@uF*jK-KWtJwN3jh$W5k=nVO9z1tE5#Y!ZKp;Yqa+!BTg6 zBtiE0AC1uG*0WWO?Zi8%O5I|L;`MV`+LmxP0M&EAw5P;|80E_uS~;9i}vQZU96#Plg-h+0WK z`sglFf(%WGRC(LL0*x3=o&deC5`F=ziALVveh4VXqM$D?kS&dDG=(jpa+GxXoP+t3 z=roSTG~7NyCy|4z-GdRted0V(Ds{=pRV5#A76@a;Csz7Xk8$KQzkSF=^q>9*H~Nne zMhLQ6CHVDe0AnFHZ1&mx`fo9T98??@^XCtn{wFQ_?=e<;8X*D39tBV%Ln$tg0t^kc z_3yOJN`k$9Gx3WJ;R^_m*dmh$njnOpoaNqd{Ud&K{Ctkx!#joRgqw+!Numf-4VSkK zQ)1Qokx04QjZwYIaVQAoi7V%B09*r&$Q3gl2*64!YYkxb$Ck)T9c3loScF25R5NN+ zBEkT-$8mc?%J$SR4g%1#WE5`5DV%mQ@*XG1vt=-{LKUsGHk{)YvUOEw4TYfmSyx3h z2vaaS>G33g0Jrb9;Hk_~NIc})m1AmpLnm#W^?=gau2u^iznsdb3A_Tp^i8_#1)#k~ zQa7wBWeAw*Ij7SKA=74Du2ve~vf*H}*6J#4q)QH~z}u+}SP~sT4i9?X{f}z)f4-p` z*`AW#Pu(5s=TZjl1UNISgn~X2K$VK7E7p&4EQ`(V#9xGXu1S~2rTvf9P?7I|>*_;K-E$agEh*KUPuwqf=Nvx*quL_%zCEp{Rnu!* zzLNj&Z@!h8O8F- zGMb82#sv!U4@B(kRwoc70HsOLel-bR6|Rlawe&pZ$PnT*??ZvG6x2?X>9ykG6mjRi zA`h`9)T&A}%wms9budLsbsQRuw`!I0G*C_yKB#7pwPFK}zzF2s8ZN`&)#dYq=2?dy zyn&2q4h=>I8pEc2hJ!2uw;f4_$)Z^$x`uL5+q<3hBDCO>wT;_g_) zU2u802hi@*QKfeWl-uDvHHc2ql)AEJ_n86Ez5UkXZ`sH&MCI0!Zt04RC-gK13^A{T z5#v(i1>}clbmi-)^9NL)M9i3;Zbd}YMZwx2g+)=f#~6BRdT24(EMQ~peXorc>_;-L zF{hjcsfqpFx%^6~G*OzPNrlFbNcR~_lFl% zw-hT4)kIFUTIl=x8!IXf=z_n=L#xmGVlRf9Ep^xX2ftX#YG*3;loa;}e_vGsVNUL; z+O%l)G2}M=j4^ap!{IKn?j$RWl71~M{=}|+gcJS6EQx2um9De=V)VtgAv)n&bX4;% zk_EH(s9H~eYtEp_O!=*_%#$0%snaw$C&m>iy2;7+Am>dC~73o16o%Ga_b5O7lhs8e{?F(MHl2B+6)UIW>W8G1G*ZVOU5m*LW+ z!z~u=`HS5BSurQVp&JgCjR03aRR41h(I_~lRw#fOmVURDQX0~47hDw0TX1_CBVyL+ z0n$*%ihqz~vO`c8#1D$_1R0P0M#b{<8okc+p!YntOCZsI1riY%W}q)*Ksfy7@^`&h zV1yW;RwNYdcad%6OpqOFiW!|A>@x*eO4f_R<%`3E?LA1yBaV8zlv7Z#0ghSR4 zu!PMHaY%PY0p(p%F5hRi!kqR%7%`TI5~lw$NZ>0@V(x!1l%D6|GzQLe>a+;=#B&iL$s+1T0_O)^QyD;t@v0^rY!S+a^Z&fb6e)QHtUg$I{U;9V_Wr3j0Wr& zJvV3kEVb?~TekI$yp+;9sg(M~n4%iL=Es*Z!!>8F)AB#d3^|eaqNo5Kv*Z&U;Wi{n z4+LdGf+^wpu%dOjdt_(@ErSft)APLyizw?r3-biY)?!9)yKLy;`Yf9k%)#k!zl6!@ z(XMSg>2!}(f}TF3hMImY)=@Kx;P=75sRhTPz~w0`G>$wkklrDR=XFFS}unP%JBU0ItaruGPITaQm@c9)d%al?3lLU@2L_x9)#oxIil!vY( z0~vE@BV_OTQNj$-EYv;>r}kKs-NSg046ID-c`HBuPx&pVs1e;12?%J2>ZknnbFxlw z${|RoT?7YYsl&J_EjH;;-*Y7Vuz+zu0X#$qqlWQ7LcTXlB1(y%43NPgZsM}>exuT< zX+ZL&`3qE!@)h}d-l}C)Dt{zGialZPUhNNm={Z{{W@t&EKlZ{8N*!1r%FA+vadWs`*B>Tx1~H&$NoHY2h0b| z8v8_9@HZIvF^lFrG4m@m^CWoJ`^<;Wps&&>;SyZh{R!S$G{mN(rs0v&kGCvlqx0Tg z%|ifYCuoL<1EeS^=W>H5`?SWxJE)Ox=vO7^0IFSXMZN|Rxh5MIyqtc=1I651tt>Rh zrlET$jj?-Hom~4U*>Sf#*`^NhP&CU9k&^pCXesBmzb0t+tys%vPIU}gLylfr0Iu?f~^PxUw^BV@M_-qnJ zU7`Y{2aGBqL#oWQMmbhqGg#ahKS{({xQl_GaD|2+f4wV zhy8DNWT`VkDYtd9x@@@TTcrBe`{T6rJsp}bqts`IS}^~yzzgmTmZY8QQQ2X@Govm- z7&3-7n_3oFTR?l3Sx0Hgd$JVU2bwoxaq?qC4Pv)93)R3gZvcajS1yS!jA1ReFdO@> zX`3vJ#YXUqjAQipL^)Y9X4P3duAu@du$oyGmgu+Z+gvW!4pUedUR7}LRqEt#=SQ|4~t&Y)Y0cV4mP@t?T<8N3ukxUE@VAq1MX72*%Cs3Nz zvg9jPmAZUUC=jsM>I34aSR47U1{li_TLw zoa4}>*kGEl+PIoEW}Ac{<*A7$P0X?|;l{29a=Ewg0MZ6>=YXT*S==nWk zN9QS}Qje0XaI6OBtmMN1Qt1EzyAeyg7pPu%E1;}#S9Z-&-vPX%ltTF`U$XlC-7x0F zJ}3a=B<38}Jh(AJi(8c)3={a3j^9i;$-XMhb#9k$pb7G1-?Tjf3H2 z2_tx@O+dC)W2mEXA1r_-6>!7+GRyWwxRn+2;5LEdmjf37GmbaV2AAAm3~W$`d23D>DGe7H(AL|dGoQfPHVyQjQcGf zS2M#2`&`e>;=d7_OG&J~wMkIw)02j?gw}pl@^Fw1jGl@&Tbu*f7!WrU6Y3JF>y3L* z8L5>=-2S;VHHXU$!67wDPlmRrP~tO9KmQ7j7&9`I%5o)1=-*fK#h>e z;C3XNe~8Z$%tZkcn7T>>p*|iTa|~{9_&XbKR41TzKuv*Dp92WQ%xmDI{bY z>GUT%ys95D!(lUn z&@!Ie-+_$p@t5xz=r;tLTOOw6Fq!=x_X!1zt9i%zj!oBGZpbNP@m^kZfZ#>k5SFg- z4c-toXuCfBwTtuR1ntleiG` zlQRNfbjK6$$`^2rF3i*8it$0fZ*lBr%8U?KOoHnuE7_(MK;X2n=EG-&d0wyd&3lXp?|$9S;=k+o9zBVDzjJa zP7zbHEJ}3JhLefK2g?tUg6$;bfjS^2{8!GdDI6bQNX~r2a7M0md1qgJT(Kmy*Bia^~3H?pUJac$@L1w7#W)g3aey5j*Dm{LWH1_qXo(41okGm6zm{-oQFh_p%+WUV=k z5T#ukm!ncKh{J;3^L&uGc-RC5!f?<85^`ovd&8fIz^X|p9F0CKt=s(O9|^X_*@PVj z5!ORTrci^Tio5fI24B#cZ$0ecjvw3k>8(Z=FJR~ zJLHp-O|sc!chC7AKg+Aa?^4LH;fm*ytgdeMe7ir48*kGd?K?4VI1=;MR=u}#rEd53 zh*X##Tlb=S%cDh5;IAB7?X${8`HIxFg*?WX00GV0=GoXk1uO9|4&o6kHddRfoa~c@ z$RV3Mgs~o0IV$o17f$)P&m2LeyBssOeKE<0-L!$h?XsW{!HoPotT!_dmuf5yrm%Hp zHq3|gA9&m_8yD^R@XMAQF0%rf$B8tABon{zVXfQ}nO$o%wMlLKSTIoz6Vq-HS*CGu zO3v287Mai|)nXm6dTy2HNE;tMTwh$&nw`-i&xw%L;7M2qP*Cnp)>m~JAwCZsE}W*D znTRIgYPZb}5%xI>&(wxsBQHfY2v&&n(aZ`U?@uEG@|qs+0UO>s7u)Dym$-QXN3&I1etf*KO4g0y!<>z;lat z$-r(i{M_UMBt-aBa1{@YxP%K9b|^q1s!uf8XHX=~=fqWvT#Cf$CcQyE+az<~{Frnb52Db`0k^60TC5M@2(hrbRdd&z#)PE-eac2 zNMrLPof9a}uE4FsKQxgi%bdXmSIoCIQ{vB|(5oojXEO`Y&^gUKbq&I-0QIcB zeU*vm`y^LOvFz&EY)qId8*rGMR`oJBx#^=)ABN$QCX9Ril}o9dQ81u(y8>q-?bRF3 z6{S5w0N=yMwCe&=LI>nF^4%nvrRyix$2#sQ*oLSyH9#l$7^12)7v3oYou$t#9+kAMWtS{; zP6}l*k=5KfJ-6ud;Ln)J0b=}1weWJyy$1Ryb-Qr&w}PTE#!ql}HHVd}&)pZeO@YO~ zNiRkr30#G?xGz;jRDacO?&pdY81UsLO3q%ZDywSydRo30b0>-Fo$fz$y&mT&5c5*; z@BtKj8oR+)3|g5KGbdiBdC2I37vwla-MXI4qt@nYT7z4OSX(FN`;8g}o4*3NPAACm zt?&2s)fflr0eGjAgwo#iCi78tZD|{qDH^RAT>19w!_8K^XgO174q{c28WW$~c(Wk9 zO4X%jD_!#ssp10`EWc*LsSvu#Oo$V;-UxseL=JwX&BF0v?oUGTA>6esd1CtM#}s=_}xjW9!Ii#)bhFEE$|t+4bd_oG`Sdt2M;6 z2?XU1FVLa)TI!;FU`noO#Yt3P({os2K$*$%6*8px8zid6;GTmAM4MEs4O@W-dv`|ATds5%OV!=q7nFO1Vk1I9SWRPAr74#p zc6K?MvXzsQS(8}-Z8IV1kKi*N=-;hxc|BkRQNEt@SM{*UToSsnp^SS%-K_y;I=)NT z5lJN|Y<*BB5rj>>reT#-7R-z*=jv(meS-I3_2YA7CC97$+C)}h0|sZa+DsrzC=2

s^q6}lz6DY#nRpP;QStGzXQUHsdE`XSvJW)q31EVjGZ ztQ+Ky#gZSLE;!p`caOw9$IK7_@4C?9>@FA#eBHQ?_+Eb&KVqHFn^X6ukgUAWGt{KX zDD}xuEkokE$joJaBs1Nk1gqTRt{4xT@D7Ha5>k`^j8eUhGxR%bi$yiz$+b+Hs&39K zw`7CPIb-39o1uIj%4$c9wvwvJkO2Vl@~DjaPhxj5yi@tRdRw$vdQcjalr!DOx;l`) z$_TXA41`7^Cph$PkG4Z;Ko>X|1Qgy3wzA=-L8BTDejYgN6~R zl%+!>ztjHMG-lBb8=A3s+w`;umFPSnCtUZIROQgex8TNG0e4dB($FQ48s)tru7LNx z^7qQBHJ);R%%UotxprB1K=`%M1j4Lg$KmMZZ*QkCxm|N|(#Be=|wf%8r>S-IBdY_Ujs4WL8oILRd^SM+w%U5$K>w zADLs+t3Zt{=(#nu(J8{X1Nl404M2&c1J@D?Rmt3_AmoIpao`W600@bqEx)8!r{NGW zED0o2jzm*r6P8fQ6-Y0hpuw*k1IO>-5?9Cq--(WSy{FMGYyb;eS@ZHp-5GPe)w?5} z9lvghT3*w?c)KDiw5hq<|G@VcsOT?;zWgS~n1{|#Viqik%EXcEp49`fl~ekAdb`*s z3W1eEORyFi)_eZIE5P+s`1IH%ZP$_O9!N}Ihe-hcWn{XoU28B zltn;Gzh#u&5V%`CpJX3#AxZR`TkSh|4*PMNcG&r4NxOps=(}QBc9|4j3tK@z!o>Lw zn7Zx^GmF`|8%6JiU|#-P=E1PLokWXlSp6wQ&5mTVHcy#bYrrOxUTsZTOnm_kHHV;b z(v%xDalse5`GVsmUzO&)<+V35_L75=UVu;}ybjo-R>RWTiC95jXe6--@f4 z;=nTcHl~*4MAUfVHy7g17ixE6RHPXv$HmsV;Zvl!hJCof?)!Xn7Af=1nVn}~F>mrV zuJgjx=v%%T90{X?FC!umGEZ|rtO$);+_95Bilrz5`~bG^at3!l(ULzYc(S-MMSezP z2ZaK$&Pj}J6v>F*TYSNlnr2N6Kti<`6#-QGyWkvn<@>vEzX?R6uUO(iF3it*6i1j> zq|AE6zR~ROVwD(!=yQ|j0&$kolIM1ER zpGQt8pGg3Z<3_9~Vr&POg;V84I>V?@LIExpN|;(l??mlp+0`^~`4vvhX)o$Z$Y8<) ztu_Y)E}qILy+g~eBMBw+8!N+%adMWT_;tDGL8EM9o>=pO7T5m$=3@rs4~+p=P0r2O zht9Z}fi6Pxqd9HF=Mok|Ay+VU+|7==VDqB^Pt=rk+yQn(V;4l9j)eX*nJR>1@!Svc zdgZcktYZNOIp@$m|Ivr-%Wtp)R4hJV162fDX}JwDBuxeb!0ht!NLUZBkSgk zT;IcRvo9q7f;#Ax)jeD?7~|6LtcIKS@RLOsZJjcx%DJ4$QGC8~zZ3AjTgfaJG&ger zWiP7=6T06?Jx@PgYq9&mq-@BncwcSBX`uGUov%EYQv21=f|?Q2c29U2G>{FNA#ZCh!%@1iK` zNVtS%aaem0H-n}dL31&q)n##5-Pgy#zHFX!=QHf$)IyyK2oY|o6j|LLthy5r*GEg0 z!Ztir>#4;J%O7&!)J(JwZ}i51!6h40S)5_?;BwZJ1LWN#eDx0b=1|VLg&#KauG?YH zrq`F%*vjrFbtflcwCmk{pjmVs%=MA6!%@-NRGt3Qc02U@wn-M`2Dw#HZ-+Be$?V3R z2!0Ru5gS}gxJxaEvl*=S6Pft@c?+<0V(?w!cV$TlxKlO(*WheD%jjCJKuh_gc6HwI zMT14(Ev=yJm=%{qyeWszo(;YdrTjF9(Ryx$Bg}tKU+3DokS<-))FW zC7oPjwuf{LbXM$3UGdi6Q#)PBCMVWFP)SC0%$KzuI6Lb)GL4iW3=)ApU`6ogWti*s zYz=C*B;F06!p9&AudY0!>}jVpFUhUNQ`olFodYI12eA$K+<*P_HiS2(HH6go8NJ7i zo3{P?%tD990^ur4vlF>7E}I5wW+U zfED^iXl@2KY%zopW9hCh9rZbHSFCy7n10RVb=t}TsA2Wg4o^~kTe%|*rcdhRg|9^z z`KZw8=q3r88Z+pKK=A4B+?`LAZG@S&l57U#D(u#`o7^(5U86r`t;QS#2(5`mA3vTc zpRlz&j=D*7O0?2f`H=Nie2EMF&2>2kncG~l%J~qKI19~{H;osKKMSlUrpWA>HY4|d ztdw?B?bKy_uzgVPT7jA}{#vq%zKqs*1$kxV7fn?kO;|7uZ9hD?F9IPyXi0iefWr*% z_G?~mW3%9xnzv*sw85P$n*79kB1v?htL1!QgXQK?g3$JfQ6M{&;;5*wG|?b&5ifX5 zN1)x8zdYC=9ZpbkDJ+mP&OK(gIswU?8|T&(qx*YdM<*2z#q+5_2_!?E)_d2ql&cXg zt6ze0Ghgw*5Zpem<)0Z4;UoUA+$+2Q)I&rwKhqt@Xs(Bx@;YQ4KMCRr7+aSem5-pVKLsSgnd zEoY|qeYK&AM?e+RF2olV9#Uob=BK@h>8^Vj{@bXGAP)OCnnPJh0xt`zwbMir-LTu%%UlDT+pmU@?@ld|Vr?N{?Hvs_ zP_-2|xU6L4-0;iZe%lt2I$ON-RL5O(g17UsciGyd>P@JMKRMZjG&& zvjVc>9p?l>_+Hyd0@}U(QQ}@r5 zN*O3!&X_Qvh*}f|1!Zm0)c~x9h*=gggMNoo80y|sH?7tn;JU3Idg4ij8|*XMu?pkU z5pQe6A|UOvOHxe^=CvV)U9U*|BQf+EyFu`C0DW(N+<8EU(>U&p_9Bcje2EkjT(x`+ zf$CoHGM8f2Y{JK63J=Z(dizWcJV6ebi+xOmXZ;|x=0Y+SO9{meTb$JQk3D; zZ`k0%HI{e#RNB1tYq%9bkuj6k7-UZb2e_u*@OFjF^xayDE0uoFmi)o}qG!+ldF=zi z_>;Xr9ySl4sajspOx3^HR7{zHHa+6t;jk*tTi76iFk{RCxo<9~6tG9AMQ z^lJQ8nN&i3QqYCV1DA#F;WswTVyc5it8$}Lf2%^PB{BXXy9b{Z9td-a2mF5?poWz`~Au5{taoc1ln*Y#{@doxPPd0jR3JxcR|Qg^EO;nKLg z_G&WgQaW{2Y?It9T!$KBP0s1oewOLqHcyp7R0d4d5!XztZt-^|U1MF)=eh(8;iqZ_ zKY%@%6SbdxWVqDYnGpmR^@HnstW{a5)pHHa>NUb+ktQMw=p@;gaS#x+qj>Ic0X%O) z)-TLZm||q)-6@gen-f?~yWf9GDXD;$W~-ZT^~R!;@ACMSssmj48P{62MUFSA^i8i- zZ(`@b9gpjFH6e;nQ4f#Un|G;(rchQ?4##x5z0DQ_=Oh*!>9_gWT6%YccJG=8&X!yx za{!W`&IoHLp7-DYJ&?XKDVi-69FRc_?~?{x6)W|XLZ@b>GSulpB4&5uH4QKt8jmxd zZ!Y*`-Ty_{oa$?!T;mW?YVN#qt6@EamlTx*Ad1s;7Rgz4FQJ8VG?AnGqQT1;^-R=To`PTTR>rdYZIBC3QUyG_WhHBE#+VI)~6)wRArS(9P@SQRN_AIo64 zmFMzFbH9s5L;~k=OLteW!|UCysIzP;lnPcMys<4EuUFes{M| znI|Ca{Woz1WCJeajkT~2$qR>_ErbX_>HQT=u0z7c^&Q^>UY%cvb+x8Ya}0y&S>EG* z{P22G+)bTL1U*f(yR;CE@C`A!7ufpmRV^O!410&(e5}|B`6UY9Xm0@aHY;W|sTaDu zh!;n6?x)rFAXZ;QaT;34;fwWs${*@EvjXHEpbcRFufJ474$Tcf}DRDf!YlepwJDZmjG%|1l?181JfaSz8vJRNZgCzbtB^VFSjkY#ev67P#o*%jY=F?AW9R9C0Jv`Tf6aGSq+E z4es8jp0zF*`@q(}b*bJmZm6NJV#^_Dqzh1cYia*U6<>)Zi8(D%oLB)>JWbHgDJ4x1 zr?o>TW%@Tm^thG%peoB3p_uq+U>Wj{CppE>Ax+uFR^tTF#U)Fhtsa_UiEvvw_WeyJ zSOSfpAq%ZE2}2I$e&FB5P1_UIB2X)oR-{sMCuv_M_0KNPG>%Gy`f_MCG^N(sFgF0Y zuiT=w?|Glu7zadE>4~k}pyCDayJXxe#@PhNIoy4n;B~q_HVZ5%aDF)z?^dO=xO(n$ z4>tYinK)Y%k~k6_d=$#C4Be(%_%)b8s(P9Cj`XXOl%=750K)T+_R=nWYmPSW_Yf+$nK%8#qQn?P(~0xb{dB~qrN5oxpV#9rzc8-B7V^h$7=0d#CG4eU)yC3s zlSH%B1X7+$?MAISyE1IYQu9)@E0GYK{aE5K^TKm77wc1@Wgxg%jrWHvszxz+_?Xyt z`zGpFeJ3x;*XRGl+UdI4C4B+)Hv7?l8Vzp&zW@Z9SWl7*0~6xa>iyA6)asKw38uWk zlw)qi5gXLIGWI$M7}n&9vF2aK2f}n^<+8_u^7Eln61>?X{PH;9sf7}_Z%#)D zKDB}}61MZR?-$-5P@dY6i+s{;pR^g3(Y1SmGN@iOJIup}WCPCx7$Wg58&#I zj@l2li;>wKnxRXp;*=BoFd=z2A|3iZ30mH5^u(+rVu$lL{HeUC$ImoPIde5R9P3ZD zy<2J2fucZpcrx$Gz$?CE!jJWqr>QC~H%^dWW0&ZcBD*g}4t}{&OY<8u?|ubaqW0SV zX%=;LMH)=$#4%B&SE1~{BMi7WIs$~%MM9Z=lrxv;Td&te6H-w`;{`pI4HIkRn}PrWw7b5u`K08}}>g1cJfIX|UPv zsh+Xus}4vDoXh%RJaq`38JgcHL3X~iyMd>z%>;m5SVvX9 zd2KFLRTHSU8X^xt0#}bmc-Mq14hm87fc=KWdA}Rn6knmP_-OKLwysGAY+*{YQ$!#) zp=6%=qE;_g?&unOe-;kFCwzUh^$Z9sZ4~=vR#|8{1C=#7+f%0fZTs8DW{Z<(>YK+d zOX!eOE_i*Ky+$4F)u0?huzNu2JNpxc)=wz3Jb{qtN7~PPcG~+jq*|R)C(3x5SgirB zre2zz*C*c4LLDenCP&-c?Dd=)m=AD538xrbJ(5SJ4{DvGCywy2Z4NwXd&QgVQ8;le zvu?U!Z7?0GJIpn)SVOB{{IMSVHAksc16(EE<*p{)V1-Cl`qRBx^;($cYucrVfScA&9fl}`h! zHKY*pfw**dlZ%yxEDg&!&Mreat-IOvTeTW~O2N@jIqu`nTA%NRVs&BD%ZXUiBsN%F z!zji5(|?a<)We7!jYir6kO`2jHK?L^ChzU*@mk$!#scP`Cw0`qSTV=y6@6F+?uK&` z22+%^NW7u~Y|cxm-hINaFNxT7h3X-Nvq4G8Lu#enox^dWr5#e+nA>nyU|e&{W-*{w zwwEJ8InlJD0f}R(xhcAoCQEPk) zG@UC!c9SmLig2Lu7vlq%O=nhx>LBBFnD|F)CCc|ZvsLn4>S=kg@h6%#nN3j|`*p}t zFm~i^!Jd>zQY|?d29naz#`pG&*YC*nt1AZZT>>3kwnn~d9JRWpj|8%I?SRvIxv*~% zJ(RDA6a$5ZUD*Jeo~&W_ED`!u)dbAzKCOXB2GlaS0uE{5zogfUBFTaa$0MXpSfBcn zsMdL+Q9s4sV7Ps8A@1r=uOt%zk6y=Jm5xD^=z{-=uyQn#r?@U7n?~prFY4v55vTxY zrnsVU8VPcYlxq)4uXRKtqb6dd?`cGR_N}mf6A3pVIKKhlIZD~TUW5yC*fxWIUgPh( z-dotEZn##E>eG7)q3{4c+ZOg|bAPMPsZRxq-^e?-4mb^YZ5>7qM-6N*zeUI$*nl71g@5 ze9wc>l`cit&$kHKj$u1=jk`vfb`Lh^#&SSuq__-yB_pu4wJ+Ze4e1TFpgK#o^zDsh z=8tN2;?{5r6KBwU=q~j^;FHcNDzsCn4xv2AuWY*jXeEuH)uq0H56CfP_gT8SJ-S#^ zExW#J#^TSsvV26MGH@|?z{&{>inb8rYvC5((J4F!bdN0TCd(a1@JufpbZ?3EwFRGABodt6sdgf+~kNi$KCp zF-md?KudgCQv`=98E1N_w-VdICU>u*4}rBJ0xi)m!UDD=#?cCOWVdQ6`9*&@*}|KH zwd&~8Pk5d8Y4n8?@tI8IH!d#xlhU8_;QMzlb%t3?IW(8tlv2LpOD2D<$sHRKl(UTI)CFq95SqXYMVS4ACk>$i$`&lmLfSL9Sfbm1* z#1_7QD`B`L1NNqk-8bEOziLW*ONJQ=3>I0V)>G6=9hB(^M6VyIqlFXbeuk4FU1N8P z){_X^0m*92)M+zCEF0T37iJC%y%zF-Yc=1xhvn@ER8Zbf93ciw5V@Q1M3a@gJFC}6 zF3~y2kdRWOr&8be)Zjc}GYgPX2GndNK43AY=&3~zRLI*6HZcXWG?cP^r(nj5Pp}ZN zaTa#+y_$Utq?%!Jz@;B&TbWj9az&ZMCp8VD2DapgelvGWRn{{H>ppJ}US2BPWRlz+ zj%@vkOC7z@m1U>pc@^z58XLjp(PO4_CDu)i9D(hHA_h&x#oN`ODI251eYij)#5 zyM4L}H7ZHOW*@p*77l)Y?ZWb&_#=-`?#E!qMzIp^6MfZ0BeNPo=E(hT#+3z#iDXN| zc$Rm*y=0n}S_%P_&}6)1e0b!C?<~nYsLu|-vc1-5y6*FW2MA4wsYhLL4Ue;HP>WP=g4|7A?_{QPr@5fgp-Y! zS=a)C!phO3PYsG+j}8!x4@yHOy~IcoA>@t7eHY0Ft5&KYs~+o347erL=N7954L0(f zpn(Pw+N11JoRE0p7CclIXtx*)Mo?!s8eI|->=)wnS17NmvDi5F_C9Dp!wVEHBk(ey z!8}d?8qly+#n8m`v8%CnsI`zr{#rjL9UyD~CsHbmiij4A9%lQ6Ypy|FYjWCT_Lofw zAX_N=xoe~@Affn5*ZeO3IaerW($V!J9CcNvL&1lm!KP8cou<(Y&#R})<4??Bc@7`? zw1?7j%nT%P9SQO9(6LP!S9QST8b?mUk{BJ!0C022cx)QHGcZ*n4yt`H@4m8z%15U~ zqSo`A5jNo69UfnLI0YLqn(KU3>s4_r)hW@NR@JW6P>otjv$JSWFRjqV6J-W{%rUc; zDSC8&kmA{1;?gs-N)`E;(t_WOXEbLuC7a?b#iafN{b#c2x8`!4<|OW@_^d%xgYl!( z1VDX(u5-&6EvGjdEkSU$_OBA1N^7Xpb4t9!{ZIvbS`y)w&`EdfV3A4Mxf0VIl3&f1 z_Ff|_o6$>Jbr!(N^dF0mm$01b>QS;KhPutlKb&+5T8mw8sNeWZm%#1uJ&ZT76A6TE zzFCA$*+!99ZG*AnCAI?6uS7cNaWy3&Ndf%8kx*7->D_A;`1iIw*(g%#EX?q7+73R( zBARv6tEPk!5*~HDM{L4M>kjodY8`Y4A#*3k>)mcas6U17PecfxGYWN+DOx3NH%_{a zbeeUuNQ#Mi`@`+1hzc5qg<8vd5LJk;qek6wilf`WTV*CC^GS5lG%p$Z2mKnk-3UN} zy6-o}BvNOkqo%7at@ab{q%lXu zVCYS-hRqu16efqiXnHQl!y;`%2b=;7a1ZQbGyKH%pXTJhHYj<_TRT;zzpcXi!6jm{z&M6oYu(!l+2>5 zi))V?lL=KQ2_l)e~=uBfkmKdcEs0cY38FJ9dMJmqgx z1l>}Na~F5qFk@0}^(e)-idBLaw!J=n$9Q)bqn+M2kdTNIp%*k06}MB0aED@V`z-N8 z(G1+1V>*p@>n`|5Fdu((u|r+6Tzn?U&Y6oNj58oxai7oeBu?8k3)@GItWl4w=A@kC zW3vfZj^GV>GELNqXb*;ek8aBfNXdyq0Y4xOh9dI*vMFPXuYyXc-R#~dP%b^?D8=NG zrSaXq8#|qa6=wpWn^kU~-Q4NNxLoAhjgLE?dJbNCy&+STu`cF&yj~fM%KQCPr+Nm$ zX$HM?`)Gu|Zg0)Wh`SLRB^p^SGiBa+$v<+(Kajj`6yTY#Ec{_XFSiz;ZKfokV^NI* zgPV|rYu}3cBw~P=v?wMX?`hRJDgY|itdtdR9YOm z*oreu`;L?Usn$StMeL?};lx5|)dCpGs)QHNAL{*0RxQ9lUW_`=RkFaB7{~21U0(|D-3=0d*S&sQ;S_Qx=W(% zSzVS?)*(ss_>P`={3AbHdm9d{_KBu=bnl9ddh2*|gYOwhYQ@R!ag^k+A)!Fx+;DJUe@5n7 z%jDVq$jSvkgQm!tfK|ybz@RBqK)Tu}NK+2;$*kGb$Ts^d*YKOaPPUQ>9jUZzXkHk6 zH5vv|*mA*P0g|@MkE4W}%8nnYu%4$7Vbp!KfG?-RF6E-uRyI%L9J`ZyhG!ubkPkuR#o(~DdHt&v#Fc-Dcj z(gty*0j@#Yp)L5=iTG0ePzsZL7cpXCMM@iX#wYit<*%|x9!!hMlTBV*(rE~j*b3b? z6mm;(xB7e=!PDX4-Z)u60{|J?gV-!0T>tXpa+#iWou*)ol}Isv4*u!&pr)rd?^m+? zSzOeF-}kq@@y&sY&tQ1LSEk))BC26Em&_%@!t>iCR1M8Ra|J4W?F$!HGl<1@oJ-So zGDNbZekKr#)fFED*4#fto381$M1SIKN7^i&79^hZMlIRMg`$o%lLqMT3|BdHTQ;kd zF=2PRF@M7)zWNTIg1hh;ztiX@*aF||F1?dYP&7bt9a~TvSr1;W#b`d+#8LNLex#rnMjdX&m0*UN_H%8-9CDH zxO{~5buXid?LkK7t2wh2&CN{3r)JG3gB9zogTVs1R&!Fa6dD^y#F1$`rpg~p0pt=s zvPCB9jni^KRW6uio9PDi>$#bEpQD(4CEFHiXg$y)al)z4r=|25{>=FNw!6F7-mTHA zqASa$+1ZMV3X;FvY#LM1ZMtAU=|{OkOIHxpBWclq3m#jH!Ck^{o7Qb|C5HRO!q)af zo`Si$rk&(K-8YA93z=b0*3CawGyw8vfq+mW++_O0Q_WUG$3kV=`>xOUrA>uM%5U*& z)-ougdYoPKc0x(r%YNNI?soCNTQ1OW(akZl(>`-!6WrA<&Y}pai>}ogV-02Jv-_yl zn-fjNSgJK%9nKjXq_h=f-cDD~xOTgx!70s-bd4zYjUS1fW6hQTZ<5>P_K@lnOO^4W2;cgSflTQ$Ph%TQgPL97+-MY@A>Cr4^!_lyfW|jS zSV4Ej02IqWg|C5LZL2RQ(pZ&$?}T3vmn_c4zHi+` z-*^J>r05zxWA<6kJ~f0qqVBq_#G zM)P=7b3k*635E)i<28d)&qQ=d?WHJyal*9B-yZap6f0@`v13e@eiFFalSqFdD$$R2 z?hgp{Ok|5L#w8dCLa0p&yV&P7M4?N}kz4I!l(DcV{;r2wm*{q1fJ80(F8OxGo;6E5 zN9Hmg3o5^TNUfuy>2%?4aIP5qi*n#c(V(pJ7)`VTpW*L=;y*vuH;N;=KO&lK2@gRL z>8D>hWlXkA9=#VaICuIt+s62T`^Us5ErG-GTWmSnQik9fn3dpZ*?J(+7LYY`thq>XP5V#qTV zxE!lw*Tg(QT^rufrt8)^K!wjEj8aE*B*(VN#hE6`HWlLlYBlXg_(607S*^IZ^?OVM z8Lf>IXLt#C0(1j*CtKWvHGnM}<>y;(m}(P}eI#$J?g`h_D9EMTGNPnByrMbI;l)|P zfMc=zU2jgk;GyMe^Er(lc7)c^+rQofsMIqq(Iuz+-Vf9&nN!d&=bz6mF6Eq5f?pUb z6C>URZZ|fwy909vNT(l&pzmT^f=Evo`zX##*Xz8SI#|$-$NZTm^Z^7g*Z4(*mNY3= zRvP-{9??eA5q?pms!d!Q%bjrs=i%x2J_u_IUCt$AeN52sYHx0wsc(B z?n}1)aZ{s9b=GGjG3ESR?3ho8^!obO{-hj5AYX4?0yXF5_=ZI)LnBY>g3pojZ z0uQ)2T9jh15m>egFSB^gn_(R3Rk&-sK^c~8%ePc} z;zaU8v{hPhR4)f$o{+&#TNtNFh~isrX!#zM_dH)FJ`m{!H>k(D4#8+d%W0$rJWC>p zvWxH+#=!lJ<41bTwao;na}>$)jY2YLxFpinV%o|1*;sJ8+R80IKw= zpqW(G*S-xOZ7i8ZfTR|T0EYTd+wzq$l&xH+g?GuK(s1Jzq9te(J)r5HSzE(IB(YpT z*GdjE$9NCPY7E=5cC1oD-GvcoR>||Y$#HvmNeBRVg0T;!fP_ETo#RC9j0Ptf{dz6x z&P(J@1WhCn?ACj!r)?U#eO8F1vvh19JRSV|6lhdAEHP;13NOPA@LJf<(CJdYhcEgu zdgPlubrSTEuWR3_$E8=UT4t8ccaff~6y<2q$XlL;WjcCam913o;Hw zf!rQe{f(>*I)*w|0I;l>ZM$E-;+4?_{pR(nr?oU;G4QT-|IpmdJyzXsU%JPaK^5X( zabRf4mGcwUiG;p3VEqdjc$i2%f09A`&;?L=-W?;%vSOlp!1kNqPogPmjFqnwA z!=B$%o!F~k*zo+H#uZ4}rwCQ@XX@Uyj~)5X4<2pjU=dJ_(*QQ~aYAMN*>uu}&z|;o zB~+3FhropVt7GwJBk12?pnCeUfX1T!UQodfMLK|j{rGmuXY&1f7t;5 zO9o6C_zO1i7+Q%3fr21UB$ih&6VVIItU&~<9Qq4RFU7r?0+pK!1q_V*6^u;&0;9Y| z0Y3cv3qJPHNjwHwo(5U|qa6OvlIKT8_W~<`AwBD z%CMT#HlgR*PKe@|2_eKy4L@dy8fRLrH}uA;p_ie z0d|6paQ_jwG)zSL$LcFB>VHgv)_H#PCO`*w3Xn84I*@(}=>>iz|N9C?HV479Kw)oW z;Of|4@GE`QS1`C02qygFvjglHMI!lAj8}4gui#nW+6WOaGYuA)JkI)33buc=*g)Lt zXFBNfOOa8=FkZk{;%BdnIyk<7<%u}Jr-{Fc`$t^skCz`J=yBwG1=~3TV`rpaEWQ%k zdIgEP0!2o-U!YeKQ?H;LH{cr!yg!OlK>3ls4*RbL49L?%=oPH%4J4bEf3f&qt&D%T zJilaL(20f-2aDT4%AD>3x>I^o7St{^D5C6Z# zdF8N#4EV6;ss_Nr+GR2zr&TFR*UTwcwPhSoDzP4{x@>< z-%BU^?ZsM2G7@Mki|_^hcdP(Z&ezwOtMLU+iX;5DX8q}*Nr7sX9H?pg^Nsb-lIKU) z0)j!{2*r!lS3Qt_D9}33?|bWuRryKs7x2GdZU3-&erw-BHH`h=+llhrt%(WR3es8m XPm5~P1^h6}gaC#H2G-W~XaD~PRg9#e diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 27b5466..710ced3 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Tue Jan 03 14:13:22 CET 2017 +#Fri Feb 15 11:59:10 CET 2019 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-5.2.1-all.zip diff --git a/gradlew b/gradlew index 4453cce..af6708f 100755 --- a/gradlew +++ b/gradlew @@ -28,16 +28,16 @@ APP_NAME="Gradle" APP_BASE_NAME=`basename "$0"` # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" +DEFAULT_JVM_OPTS='"-Xmx64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" -warn ( ) { +warn () { echo "$*" } -die ( ) { +die () { echo echo "$*" echo @@ -155,7 +155,7 @@ if $cygwin ; then fi # Escape application args -save ( ) { +save () { for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done echo " " } diff --git a/gradlew.bat b/gradlew.bat index e95643d..0f8d593 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -14,7 +14,7 @@ set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= +set DEFAULT_JVM_OPTS="-Xmx64m" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome diff --git a/settings.gradle b/settings.gradle index ef50653..57d6828 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1 +1,5 @@ -rootProject.name = 'elasticsearch-extras-client' +include 'elx-api' +include 'elx-common' +include 'elx-node' +include 'elx-transport' +include 'elx-http' diff --git a/src/integration-test/java/org/elasticsearch/node/package-info.java b/src/integration-test/java/org/elasticsearch/node/package-info.java deleted file mode 100644 index f299cbc..0000000 --- a/src/integration-test/java/org/elasticsearch/node/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Classes to support Elasticsearch node creation. - */ -package org.elasticsearch.node; diff --git a/src/integration-test/java/org/xbib/elasticsearch/SearchTest.java b/src/integration-test/java/org/xbib/elasticsearch/SearchTest.java deleted file mode 100644 index 8d1276a..0000000 --- a/src/integration-test/java/org/xbib/elasticsearch/SearchTest.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.xbib.elasticsearch; - -import static org.elasticsearch.client.Requests.indexRequest; -import static org.elasticsearch.client.Requests.refreshRequest; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; - -import org.elasticsearch.action.bulk.BulkAction; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.sort.SortOrder; -import org.junit.Test; - -/** - * - */ -public class SearchTest extends NodeTestUtils { - - private static final ESLogger logger = ESLoggerFactory.getLogger("test"); - - @Test - public void testSearch() throws Exception { - Client client = client("1"); - long t0 = System.currentTimeMillis(); - BulkRequestBuilder builder = new BulkRequestBuilder(client, BulkAction.INSTANCE); - for (int i = 0; i < 1000; i++) { - builder.add(indexRequest() - .index("pages").type("row") - .source(jsonBuilder() - .startObject() - .field("user1", "kimchy") - .field("user2", "kimchy") - .field("user3", "kimchy") - .field("user4", "kimchy") - .field("user5", "kimchy") - .field("user6", "kimchy") - .field("user7", "kimchy") - .field("user8", "kimchy") - .field("user9", "kimchy") - .field("rowcount", i) - .field("rs", 1234))); - } - client.bulk(builder.request()).actionGet(); - - client.admin().indices().refresh(refreshRequest()).actionGet(); - - long t1 = System.currentTimeMillis(); - logger.info("t1-t0 = {}", t1 - t0); - - for (int i = 0; i < 100; i++) { - t1 = System.currentTimeMillis(); - QueryBuilder queryStringBuilder = - QueryBuilders.queryStringQuery("rs:" + 1234); - SearchRequestBuilder requestBuilder = client.prepareSearch() - .setIndices("pages") - .setTypes("row") - .setQuery(queryStringBuilder) - .addSort("rowcount", SortOrder.DESC) - .setFrom(i * 10).setSize(10); - SearchResponse response = requestBuilder.execute().actionGet(); - long t2 = System.currentTimeMillis(); - logger.info("t2-t1 = {}", t2 - t1); - } - } -} diff --git a/src/integration-test/java/org/xbib/elasticsearch/WildcardTest.java b/src/integration-test/java/org/xbib/elasticsearch/WildcardTest.java deleted file mode 100644 index 6e252d1..0000000 --- a/src/integration-test/java/org/xbib/elasticsearch/WildcardTest.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.xbib.elasticsearch; - -import static org.elasticsearch.client.Requests.indexRequest; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; - -import org.elasticsearch.client.Client; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.query.QueryBuilder; -import org.junit.Test; - -import java.io.IOException; - -/** - * - */ -public class WildcardTest extends NodeTestUtils { - - protected Settings getNodeSettings() { - return settingsBuilder() - .put("cluster.name", getClusterName()) - .put("cluster.routing.allocation.disk.threshold_enabled", false) - .put("discovery.zen.multicast.enabled", false) - .put("http.enabled", false) - .put("path.home", System.getProperty("path.home")) - .put("index.number_of_shards", 1) - .put("index.number_of_replicas", 0) - .build(); - } - - @Test - public void testWildcard() throws Exception { - index(client("1"), "1", "010"); - index(client("1"), "2", "0*0"); - // exact - validateCount(client("1"), queryStringQuery("010").defaultField("field"), 1); - validateCount(client("1"), queryStringQuery("0\\*0").defaultField("field"), 1); - // pattern - validateCount(client("1"), queryStringQuery("0*0").defaultField("field"), 1); // 2? - validateCount(client("1"), queryStringQuery("0?0").defaultField("field"), 1); // 2? - validateCount(client("1"), queryStringQuery("0**0").defaultField("field"), 1); // 2? - validateCount(client("1"), queryStringQuery("0??0").defaultField("field"), 0); - validateCount(client("1"), queryStringQuery("*10").defaultField("field"), 1); - validateCount(client("1"), queryStringQuery("*1*").defaultField("field"), 1); - validateCount(client("1"), queryStringQuery("*\\*0").defaultField("field"), 0); // 1? - validateCount(client("1"), queryStringQuery("*\\**").defaultField("field"), 0); // 1? - } - - private void index(Client client, String id, String fieldValue) throws IOException { - client.index(indexRequest() - .index("index").type("type").id(id) - .source(jsonBuilder().startObject().field("field", fieldValue).endObject()) - .refresh(true)).actionGet(); - } - - private long count(Client client, QueryBuilder queryBuilder) { - return client.prepareSearch("index").setTypes("type") - .setQuery(queryBuilder) - .execute().actionGet().getHits().getTotalHits(); - } - - private void validateCount(Client client, QueryBuilder queryBuilder, long expectedHits) { - final long actualHits = count(client, queryBuilder); - if (actualHits != expectedHits) { - throw new RuntimeException("actualHits=" + actualHits + ", expectedHits=" + expectedHits); - } - } -} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeDuplicateIDTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeDuplicateIDTest.java deleted file mode 100644 index 7c11526..0000000 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeDuplicateIDTest.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.xbib.elasticsearch.extras.client.node; - -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.junit.Test; -import org.xbib.elasticsearch.NodeTestUtils; -import org.xbib.elasticsearch.extras.client.Clients; -import org.xbib.elasticsearch.extras.client.SimpleBulkControl; -import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; - -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.junit.Assert.*; - -/** - * - */ -public class BulkNodeDuplicateIDTest extends NodeTestUtils { - - private static final ESLogger logger = ESLoggerFactory.getLogger(BulkNodeDuplicateIDTest.class.getSimpleName()); - - private static final Long MAX_ACTIONS = 1000L; - - private static final Long NUM_ACTIONS = 12345L; - - @Test - public void testDuplicateDocIDs() throws Exception { - long numactions = NUM_ACTIONS; - final BulkNodeClient client = Clients.builder() - .put(Clients.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkNodeClient(client("1")); - try { - client.newIndex("test"); - for (int i = 0; i < NUM_ACTIONS; i++) { - client.index("test", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}"); - } - client.flushIngest(); - client.waitForResponses("30s"); - client.refreshIndex("test"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE) - .setIndices("test") - .setTypes("test") - .setQuery(matchAllQuery()); - long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); - logger.info("hits = {}", hits); - assertTrue(hits < NUM_ACTIONS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - client.shutdown(); - assertEquals(numactions, client.getMetric().getSucceeded().getCount()); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); - } - assertFalse(client.hasThrowable()); - } - } -} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/package-info.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/package-info.java deleted file mode 100644 index 873ebae..0000000 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Classes for testing Elasticsearch node client extras. - */ -package org.xbib.elasticsearch.extras.client.node; diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/package-info.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/package-info.java deleted file mode 100644 index 2bfc45c..0000000 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Classes to test Elasticsearch clients. - */ -package org.xbib.elasticsearch.extras.client; diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportDuplicateIDTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportDuplicateIDTest.java deleted file mode 100644 index c087601..0000000 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportDuplicateIDTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.xbib.elasticsearch.extras.client.transport; - -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.junit.Test; -import org.xbib.elasticsearch.NodeTestUtils; -import org.xbib.elasticsearch.extras.client.Clients; -import org.xbib.elasticsearch.extras.client.SimpleBulkControl; -import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; - -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.junit.Assert.*; - -public class BulkTransportDuplicateIDTest extends NodeTestUtils { - - private final static ESLogger logger = ESLoggerFactory.getLogger(BulkTransportDuplicateIDTest.class.getSimpleName()); - - private final static Long MAX_ACTIONS = 1000L; - - private final static Long NUM_ACTIONS = 12345L; - - @Test - public void testDuplicateDocIDs() throws Exception { - long numactions = NUM_ACTIONS; - final BulkTransportClient client = Clients.builder() - .put(getSettings()) - .put(Clients.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkTransportClient(); - try { - client.newIndex("test"); - for (int i = 0; i < NUM_ACTIONS; i++) { - client.index("test", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}"); - } - client.flushIngest(); - client.waitForResponses("30s"); - client.refreshIndex("test"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE) - .setIndices("test") - .setTypes("test") - .setQuery(matchAllQuery()); - long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); - logger.info("hits = {}", hits); - assertTrue(hits < NUM_ACTIONS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - client.shutdown(); - assertEquals(numactions, client.getMetric().getSucceeded().getCount()); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); - } - assertFalse(client.hasThrowable()); - } - } -} diff --git a/src/integration-test/java/org/xbib/elasticsearch/package-info.java b/src/integration-test/java/org/xbib/elasticsearch/package-info.java deleted file mode 100644 index 2958ce1..0000000 --- a/src/integration-test/java/org/xbib/elasticsearch/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Test classes for testing Elasticsearch. - */ -package org.xbib.elasticsearch; \ No newline at end of file diff --git a/src/integration-test/java/suites/BulkNodeTestSuite.java b/src/integration-test/java/suites/BulkNodeTestSuite.java deleted file mode 100644 index caac820..0000000 --- a/src/integration-test/java/suites/BulkNodeTestSuite.java +++ /dev/null @@ -1,23 +0,0 @@ -package suites; - -import org.junit.runner.RunWith; -import org.junit.runners.Suite; -import org.xbib.elasticsearch.extras.client.node.BulkNodeClientTest; -import org.xbib.elasticsearch.extras.client.node.BulkNodeDuplicateIDTest; -import org.xbib.elasticsearch.extras.client.node.BulkNodeIndexAliasTest; -import org.xbib.elasticsearch.extras.client.node.BulkNodeReplicaTest; -import org.xbib.elasticsearch.extras.client.node.BulkNodeUpdateReplicaLevelTest; - -/** - * - */ -@RunWith(ListenerSuite.class) -@Suite.SuiteClasses({ - BulkNodeClientTest.class, - BulkNodeDuplicateIDTest.class, - BulkNodeReplicaTest.class, - BulkNodeUpdateReplicaLevelTest.class, - BulkNodeIndexAliasTest.class -}) -public class BulkNodeTestSuite { -} diff --git a/src/integration-test/java/suites/BulkTransportTestSuite.java b/src/integration-test/java/suites/BulkTransportTestSuite.java deleted file mode 100644 index f429dfc..0000000 --- a/src/integration-test/java/suites/BulkTransportTestSuite.java +++ /dev/null @@ -1,22 +0,0 @@ -package suites; - -import org.junit.runner.RunWith; -import org.junit.runners.Suite; -import org.xbib.elasticsearch.extras.client.transport.BulkTransportClientTest; -import org.xbib.elasticsearch.extras.client.transport.BulkTransportDuplicateIDTest; -import org.xbib.elasticsearch.extras.client.transport.BulkTransportReplicaTest; -import org.xbib.elasticsearch.extras.client.transport.BulkTransportUpdateReplicaLevelTest; - -/** - * - */ -@RunWith(ListenerSuite.class) -@Suite.SuiteClasses({ - BulkTransportClientTest.class, - BulkTransportDuplicateIDTest.class, - BulkTransportReplicaTest.class, - BulkTransportUpdateReplicaLevelTest.class -}) -public class BulkTransportTestSuite { - -} diff --git a/src/integration-test/java/suites/ListenerSuite.java b/src/integration-test/java/suites/ListenerSuite.java deleted file mode 100644 index c02d371..0000000 --- a/src/integration-test/java/suites/ListenerSuite.java +++ /dev/null @@ -1,23 +0,0 @@ -package suites; - -import org.junit.runner.Runner; -import org.junit.runner.notification.RunNotifier; -import org.junit.runners.Suite; -import org.junit.runners.model.InitializationError; -import org.junit.runners.model.RunnerBuilder; - -public class ListenerSuite extends Suite { - - private final TestListener listener = new TestListener(); - - public ListenerSuite(Class klass, RunnerBuilder builder) throws InitializationError { - super(klass, builder); - } - - @Override - protected void runChild(Runner runner, RunNotifier notifier) { - notifier.addListener(listener); - runner.run(notifier); - notifier.removeListener(listener); - } -} diff --git a/src/integration-test/java/suites/MiscTestSuite.java b/src/integration-test/java/suites/MiscTestSuite.java deleted file mode 100644 index ea23630..0000000 --- a/src/integration-test/java/suites/MiscTestSuite.java +++ /dev/null @@ -1,21 +0,0 @@ -package suites; - -import org.junit.runner.RunWith; -import org.junit.runners.Suite; -import org.xbib.elasticsearch.AliasTest; -import org.xbib.elasticsearch.SearchTest; -import org.xbib.elasticsearch.SimpleTest; -import org.xbib.elasticsearch.WildcardTest; - -/** - * - */ -@RunWith(ListenerSuite.class) -@Suite.SuiteClasses({ - SimpleTest.class, - AliasTest.class, - SearchTest.class, - WildcardTest.class -}) -public class MiscTestSuite { -} diff --git a/src/integration-test/java/suites/TestListener.java b/src/integration-test/java/suites/TestListener.java deleted file mode 100644 index 7e24527..0000000 --- a/src/integration-test/java/suites/TestListener.java +++ /dev/null @@ -1,44 +0,0 @@ -package suites; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.junit.runner.Description; -import org.junit.runner.Result; -import org.junit.runner.notification.Failure; -import org.junit.runner.notification.RunListener; - -/** - * - */ -public class TestListener extends RunListener { - - private static final Logger logger = LogManager.getLogger("test.listener"); - - public void testRunStarted(Description description) throws java.lang.Exception { - logger.info("number of tests to execute: {}", description.testCount()); - } - - public void testRunFinished(Result result) throws java.lang.Exception { - logger.info("number of tests executed: {}", result.getRunCount()); - } - - public void testStarted(Description description) throws java.lang.Exception { - logger.info("starting execution of {} {}", - description.getClassName(), description.getMethodName()); - } - - public void testFinished(Description description) throws java.lang.Exception { - logger.info("finished execution of {} {}", - description.getClassName(), description.getMethodName()); - } - - public void testFailure(Failure failure) throws java.lang.Exception { - logger.info("failed execution of tests: {}", - failure.getMessage()); - } - - public void testIgnored(Description description) throws java.lang.Exception { - logger.info("execution of test ignored: {}", - description.getClassName(), description.getMethodName()); - } -} diff --git a/src/integration-test/resources/org/xbib/elasticsearch/extras/client/settings.json b/src/integration-test/resources/org/xbib/elasticsearch/extras/client/settings.json deleted file mode 100644 index 86f5118..0000000 --- a/src/integration-test/resources/org/xbib/elasticsearch/extras/client/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "index.analysis.analyzer.default.type" : "keyword" -} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/AbstractClient.java b/src/main/java/org/xbib/elasticsearch/extras/client/AbstractClient.java deleted file mode 100644 index aed7be0..0000000 --- a/src/main/java/org/xbib/elasticsearch/extras/client/AbstractClient.java +++ /dev/null @@ -1,496 +0,0 @@ -package org.xbib.elasticsearch.extras.client; - -import com.carrotsearch.hppc.cursors.ObjectCursor; -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import org.elasticsearch.ElasticsearchTimeoutException; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; -import org.elasticsearch.action.admin.indices.flush.FlushAction; -import org.elasticsearch.action.admin.indices.flush.FlushRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexAction; -import org.elasticsearch.action.admin.indices.get.GetIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.get.GetIndexResponse; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.action.admin.indices.recovery.RecoveryAction; -import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest; -import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; -import org.elasticsearch.action.admin.indices.refresh.RefreshAction; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.cluster.metadata.AliasMetaData; -import org.elasticsearch.common.io.Streams; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.sort.SortBuilder; -import org.elasticsearch.search.sort.SortBuilders; -import org.elasticsearch.search.sort.SortOrder; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * - */ -public abstract class AbstractClient { - - private static final ESLogger logger = ESLoggerFactory.getLogger(AbstractClient.class.getName()); - - private Settings.Builder settingsBuilder; - - private Settings settings; - - private Map mappings = new HashMap<>(); - - public abstract ElasticsearchClient client(); - - protected abstract void createClient(Settings settings) throws IOException; - - public abstract void shutdown(); - - public Settings.Builder getSettingsBuilder() { - return settingsBuilder(); - } - - public void resetSettings() { - this.settingsBuilder = Settings.settingsBuilder(); - settings = null; - mappings = new HashMap<>(); - } - - public void setSettings(Settings settings) { - this.settings = settings; - } - - public void setting(String key, String value) { - if (settingsBuilder == null) { - settingsBuilder = Settings.settingsBuilder(); - } - settingsBuilder.put(key, value); - } - - public void setting(String key, Boolean value) { - if (settingsBuilder == null) { - settingsBuilder = Settings.settingsBuilder(); - } - settingsBuilder.put(key, value); - } - - public void setting(String key, Integer value) { - if (settingsBuilder == null) { - settingsBuilder = Settings.settingsBuilder(); - } - settingsBuilder.put(key, value); - } - - public void setting(InputStream in) throws IOException { - settingsBuilder = Settings.settingsBuilder().loadFromStream(".json", in); - } - - public Settings.Builder settingsBuilder() { - return settingsBuilder != null ? settingsBuilder : Settings.settingsBuilder(); - } - - public Settings settings() { - if (settings != null) { - return settings; - } - if (settingsBuilder == null) { - settingsBuilder = Settings.settingsBuilder(); - } - return settingsBuilder.build(); - } - - public void mapping(String type, String mapping) throws IOException { - mappings.put(type, mapping); - } - - public void mapping(String type, InputStream in) throws IOException { - if (type == null) { - return; - } - StringWriter sw = new StringWriter(); - Streams.copy(new InputStreamReader(in), sw); - mappings.put(type, sw.toString()); - } - - public Map mappings() { - return mappings.isEmpty() ? null : mappings; - } - - - public void updateIndexSetting(String index, String key, Object value) throws IOException { - if (client() == null) { - return; - } - if (index == null) { - throw new IOException("no index name given"); - } - if (key == null) { - throw new IOException("no key given"); - } - if (value == null) { - throw new IOException("no value given"); - } - Settings.Builder updateSettingsBuilder = Settings.settingsBuilder(); - updateSettingsBuilder.put(key, value.toString()); - UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index) - .settings(updateSettingsBuilder); - client().execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet(); - } - - public void waitForRecovery() throws IOException { - if (client() == null) { - return; - } - client().execute(RecoveryAction.INSTANCE, new RecoveryRequest()).actionGet(); - } - - public int waitForRecovery(String index) throws IOException { - if (client() == null) { - return -1; - } - if (index == null) { - throw new IOException("unable to waitfor recovery, index not set"); - } - RecoveryResponse response = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest(index)).actionGet(); - int shards = response.getTotalShards(); - client().execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest(index) - .waitForActiveShards(shards)).actionGet(); - return shards; - } - - public void waitForCluster(String statusString, String timeout) throws IOException { - if (client() == null) { - return; - } - ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString); - ClusterHealthResponse healthResponse = - client().execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest() - .waitForStatus(status).timeout(timeout)).actionGet(); - if (healthResponse != null && healthResponse.isTimedOut()) { - throw new IOException("cluster state is " + healthResponse.getStatus().name() - + " and not " + status.name() - + ", from here on, everything will fail!"); - } - } - - public String fetchClusterName() { - if (client() == null) { - return null; - } - try { - ClusterStateRequestBuilder clusterStateRequestBuilder = - new ClusterStateRequestBuilder(client(), ClusterStateAction.INSTANCE).all(); - ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); - String name = clusterStateResponse.getClusterName().value(); - int nodeCount = clusterStateResponse.getState().getNodes().size(); - return name + " (" + nodeCount + " nodes connected)"; - } catch (ElasticsearchTimeoutException e) { - logger.warn(e.getMessage(), e); - return "TIMEOUT"; - } catch (NoNodeAvailableException e) { - logger.warn(e.getMessage(), e); - return "DISCONNECTED"; - } catch (Exception e) { - logger.warn(e.getMessage(), e); - return "[" + e.getMessage() + "]"; - } - } - - public String healthColor() { - if (client() == null) { - return null; - } - try { - ClusterHealthResponse healthResponse = - client().execute(ClusterHealthAction.INSTANCE, - new ClusterHealthRequest().timeout(TimeValue.timeValueSeconds(30))).actionGet(); - ClusterHealthStatus status = healthResponse.getStatus(); - return status.name(); - } catch (ElasticsearchTimeoutException e) { - logger.warn(e.getMessage(), e); - return "TIMEOUT"; - } catch (NoNodeAvailableException e) { - logger.warn(e.getMessage(), e); - return "DISCONNECTED"; - } catch (Exception e) { - logger.warn(e.getMessage(), e); - return "[" + e.getMessage() + "]"; - } - } - - public int updateReplicaLevel(String index, int level) throws IOException { - waitForCluster("YELLOW", "30s"); - updateIndexSetting(index, "number_of_replicas", level); - return waitForRecovery(index); - } - - public void flushIndex(String index) { - if (client() == null) { - return; - } - if (index != null) { - client().execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet(); - } - } - - public void refreshIndex(String index) { - if (client() == null) { - return; - } - if (index != null) { - client().execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet(); - } - } - - public void putMapping(String index) { - if (client() == null) { - return; - } - if (!mappings().isEmpty()) { - for (Map.Entry me : mappings().entrySet()) { - client().execute(PutMappingAction.INSTANCE, - new PutMappingRequest(index).type(me.getKey()).source(me.getValue())).actionGet(); - } - } - } - - public String resolveAlias(String alias) { - if (client() == null) { - return alias; - } - GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client(), GetAliasesAction.INSTANCE); - GetAliasesResponse getAliasesResponse = getAliasesRequestBuilder.setAliases(alias).execute().actionGet(); - if (!getAliasesResponse.getAliases().isEmpty()) { - return getAliasesResponse.getAliases().keys().iterator().next().value; - } - return alias; - } - - public String resolveMostRecentIndex(String alias) { - if (client() == null) { - return alias; - } - if (alias == null) { - return null; - } - GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client(), GetAliasesAction.INSTANCE); - GetAliasesResponse getAliasesResponse = getAliasesRequestBuilder.setAliases(alias).execute().actionGet(); - Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); - Set indices = new TreeSet<>(Collections.reverseOrder()); - for (ObjectCursor indexName : getAliasesResponse.getAliases().keys()) { - Matcher m = pattern.matcher(indexName.value); - if (m.matches() && alias.equals(m.group(1))) { - indices.add(indexName.value); - } - } - return indices.isEmpty() ? alias : indices.iterator().next(); - } - - public Map getAliasFilters(String alias) { - GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client(), GetAliasesAction.INSTANCE); - return getFilters(getAliasesRequestBuilder.setIndices(resolveAlias(alias)).execute().actionGet()); - } - - public Map getIndexFilters(String index) { - GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client(), GetAliasesAction.INSTANCE); - return getFilters(getAliasesRequestBuilder.setIndices(index).execute().actionGet()); - } - - private Map getFilters(GetAliasesResponse getAliasesResponse) { - Map result = new HashMap<>(); - for (ObjectObjectCursor> object : getAliasesResponse.getAliases()) { - List aliasMetaDataList = object.value; - for (AliasMetaData aliasMetaData : aliasMetaDataList) { - if (aliasMetaData.filteringRequired()) { - result.put(aliasMetaData.alias(), new String(aliasMetaData.getFilter().uncompressed())); - } else { - result.put(aliasMetaData.alias(), null); - } - } - } - return result; - } - - public void switchAliases(String index, String concreteIndex, List extraAliases) { - switchAliases(index, concreteIndex, extraAliases, null); - } - - public void switchAliases(String index, String concreteIndex, - List extraAliases, IndexAliasAdder adder) { - if (client() == null) { - return; - } - if (index.equals(concreteIndex)) { - return; - } - // two situations: 1. there is a new alias 2. there is already an old index with the alias - String oldIndex = resolveAlias(index); - final Map oldFilterMap = oldIndex.equals(index) ? null : getIndexFilters(oldIndex); - final List newAliases = new LinkedList<>(); - final List switchAliases = new LinkedList<>(); - IndicesAliasesRequestBuilder requestBuilder = new IndicesAliasesRequestBuilder(client(), IndicesAliasesAction.INSTANCE); - if (oldFilterMap == null || !oldFilterMap.containsKey(index)) { - // never apply a filter for trunk index name - requestBuilder.addAlias(concreteIndex, index); - newAliases.add(index); - } - // switch existing aliases - if (oldFilterMap != null) { - for (Map.Entry entry : oldFilterMap.entrySet()) { - String alias = entry.getKey(); - String filter = entry.getValue(); - requestBuilder.removeAlias(oldIndex, alias); - if (filter != null) { - requestBuilder.addAlias(concreteIndex, alias, filter); - } else { - requestBuilder.addAlias(concreteIndex, alias); - } - switchAliases.add(alias); - } - } - // a list of aliases that should be added, check if new or old - if (extraAliases != null) { - for (String extraAlias : extraAliases) { - if (oldFilterMap == null || !oldFilterMap.containsKey(extraAlias)) { - // index alias adder only active on extra aliases, and if alias is new - if (adder != null) { - adder.addIndexAlias(requestBuilder, concreteIndex, extraAlias); - } else { - requestBuilder.addAlias(concreteIndex, extraAlias); - } - newAliases.add(extraAlias); - } else { - String filter = oldFilterMap.get(extraAlias); - requestBuilder.removeAlias(oldIndex, extraAlias); - if (filter != null) { - requestBuilder.addAlias(concreteIndex, extraAlias, filter); - } else { - requestBuilder.addAlias(concreteIndex, extraAlias); - } - switchAliases.add(extraAlias); - } - } - } - if (!newAliases.isEmpty() || !switchAliases.isEmpty()) { - logger.info("new aliases = {}, switch aliases = {}", newAliases, switchAliases); - requestBuilder.execute().actionGet(); - } - } - - public void performRetentionPolicy(String index, String concreteIndex, int timestampdiff, int mintokeep) { - if (client() == null) { - return; - } - if (index.equals(concreteIndex)) { - return; - } - GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client(), GetIndexAction.INSTANCE); - GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet(); - Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); - Set indices = new TreeSet<>(); - logger.info("{} indices", getIndexResponse.getIndices().length); - for (String s : getIndexResponse.getIndices()) { - Matcher m = pattern.matcher(s); - if (m.matches() && index.equals(m.group(1)) && !s.equals(concreteIndex)) { - indices.add(s); - } - } - if (indices.isEmpty()) { - logger.info("no indices found, retention policy skipped"); - return; - } - if (mintokeep > 0 && indices.size() <= mintokeep) { - logger.info("{} indices found, not enough for retention policy ({}), skipped", - indices.size(), mintokeep); - return; - } else { - logger.info("candidates for deletion = {}", indices); - } - List indicesToDelete = new ArrayList<>(); - // our index - Matcher m1 = pattern.matcher(concreteIndex); - if (m1.matches()) { - Integer i1 = Integer.parseInt(m1.group(2)); - for (String s : indices) { - Matcher m2 = pattern.matcher(s); - if (m2.matches()) { - Integer i2 = Integer.parseInt(m2.group(2)); - int kept = indices.size() - indicesToDelete.size(); - if ((timestampdiff == 0 || (timestampdiff > 0 && i1 - i2 > timestampdiff)) && mintokeep <= kept) { - indicesToDelete.add(s); - } - } - } - } - logger.info("indices to delete = {}", indicesToDelete); - if (indicesToDelete.isEmpty()) { - logger.info("not enough indices found to delete, retention policy complete"); - return; - } - String[] s = indicesToDelete.toArray(new String[indicesToDelete.size()]); - DeleteIndexRequestBuilder requestBuilder = new DeleteIndexRequestBuilder(client(), DeleteIndexAction.INSTANCE, s); - DeleteIndexResponse response = requestBuilder.execute().actionGet(); - if (!response.isAcknowledged()) { - logger.warn("retention delete index operation was not acknowledged"); - } - } - - public Long mostRecentDocument(String index, String timestampfieldname) { - if (client() == null) { - return null; - } - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client(), SearchAction.INSTANCE); - SortBuilder sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC); - SearchResponse searchResponse = searchRequestBuilder.setIndices(index) - .addField(timestampfieldname) - .setSize(1) - .addSort(sort) - .execute().actionGet(); - if (searchResponse.getHits().getHits().length == 1) { - SearchHit hit = searchResponse.getHits().getHits()[0]; - if (hit.getFields().get(timestampfieldname) != null) { - return hit.getFields().get(timestampfieldname).getValue(); - } else { - return 0L; - } - } - return null; - } - -} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/Clients.java b/src/main/java/org/xbib/elasticsearch/extras/client/Clients.java deleted file mode 100644 index daa4981..0000000 --- a/src/main/java/org/xbib/elasticsearch/extras/client/Clients.java +++ /dev/null @@ -1,105 +0,0 @@ -package org.xbib.elasticsearch.extras.client; - -import org.elasticsearch.client.Client; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.xbib.elasticsearch.extras.client.node.BulkNodeClient; -import org.xbib.elasticsearch.extras.client.transport.BulkTransportClient; -import org.xbib.elasticsearch.extras.client.transport.MockTransportClient; - -/** - * - */ -public final class Clients implements Parameters { - - private final Settings.Builder settingsBuilder; - - private BulkMetric metric; - - private BulkControl control; - - public Clients() { - settingsBuilder = Settings.builder(); - } - - public static Clients builder() { - return new Clients(); - } - - public Clients put(String key, String value) { - settingsBuilder.put(key, value); - return this; - } - - public Clients put(String key, Integer value) { - settingsBuilder.put(key, value); - return this; - } - - public Clients put(String key, Long value) { - settingsBuilder.put(key, value); - return this; - } - - public Clients put(String key, Double value) { - settingsBuilder.put(key, value); - return this; - } - - public Clients put(String key, ByteSizeValue value) { - settingsBuilder.put(key, value); - return this; - } - - public Clients put(String key, TimeValue value) { - settingsBuilder.put(key, value); - return this; - } - - public Clients put(Settings settings) { - settingsBuilder.put(settings); - return this; - } - - public Clients setMetric(BulkMetric metric) { - this.metric = metric; - return this; - } - - public Clients setControl(BulkControl control) { - this.control = control; - return this; - } - - public BulkNodeClient toBulkNodeClient(Client client) { - Settings settings = settingsBuilder.build(); - return new BulkNodeClient() - .maxActionsPerRequest(settings.getAsInt(MAX_ACTIONS_PER_REQUEST, DEFAULT_MAX_ACTIONS_PER_REQUEST)) - .maxConcurrentRequests(settings.getAsInt(MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_REQUESTS)) - .maxVolumePerRequest(settings.get(MAX_VOLUME_PER_REQUEST, DEFAULT_MAX_VOLUME_PER_REQUEST)) - .flushIngestInterval(settings.get(FLUSH_INTERVAL, DEFAULT_FLUSH_INTERVAL)) - .init(client, metric, control); - } - - public BulkTransportClient toBulkTransportClient() { - Settings settings = settingsBuilder.build(); - return new BulkTransportClient() - .maxActionsPerRequest(settings.getAsInt(MAX_ACTIONS_PER_REQUEST, DEFAULT_MAX_ACTIONS_PER_REQUEST)) - .maxConcurrentRequests(settings.getAsInt(MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_REQUESTS)) - .maxVolumePerRequest(settings.get(MAX_VOLUME_PER_REQUEST, DEFAULT_MAX_VOLUME_PER_REQUEST)) - .flushIngestInterval(settings.get(FLUSH_INTERVAL, DEFAULT_FLUSH_INTERVAL)) - .init(settings, metric, control); - } - - public MockTransportClient toMockTransportClient() { - Settings settings = settingsBuilder.build(); - return new MockTransportClient() - .maxActionsPerRequest(settings.getAsInt(MAX_ACTIONS_PER_REQUEST, DEFAULT_MAX_ACTIONS_PER_REQUEST)) - .maxConcurrentRequests(settings.getAsInt(MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_REQUESTS)) - .maxVolumePerRequest(settings.get(MAX_VOLUME_PER_REQUEST, DEFAULT_MAX_VOLUME_PER_REQUEST)) - .flushIngestInterval(settings.get(FLUSH_INTERVAL, DEFAULT_FLUSH_INTERVAL)) - .init(settings, metric, control); - } - -} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/Parameters.java b/src/main/java/org/xbib/elasticsearch/extras/client/Parameters.java deleted file mode 100644 index d77ce24..0000000 --- a/src/main/java/org/xbib/elasticsearch/extras/client/Parameters.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.xbib.elasticsearch.extras.client; - -/** - * - */ -public interface Parameters { - - int DEFAULT_MAX_ACTIONS_PER_REQUEST = 1000; - - int DEFAULT_MAX_CONCURRENT_REQUESTS = Runtime.getRuntime().availableProcessors() * 4; - - String DEFAULT_MAX_VOLUME_PER_REQUEST = "10mb"; - - String DEFAULT_FLUSH_INTERVAL = "30s"; - - String MAX_ACTIONS_PER_REQUEST = "max_actions_per_request"; - - String MAX_CONCURRENT_REQUESTS = "max_concurrent_requests"; - - String MAX_VOLUME_PER_REQUEST = "max_volume_per_request"; - - String FLUSH_INTERVAL = "flush_interval"; - -} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClient.java b/src/main/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClient.java deleted file mode 100644 index 0f387b6..0000000 --- a/src/main/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClient.java +++ /dev/null @@ -1,513 +0,0 @@ -package org.xbib.elasticsearch.extras.client.node; - -import com.google.common.collect.ImmutableSet; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.indices.create.CreateIndexAction; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; -import org.elasticsearch.action.bulk.BulkItemResponse; -import org.elasticsearch.action.bulk.BulkProcessor; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.env.Environment; -import org.elasticsearch.node.Node; -import org.elasticsearch.plugins.Plugin; -import org.xbib.elasticsearch.extras.client.AbstractClient; -import org.xbib.elasticsearch.extras.client.BulkControl; -import org.xbib.elasticsearch.extras.client.BulkMetric; -import org.xbib.elasticsearch.extras.client.ClientMethods; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Collection; -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; - -/** - * - */ -public class BulkNodeClient extends AbstractClient implements ClientMethods { - - private static final ESLogger logger = ESLoggerFactory.getLogger(BulkNodeClient.class.getName()); - - private int maxActionsPerRequest = DEFAULT_MAX_ACTIONS_PER_REQUEST; - - private int maxConcurrentRequests = DEFAULT_MAX_CONCURRENT_REQUESTS; - - private ByteSizeValue maxVolume; - - private TimeValue flushInterval; - - private Node node; - - private ElasticsearchClient client; - - private BulkProcessor bulkProcessor; - - private BulkMetric metric; - - private BulkControl control; - - private Throwable throwable; - - private boolean closed; - - @Override - public BulkNodeClient maxActionsPerRequest(int maxActionsPerRequest) { - this.maxActionsPerRequest = maxActionsPerRequest; - return this; - } - - @Override - public BulkNodeClient maxConcurrentRequests(int maxConcurrentRequests) { - this.maxConcurrentRequests = maxConcurrentRequests; - return this; - } - - @Override - public BulkNodeClient maxVolumePerRequest(String maxVolume) { - this.maxVolume = ByteSizeValue.parseBytesSizeValue(maxVolume, "maxVolumePerRequest"); - return this; - } - - @Override - public BulkNodeClient flushIngestInterval(String flushInterval) { - this.flushInterval = TimeValue.parseTimeValue(flushInterval, TimeValue.timeValueSeconds(5), "flushIngestInterval"); - return this; - } - - @Override - public BulkNodeClient init(ElasticsearchClient client, - final BulkMetric metric, final BulkControl control) { - this.client = client; - this.metric = metric; - this.control = control; - if (metric != null) { - metric.start(); - } - BulkProcessor.Listener listener = new BulkProcessor.Listener() { - @Override - public void beforeBulk(long executionId, BulkRequest request) { - long l = -1; - if (metric != null) { - metric.getCurrentIngest().inc(); - l = metric.getCurrentIngest().getCount(); - int n = request.numberOfActions(); - metric.getSubmitted().inc(n); - metric.getCurrentIngestNumDocs().inc(n); - metric.getTotalIngestSizeInBytes().inc(request.estimatedSizeInBytes()); - } - logger.debug("before bulk [{}] [actions={}] [bytes={}] [concurrent requests={}]", - executionId, - request.numberOfActions(), - request.estimatedSizeInBytes(), - l); - } - - @Override - public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { - long l = -1; - if (metric != null) { - metric.getCurrentIngest().dec(); - l = metric.getCurrentIngest().getCount(); - metric.getSucceeded().inc(response.getItems().length); - } - int n = 0; - for (BulkItemResponse itemResponse : response.getItems()) { - if (metric != null) { - metric.getCurrentIngest().dec(itemResponse.getIndex(), itemResponse.getType(), itemResponse.getId()); - } - if (itemResponse.isFailed()) { - n++; - if (metric != null) { - metric.getSucceeded().dec(1); - metric.getFailed().inc(1); - } - } - } - if (metric != null) { - logger.debug("after bulk [{}] [succeeded={}] [failed={}] [{}ms] {} concurrent requests", - executionId, - metric.getSucceeded().getCount(), - metric.getFailed().getCount(), - response.getTook().millis(), - l); - } - if (n > 0) { - logger.error("bulk [{}] failed with {} failed items, failure message = {}", - executionId, n, response.buildFailureMessage()); - } else { - if (metric != null) { - metric.getCurrentIngestNumDocs().dec(response.getItems().length); - } - } - } - - @Override - public void afterBulk(long executionId, BulkRequest request, Throwable failure) { - if (metric != null) { - metric.getCurrentIngest().dec(); - } - throwable = failure; - closed = true; - logger.error("after bulk [" + executionId + "] error", failure); - } - }; - BulkProcessor.Builder builder = BulkProcessor.builder((Client) client, listener) - .setBulkActions(maxActionsPerRequest) - .setConcurrentRequests(maxConcurrentRequests) - .setFlushInterval(flushInterval); - if (maxVolume != null) { - builder.setBulkSize(maxVolume); - } - this.bulkProcessor = builder.build(); - this.closed = false; - return this; - } - - @Override - public BulkNodeClient init(Settings settings, BulkMetric metric, BulkControl control) throws IOException { - createClient(settings); - this.metric = metric; - this.control = control; - return this; - } - - @Override - public ElasticsearchClient client() { - return client; - } - - @Override - protected synchronized void createClient(Settings settings) throws IOException { - if (client != null) { - logger.warn("client is open, closing..."); - client.threadPool().shutdown(); - client = null; - node.close(); - } - if (settings != null) { - String version = System.getProperty("os.name") - + " " + System.getProperty("java.vm.name") - + " " + System.getProperty("java.vm.vendor") - + " " + System.getProperty("java.runtime.version") - + " " + System.getProperty("java.vm.version"); - Settings effectiveSettings = Settings.builder().put(settings) - .put("node.client", true) - .put("node.master", false) - .put("node.data", false).build(); - logger.info("creating node client on {} with effective settings {}", - version, effectiveSettings.getAsMap()); - Collection> plugins = Collections.emptyList(); - this.node = new BulkNode(new Environment(effectiveSettings), plugins); - node.start(); - this.client = node.client(); - } - } - - @Override - public BulkMetric getMetric() { - return metric; - } - - @Override - public BulkNodeClient index(String index, String type, String id, String source) { - if (closed) { - throwClose(); - } - try { - if (metric != null) { - metric.getCurrentIngest().inc(index, type, id); - } - bulkProcessor.add(new IndexRequest(index).type(type).id(id).create(false).source(source)); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of index request failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkNodeClient bulkIndex(IndexRequest indexRequest) { - if (closed) { - throwClose(); - } - try { - if (metric != null) { - metric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id()); - } - bulkProcessor.add(indexRequest); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of index request failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkNodeClient delete(String index, String type, String id) { - if (closed) { - throwClose(); - } - try { - if (metric != null) { - metric.getCurrentIngest().inc(index, type, id); - } - bulkProcessor.add(new DeleteRequest(index).type(type).id(id)); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of delete failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkNodeClient bulkDelete(DeleteRequest deleteRequest) { - if (closed) { - throwClose(); - } - try { - if (metric != null) { - metric.getCurrentIngest().inc(deleteRequest.index(), deleteRequest.type(), deleteRequest.id()); - } - bulkProcessor.add(deleteRequest); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of delete failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkNodeClient update(String index, String type, String id, String source) { - if (closed) { - throwClose(); - } - try { - if (metric != null) { - metric.getCurrentIngest().inc(index, type, id); - } - bulkProcessor.add(new UpdateRequest().index(index).type(type).id(id).upsert(source)); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of update request failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkNodeClient bulkUpdate(UpdateRequest updateRequest) { - if (closed) { - throwClose(); - } - try { - if (metric != null) { - metric.getCurrentIngest().inc(updateRequest.index(), updateRequest.type(), updateRequest.id()); - } - bulkProcessor.add(updateRequest); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of update request failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkNodeClient flushIngest() { - if (closed) { - throwClose(); - } - logger.debug("flushing bulk processor"); - bulkProcessor.flush(); - return this; - } - - @Override - public BulkNodeClient waitForResponses(String maxWaitTime) throws InterruptedException, ExecutionException { - if (closed) { - throwClose(); - } - while (!bulkProcessor.awaitClose(TimeValue.parseTimeValue(maxWaitTime, TimeValue.timeValueSeconds(30), - "maxWaitTime").getMillis(), TimeUnit.MILLISECONDS)) { - logger.warn("still waiting for responses"); - } - return this; - } - - @Override - public BulkNodeClient startBulk(String index, long startRefreshIntervalMillis, long stopRefreshItervalMillis) - throws IOException { - if (control == null) { - return this; - } - if (!control.isBulk(index)) { - control.startBulk(index, startRefreshIntervalMillis, stopRefreshItervalMillis); - updateIndexSetting(index, "refresh_interval", startRefreshIntervalMillis + "ms"); - } - return this; - } - - @Override - public BulkNodeClient stopBulk(String index) throws IOException { - if (control == null) { - return this; - } - if (control.isBulk(index)) { - updateIndexSetting(index, "refresh_interval", control.getStopBulkRefreshIntervals().get(index) + "ms"); - control.finishBulk(index); - } - return this; - } - - @Override - public synchronized void shutdown() { - try { - if (bulkProcessor != null) { - logger.debug("closing bulk processor..."); - bulkProcessor.close(); - } - if (control != null && control.indices() != null && !control.indices().isEmpty()) { - logger.debug("stopping bulk mode for indices {}...", control.indices()); - for (String index : ImmutableSet.copyOf(control.indices())) { - stopBulk(index); - } - metric.stop(); - } - if (node != null) { - logger.debug("closing node..."); - node.close(); - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - @Override - public BulkNodeClient newIndex(String index) { - return newIndex(index, null, null); - } - - @Override - public BulkNodeClient newIndex(String index, String type, InputStream settings, InputStream mappings) throws IOException { - resetSettings(); - setting(settings); - mapping(type, mappings); - return newIndex(index, settings(), mappings()); - } - - @Override - public BulkNodeClient newIndex(String index, Settings settings, Map mappings) { - if (closed) { - throwClose(); - } - if (client == null) { - logger.warn("no client for create index"); - return this; - } - if (index == null) { - logger.warn("no index name given to create index"); - return this; - } - CreateIndexRequestBuilder createIndexRequestBuilder = - new CreateIndexRequestBuilder(client(), CreateIndexAction.INSTANCE).setIndex(index); - if (settings != null) { - logger.info("settings = {}", settings.getAsStructuredMap()); - createIndexRequestBuilder.setSettings(settings); - } - if (mappings != null) { - for (Map.Entry entry : mappings.entrySet()) { - String type = entry.getKey(); - String mapping = entry.getValue(); - logger.info("found mapping for {}", type); - createIndexRequestBuilder.addMapping(type, mapping); - } - } - createIndexRequestBuilder.execute().actionGet(); - logger.info("index {} created", index); - return this; - } - - @Override - public BulkNodeClient newMapping(String index, String type, Map mapping) { - PutMappingRequestBuilder putMappingRequestBuilder = - new PutMappingRequestBuilder(client(), PutMappingAction.INSTANCE) - .setIndices(index) - .setType(type) - .setSource(mapping); - putMappingRequestBuilder.execute().actionGet(); - logger.info("mapping created for index {} and type {}", index, type); - return this; - } - - @Override - public BulkNodeClient deleteIndex(String index) { - if (closed) { - throwClose(); - } - if (client == null) { - logger.warn("no client"); - return this; - } - if (index == null) { - logger.warn("no index name given to delete index"); - return this; - } - DeleteIndexRequestBuilder deleteIndexRequestBuilder = - new DeleteIndexRequestBuilder(client(), DeleteIndexAction.INSTANCE, index); - deleteIndexRequestBuilder.execute().actionGet(); - return this; - } - - @Override - public boolean hasThrowable() { - return throwable != null; - } - - @Override - public Throwable getThrowable() { - return throwable; - } - - public Settings getSettings() { - return settings(); - } - - @Override - public Settings.Builder getSettingsBuilder() { - return settingsBuilder(); - } - - private static void throwClose() { - throw new ElasticsearchException("client is closed"); - } - - private class BulkNode extends Node { - - BulkNode(Environment env, Collection> classpathPlugins) { - super(env, Version.CURRENT, classpathPlugins); - } - } - -} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/node/package-info.java b/src/main/java/org/xbib/elasticsearch/extras/client/node/package-info.java deleted file mode 100644 index c5c0895..0000000 --- a/src/main/java/org/xbib/elasticsearch/extras/client/node/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Classes for Elasticsearch node client extras. - */ -package org.xbib.elasticsearch.extras.client.node; diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/package-info.java b/src/main/java/org/xbib/elasticsearch/extras/client/package-info.java deleted file mode 100644 index c231c60..0000000 --- a/src/main/java/org/xbib/elasticsearch/extras/client/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Classes for Elasticsearch client extras. - */ -package org.xbib.elasticsearch.extras.client; diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClient.java b/src/main/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClient.java deleted file mode 100644 index ac37781..0000000 --- a/src/main/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClient.java +++ /dev/null @@ -1,564 +0,0 @@ -package org.xbib.elasticsearch.extras.client.transport; - -import com.google.common.collect.ImmutableSet; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.action.admin.indices.create.CreateIndexAction; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; -import org.elasticsearch.action.bulk.BulkItemResponse; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.InetSocketTransportAddress; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.xbib.elasticsearch.extras.client.AbstractClient; -import org.xbib.elasticsearch.extras.client.BulkControl; -import org.xbib.elasticsearch.extras.client.BulkMetric; -import org.xbib.elasticsearch.extras.client.BulkProcessor; -import org.xbib.elasticsearch.extras.client.ClientMethods; -import org.xbib.elasticsearch.extras.client.NetworkUtils; - -import java.io.IOException; -import java.io.InputStream; -import java.net.InetAddress; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; - -/** - * Transport client with addtitional methods using the BulkProcessor. - */ -public class BulkTransportClient extends AbstractClient implements ClientMethods { - - private static final ESLogger logger = ESLoggerFactory.getLogger(BulkTransportClient.class.getName()); - - private int maxActionsPerRequest = DEFAULT_MAX_ACTIONS_PER_REQUEST; - - private int maxConcurrentRequests = DEFAULT_MAX_CONCURRENT_REQUESTS; - - private ByteSizeValue maxVolumePerRequest; - - private TimeValue flushInterval; - - private BulkProcessor bulkProcessor; - - private Throwable throwable; - - private boolean closed; - - private TransportClient client; - - private BulkMetric metric; - - private BulkControl control; - - private boolean ignoreBulkErrors; - - private boolean isShutdown; - - @Override - public BulkTransportClient init(ElasticsearchClient client, BulkMetric metric, BulkControl control) throws IOException { - return init(findSettings(), metric, control); - } - - @Override - public BulkTransportClient init(Settings settings, final BulkMetric metric, final BulkControl control) { - createClient(settings); - this.metric = metric; - this.control = control; - if (metric != null) { - metric.start(); - } - resetSettings(); - BulkProcessor.Listener listener = new BulkProcessor.Listener() { - @Override - public void beforeBulk(long executionId, BulkRequest request) { - long l = -1L; - if (metric != null) { - metric.getCurrentIngest().inc(); - l = metric.getCurrentIngest().getCount(); - int n = request.numberOfActions(); - metric.getSubmitted().inc(n); - metric.getCurrentIngestNumDocs().inc(n); - metric.getTotalIngestSizeInBytes().inc(request.estimatedSizeInBytes()); - } - logger.debug("before bulk [{}] [actions={}] [bytes={}] [concurrent requests={}]", - executionId, - request.numberOfActions(), - request.estimatedSizeInBytes(), - l); - } - - @Override - public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { - long l = -1L; - if (metric != null) { - metric.getCurrentIngest().dec(); - l = metric.getCurrentIngest().getCount(); - metric.getSucceeded().inc(response.getItems().length); - } - int n = 0; - for (BulkItemResponse itemResponse : response.getItems()) { - if (metric != null) { - metric.getCurrentIngest().dec(itemResponse.getIndex(), itemResponse.getType(), itemResponse.getId()); - if (itemResponse.isFailed()) { - n++; - metric.getSucceeded().dec(1); - metric.getFailed().inc(1); - } - } - } - if (metric != null) { - logger.debug("after bulk [{}] [succeeded={}] [failed={}] [{}ms] [concurrent requests={}]", - executionId, - metric.getSucceeded().getCount(), - metric.getFailed().getCount(), - response.getTook().millis(), - l); - } - if (n > 0) { - logger.error("bulk [{}] failed with {} failed items, failure message = {}", - executionId, n, response.buildFailureMessage()); - } else { - if (metric != null) { - metric.getCurrentIngestNumDocs().dec(response.getItems().length); - } - } - } - - @Override - public void afterBulk(long executionId, BulkRequest requst, Throwable failure) { - if (metric != null) { - metric.getCurrentIngest().dec(); - } - throwable = failure; - if (!ignoreBulkErrors) { - closed = true; - } - logger.error("bulk [" + executionId + "] error", failure); - } - }; - BulkProcessor.Builder builder = BulkProcessor.builder(client, listener) - .setBulkActions(maxActionsPerRequest) - .setConcurrentRequests(maxConcurrentRequests) - .setFlushInterval(flushInterval); - if (maxVolumePerRequest != null) { - builder.setBulkSize(maxVolumePerRequest); - } - this.bulkProcessor = builder.build(); - try { - Collection addrs = findAddresses(settings); - if (!connect(addrs, settings.getAsBoolean("autodiscover", false))) { - throw new NoNodeAvailableException("no cluster nodes available, check settings " - + settings.getAsMap()); - } - } catch (IOException e) { - logger.error(e.getMessage(), e); - } - this.closed = false; - return this; - } - - @Override - public ClientMethods newMapping(String index, String type, Map mapping) { - new PutMappingRequestBuilder(client(), PutMappingAction.INSTANCE) - .setIndices(index) - .setType(type) - .setSource(mapping) - .execute().actionGet(); - logger.info("mapping created for index {} and type {}", index, type); - return this; - } - - @Override - protected void createClient(Settings settings) { - if (client != null) { - logger.warn("client is open, closing..."); - client.close(); - client.threadPool().shutdown(); - client = null; - } - if (settings != null) { - String version = System.getProperty("os.name") - + " " + System.getProperty("java.vm.name") - + " " + System.getProperty("java.vm.vendor") - + " " + System.getProperty("java.runtime.version") - + " " + System.getProperty("java.vm.version"); - logger.info("creating transport client on {} with effective settings {}", - version, settings.getAsMap()); - this.client = TransportClient.builder() - .settings(settings) - .build(); - this.ignoreBulkErrors = settings.getAsBoolean("ignoreBulkErrors", true); - } - } - - public boolean isShutdown() { - return isShutdown; - } - - @Override - public BulkTransportClient maxActionsPerRequest(int maxActionsPerRequest) { - this.maxActionsPerRequest = maxActionsPerRequest; - return this; - } - - @Override - public BulkTransportClient maxConcurrentRequests(int maxConcurrentRequests) { - this.maxConcurrentRequests = maxConcurrentRequests; - return this; - } - - @Override - public BulkTransportClient maxVolumePerRequest(String maxVolumePerRequest) { - this.maxVolumePerRequest = ByteSizeValue.parseBytesSizeValue(maxVolumePerRequest, "maxVolumePerRequest"); - return this; - } - - @Override - public BulkTransportClient flushIngestInterval(String flushInterval) { - this.flushInterval = TimeValue.parseTimeValue(flushInterval, TimeValue.timeValueSeconds(5), "flushIngestInterval"); - return this; - } - - @Override - public ElasticsearchClient client() { - return client; - } - - @Override - public BulkMetric getMetric() { - return metric; - } - - @Override - public ClientMethods newIndex(String index) { - if (closed) { - throwClose(); - } - return newIndex(index, null, null); - } - - @Override - public ClientMethods newIndex(String index, String type, InputStream settings, InputStream mappings) throws IOException { - resetSettings(); - setting(settings); - mapping(type, mappings); - return newIndex(index, settings(), mappings()); - } - - @Override - public ClientMethods newIndex(String index, Settings settings, Map mappings) { - if (closed) { - throwClose(); - } - if (index == null) { - logger.warn("no index name given to create index"); - return this; - } - CreateIndexRequestBuilder createIndexRequestBuilder = - new CreateIndexRequestBuilder(client(), CreateIndexAction.INSTANCE).setIndex(index); - if (settings != null) { - logger.info("settings = {}", settings.getAsStructuredMap()); - createIndexRequestBuilder.setSettings(settings); - } - if (mappings != null) { - for (Map.Entry entry : mappings.entrySet()) { - String type = entry.getKey(); - String mapping = entry.getValue(); - logger.info("found mapping for {}", type); - createIndexRequestBuilder.addMapping(type, mapping); - } - } - createIndexRequestBuilder.execute().actionGet(); - logger.info("index {} created", index); - return this; - } - - @Override - public ClientMethods deleteIndex(String index) { - if (closed) { - throwClose(); - } - if (index == null) { - logger.warn("no index name given to delete index"); - return this; - } - new DeleteIndexRequestBuilder(client(), DeleteIndexAction.INSTANCE, index).execute().actionGet(); - return this; - } - - @Override - public ClientMethods startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds) - throws IOException { - if (control == null) { - return this; - } - if (!control.isBulk(index)) { - control.startBulk(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds); - updateIndexSetting(index, "refresh_interval", startRefreshIntervalSeconds + "s"); - } - return this; - } - - @Override - public ClientMethods stopBulk(String index) throws IOException { - if (control == null) { - return this; - } - if (control.isBulk(index)) { - updateIndexSetting(index, "refresh_interval", control.getStopBulkRefreshIntervals().get(index) + "s"); - control.finishBulk(index); - } - return this; - } - - @Override - public BulkTransportClient index(String index, String type, String id, String source) { - if (closed) { - throwClose(); - } - try { - metric.getCurrentIngest().inc(index, type, id); - bulkProcessor.add(new IndexRequest().index(index).type(type).id(id).create(false).source(source)); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of index request failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkTransportClient bulkIndex(IndexRequest indexRequest) { - if (closed) { - throwClose(); - } - try { - metric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id()); - bulkProcessor.add(indexRequest); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of index request failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkTransportClient delete(String index, String type, String id) { - if (closed) { - throwClose(); - } - try { - metric.getCurrentIngest().inc(index, type, id); - bulkProcessor.add(new DeleteRequest().index(index).type(type).id(id)); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of delete request failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkTransportClient bulkDelete(DeleteRequest deleteRequest) { - if (closed) { - throwClose(); - } - try { - metric.getCurrentIngest().inc(deleteRequest.index(), deleteRequest.type(), deleteRequest.id()); - bulkProcessor.add(deleteRequest); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of delete request failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkTransportClient update(String index, String type, String id, String source) { - if (closed) { - throwClose(); - } - try { - metric.getCurrentIngest().inc(index, type, id); - bulkProcessor.add(new UpdateRequest().index(index).type(type).id(id).upsert(source)); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of update request failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkTransportClient bulkUpdate(UpdateRequest updateRequest) { - if (closed) { - throwClose(); - } - try { - metric.getCurrentIngest().inc(updateRequest.index(), updateRequest.type(), updateRequest.id()); - bulkProcessor.add(updateRequest); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of update request failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public synchronized BulkTransportClient flushIngest() { - if (closed) { - throwClose(); - } - logger.debug("flushing bulk processor"); - bulkProcessor.flush(); - return this; - } - - @Override - public synchronized BulkTransportClient waitForResponses(String maxWaitTime) - throws InterruptedException, ExecutionException { - if (closed) { - throwClose(); - } - bulkProcessor.awaitClose(TimeValue.parseTimeValue(maxWaitTime, - TimeValue.timeValueSeconds(30), "maxWaitTime").getMillis(), TimeUnit.MILLISECONDS); - return this; - } - - @Override - public synchronized void shutdown() { - if (closed) { - shutdownClient(); - throwClose(); - } - try { - if (bulkProcessor != null) { - logger.debug("closing bulk processor..."); - bulkProcessor.close(); - } - if (control != null && control.indices() != null && !control.indices().isEmpty()) { - logger.debug("stopping bulk mode for indices {}...", control.indices()); - for (String index : ImmutableSet.copyOf(control.indices())) { - stopBulk(index); - } - metric.stop(); - } - logger.debug("shutting down..."); - shutdownClient(); - logger.debug("shutting down completed"); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - @Override - public boolean hasThrowable() { - return throwable != null; - } - - @Override - public Throwable getThrowable() { - return throwable; - } - - private Settings findSettings() { - Settings.Builder settingsBuilder = Settings.settingsBuilder(); - settingsBuilder.put("host", "localhost"); - try { - String hostname = NetworkUtils.getLocalAddress().getHostName(); - logger.debug("the hostname is {}", hostname); - settingsBuilder.put("host", hostname) - .put("port", 9300); - } catch (Exception e) { - logger.warn(e.getMessage(), e); - } - return settingsBuilder.build(); - } - - private Collection findAddresses(Settings settings) throws IOException { - String[] hostnames = settings.getAsArray("host", new String[]{"localhost"}); - int port = settings.getAsInt("port", 9300); - Collection addresses = new ArrayList<>(); - for (String hostname : hostnames) { - String[] splitHost = hostname.split(":", 2); - if (splitHost.length == 2) { - String host = splitHost[0]; - InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null); - try { - port = Integer.parseInt(splitHost[1]); - } catch (Exception e) { - logger.warn(e.getMessage(), e); - } - addresses.add(new InetSocketTransportAddress(inetAddress, port)); - } - if (splitHost.length == 1) { - String host = splitHost[0]; - InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null); - addresses.add(new InetSocketTransportAddress(inetAddress, port)); - } - } - return addresses; - } - - private static void throwClose() { - throw new ElasticsearchException("client is closed"); - } - - private void shutdownClient() { - if (client != null) { - logger.debug("shutdown started"); - client.close(); - client.threadPool().shutdown(); - client = null; - logger.debug("shutdown complete"); - } - isShutdown = true; - } - - private boolean connect(Collection addresses, boolean autodiscover) { - logger.info("trying to connect to {}", addresses); - client.addTransportAddresses(addresses); - if (client.connectedNodes() != null) { - List nodes = client.connectedNodes(); - if (!nodes.isEmpty()) { - logger.info("connected to {}", nodes); - if (autodiscover) { - logger.info("trying to auto-discover all cluster nodes..."); - ClusterStateRequestBuilder clusterStateRequestBuilder = - new ClusterStateRequestBuilder(client, ClusterStateAction.INSTANCE); - ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); - DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes(); - client.addDiscoveryNodes(discoveryNodes); - logger.info("after auto-discovery connected to {}", client.connectedNodes()); - } - return true; - } - return false; - } - return false; - } -} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/transport/MockTransportClient.java b/src/main/java/org/xbib/elasticsearch/extras/client/transport/MockTransportClient.java deleted file mode 100644 index ed0fcc7..0000000 --- a/src/main/java/org/xbib/elasticsearch/extras/client/transport/MockTransportClient.java +++ /dev/null @@ -1,155 +0,0 @@ -package org.xbib.elasticsearch.extras.client.transport; - -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.settings.Settings; -import org.xbib.elasticsearch.extras.client.BulkControl; -import org.xbib.elasticsearch.extras.client.BulkMetric; - -import java.io.IOException; -import java.util.Map; - -/** - * Mock client, it does not perform actions on a cluster. - * Useful for testing or dry runs. - */ -public class MockTransportClient extends BulkTransportClient { - - @Override - public ElasticsearchClient client() { - return null; - } - - @Override - public MockTransportClient init(ElasticsearchClient client, BulkMetric metric, BulkControl control) { - return this; - } - - @Override - public MockTransportClient init(Settings settings, BulkMetric metric, BulkControl control) { - return this; - } - - @Override - public MockTransportClient maxActionsPerRequest(int maxActions) { - return this; - } - - @Override - public MockTransportClient maxConcurrentRequests(int maxConcurrentRequests) { - return this; - } - - @Override - public MockTransportClient maxVolumePerRequest(String maxVolumePerRequest) { - return this; - } - - @Override - public MockTransportClient flushIngestInterval(String interval) { - return this; - } - - @Override - public MockTransportClient index(String index, String type, String id, String source) { - return this; - } - - @Override - public MockTransportClient delete(String index, String type, String id) { - return this; - } - - @Override - public MockTransportClient update(String index, String type, String id, String source) { - return this; - } - - @Override - public MockTransportClient bulkIndex(IndexRequest indexRequest) { - return this; - } - - @Override - public MockTransportClient bulkDelete(DeleteRequest deleteRequest) { - return this; - } - - @Override - public MockTransportClient bulkUpdate(UpdateRequest updateRequest) { - return this; - } - - @Override - public MockTransportClient flushIngest() { - return this; - } - - @Override - public MockTransportClient waitForResponses(String timeValue) throws InterruptedException { - return this; - } - - @Override - public MockTransportClient startBulk(String index, long startRefreshInterval, long stopRefreshIterval) { - return this; - } - - @Override - public MockTransportClient stopBulk(String index) { - return this; - } - - @Override - public MockTransportClient deleteIndex(String index) { - return this; - } - - @Override - public MockTransportClient newIndex(String index) { - return this; - } - - @Override - public MockTransportClient newMapping(String index, String type, Map mapping) { - return this; - } - - @Override - public void putMapping(String index) { - // mockup method - } - - @Override - public void refreshIndex(String index) { - // mockup method - } - - @Override - public void flushIndex(String index) { - // mockup method - } - - @Override - public void waitForCluster(String healthColor, String timeValue) throws IOException { - // mockup method - } - - @Override - public int waitForRecovery(String index) throws IOException { - return -1; - } - - @Override - public int updateReplicaLevel(String index, int level) throws IOException { - return -1; - } - - @Override - public void shutdown() { - // mockup method - } - -} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/transport/package-info.java b/src/main/java/org/xbib/elasticsearch/extras/client/transport/package-info.java deleted file mode 100644 index ac6a50d..0000000 --- a/src/main/java/org/xbib/elasticsearch/extras/client/transport/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Classes for Elasticsearch transport client extras. - */ -package org.xbib.elasticsearch.extras.client.transport; From 4701447d3c1e9c0bfa06b2d1e0ad4fe4c16b1a7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Tue, 19 Feb 2019 17:29:51 +0100 Subject: [PATCH 2/3] clean up extended client --- build.gradle | 2 +- .../java/org/xbib/elx/api/BulkControl.java | 2 + .../java/org/xbib/elx/api/ExtendedClient.java | 421 ++++++---- .../org/xbib/elx/api}/IndexDefinition.java | 28 +- .../org/xbib/elx/api}/IndexRetention.java | 6 +- .../elx/common/AbstractExtendedClient.java | 757 ++++++++---------- .../org/xbib/elx/common/BulkProcessor.java | 42 +- .../xbib/elx/common/MockExtendedClient.java | 73 +- .../java/org/xbib/elx/common/Parameters.java | 2 +- .../xbib/elx/common/SimpleBulkControl.java | 32 +- .../elx/common/management/package-info.java | 1 - .../java/org/xbib/elx/common/AliasTest.java | 22 +- .../java/org/elasticsearch/node/MockNode.java | 4 - .../elx/node/ExtendeNodeDuplicateIDTest.java | 2 +- .../ExtendedNodeClientSingleNodeTest.java | 39 - .../xbib/elx/node/ExtendedNodeClientTest.java | 28 +- .../elx/node/ExtendedNodeIndexAliasTest.java | 8 +- .../elx/node/ExtendedNodeReplicaTest.java | 9 +- .../xbib/elx/node/ExtendedNodeSmokeTest.java | 69 ++ .../ExtendedNodeUpdateReplicaLevelTest.java | 10 +- .../java/org/xbib/elx/node/NodeTestUtils.java | 14 +- .../transport/ExtendedTransportClient.java | 26 +- ...ExtendedTransportClientSingleNodeTest.java | 4 +- .../ExtendedTransportClientTest.java | 63 +- .../ExtendedTransportDuplicateIDTest.java | 2 +- .../ExtendedTransportIndexAliasTest.java | 35 +- .../ExtendedTransportReplicaTest.java | 24 +- ...tendedTransportUpdateReplicaLevelTest.java | 10 +- gradle.properties | 2 +- gradle/sonarqube.gradle | 41 - 30 files changed, 929 insertions(+), 849 deletions(-) rename {elx-common/src/main/java/org/xbib/elx/common/management => elx-api/src/main/java/org/xbib/elx/api}/IndexDefinition.java (79%) rename {elx-common/src/main/java/org/xbib/elx/common/management => elx-api/src/main/java/org/xbib/elx/api}/IndexRetention.java (73%) delete mode 100644 elx-common/src/main/java/org/xbib/elx/common/management/package-info.java delete mode 100644 elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientSingleNodeTest.java create mode 100644 elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeSmokeTest.java delete mode 100644 gradle/sonarqube.gradle diff --git a/build.gradle b/build.gradle index 01e2104..090181a 100644 --- a/build.gradle +++ b/build.gradle @@ -67,7 +67,7 @@ subprojects { } clean { - delete "plugins" + delete "data" delete "logs" delete "out" } diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkControl.java b/elx-api/src/main/java/org/xbib/elx/api/BulkControl.java index e0fcf84..c43c137 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/BulkControl.java +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkControl.java @@ -16,4 +16,6 @@ public interface BulkControl { Map getStartBulkRefreshIntervals(); Map getStopBulkRefreshIntervals(); + + String getMaxWaitTime(); } diff --git a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java index 4ba2496..9c2f0e1 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java +++ b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java @@ -10,17 +10,16 @@ import java.io.IOException; import java.io.InputStream; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutionException; /** - * Interface for providing extended administrative methods for managing and ingesting Elasticsearch. + * Interface for extended managing and indexing methods of an Elasticsearch client. */ public interface ExtendedClient { /** * Set an Elasticsearch client to extend from it. May be null for TransportClient. * @param client client - * @return an ELasticsearch client + * @return this client */ ExtendedClient setClient(ElasticsearchClient client); @@ -31,16 +30,8 @@ public interface ExtendedClient { */ ElasticsearchClient getClient(); - ExtendedClient setBulkMetric(BulkMetric bulkMetric); - - BulkMetric getBulkMetric(); - - ExtendedClient setBulkControl(BulkControl bulkControl); - - BulkControl getBulkControl(); - /** - * Create new Elasticsearch client, wrap an existing Elasticsearch client. + * Initiative the extended client, cerates instances and connect to cluster, if required. * * @param settings settings * @return this client @@ -49,57 +40,91 @@ public interface ExtendedClient { ExtendedClient init(Settings settings) throws IOException; /** - * Bulked index request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when bulk limits are exceeded. + * Set bulk metric. + * @param bulkMetric the bulk metric + * @return this client + */ + ExtendedClient setBulkMetric(BulkMetric bulkMetric); + + /** + * Get bulk metric. + * @return the bulk metric + */ + BulkMetric getBulkMetric(); + + /** + * Set bulk control. + * @param bulkControl the bulk control + * @return this + */ + ExtendedClient setBulkControl(BulkControl bulkControl); + + /** + * Get buulk control. + * @return the bulk control + */ + BulkControl getBulkControl(); + + /** + * Build index definition from settings. + * + * @param index the index name + * @param settings the settings for the index + * @return index definition + * @throws IOException if settings/mapping URL is invalid/malformed + */ + IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException; + + /** + * Add index request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when limits are exceeded. * * @param index the index - * @param type the type * @param id the id * @param create true if document must be created * @param source the source * @return this */ - ExtendedClient index(String index, String type, String id, boolean create, BytesReference source); + ExtendedClient index(String index, String id, boolean create, BytesReference source); /** - * Index document. + * Index request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when limits are exceeded. * * @param index the index - * @param type the type * @param id the id * @param create true if document is to be created, false otherwise * @param source the source * @return this client methods */ - ExtendedClient index(String index, String type, String id, boolean create, String source); + ExtendedClient index(String index, String id, boolean create, String source); /** - * Bulked index request. Each request will be added to a queue for bulking requests. + * Index request. Each request will be added to a queue for bulking requests. * Submitting request will be done when bulk limits are exceeded. * * @param indexRequest the index request to add - * @return this ingest + * @return this */ - ExtendedClient indexRequest(IndexRequest indexRequest); + ExtendedClient index(IndexRequest indexRequest); /** - * Delete document. + * Delete request. * * @param index the index - * @param type the type * @param id the id - * @return this ingest + * @return this */ - ExtendedClient delete(String index, String type, String id); + ExtendedClient delete(String index, String id); /** - * Bulked delete request. Each request will be added to a queue for bulking requests. + * Delete request. Each request will be added to a queue for bulking requests. * Submitting request will be done when bulk limits are exceeded. * * @param deleteRequest the delete request to add - * @return this ingest + * @return this */ - ExtendedClient deleteRequest(DeleteRequest deleteRequest); + ExtendedClient delete(DeleteRequest deleteRequest); /** * Bulked update request. Each request will be added to a queue for bulking requests. @@ -107,23 +132,21 @@ public interface ExtendedClient { * Note that updates only work correctly when all operations between nodes are synchronized. * * @param index the index - * @param type the type * @param id the id * @param source the source * @return this */ - ExtendedClient update(String index, String type, String id, BytesReference source); + ExtendedClient update(String index, String id, BytesReference source); /** * Update document. Use with precaution! Does not work in all cases. * * @param index the index - * @param type the type * @param id the id * @param source the source * @return this */ - ExtendedClient update(String index, String type, String id, String source); + ExtendedClient update(String index, String id, String source); /** * Bulked update request. Each request will be added to a queue for bulking requests. @@ -131,207 +154,225 @@ public interface ExtendedClient { * Note that updates only work correctly when all operations between nodes are synchronized. * * @param updateRequest the update request to add - * @return this ingest + * @return this */ - ExtendedClient updateRequest(UpdateRequest updateRequest); - - /** - * Set the maximum number of actions per request. - * - * @param maxActionsPerRequest maximum number of actions per request - * @return this ingest - */ - ExtendedClient maxActionsPerRequest(int maxActionsPerRequest); - - /** - * Set the maximum concurent requests. - * - * @param maxConcurentRequests maximum number of concurrent ingest requests - * @return this Ingest - */ - ExtendedClient maxConcurrentRequests(int maxConcurentRequests); - - /** - * Set the maximum volume for request before flush. - * - * @param maxVolume maximum volume - * @return this ingest - */ - ExtendedClient maxVolumePerRequest(String maxVolume); - - /** - * Set the flush interval for automatic flushing outstanding ingest requests. - * - * @param flushInterval the flush interval, default is 30 seconds - * @return this ingest - */ - ExtendedClient flushIngestInterval(String flushInterval); - - /** - * Set mapping. - * - * @param type mapping type - * @param in mapping definition as input stream - * @throws IOException if mapping could not be added - */ - void mapping(String type, InputStream in) throws IOException; - - /** - * Set mapping. - * - * @param type mapping type - * @param mapping mapping definition as input stream - * @throws IOException if mapping could not be added - */ - void mapping(String type, String mapping) throws IOException; - - /** - * Put mapping. - * - * @param index index - */ - void putMapping(String index); + ExtendedClient update(UpdateRequest updateRequest); /** * Create a new index. * * @param index index - * @return this ingest - */ - ExtendedClient newIndex(String index); - - /** - * Create a new index. - * - * @param index index - * @param type type - * @param settings settings - * @param mappings mappings - * @return this ingest + * @return this * @throws IOException if new index creation fails */ - ExtendedClient newIndex(String index, String type, InputStream settings, InputStream mappings) throws IOException; + ExtendedClient newIndex(String index) throws IOException; /** * Create a new index. * - * @param index index + * @param index index * @param settings settings - * @param mappings mappings - * @return this ingest - */ - ExtendedClient newIndex(String index, Settings settings, Map mappings); - - /** - * Create new mapping. - * - * @param index index - * @param type index type * @param mapping mapping - * @return this ingest + * @return this + * @throws IOException if settings/mapping is invalid or index creation fails */ - ExtendedClient newMapping(String index, String type, Map mapping); + ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException; /** - * Delete index. + * Create a new index. * * @param index index - * @return this ingest + * @param settings settings + * @param mapping mapping + * @return this + * @throws IOException if settings/mapping is invalid or index creation fails + */ + ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException; + + /** + * Create a new index. + * + * @param index index + * @param settings settings + * @param mapping mapping + * @return this + * @throws IOException if settings/mapping is invalid or index creation fails + */ + ExtendedClient newIndex(String index, Settings settings, Map mapping) throws IOException; + + /** + * Create a new index. + * @param indexDefinition the index definition + * @return this + * @throws IOException if settings/mapping is invalid or index creation fails + */ + ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException; + + /** + * Delete an index. + * @param indexDefinition the index definition + * @return this + */ + ExtendedClient deleteIndex(IndexDefinition indexDefinition); + + /** + * Delete an index. + * + * @param index index + * @return this */ ExtendedClient deleteIndex(String index); + /** + * Start bulk mode for indexes. + * @param indexDefinition index definition + * @return this + * @throws IOException if bulk could not be started + */ + ExtendedClient startBulk(IndexDefinition indexDefinition) throws IOException; + /** * Start bulk mode. * - * @param index index + * @param index index * @param startRefreshIntervalSeconds refresh interval before bulk * @param stopRefreshIntervalSeconds refresh interval after bulk - * @return this ingest + * @return this * @throws IOException if bulk could not be started */ ExtendedClient startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds) throws IOException; + /** + * Stop bulk mode. + * + * @param indexDefinition index definition + * @return this + * @throws IOException if bulk could not be startet + */ + ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException; + /** * Stops bulk mode. * * @param index index - * @return this Ingest + * @param maxWaitTime maximum wait time + * @return this * @throws IOException if bulk could not be stopped */ - ExtendedClient stopBulk(String index) throws IOException; + ExtendedClient stopBulk(String index, String maxWaitTime) throws IOException; /** - * Flush ingest, move all pending documents to the cluster. + * Flush bulk indexing, move all pending documents to the cluster. * * @return this */ ExtendedClient flushIngest(); /** - * Wait for all outstanding responses. - * - * @param maxWaitTime maximum wait time - * @return this ingest - * @throws InterruptedException if wait is interrupted - * @throws ExecutionException if execution failed + * Update replica level. + * @param indexDefinition the index definition + * @param level the replica level + * @return this + * @throws IOException if replica setting could not be updated */ - ExtendedClient waitForResponses(String maxWaitTime) throws InterruptedException, ExecutionException; - - /** - * Refresh the index. - * - * @param index index - */ - void refreshIndex(String index); - - /** - * Flush the index. - * - * @param index index - */ - void flushIndex(String index); + ExtendedClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException; /** * Update replica level. * * @param index index * @param level the replica level - * @return number of shards after updating replica level - * @throws IOException if replica could not be updated + * @param maxWaitTime maximum wait time + * @return this + * @throws IOException if replica setting could not be updated */ - int updateReplicaLevel(String index, int level) throws IOException; + ExtendedClient updateReplicaLevel(String index, int level, String maxWaitTime) throws IOException; + + /** + * Get replica level. + * @param indexDefinition the index name + * @return the replica level of the index + */ + int getReplicaLevel(IndexDefinition indexDefinition); + + /** + * Get replica level. + * @param index the index name + * @return the replica level of the index + */ + int getReplicaLevel(String index); + + /** + * Refresh the index. + * + * @param index index + * @return this + */ + ExtendedClient refreshIndex(String index); + + /** + * Flush the index. The cluster clears cache and completes indexing. + * + * @param index index + * @return this + */ + ExtendedClient flushIndex(String index); + + /** + * Force segment merge of an index. + * @param indexDefinition th eindex definition + * @return this + */ + boolean forceMerge(IndexDefinition indexDefinition); + + /** + * Force segment merge of an index. + * @param index the index + * @param maxWaitTime maximum wait time + * @return this + */ + boolean forceMerge(String index, String maxWaitTime); + + /** + * Wait for all outstanding bulk responses. + * + * @param maxWaitTime maximum wait time + * @return true if wait succeeded, false if wait timed out + */ + boolean waitForResponses(String maxWaitTime); /** * Wait for cluster being healthy. * * @param healthColor cluster health color to wait for - * @param timeValue time value - * @throws IOException if wait failed + * @param maxWaitTime time value + * @return true if wait succeeded, false if wait timed out */ - void waitForCluster(String healthColor, String timeValue) throws IOException; + boolean waitForCluster(String healthColor, String maxWaitTime); /** * Get current health color. * + * @param maxWaitTime maximum wait time * @return the cluster health color */ - String healthColor(); + String getHealthColor(String maxWaitTime); /** * Wait for index recovery (after replica change). * * @param index index - * @return number of shards found - * @throws IOException if wait failed + * @param maxWaitTime maximum wait time + * @return true if wait succeeded, false if wait timed out */ - int waitForRecovery(String index) throws IOException; + boolean waitForRecovery(String index, String maxWaitTime); /** * Resolve alias. * * @param alias the alias - * @return one index name behind the alias or the alias if there is no index + * @return this index name behind the alias or the alias if there is no index */ String resolveAlias(String alias); @@ -347,40 +388,72 @@ public interface ExtendedClient { /** * Get all alias filters. * - * @param index index + * @param alias the alias * @return map of alias filters */ - Map getAliasFilters(String index); + Map getAliasFilters(String alias); /** - * Switch aliases from one index to another. - * - * @param index the index name - * @param concreteIndex the index name with timestamp - * @param extraAliases a list of names that should be set as index aliases + * Get all index filters. + * @param index the index + * @return map of index filters */ - void switchAliases(String index, String concreteIndex, List extraAliases); + Map getIndexFilters(String index); /** - * Switch aliases from one index to another. + * Switch from one index to another. + * @param indexDefinition the index definition + * @param extraAliases new aliases + * @return this + */ + ExtendedClient switchIndex(IndexDefinition indexDefinition, List extraAliases); + + /** + * Switch from one index to another. + * @param indexDefinition the index definition + * @param extraAliases new aliases + * @param indexAliasAdder method to add aliases + * @return this + */ + ExtendedClient switchIndex(IndexDefinition indexDefinition, List extraAliases, IndexAliasAdder indexAliasAdder); + + /** + * Switch from one index to another. * * @param index the index name - * @param concreteIndex the index name with timestamp + * @param fullIndexName the index name with timestamp + * @param extraAliases a list of names that should be set as index aliases + * @return this + */ + ExtendedClient switchIndex(String index, String fullIndexName, List extraAliases); + + /** + * Switch from one index to another. + * + * @param index the index name + * @param fullIndexName the index name with timestamp * @param extraAliases a list of names that should be set as index aliases * @param adder an adder method to create alias term queries + * @return this */ - void switchAliases(String index, String concreteIndex, List extraAliases, IndexAliasAdder adder); + ExtendedClient switchIndex(String index, String fullIndexName, List extraAliases, IndexAliasAdder adder); /** - * Retention policy for an index. All indices before timestampdiff should be deleted, - * but mintokeep indices must be kept. + * Prune index. + * @param indexDefinition the index definition + */ + void pruneIndex(IndexDefinition indexDefinition); + + /** + * Apply retention policy to prune indices. All indices before delta should be deleted, + * but the number of mintokeep indices must be kept. * * @param index index name - * @param concreteIndex index name with timestamp - * @param timestampdiff timestamp delta (for index timestamps) + * @param fullIndexName index name with timestamp + * @param delta timestamp delta (for index timestamps) * @param mintokeep minimum number of indices to keep */ - void performRetentionPolicy(String index, String concreteIndex, int timestampdiff, int mintokeep); + void pruneIndex(String index, String fullIndexName, int delta, int mintokeep); /** * Find the timestamp of the most recently indexed document in the index. @@ -413,7 +486,7 @@ public interface ExtendedClient { Throwable getThrowable(); /** - * Shutdown the ingesting. + * Shutdown the client. * @throws IOException if shutdown fails */ void shutdown() throws IOException; diff --git a/elx-common/src/main/java/org/xbib/elx/common/management/IndexDefinition.java b/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java similarity index 79% rename from elx-common/src/main/java/org/xbib/elx/common/management/IndexDefinition.java rename to elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java index 37dcc45..7e12592 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/management/IndexDefinition.java +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java @@ -1,13 +1,12 @@ -package org.xbib.elx.common.management; +package org.xbib.elx.api; +import java.net.MalformedURLException; import java.net.URL; public class IndexDefinition { private String index; - private String type; - private String fullIndexName; private String dateTimePattern; @@ -28,6 +27,8 @@ public class IndexDefinition { private IndexRetention indexRetention; + private String maxWaitTime; + public IndexDefinition setIndex(String index) { this.index = index; return this; @@ -46,15 +47,11 @@ public class IndexDefinition { return fullIndexName; } - public IndexDefinition setType(String type) { - this.type = type; + public IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException { + this.settingsUrl = settingsUrlString != null ? new URL(settingsUrlString) : null; return this; } - public String getType() { - return type; - } - public IndexDefinition setSettingsUrl(URL settingsUrl) { this.settingsUrl = settingsUrl; return this; @@ -64,6 +61,11 @@ public class IndexDefinition { return settingsUrl; } + public IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException { + this.mappingsUrl = mappingsUrlString != null ? new URL(mappingsUrlString) : null; + return this; + } + public IndexDefinition setMappingsUrl(URL mappingsUrl) { this.mappingsUrl = mappingsUrl; return this; @@ -136,4 +138,12 @@ public class IndexDefinition { return indexRetention; } + public IndexDefinition setMaxWaitTime(String maxWaitTime) { + this.maxWaitTime = maxWaitTime; + return this; + } + + public String getMaxWaitTime() { + return maxWaitTime; + } } diff --git a/elx-common/src/main/java/org/xbib/elx/common/management/IndexRetention.java b/elx-api/src/main/java/org/xbib/elx/api/IndexRetention.java similarity index 73% rename from elx-common/src/main/java/org/xbib/elx/common/management/IndexRetention.java rename to elx-api/src/main/java/org/xbib/elx/api/IndexRetention.java index 8024ef4..e1445fc 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/management/IndexRetention.java +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexRetention.java @@ -1,4 +1,4 @@ -package org.xbib.elx.common.management; +package org.xbib.elx.api; public class IndexRetention { @@ -6,12 +6,12 @@ public class IndexRetention { private int minToKeep; - public IndexRetention setTimestampDiff(int timestampDiff) { + public IndexRetention setDelta(int timestampDiff) { this.timestampDiff = timestampDiff; return this; } - public int getTimestampDiff() { + public int getDelta() { return timestampDiff; } diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java index 69f2608..e9a2230 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; import org.elasticsearch.action.admin.indices.flush.FlushAction; @@ -33,14 +34,14 @@ import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequestBuilder; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; import org.elasticsearch.action.admin.indices.recovery.RecoveryAction; import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshAction; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.bulk.BulkItemResponse; @@ -60,13 +61,12 @@ import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortBuilders; @@ -75,15 +75,11 @@ import org.xbib.elx.api.BulkControl; import org.xbib.elx.api.BulkMetric; import org.xbib.elx.api.ExtendedClient; import org.xbib.elx.api.IndexAliasAdder; -import org.xbib.elx.common.management.IndexDefinition; -import org.xbib.elx.common.management.IndexRetention; +import org.xbib.elx.api.IndexDefinition; +import org.xbib.elx.api.IndexRetention; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.io.StringWriter; -import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.StandardCharsets; import java.time.LocalDate; @@ -100,25 +96,33 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; -import static org.elasticsearch.index.query.QueryBuilders.existsQuery; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; - public abstract class AbstractExtendedClient implements ExtendedClient { private static final Logger logger = LogManager.getLogger(AbstractExtendedClient.class.getName()); - private Map mappings; + /** + * The one and only index type name used in the extended client. + * Notr that all Elasticsearch version < 6.2.0 do not allow a prepending "_". + */ + private static final String TYPE_NAME = "doc"; + /** + * The Elasticsearch client. + */ private ElasticsearchClient client; + /** + * Our replacement for the buk processor. + */ private BulkProcessor bulkProcessor; private BulkMetric bulkMetric; @@ -129,22 +133,9 @@ public abstract class AbstractExtendedClient implements ExtendedClient { private boolean closed; - private int maxActionsPerRequest; - - private int maxConcurrentRequests; - - private String maxVolumePerRequest; - - private String flushIngestInterval; - protected abstract ElasticsearchClient createClient(Settings settings) throws IOException; protected AbstractExtendedClient() { - maxActionsPerRequest = Parameters.DEFAULT_MAX_ACTIONS_PER_REQUEST.getNum(); - maxConcurrentRequests = Parameters.DEFAULT_MAX_CONCURRENT_REQUESTS.getNum(); - maxVolumePerRequest = Parameters.DEFAULT_MAX_VOLUME_PER_REQUEST.getString(); - flushIngestInterval = Parameters.DEFAULT_FLUSH_INTERVAL.getString(); - mappings = new HashMap<>(); } @Override @@ -161,6 +152,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public AbstractExtendedClient setBulkMetric(BulkMetric metric) { this.bulkMetric = metric; + // you must start bulk metric or it will bail out at stop() + bulkMetric.start(); return this; } @@ -183,7 +176,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public AbstractExtendedClient init(Settings settings) throws IOException { if (client == null) { - this.client = createClient(settings); + client = createClient(settings); } if (bulkMetric != null) { bulkMetric.start(); @@ -259,17 +252,28 @@ public abstract class AbstractExtendedClient implements ExtendedClient { logger.error("after bulk [" + executionId + "] error", failure); } }; - if (this.client != null) { - BulkProcessor.Builder builder = BulkProcessor.builder((Client)this.client, listener) + if (client != null) { + int maxActionsPerRequest = settings.getAsInt(Parameters.MAX_ACTIONS_PER_REQUEST.name(), + Parameters.DEFAULT_MAX_ACTIONS_PER_REQUEST.getNum()); + int maxConcurrentRequests = settings.getAsInt(Parameters.MAX_CONCURRENT_REQUESTS.name(), + Parameters.DEFAULT_MAX_CONCURRENT_REQUESTS.getNum()); + TimeValue flushIngestInterval = settings.getAsTime(Parameters.FLUSH_INTERVAL.name(), + TimeValue.timeValueSeconds(Parameters.DEFAULT_FLUSH_INTERVAL.getNum())); + ByteSizeValue maxVolumePerRequest = settings.getAsBytesSize(Parameters.MAX_VOLUME_PER_REQUEST.name(), + ByteSizeValue.parseBytesSizeValue(Parameters.DEFAULT_MAX_VOLUME_PER_REQUEST.getString(), + "maxVolumePerRequest")); + logger.info("bulk processor up with maxActionsPerRequest = {} maxConcurrentRequests = {} " + + "flushIngestInterval = {} maxVolumePerRequest = {}", + maxActionsPerRequest, maxConcurrentRequests, flushIngestInterval, maxVolumePerRequest); + BulkProcessor.Builder builder = BulkProcessor.builder((Client) client, listener) .setBulkActions(maxActionsPerRequest) .setConcurrentRequests(maxConcurrentRequests) - .setFlushInterval(TimeValue.parseTimeValue(flushIngestInterval, null, "flushIngestInterval")); - if (maxVolumePerRequest != null) { - builder.setBulkSize(ByteSizeValue.parseBytesSizeValue(maxVolumePerRequest, "maxVolumePerRequest")); - } + .setFlushInterval(flushIngestInterval) + .setBulkSize(maxVolumePerRequest); this.bulkProcessor = builder.build(); } this.closed = false; + this.throwable = null; return this; } @@ -277,60 +281,97 @@ public abstract class AbstractExtendedClient implements ExtendedClient { public synchronized void shutdown() throws IOException { ensureActive(); if (bulkProcessor != null) { - logger.info("closing bulk processor..."); + logger.info("closing bulk processor"); bulkProcessor.close(); } if (bulkMetric != null) { - logger.info("stopping metric"); + logger.info("stopping metric before bulk stop (for precise measurement)"); bulkMetric.stop(); } if (bulkControl != null && bulkControl.indices() != null && !bulkControl.indices().isEmpty()) { logger.info("stopping bulk mode for indices {}...", bulkControl.indices()); for (String index : bulkControl.indices()) { - stopBulk(index); + stopBulk(index, bulkControl.getMaxWaitTime()); } } + logger.info("shutdown complete"); + } + + @Override + public String getClusterName() { + ensureActive(); + try { + ClusterStateRequestBuilder clusterStateRequestBuilder = + new ClusterStateRequestBuilder(client, ClusterStateAction.INSTANCE).all(); + ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); + return clusterStateResponse.getClusterName().value(); + } catch (ElasticsearchTimeoutException e) { + logger.warn(e.getMessage(), e); + return "TIMEOUT"; + } catch (NoNodeAvailableException e) { + logger.warn(e.getMessage(), e); + return "DISCONNECTED"; + } catch (Exception e) { + logger.warn(e.getMessage(), e); + return "[" + e.getMessage() + "]"; + } } @Override - public ExtendedClient maxActionsPerRequest(int maxActionsPerRequest) { - this.maxActionsPerRequest = maxActionsPerRequest; - return this; - } - - @Override - public ExtendedClient maxConcurrentRequests(int maxConcurrentRequests) { - this.maxConcurrentRequests = maxConcurrentRequests; - return this; - } - - @Override - public ExtendedClient maxVolumePerRequest(String maxVolumePerRequest) { - this.maxVolumePerRequest = maxVolumePerRequest; - return this; - } - - @Override - public ExtendedClient flushIngestInterval(String flushIngestInterval) { - this.flushIngestInterval = flushIngestInterval; + public ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException { + ensureActive(); + waitForCluster("YELLOW", "30s"); + URL indexSettings = indexDefinition.getSettingsUrl(); + if (indexSettings == null) { + logger.warn("warning while creating index '{}', no settings/mappings", + indexDefinition.getFullIndexName()); + newIndex(indexDefinition.getFullIndexName()); + return this; + } + URL indexMappings = indexDefinition.getMappingsUrl(); + if (indexMappings == null) { + logger.warn("warning while creating index '{}', no mappings", + indexDefinition.getFullIndexName()); + newIndex(indexDefinition.getFullIndexName(), indexSettings.openStream(), null); + return this; + } + try (InputStream indexSettingsInput = indexSettings.openStream(); + InputStream indexMappingsInput = indexMappings.openStream()) { + newIndex(indexDefinition.getFullIndexName(), indexSettingsInput, indexMappingsInput); + } catch (IOException e) { + if (indexDefinition.ignoreErrors()) { + logger.warn(e.getMessage(), e); + logger.warn("warning while creating index '{}' with settings at {} and mappings at {}", + indexDefinition.getFullIndexName(), indexSettings, indexMappings); + } else { + logger.error("error while creating index '{}' with settings at {} and mappings at {}", + indexDefinition.getFullIndexName(), indexSettings, indexMappings); + throw new IOException(e); + } + } return this; } @Override public ExtendedClient newIndex(String index) { - ensureActive(); - return newIndex(index, null, null); + return newIndex(index, Settings.EMPTY, (Map) null); } @Override - public ExtendedClient newIndex(String index, String type, InputStream settings, InputStream mappings) throws IOException { - mapping(type, mappings); - return newIndex(index, Settings.settingsBuilder().loadFromStream(".json", settings).build(), - this.mappings); + public ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException { + return newIndex(index, + Settings.settingsBuilder().loadFromStream(".json", settings).build(), + JsonXContent.jsonXContent.createParser(mapping).mapOrdered()); } @Override - public ExtendedClient newIndex(String index, Settings settings, Map mappings) { + public ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException { + return newIndex(index, settings, + JsonXContent.jsonXContent.createParser(mapping).mapOrdered()); + } + + @Override + public ExtendedClient newIndex(String index, Settings settings, Map mapping) { ensureActive(); if (index == null) { logger.warn("no index name given to create index"); @@ -339,16 +380,10 @@ public abstract class AbstractExtendedClient implements ExtendedClient { CreateIndexRequestBuilder createIndexRequestBuilder = new CreateIndexRequestBuilder(client, CreateIndexAction.INSTANCE).setIndex(index); if (settings != null) { - logger.info("found settings {}", settings.getAsMap()); createIndexRequestBuilder.setSettings(settings); } - if (mappings != null) { - for (Map.Entry entry : mappings.entrySet()) { - String type = entry.getKey(); - String mapping = entry.getValue(); - logger.info("found mapping for {}", type); - createIndexRequestBuilder.addMapping(type, mapping, XContentType.JSON); - } + if (mapping != null) { + createIndexRequestBuilder.addMapping(TYPE_NAME, mapping); } CreateIndexResponse createIndexResponse = createIndexRequestBuilder.execute().actionGet(); logger.info("index {} created: {}", index, createIndexResponse); @@ -356,15 +391,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public ExtendedClient newMapping(String index, String type, Map mapping) { - PutMappingRequestBuilder putMappingRequestBuilder = - new PutMappingRequestBuilder(client, PutMappingAction.INSTANCE) - .setIndices(index) - .setType(type) - .setSource(mapping); - putMappingRequestBuilder.execute().actionGet(); - logger.info("mapping created for index {} and type {}", index, type); - return this; + public ExtendedClient deleteIndex(IndexDefinition indexDefinition) { + return deleteIndex(indexDefinition.getFullIndexName()); } @Override @@ -380,6 +408,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient { return this; } + @Override + public ExtendedClient startBulk(IndexDefinition indexDefinition) throws IOException { + startBulk(indexDefinition.getFullIndexName(), -1, 1); + return this; + } + @Override public ExtendedClient startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds) throws IOException { @@ -395,52 +429,41 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public ExtendedClient stopBulk(String index) throws IOException { + public ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException { + return stopBulk(indexDefinition.getFullIndexName(), indexDefinition.getMaxWaitTime()); + } + + @Override + public ExtendedClient stopBulk(String index, String maxWaitTime) throws IOException { ensureActive(); if (bulkControl == null) { return this; } - if (bulkControl.isBulk(index)) { - long secs = bulkControl.getStopBulkRefreshIntervals().get(index); - if (secs > 0L) { - updateIndexSetting(index, "refresh_interval", secs + "s"); + flushIngest(); + if (waitForResponses(maxWaitTime)) { + if (bulkControl.isBulk(index)) { + long secs = bulkControl.getStopBulkRefreshIntervals().get(index); + if (secs > 0L) { + updateIndexSetting(index, "refresh_interval", secs + "s"); + } + bulkControl.finishBulk(index); } - bulkControl.finishBulk(index); } return this; } @Override - public ExtendedClient flushIngest() { - ensureActive(); - logger.debug("flushing bulk processor"); - bulkProcessor.flush(); - return this; + public ExtendedClient index(String index, String id, boolean create, BytesReference source) { + return index(new IndexRequest(index, TYPE_NAME, id).create(create).source(source)); } @Override - public ExtendedClient waitForResponses(String maxWaitTime) throws InterruptedException { - ensureActive(); - long millis = TimeValue.parseTimeValue(maxWaitTime, TimeValue.timeValueMinutes(1),"millis").getMillis(); - logger.debug("waiting for " + millis + " millis"); - while (!bulkProcessor.awaitClose(millis, TimeUnit.MILLISECONDS)) { - logger.warn("still waiting for responses"); - } - return this; + public ExtendedClient index(String index, String id, boolean create, String source) { + return index(new IndexRequest(index, TYPE_NAME, id).create(create).source(source.getBytes(StandardCharsets.UTF_8))); } @Override - public ExtendedClient index(String index, String type, String id, boolean create, BytesReference source) { - return indexRequest(new IndexRequest(index).type(type).id(id).create(create).source(source)); - } - - @Override - public ExtendedClient index(String index, String type, String id, boolean create, String source) { - return indexRequest(new IndexRequest(index).type(type).id(id).create(create).source(source.getBytes(StandardCharsets.UTF_8))); - } - - @Override - public ExtendedClient indexRequest(IndexRequest indexRequest) { + public ExtendedClient index(IndexRequest indexRequest) { ensureActive(); try { if (bulkMetric != null) { @@ -456,12 +479,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public ExtendedClient delete(String index, String type, String id) { - return deleteRequest(new DeleteRequest(index).type(type).id(id)); + public ExtendedClient delete(String index, String id) { + return delete(new DeleteRequest(index, TYPE_NAME, id)); } @Override - public ExtendedClient deleteRequest(DeleteRequest deleteRequest) { + public ExtendedClient delete(DeleteRequest deleteRequest) { ensureActive(); try { if (bulkMetric != null) { @@ -477,17 +500,17 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public ExtendedClient update(String index, String type, String id, BytesReference source) { - return updateRequest(new UpdateRequest().index(index).type(type).id(id).upsert(source)); + public ExtendedClient update(String index, String id, BytesReference source) { + return update(new UpdateRequest(index, TYPE_NAME, id).doc(source)); } @Override - public ExtendedClient update(String index, String type, String id, String source) { - return updateRequest(new UpdateRequest().index(index).type(type).id(id).upsert(source.getBytes(StandardCharsets.UTF_8))); + public ExtendedClient update(String index, String id, String source) { + return update(new UpdateRequest(index, TYPE_NAME, id).doc(source.getBytes(StandardCharsets.UTF_8))); } @Override - public ExtendedClient updateRequest(UpdateRequest updateRequest) { + public ExtendedClient update(UpdateRequest updateRequest) { ensureActive(); try { if (bulkMetric != null) { @@ -503,54 +526,68 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public void mapping(String type, String mapping) { - mappings.put(type, mapping); - } - - @Override - public void mapping(String type, InputStream in) throws IOException { - if (type == null) { - return; - } - StringWriter sw = new StringWriter(); - Streams.copy(new InputStreamReader(in, StandardCharsets.UTF_8), sw); - mappings.put(type, sw.toString()); - } - - @Override - public int waitForRecovery(String index) throws IOException { + public ExtendedClient flushIngest() { ensureActive(); - if (index == null) { - throw new IOException("unable to wait for recovery, no index no given"); + logger.debug("flushing bulk processor"); + bulkProcessor.flush(); + return this; + } + + @Override + public boolean waitForResponses(String maxWaitTime) { + ensureActive(); + long millis = TimeValue.parseTimeValue(maxWaitTime, TimeValue.timeValueMinutes(1),"millis").getMillis(); + logger.debug("waiting for " + millis + " millis"); + try { + return bulkProcessor.awaitFlush(millis, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.error("interrupted"); + return false; } + } + + @Override + public boolean waitForRecovery(String index, String maxWaitTime) { + ensureActive(); + ensureIndexGiven(index); RecoveryResponse response = client.execute(RecoveryAction.INSTANCE, new RecoveryRequest(index)).actionGet(); int shards = response.getTotalShards(); - client.execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest(index) - .waitForActiveShards(shards)).actionGet(); - return shards; + TimeValue timeout = TimeValue.parseTimeValue(maxWaitTime, TimeValue.timeValueSeconds(10), + getClass().getSimpleName() + ".timeout"); + ClusterHealthResponse healthResponse = + client.execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest(index) + .waitForActiveShards(shards).timeout(timeout)).actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + logger.error("timeout waiting for recovery"); + return false; + } + return true; } @Override - public void waitForCluster(String statusString, String timeout) throws IOException { + public boolean waitForCluster(String statusString, String maxWaitTime) { ensureActive(); ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString); - ClusterHealthResponse healthResponse = - client.execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest() - .waitForStatus(status).timeout(timeout)).actionGet(); + TimeValue timeout = TimeValue.parseTimeValue(maxWaitTime, TimeValue.timeValueSeconds(10), + getClass().getSimpleName() + ".timeout"); + ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet(); if (healthResponse != null && healthResponse.isTimedOut()) { - throw new IOException("cluster state is " + healthResponse.getStatus().name() - + " and not " + status.name() - + ", from here on, everything will fail!"); + logger.error("timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name()); + return false; } + return true; } @Override - public String healthColor() { + public String getHealthColor(String maxWaitTime) { ensureActive(); try { - ClusterHealthResponse healthResponse = - client.execute(ClusterHealthAction.INSTANCE, - new ClusterHealthRequest().timeout(TimeValue.timeValueSeconds(30))).actionGet(); + TimeValue timeout = TimeValue.parseTimeValue(maxWaitTime, TimeValue.timeValueSeconds(10), + getClass().getSimpleName() + ".timeout"); + ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().timeout(timeout)).actionGet(); ClusterHealthStatus status = healthResponse.getStatus(); return status.name(); } catch (ElasticsearchTimeoutException e) { @@ -566,37 +603,55 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public int updateReplicaLevel(String index, int level) throws IOException { - waitForCluster("YELLOW", "30s"); - updateIndexSetting(index, "number_of_replicas", level); - return waitForRecovery(index); + public ExtendedClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException { + return updateReplicaLevel(indexDefinition.getFullIndexName(), level, indexDefinition.getMaxWaitTime()); } @Override - public void flushIndex(String index) { - ensureActive(); - if (index != null) { - client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet(); + public ExtendedClient updateReplicaLevel(String index, int level, String maxWaitTime) throws IOException { + waitForCluster("YELLOW", maxWaitTime); // let cluster settle down from critical operations + if (level > 0) { + updateIndexSetting(index, "number_of_replicas", level); + waitForRecovery(index, maxWaitTime); } + return this; } @Override - public void refreshIndex(String index) { - ensureActive(); - if (index != null) { - client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet(); - } + public int getReplicaLevel(IndexDefinition indexDefinition) { + return getReplicaLevel(indexDefinition.getFullIndexName()); } @Override - public void putMapping(String index) { - ensureActive(); - if (mappings != null && !mappings.isEmpty()) { - for (Map.Entry me : mappings.entrySet()) { - client.execute(PutMappingAction.INSTANCE, - new PutMappingRequest(index).type(me.getKey()).source(me.getValue())).actionGet(); + public int getReplicaLevel(String index) { + GetSettingsRequest request = new GetSettingsRequest().indices(index); + GetSettingsResponse response = client.execute(GetSettingsAction.INSTANCE, request).actionGet(); + int replica = -1; + for (ObjectObjectCursor cursor : response.getIndexToSettings()) { + Settings settings = cursor.value; + if (index.equals(cursor.key)) { + replica = settings.getAsInt("index.number_of_replicas", null); } } + return replica; + } + + @Override + public ExtendedClient flushIndex(String index) { + if (index != null) { + ensureActive(); + client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet(); + } + return this; + } + + @Override + public ExtendedClient refreshIndex(String index) { + if (index != null) { + ensureActive(); + client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet(); + } + return this; } @Override @@ -636,16 +691,43 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public void switchAliases(String index, String concreteIndex, List extraAliases) { - switchAliases(index, concreteIndex, extraAliases, null); + public Map getIndexFilters(String index) { + GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client, GetAliasesAction.INSTANCE); + return getFilters(getAliasesRequestBuilder.setIndices(index).execute().actionGet()); } @Override - public void switchAliases(String index, String concreteIndex, - List extraAliases, IndexAliasAdder adder) { + public ExtendedClient switchIndex(IndexDefinition indexDefinition, List extraAliases) { + return switchIndex(indexDefinition, extraAliases, null); + } + + + @Override + public ExtendedClient switchIndex(IndexDefinition indexDefinition, + List extraAliases, IndexAliasAdder indexAliasAdder) { + if (extraAliases == null) { + return this; + } + if (indexDefinition.isSwitchAliases()) { + switchIndex(indexDefinition.getIndex(), + indexDefinition.getFullIndexName(), extraAliases.stream() + .filter(a -> a != null && !a.isEmpty()) + .collect(Collectors.toList()), indexAliasAdder); + } + return this; + } + + @Override + public ExtendedClient switchIndex(String index, String fullIndexName, List extraAliases) { + return switchIndex(index, fullIndexName, extraAliases, null); + } + + @Override + public ExtendedClient switchIndex(String index, String fullIndexName, + List extraAliases, IndexAliasAdder adder) { ensureActive(); - if (index.equals(concreteIndex)) { - return; + if (index.equals(fullIndexName)) { + return this; // nothing to switch to } // two situations: 1. there is a new alias 2. there is already an old index with the alias String oldIndex = resolveAlias(index); @@ -655,7 +737,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { IndicesAliasesRequestBuilder requestBuilder = new IndicesAliasesRequestBuilder(client, IndicesAliasesAction.INSTANCE); if (oldFilterMap == null || !oldFilterMap.containsKey(index)) { // never apply a filter for trunk index name - requestBuilder.addAlias(concreteIndex, index); + requestBuilder.addAlias(fullIndexName, index); newAliases.add(index); } // switch existing aliases @@ -665,9 +747,9 @@ public abstract class AbstractExtendedClient implements ExtendedClient { String filter = entry.getValue(); requestBuilder.removeAlias(oldIndex, alias); if (filter != null) { - requestBuilder.addAlias(concreteIndex, alias, filter); + requestBuilder.addAlias(fullIndexName, alias, filter); } else { - requestBuilder.addAlias(concreteIndex, alias); + requestBuilder.addAlias(fullIndexName, alias); } switchAliases.add(alias); } @@ -678,18 +760,18 @@ public abstract class AbstractExtendedClient implements ExtendedClient { if (oldFilterMap == null || !oldFilterMap.containsKey(extraAlias)) { // index alias adder only active on extra aliases, and if alias is new if (adder != null) { - adder.addIndexAlias(requestBuilder, concreteIndex, extraAlias); + adder.addIndexAlias(requestBuilder, fullIndexName, extraAlias); } else { - requestBuilder.addAlias(concreteIndex, extraAlias); + requestBuilder.addAlias(fullIndexName, extraAlias); } newAliases.add(extraAlias); } else { String filter = oldFilterMap.get(extraAlias); requestBuilder.removeAlias(oldIndex, extraAlias); if (filter != null) { - requestBuilder.addAlias(concreteIndex, extraAlias, filter); + requestBuilder.addAlias(fullIndexName, extraAlias, filter); } else { - requestBuilder.addAlias(concreteIndex, extraAlias); + requestBuilder.addAlias(fullIndexName, extraAlias); } switchAliases.add(extraAlias); } @@ -699,15 +781,22 @@ public abstract class AbstractExtendedClient implements ExtendedClient { logger.info("new aliases = {}, switch aliases = {}", newAliases, switchAliases); requestBuilder.execute().actionGet(); } + return this; } @Override - public void performRetentionPolicy(String index, String concreteIndex, int timestampdiff, int mintokeep) { - if (timestampdiff == 0 && mintokeep == 0) { + public void pruneIndex(IndexDefinition indexDefinition) { + pruneIndex(indexDefinition.getIndex(), indexDefinition.getFullIndexName(), + indexDefinition.getRetention().getDelta(), indexDefinition.getRetention().getMinToKeep()); + } + + @Override + public void pruneIndex(String index, String fullIndexName, int delta, int mintokeep) { + if (delta == 0 && mintokeep == 0) { return; } ensureActive(); - if (index.equals(concreteIndex)) { + if (index.equals(fullIndexName)) { return; } GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE); @@ -717,7 +806,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { logger.info("{} indices", getIndexResponse.getIndices().length); for (String s : getIndexResponse.getIndices()) { Matcher m = pattern.matcher(s); - if (m.matches() && index.equals(m.group(1)) && !s.equals(concreteIndex)) { + if (m.matches() && index.equals(m.group(1)) && !s.equals(fullIndexName)) { indices.add(s); } } @@ -734,7 +823,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } List indicesToDelete = new ArrayList<>(); // our index - Matcher m1 = pattern.matcher(concreteIndex); + Matcher m1 = pattern.matcher(fullIndexName); if (m1.matches()) { Integer i1 = Integer.parseInt(m1.group(2)); for (String s : indices) { @@ -742,7 +831,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { if (m2.matches()) { Integer i2 = Integer.parseInt(m2.group(2)); int kept = indices.size() - indicesToDelete.size(); - if ((timestampdiff == 0 || (timestampdiff > 0 && i1 - i2 > timestampdiff)) && mintokeep <= kept) { + if ((delta == 0 || (delta > 0 && i1 - i2 > delta)) && mintokeep <= kept) { indicesToDelete.add(s); } } @@ -753,9 +842,10 @@ public abstract class AbstractExtendedClient implements ExtendedClient { logger.info("not enough indices found to delete, retention policy complete"); return; } - String[] s = indicesToDelete.toArray(new String[indicesToDelete.size()]); - DeleteIndexRequestBuilder requestBuilder = new DeleteIndexRequestBuilder(client, DeleteIndexAction.INSTANCE, s); - DeleteIndexResponse response = requestBuilder.execute().actionGet(); + String[] s = new String[indicesToDelete.size()]; + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest() + .indices(indicesToDelete.toArray(s)); + DeleteIndexResponse response = client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet(); if (!response.isAcknowledged()) { logger.warn("retention delete index operation was not acknowledged"); } @@ -782,6 +872,70 @@ public abstract class AbstractExtendedClient implements ExtendedClient { return null; } + @Override + public boolean forceMerge(IndexDefinition indexDefinition) { + if (indexDefinition.hasForceMerge()) { + return forceMerge(indexDefinition.getFullIndexName(), indexDefinition.getMaxWaitTime()); + } + return false; + } + + @Override + public boolean forceMerge(String index, String maxWaitTime) { + TimeValue timeout = TimeValue.parseTimeValue(maxWaitTime, TimeValue.timeValueSeconds(10), + getClass().getSimpleName() + ".timeout"); + ForceMergeRequestBuilder forceMergeRequestBuilder = + new ForceMergeRequestBuilder(client, ForceMergeAction.INSTANCE); + forceMergeRequestBuilder.setIndices(index); + try { + forceMergeRequestBuilder.execute().get(timeout.getMillis(), TimeUnit.MILLISECONDS); + return true; + } catch (TimeoutException e) { + logger.error("timeout"); + } catch (ExecutionException e) { + logger.error(e.getMessage(), e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.error(e.getMessage(), e); + } + return false; + } + + @Override + public IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) + throws IOException { + boolean isEnabled = settings.getAsBoolean("enabled", !(client instanceof MockExtendedClient)); + String indexName = settings.get("name", index); + String fullIndexName; + String dateTimePattern = settings.get("dateTimePattern"); + if (dateTimePattern != null) { + fullIndexName = resolveAlias(indexName + + DateTimeFormatter.ofPattern(dateTimePattern) + .withZone(ZoneId.systemDefault()) // not GMT + .format(LocalDate.now())); + logger.info("index name {} resolved to full index name = {}", indexName, fullIndexName); + } else { + fullIndexName = resolveMostRecentIndex(indexName); + logger.info("index name {} resolved to full index name = {}", indexName, fullIndexName); + } + IndexRetention indexRetention = new IndexRetention() + .setMinToKeep(settings.getAsInt("retention.mintokeep", 0)) + .setDelta(settings.getAsInt("retention.delta", 0)); + + return new IndexDefinition() + .setIndex(indexName) + .setFullIndexName(fullIndexName) + .setSettingsUrl(settings.get("settings")) + .setMappingsUrl(settings.get("mapping")) + .setDateTimePattern(dateTimePattern) + .setEnabled(isEnabled) + .setIgnoreErrors(settings.getAsBoolean("skiperrors", false)) + .setSwitchAliases(settings.getAsBoolean("aliases", true)) + .setReplicaLevel(settings.getAsInt("replica", 0)) + .setMaxWaitTime(settings.get("timout", "30s")) + .setRetention(indexRetention); + } + @Override public boolean hasThrowable() { return throwable != null; @@ -822,9 +976,10 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } } - public Map getIndexFilters(String index) { - GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client, GetAliasesAction.INSTANCE); - return getFilters(getAliasesRequestBuilder.setIndices(index).execute().actionGet()); + private void ensureIndexGiven(String index) { + if (index == null) { + throw new IllegalArgumentException("no index given"); + } } private Map getFilters(GetAliasesResponse getAliasesResponse) { @@ -834,7 +989,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { for (AliasMetaData aliasMetaData : aliasMetaDataList) { if (aliasMetaData.filteringRequired()) { result.put(aliasMetaData.alias(), - new String(aliasMetaData.getFilter().uncompressed(), StandardCharsets.UTF_8) ); + new String(aliasMetaData.getFilter().uncompressed(), StandardCharsets.UTF_8)); } else { result.put(aliasMetaData.alias(), null); } @@ -843,61 +998,6 @@ public abstract class AbstractExtendedClient implements ExtendedClient { return result; } - @Override - public String getClusterName() { - ensureActive(); - try { - ClusterStateRequestBuilder clusterStateRequestBuilder = - new ClusterStateRequestBuilder(client, ClusterStateAction.INSTANCE).all(); - ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); - String name = clusterStateResponse.getClusterName().value(); - int nodeCount = clusterStateResponse.getState().getNodes().size(); - return name + " (" + nodeCount + " nodes connected)"; - } catch (ElasticsearchTimeoutException e) { - logger.warn(e.getMessage(), e); - return "TIMEOUT"; - } catch (NoNodeAvailableException e) { - logger.warn(e.getMessage(), e); - return "DISCONNECTED"; - } catch (Exception e) { - logger.warn(e.getMessage(), e); - return "[" + e.getMessage() + "]"; - } - } - - public IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) - throws MalformedURLException { - boolean isEnabled = settings.getAsBoolean("enabled", !(client instanceof MockExtendedClient)); - String indexName = settings.get("name", index); - String fullIndexName; - String dateTimePattern = settings.get("dateTimePattern"); - if (dateTimePattern != null) { - fullIndexName = resolveAlias(indexName + - DateTimeFormatter.ofPattern(dateTimePattern) - .withZone(ZoneId.systemDefault()) // not GMT - .format(LocalDate.now())); - logger.info("index name {} resolved to full index name = {}", indexName, fullIndexName); - } else { - fullIndexName = resolveMostRecentIndex(indexName); - logger.info("index name {} resolved to full index name = {}", indexName, fullIndexName); - } - IndexRetention indexRetention = new IndexRetention() - .setMinToKeep(settings.getAsInt("retention.mintokeep", 0)) - .setTimestampDiff(settings.getAsInt("retention.diff", 0)); - return new IndexDefinition() - .setIndex(indexName) - .setFullIndexName(fullIndexName) - .setType(settings.get("type")) - .setSettingsUrl(new URL(settings.get("settings"))) - .setMappingsUrl(new URL(settings.get("mapping"))) - .setDateTimePattern(dateTimePattern) - .setEnabled(isEnabled) - .setIgnoreErrors(settings.getAsBoolean("skiperrors", false)) - .setSwitchAliases(settings.getAsBoolean("aliases", true)) - .setReplicaLevel(settings.getAsInt("replica", 0)) - .setRetention(indexRetention); - } - public void checkMapping(String index) { ensureActive(); GetMappingsRequestBuilder getMappingsRequestBuilder = new GetMappingsRequestBuilder(client, GetMappingsAction.INSTANCE) @@ -914,14 +1014,13 @@ public abstract class AbstractExtendedClient implements ExtendedClient { }); } - @SuppressWarnings("unchecked") private void checkMapping(String index, String type, MappingMetaData mappingMetaData) { try { SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); SearchResponse searchResponse = searchRequestBuilder.setSize(0) .setIndices(index) .setTypes(type) - .setQuery(matchAllQuery()) + .setQuery(QueryBuilders.matchAllQuery()) .execute() .actionGet(); long total = searchResponse.getHits().getTotalHits(); @@ -977,8 +1076,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient { checkMapping(index, type, path, key, child, fields); } } else if ("type".equals(key)) { - QueryBuilder filterBuilder = existsQuery(path); - QueryBuilder queryBuilder = constantScoreQuery(filterBuilder); + QueryBuilder filterBuilder = QueryBuilders.existsQuery(path); + QueryBuilder queryBuilder = QueryBuilders.constantScoreQuery(filterBuilder); SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); SearchResponse searchResponse = searchRequestBuilder.setSize(0) .setIndices(index) @@ -997,148 +1096,4 @@ public abstract class AbstractExtendedClient implements ExtendedClient { .forEachOrdered(e -> result.put(e.getKey(), e.getValue())); return result; } - - @SuppressWarnings("unchecked") - public void createIndex(IndexDefinition indexDefinition) - throws IOException { - ensureActive(); - waitForCluster("YELLOW", "30s"); - URL indexSettings = indexDefinition.getSettingsUrl(); - if (indexSettings == null) { - throw new IllegalArgumentException("no settings defined for index " + indexDefinition.getIndex()); - } - URL indexMappings = indexDefinition.getMappingsUrl(); - if (indexMappings == null) { - throw new IllegalArgumentException("no mappings defined for index " + indexDefinition.getIndex()); - } - try (InputStream indexSettingsInput = indexSettings.openStream(); - InputStream indexMappingsInput = indexMappings.openStream()) { - // multiple type? - if (indexDefinition.getType() == null) { - Map mapping = new HashMap<>(); - // get type names from input stream - Reader reader = new InputStreamReader(indexMappingsInput, StandardCharsets.UTF_8); - Map map = JsonXContent.jsonXContent.createParser(reader).mapOrdered(); - for (Map.Entry entry : map.entrySet()) { - mapping.put(entry.getKey(), JsonXContent.contentBuilder().map((Map) entry.getValue()).string()); - } - Settings settings = Settings.settingsBuilder() - .loadFromStream("", indexSettingsInput) - .build(); - newIndex(indexDefinition.getFullIndexName(), settings, mapping); - } else { - newIndex(indexDefinition.getFullIndexName(), - indexDefinition.getType(), indexSettingsInput, indexMappingsInput); - } - } catch (IOException e) { - if (indexDefinition.ignoreErrors()) { - logger.warn(e.getMessage(), e); - logger.warn("warning while creating index '{}' with settings at {} and mappings at {}", - indexDefinition.getFullIndexName(), indexSettings, indexMappings); - } else { - logger.error("error while creating index '{}' with settings at {} and mappings at {}", - indexDefinition.getFullIndexName(), indexSettings, indexMappings); - throw new IOException(e); - } - } - } - - public void startBulk(Map defs) throws IOException { - ensureActive(); - for (Map.Entry entry : defs.entrySet()) { - IndexDefinition def = entry.getValue(); - startBulk(def.getFullIndexName(), -1, 1); - } - } - - public void stopBulk(Map defs) throws IOException { - ensureActive(); - if (defs == null) { - return; - } - try { - logger.info("flush bulk"); - flushIngest(); - logger.info("waiting for all bulk responses from cluster"); - waitForResponses("120s"); - logger.info("all bulk responses received"); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.error(e.getMessage(), e); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } finally { - logger.info("updating cluster settings of {}", defs.keySet()); - for (Map.Entry entry : defs.entrySet()) { - IndexDefinition def = entry.getValue(); - stopBulk(def.getFullIndexName()); - } - } - } - - public void forceMerge(Map defs) { - for (Map.Entry entry : defs.entrySet()) { - if (entry.getValue().hasForceMerge()) { - logger.info("force merge of {}", entry.getKey()); - try { - ForceMergeRequestBuilder forceMergeRequestBuilder = - new ForceMergeRequestBuilder(client, ForceMergeAction.INSTANCE); - forceMergeRequestBuilder.setIndices(entry.getValue().getFullIndexName()); - forceMergeRequestBuilder.execute().get(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.error(e.getMessage(), e); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - } - } - - public void switchIndex(IndexDefinition indexDefinition, List extraAliases) { - if (extraAliases == null) { - return; - } - if (indexDefinition.isSwitchAliases()) { - // filter out null/empty values - List validAliases = extraAliases.stream() - .filter(a -> a != null && !a.isEmpty()) - .collect(Collectors.toList()); - try { - switchAliases(indexDefinition.getIndex(), - indexDefinition.getFullIndexName(), validAliases); - } catch (Exception e) { - logger.warn("switching index failed: " + e.getMessage(), e); - } - } - } - - public void switchIndex(IndexDefinition indexDefinition, - List extraAliases, IndexAliasAdder indexAliasAdder) { - if (extraAliases == null) { - return; - } - if (indexDefinition.isSwitchAliases()) { - // filter out null/empty values - List validAliases = extraAliases.stream() - .filter(a -> a != null && !a.isEmpty()) - .collect(Collectors.toList()); - try { - switchAliases(indexDefinition.getIndex(), - indexDefinition.getFullIndexName(), validAliases, indexAliasAdder); - } catch (Exception e) { - logger.warn("switching index failed: " + e.getMessage(), e); - } - } - } - - public void replica(IndexDefinition indexDefinition) { - if (indexDefinition.getReplicaLevel() > 0) { - try { - updateReplicaLevel(indexDefinition.getFullIndexName(), indexDefinition.getReplicaLevel()); - } catch (Exception e) { - logger.warn("setting replica failed: " + e.getMessage(), e); - } - } - } } diff --git a/elx-common/src/main/java/org/xbib/elx/common/BulkProcessor.java b/elx-common/src/main/java/org/xbib/elx/common/BulkProcessor.java index 10f6e62..1aeeed4 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/BulkProcessor.java +++ b/elx-common/src/main/java/org/xbib/elx/common/BulkProcessor.java @@ -38,24 +38,24 @@ public class BulkProcessor implements Closeable { private final ScheduledFuture scheduledFuture; - private final AtomicLong executionIdGen = new AtomicLong(); + private final AtomicLong executionIdGen; private final BulkRequestHandler bulkRequestHandler; private BulkRequest bulkRequest; - private volatile boolean closed = false; + private volatile boolean closed; private BulkProcessor(Client client, Listener listener, String name, int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval) { + this.executionIdGen = new AtomicLong(); + this.closed = false; this.bulkActions = bulkActions; this.bulkSize = bulkSize.bytes(); - this.bulkRequest = new BulkRequest(); this.bulkRequestHandler = concurrentRequests == 0 ? new SyncBulkRequestHandler(client, listener) : new AsyncBulkRequestHandler(client, listener, concurrentRequests); - if (flushInterval != null) { this.scheduler = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(1, EsExecutors.daemonThreadFactory(client.settings(), @@ -83,6 +83,7 @@ public class BulkProcessor implements Closeable { @Override public void close() { try { + // 0 = immediate close awaitClose(0, TimeUnit.NANOSECONDS); } catch (InterruptedException exc) { Thread.currentThread().interrupt(); @@ -90,8 +91,27 @@ public class BulkProcessor implements Closeable { } /** - * Closes the processor. If flushing by time is enabled, then it's shutdown. Any remaining bulk actions are - * flushed. + * Wait for bulk request handler with flush. + * @param timeout the timeout value + * @param unit the timeout unit + * @return true is method was successful, false if timeout + * @throws InterruptedException if timeout + */ + public boolean awaitFlush(long timeout, TimeUnit unit) throws InterruptedException { + if (closed) { + return true; + } + // flush + if (bulkRequest.numberOfActions() > 0) { + execute(); + } + // wait for all bulk responses + return this.bulkRequestHandler.close(timeout, unit); + } + + /** + * Closes the processor. Any remaining bulk actions are flushed and then closed. This emthod can only be called + * once as the last action of a bulk processor. * * If concurrent requests are not enabled, returns {@code true} immediately. * If concurrent requests are enabled, waits for up to the specified timeout for all bulk requests to complete then @@ -116,7 +136,7 @@ public class BulkProcessor implements Closeable { if (bulkRequest.numberOfActions() > 0) { execute(); } - return this.bulkRequestHandler.awaitClose(timeout, unit); + return this.bulkRequestHandler.close(timeout, unit); } /** @@ -257,7 +277,7 @@ public class BulkProcessor implements Closeable { private int bulkActions = 1000; - private ByteSizeValue bulkSize = new ByteSizeValue(5, ByteSizeUnit.MB); + private ByteSizeValue bulkSize = new ByteSizeValue(10, ByteSizeUnit.MB); private TimeValue flushInterval = null; @@ -367,7 +387,7 @@ public class BulkProcessor implements Closeable { void execute(BulkRequest bulkRequest, long executionId); - boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException; + boolean close(long timeout, TimeUnit unit) throws InterruptedException; } @@ -398,7 +418,7 @@ public class BulkProcessor implements Closeable { } @Override - public boolean awaitClose(long timeout, TimeUnit unit) { + public boolean close(long timeout, TimeUnit unit) { return true; } } @@ -461,7 +481,7 @@ public class BulkProcessor implements Closeable { } @Override - public boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException { + public boolean close(long timeout, TimeUnit unit) throws InterruptedException { if (semaphore.tryAcquire(concurrentRequests, timeout, unit)) { semaphore.release(concurrentRequests); return true; diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java index dc01807..7e5a1e4 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java @@ -6,8 +6,6 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; -import java.util.Map; - /** * Mock client, it does not perform actions on a cluster. Useful for testing or dry runs. */ @@ -29,52 +27,32 @@ public class MockExtendedClient extends AbstractExtendedClient { } @Override - public MockExtendedClient maxActionsPerRequest(int maxActions) { + public MockExtendedClient index(String index, String id, boolean create, String source) { return this; } @Override - public MockExtendedClient maxConcurrentRequests(int maxConcurrentRequests) { + public MockExtendedClient delete(String index, String id) { return this; } @Override - public MockExtendedClient maxVolumePerRequest(String maxVolumePerRequest) { + public MockExtendedClient update(String index, String id, String source) { return this; } @Override - public MockExtendedClient flushIngestInterval(String interval) { + public MockExtendedClient index(IndexRequest indexRequest) { return this; } @Override - public MockExtendedClient index(String index, String type, String id, boolean create, String source) { + public MockExtendedClient delete(DeleteRequest deleteRequest) { return this; } @Override - public MockExtendedClient delete(String index, String type, String id) { - return this; - } - - @Override - public MockExtendedClient update(String index, String type, String id, String source) { - return this; - } - - @Override - public MockExtendedClient indexRequest(IndexRequest indexRequest) { - return this; - } - - @Override - public MockExtendedClient deleteRequest(DeleteRequest deleteRequest) { - return this; - } - - @Override - public MockExtendedClient updateRequest(UpdateRequest updateRequest) { + public MockExtendedClient update(UpdateRequest updateRequest) { return this; } @@ -83,23 +61,13 @@ public class MockExtendedClient extends AbstractExtendedClient { return this; } - @Override - public MockExtendedClient waitForResponses(String timeValue) { - return this; - } - @Override public MockExtendedClient startBulk(String index, long startRefreshInterval, long stopRefreshIterval) { return this; } @Override - public MockExtendedClient stopBulk(String index) { - return this; - } - - @Override - public MockExtendedClient deleteIndex(String index) { + public MockExtendedClient stopBulk(String index, String maxWaitTime) { return this; } @@ -109,34 +77,43 @@ public class MockExtendedClient extends AbstractExtendedClient { } @Override - public MockExtendedClient newMapping(String index, String type, Map mapping) { + public MockExtendedClient deleteIndex(String index) { return this; } @Override - public void putMapping(String index) { + public MockExtendedClient refreshIndex(String index) { + return this; } @Override - public void refreshIndex(String index) { + public MockExtendedClient flushIndex(String index) { + return this; } @Override - public void flushIndex(String index) { + public boolean forceMerge(String index, String maxWaitTime) { + return true; } @Override - public void waitForCluster(String healthColor, String timeValue) { + public boolean waitForCluster(String healthColor, String timeValue) { + return true; } @Override - public int waitForRecovery(String index) { - return -1; + public boolean waitForResponses(String maxWaitTime) { + return true; } @Override - public int updateReplicaLevel(String index, int level) { - return -1; + public boolean waitForRecovery(String index, String maxWaitTime) { + return true; + } + + @Override + public MockExtendedClient updateReplicaLevel(String index, int level, String maxWaitTime) { + return this; } @Override diff --git a/elx-common/src/main/java/org/xbib/elx/common/Parameters.java b/elx-common/src/main/java/org/xbib/elx/common/Parameters.java index 017c780..28d10d7 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/Parameters.java +++ b/elx-common/src/main/java/org/xbib/elx/common/Parameters.java @@ -8,7 +8,7 @@ public enum Parameters { DEFAULT_MAX_VOLUME_PER_REQUEST("10mb"), - DEFAULT_FLUSH_INTERVAL("30s"), + DEFAULT_FLUSH_INTERVAL(30), MAX_ACTIONS_PER_REQUEST ("max_actions_per_request"), diff --git a/elx-common/src/main/java/org/xbib/elx/common/SimpleBulkControl.java b/elx-common/src/main/java/org/xbib/elx/common/SimpleBulkControl.java index 624cec5..d606ecc 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/SimpleBulkControl.java +++ b/elx-common/src/main/java/org/xbib/elx/common/SimpleBulkControl.java @@ -11,19 +11,26 @@ import java.util.Set; */ public class SimpleBulkControl implements BulkControl { - private final Set indexNames = new HashSet<>(); + private final Set indexNames; - private final Map startBulkRefreshIntervals = new HashMap<>(); + private final Map startBulkRefreshIntervals; - private final Map stopBulkRefreshIntervals = new HashMap<>(); + private final Map stopBulkRefreshIntervals; + + private String maxWaitTime; + + public SimpleBulkControl() { + indexNames = new HashSet<>(); + startBulkRefreshIntervals = new HashMap<>(); + stopBulkRefreshIntervals = new HashMap<>(); + maxWaitTime = "30s"; + } @Override public void startBulk(String indexName, long startRefreshInterval, long stopRefreshInterval) { - synchronized (indexNames) { - indexNames.add(indexName); - startBulkRefreshIntervals.put(indexName, startRefreshInterval); - stopBulkRefreshIntervals.put(indexName, stopRefreshInterval); - } + indexNames.add(indexName); + startBulkRefreshIntervals.put(indexName, startRefreshInterval); + stopBulkRefreshIntervals.put(indexName, stopRefreshInterval); } @Override @@ -33,9 +40,7 @@ public class SimpleBulkControl implements BulkControl { @Override public void finishBulk(String indexName) { - synchronized (indexNames) { - indexNames.remove(indexName); - } + indexNames.remove(indexName); } @Override @@ -53,4 +58,9 @@ public class SimpleBulkControl implements BulkControl { return stopBulkRefreshIntervals; } + @Override + public String getMaxWaitTime() { + return maxWaitTime; + } + } diff --git a/elx-common/src/main/java/org/xbib/elx/common/management/package-info.java b/elx-common/src/main/java/org/xbib/elx/common/management/package-info.java deleted file mode 100644 index 0d98623..0000000 --- a/elx-common/src/main/java/org/xbib/elx/common/management/package-info.java +++ /dev/null @@ -1 +0,0 @@ -package org.xbib.elx.common.management; \ No newline at end of file diff --git a/elx-common/src/test/java/org/xbib/elx/common/AliasTest.java b/elx-common/src/test/java/org/xbib/elx/common/AliasTest.java index e9106d0..419da0e 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/AliasTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/AliasTest.java @@ -12,11 +12,11 @@ import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.AliasAction; import org.elasticsearch.common.Strings; import org.junit.Test; -import java.io.IOException; import java.util.Collections; import java.util.Iterator; import java.util.Set; @@ -32,9 +32,10 @@ public class AliasTest extends NodeTestUtils { private static final Logger logger = LogManager.getLogger(AliasTest.class.getName()); @Test - public void testAlias() throws IOException { + public void testAlias() { + Client client = client("1"); CreateIndexRequest indexRequest = new CreateIndexRequest("test"); - client("1").admin().indices().create(indexRequest).actionGet(); + client.admin().indices().create(indexRequest).actionGet(); // put alias IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest(); String[] indices = new String[]{"test"}; @@ -42,11 +43,11 @@ public class AliasTest extends NodeTestUtils { IndicesAliasesRequest.AliasActions aliasAction = new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, indices, aliases); indicesAliasesRequest.addAliasAction(aliasAction); - client("1").admin().indices().aliases(indicesAliasesRequest).actionGet(); + client.admin().indices().aliases(indicesAliasesRequest).actionGet(); // get alias GetAliasesRequest getAliasesRequest = new GetAliasesRequest(Strings.EMPTY_ARRAY); long t0 = System.nanoTime(); - GetAliasesResponse getAliasesResponse = client("1").admin().indices().getAliases(getAliasesRequest).actionGet(); + GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(getAliasesRequest).actionGet(); long t1 = (System.nanoTime() - t0) / 1000000; logger.info("{} time(ms) = {}", getAliasesResponse.getAliases(), t1); assertTrue(t1 >= 0); @@ -54,22 +55,23 @@ public class AliasTest extends NodeTestUtils { @Test public void testMostRecentIndex() { + Client client = client("1"); String alias = "test"; CreateIndexRequest indexRequest = new CreateIndexRequest("test20160101"); - client("1").admin().indices().create(indexRequest).actionGet(); + client.admin().indices().create(indexRequest).actionGet(); indexRequest = new CreateIndexRequest("test20160102"); - client("1").admin().indices().create(indexRequest).actionGet(); + client.admin().indices().create(indexRequest).actionGet(); indexRequest = new CreateIndexRequest("test20160103"); - client("1").admin().indices().create(indexRequest).actionGet(); + client.admin().indices().create(indexRequest).actionGet(); IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest(); String[] indices = new String[]{"test20160101", "test20160102", "test20160103"}; String[] aliases = new String[]{alias}; IndicesAliasesRequest.AliasActions aliasAction = new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, indices, aliases); indicesAliasesRequest.addAliasAction(aliasAction); - client("1").admin().indices().aliases(indicesAliasesRequest).actionGet(); + client.admin().indices().aliases(indicesAliasesRequest).actionGet(); - GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client("1"), + GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client, GetAliasesAction.INSTANCE); GetAliasesResponse getAliasesResponse = getAliasesRequestBuilder.setAliases(alias).execute().actionGet(); Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); diff --git a/elx-node/src/test/java/org/elasticsearch/node/MockNode.java b/elx-node/src/test/java/org/elasticsearch/node/MockNode.java index aad8b8b..1de4c2f 100644 --- a/elx-node/src/test/java/org/elasticsearch/node/MockNode.java +++ b/elx-node/src/test/java/org/elasticsearch/node/MockNode.java @@ -10,10 +10,6 @@ import java.util.Collection; public class MockNode extends Node { - public MockNode() { - super(Settings.EMPTY); - } - public MockNode(Settings settings) { super(settings); } diff --git a/elx-node/src/test/java/org/xbib/elx/node/ExtendeNodeDuplicateIDTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendeNodeDuplicateIDTest.java index 97cb185..ad75a95 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/ExtendeNodeDuplicateIDTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/ExtendeNodeDuplicateIDTest.java @@ -32,7 +32,7 @@ public class ExtendeNodeDuplicateIDTest extends NodeTestUtils { try { client.newIndex("test"); for (int i = 0; i < ACTIONS; i++) { - client.index("test", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.waitForResponses("30s"); diff --git a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientSingleNodeTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientSingleNodeTest.java deleted file mode 100644 index 0d7335d..0000000 --- a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientSingleNodeTest.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.xbib.elx.node; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.junit.Test; -import org.xbib.elx.common.ClientBuilder; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; - -public class ExtendedNodeClientSingleNodeTest extends NodeTestUtils { - - private static final Logger logger = LogManager.getLogger(ExtendedNodeClientSingleNodeTest.class.getSimpleName()); - - @Test - public void testSingleDocNodeClient() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) - .provider(ExtendedNodeClientProvider.class) - .build(); - try { - client.newIndex("test"); - client.index("test", "test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest - client.flushIngest(); - client.waitForResponses("30s"); - } catch (InterruptedException e) { - // ignore - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); - } - assertFalse(client.hasThrowable()); - client.shutdown(); - } - } -} diff --git a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientTest.java index 957972d..de28b4b 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientTest.java @@ -3,6 +3,7 @@ package org.xbib.elx.node; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -12,6 +13,7 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -44,7 +46,7 @@ public class ExtendedNodeClientTest extends NodeTestUtils { } @Test - public void testSingleDocNodeClient() throws Exception { + public void testSingleDoc() throws Exception { final ExtendedNodeClient client = ClientBuilder.builder(client("1")) .provider(ExtendedNodeClientProvider.class) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) @@ -52,11 +54,9 @@ public class ExtendedNodeClientTest extends NodeTestUtils { .build(); try { client.newIndex("test"); - client.index("test", "test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest client.flushIngest(); client.waitForResponses("30s"); - } catch (InterruptedException e) { - // ignore } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { @@ -70,7 +70,7 @@ public class ExtendedNodeClientTest extends NodeTestUtils { } @Test - public void testNewIndexNodeClient() throws Exception { + public void testNewIndex() throws Exception { final ExtendedNodeClient client = ClientBuilder.builder(client("1")) .provider(ExtendedNodeClientProvider.class) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) @@ -84,14 +84,14 @@ public class ExtendedNodeClientTest extends NodeTestUtils { } @Test - public void testMappingNodeClient() throws Exception { + public void testMapping() throws Exception { final ExtendedNodeClient client = ClientBuilder.builder(client("1")) .provider(ExtendedNodeClientProvider.class) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) .build(); XContentBuilder builder = jsonBuilder() .startObject() - .startObject("test") + .startObject("doc") .startObject("properties") .startObject("location") .field("type", "geo_point") @@ -99,12 +99,12 @@ public class ExtendedNodeClientTest extends NodeTestUtils { .endObject() .endObject() .endObject(); - client.mapping("test", builder.string()); - client.newIndex("test"); + client.newIndex("test", Settings.EMPTY, builder.string()); GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test"); GetMappingsResponse getMappingsResponse = client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); logger.info("mappings={}", getMappingsResponse.getMappings()); + assertTrue(getMappingsResponse.getMappings().get("test").containsKey("doc")); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); } @@ -113,7 +113,7 @@ public class ExtendedNodeClientTest extends NodeTestUtils { } @Test - public void testRandomDocsNodeClient() throws Exception { + public void testRandomDocs() throws Exception { long numactions = ACTIONS; final ExtendedNodeClient client = ClientBuilder.builder(client("1")) .provider(ExtendedNodeClientProvider.class) @@ -123,7 +123,7 @@ public class ExtendedNodeClientTest extends NodeTestUtils { try { client.newIndex("test"); for (int i = 0; i < ACTIONS; i++) { - client.index("test", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.waitForResponses("30s"); @@ -145,7 +145,7 @@ public class ExtendedNodeClientTest extends NodeTestUtils { } @Test - public void testThreadedRandomDocsNodeClient() throws Exception { + public void testThreadedRandomDocs() throws Exception { int maxthreads = Runtime.getRuntime().availableProcessors(); Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; final Long actions = ACTIONS; @@ -165,7 +165,7 @@ public class ExtendedNodeClientTest extends NodeTestUtils { for (int i = 0; i < maxthreads; i++) { pool.execute(() -> { for (int i1 = 0; i1 < actions; i1++) { - client.index("test", "test", null, false,"{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test", null, false,"{ \"name\" : \"" + randomString(32) + "\"}"); } latch.countDown(); }); @@ -184,7 +184,7 @@ public class ExtendedNodeClientTest extends NodeTestUtils { } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - client.stopBulk("test"); + client.stopBulk("test", "30s"); assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount()); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); diff --git a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeIndexAliasTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeIndexAliasTest.java index 1503fee..222b261 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeIndexAliasTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeIndexAliasTest.java @@ -27,23 +27,23 @@ public class ExtendedNodeIndexAliasTest extends NodeTestUtils { try { client.newIndex("test1234"); for (int i = 0; i < 1; i++) { - client.index("test1234", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test1234", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.refreshIndex("test1234"); List simpleAliases = Arrays.asList("a", "b", "c"); - client.switchAliases("test", "test1234", simpleAliases); + client.switchIndex("test", "test1234", simpleAliases); client.newIndex("test5678"); for (int i = 0; i < 1; i++) { - client.index("test5678", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test5678", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.refreshIndex("test5678"); simpleAliases = Arrays.asList("d", "e", "f"); - client.switchAliases("test", "test5678", simpleAliases, (builder, index, alias) -> + client.switchIndex("test", "test5678", simpleAliases, (builder, index, alias) -> builder.addAlias(index, alias, QueryBuilders.termQuery("my_key", alias))); Map aliases = client.getIndexFilters("test5678"); logger.info("aliases of index test5678 = {}", aliases); diff --git a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeReplicaTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeReplicaTest.java index 89de2df..ed53cf5 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeReplicaTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeReplicaTest.java @@ -17,6 +17,7 @@ import org.junit.Ignore; import org.junit.Test; import org.xbib.elx.common.ClientBuilder; +import java.util.HashMap; import java.util.Map; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; @@ -51,14 +52,14 @@ public class ExtendedNodeReplicaTest extends NodeTestUtils { .build(); try { - client.newIndex("test1", settingsTest1, null) - .newIndex("test2", settingsTest2, null); + client.newIndex("test1", settingsTest1, new HashMap<>()) + .newIndex("test2", settingsTest2, new HashMap<>()); client.waitForCluster("GREEN", "30s"); for (int i = 0; i < 1234; i++) { - client.index("test1", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test1", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } for (int i = 0; i < 1234; i++) { - client.index("test2", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test2", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.waitForResponses("30s"); diff --git a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeSmokeTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeSmokeTest.java new file mode 100644 index 0000000..adac4c6 --- /dev/null +++ b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeSmokeTest.java @@ -0,0 +1,69 @@ +package org.xbib.elx.node; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.common.settings.Settings; +import org.junit.Test; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.SimpleBulkControl; +import org.xbib.elx.common.SimpleBulkMetric; +import org.xbib.elx.api.IndexDefinition; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +public class ExtendedNodeSmokeTest extends NodeTestUtils { + + private static final Logger logger = LogManager.getLogger(ExtendedNodeSmokeTest.class.getSimpleName()); + + @Test + public void smokeTest() throws Exception { + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .build(); + try { + client.setBulkControl(new SimpleBulkControl()); + client.setBulkMetric(new SimpleBulkMetric()); + client.newIndex("test"); + client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + client.flushIngest(); + client.waitForResponses("30s"); + + assertEquals(clusterName, client.getClusterName()); + + client.checkMapping("test"); + + client.update("test", "1", "{ \"name\" : \"Another name\"}"); + client.flushIngest(); + + client.waitForRecovery("test", "10s"); + + client.delete("test", "1"); + client.deleteIndex("test"); + + IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test2", Settings.settingsBuilder() + .build()); + assertEquals(0, indexDefinition.getReplicaLevel()); + client.newIndex(indexDefinition); + client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); + client.flushIngest(); + client.updateReplicaLevel(indexDefinition, 2); + + int replica = client.getReplicaLevel(indexDefinition); + assertEquals(2, replica); + + client.deleteIndex(indexDefinition); + assertEquals(0, client.getBulkMetric().getFailed().getCount()); + assertEquals(4, client.getBulkMetric().getSucceeded().getCount()); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + client.shutdown(); + } + } +} diff --git a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeUpdateReplicaLevelTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeUpdateReplicaLevelTest.java index a1b29a4..01e1df4 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeUpdateReplicaLevelTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeUpdateReplicaLevelTest.java @@ -8,6 +8,8 @@ import org.junit.Ignore; import org.junit.Test; import org.xbib.elx.common.ClientBuilder; +import java.util.HashMap; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -38,15 +40,15 @@ public class ExtendedNodeUpdateReplicaLevelTest extends NodeTestUtils { .build(); try { - client.newIndex("replicatest", settings, null); + client.newIndex("replicatest", settings, new HashMap<>()); client.waitForCluster("GREEN", "30s"); for (int i = 0; i < 12345; i++) { - client.index("replicatest", "replicatest", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("replicatest",null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.waitForResponses("30s"); - shardsAfterReplica = client.updateReplicaLevel("replicatest", replicaLevel); - assertEquals(shardsAfterReplica, numberOfShards * (replicaLevel + 1)); + client.updateReplicaLevel("replicatest", replicaLevel, "30s"); + //assertEquals(shardsAfterReplica, numberOfShards * (replicaLevel + 1)); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { diff --git a/elx-node/src/test/java/org/xbib/elx/node/NodeTestUtils.java b/elx-node/src/test/java/org/xbib/elx/node/NodeTestUtils.java index 9a4750e..05bf386 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/NodeTestUtils.java +++ b/elx-node/src/test/java/org/xbib/elx/node/NodeTestUtils.java @@ -47,11 +47,7 @@ public class NodeTestUtils { private AtomicInteger counter = new AtomicInteger(); - private String cluster; - - private String host; - - private int port; + protected String clusterName; private static void deleteFiles() throws IOException { Path directory = Paths.get(getHome() + "/data"); @@ -115,7 +111,7 @@ public class NodeTestUtils { } protected void setClusterName() { - this.cluster = "test-helper-cluster-" + this.clusterName = "test-helper-cluster-" + NetworkUtils.getLocalAddress().getHostName() + "-" + System.getProperty("user.name") + "-" + counter.incrementAndGet(); @@ -123,7 +119,7 @@ public class NodeTestUtils { protected Settings getNodeSettings() { return settingsBuilder() - .put("cluster.name", cluster) + .put("cluster.name", clusterName) .put("cluster.routing.schedule", "50ms") .put("cluster.routing.allocation.disk.threshold_enabled", false) .put("discovery.zen.multicast.enabled", true) @@ -171,8 +167,8 @@ public class NodeTestUtils { .publishAddress(); if (obj instanceof InetSocketTransportAddress) { InetSocketTransportAddress address = (InetSocketTransportAddress) obj; - host = address.address().getHostName(); - port = address.address().getPort(); + String host = address.address().getHostName(); + int port = address.address().getPort(); } } diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java index f66ac58..e883091 100644 --- a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java +++ b/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java @@ -12,6 +12,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.jboss.netty.channel.DefaultChannelFuture; import org.xbib.elx.common.AbstractExtendedClient; import org.xbib.elx.common.util.NetworkUtils; @@ -36,15 +37,22 @@ public class ExtendedTransportClient extends AbstractExtendedClient { + " " + System.getProperty("java.vm.vendor") + " " + System.getProperty("java.vm.version") + " Elasticsearch " + Version.CURRENT.toString(); - logger.info("creating transport client on {} with effective settings {}", - systemIdentifier, settings.getAsMap()); - TransportClient.Builder builder = TransportClient.builder() - .settings(Settings.builder() - .put("cluster.name", settings.get("cluster.name")) - .put("processors", settings.getAsInt("processors", Runtime.getRuntime().availableProcessors())) - .put("client.transport.ignore_cluster_name", true) - .build()); - return builder.build(); + Settings effectiveSettings = Settings.builder() + // for thread pool size + .put("processors", + settings.getAsInt("processors", Runtime.getRuntime().availableProcessors())) + .put("client.transport.sniff", false) // do not sniff + .put("client.transport.nodes_sampler_interval", "1m") // do not ping + .put("client.transport.ping_timeout", "1m") // wait for unresponsive nodes a very long time before disconnect + .put("client.transport.ignore_cluster_name", true) // connect to any cluster + // custom settings may override defaults + .put(settings) + .build(); + logger.info("creating transport client on {} with custom settings {} and effective settings {}", + systemIdentifier, settings.getAsMap(), effectiveSettings.getAsMap()); + // we need to disable dead lock check because we may have mixed node/transport clients + DefaultChannelFuture.setUseDeadLockChecker(false); + return TransportClient.builder().settings(effectiveSettings).build(); } return null; } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientSingleNodeTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientSingleNodeTest.java index cb9dba9..3422762 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientSingleNodeTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientSingleNodeTest.java @@ -21,11 +21,9 @@ public class ExtendedTransportClientSingleNodeTest extends NodeTestUtils { .build(); try { client.newIndex("test"); - client.index("test", "test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest client.flushIngest(); client.waitForResponses("30s"); - } catch (InterruptedException e) { - // ignore } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientTest.java index cfca3da..5085df6 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientTest.java @@ -2,25 +2,31 @@ package org.xbib.elx.transport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.junit.Before; import org.junit.Test; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; +import java.util.HashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; - +import static org.junit.Assert.assertTrue; public class ExtendedTransportClientTest extends NodeTestUtils { @@ -41,7 +47,7 @@ public class ExtendedTransportClientTest extends NodeTestUtils { } @Test - public void testBulkClient() throws Exception { + public void testClientIndexOp() throws Exception { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) .put(getSettings()) @@ -68,7 +74,7 @@ public class ExtendedTransportClientTest extends NodeTestUtils { } @Test - public void testSingleDocBulkClient() throws Exception { + public void testSingleDoc() throws Exception { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) .put(getSettings()) @@ -77,11 +83,9 @@ public class ExtendedTransportClientTest extends NodeTestUtils { .build(); try { client.newIndex("test"); - client.index("test", "test", "1", true, "{ \"name\" : \"Hello World\"}"); + client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); client.flushIngest(); client.waitForResponses("30s"); - } catch (InterruptedException e) { - // ignore } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { @@ -95,7 +99,37 @@ public class ExtendedTransportClientTest extends NodeTestUtils { } @Test - public void testRandomDocsBulkClient() throws Exception { + public void testMapping() throws Exception { + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) + .put(getSettings()) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) + .build(); + XContentBuilder builder = jsonBuilder() + .startObject() + .startObject("doc") + .startObject("properties") + .startObject("location") + .field("type", "geo_point") + .endObject() + .endObject() + .endObject() + .endObject(); + client.newIndex("test", Settings.EMPTY, builder.string()); + GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test"); + GetMappingsResponse getMappingsResponse = + client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); + logger.info("mappings={}", getMappingsResponse.getMappings()); + assertTrue(getMappingsResponse.getMappings().get("test").containsKey("doc")); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + client.shutdown(); + } + + @Test + public void testRandomDocs() throws Exception { long numactions = ACTIONS; final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) @@ -106,12 +140,10 @@ public class ExtendedTransportClientTest extends NodeTestUtils { try { client.newIndex("test"); for (int i = 0; i < ACTIONS; i++) { - client.index("test", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.waitForResponses("30s"); - } catch (InterruptedException e) { - // ignore } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { @@ -125,7 +157,7 @@ public class ExtendedTransportClientTest extends NodeTestUtils { } @Test - public void testThreadedRandomDocsBulkClient() throws Exception { + public void testThreadedRandomDocs() throws Exception { int maxthreads = Runtime.getRuntime().availableProcessors(); long maxactions = MAX_ACTIONS_PER_REQUEST; final long maxloop = ACTIONS; @@ -142,7 +174,7 @@ public class ExtendedTransportClientTest extends NodeTestUtils { .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) .build(); try { - client.newIndex("test", settingsForIndex, null) + client.newIndex("test", settingsForIndex, new HashMap<>()) .startBulk("test", -1, 1000); ThreadPoolExecutor pool = EsExecutors.newFixed("bulkclient-test", maxthreads, 30, EsExecutors.daemonThreadFactory("bulkclient-test")); @@ -150,7 +182,7 @@ public class ExtendedTransportClientTest extends NodeTestUtils { for (int i = 0; i < maxthreads; i++) { pool.execute(() -> { for (int i1 = 0; i1 < maxloop; i1++) { - client.index("test", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test",null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } latch.countDown(); }); @@ -164,15 +196,16 @@ public class ExtendedTransportClientTest extends NodeTestUtils { pool.shutdown(); logger.info("poot shut down"); } + client.stopBulk("test", "30s"); + assertEquals(maxthreads * maxloop, client.getBulkMetric().getSucceeded().getCount()); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - client.stopBulk("test"); - assertEquals(maxthreads * maxloop, client.getBulkMetric().getSucceeded().getCount()); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); } assertFalse(client.hasThrowable()); + // extra search lookup client.refreshIndex("test"); SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) // to avoid NPE at org.elasticsearch.action.search.SearchRequest.writeTo(SearchRequest.java:580) diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportDuplicateIDTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportDuplicateIDTest.java index 1fa73c9..ea1cd1d 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportDuplicateIDTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportDuplicateIDTest.java @@ -31,7 +31,7 @@ public class ExtendedTransportDuplicateIDTest extends NodeTestUtils { try { client.newIndex("test"); for (int i = 0; i < ACTIONS; i++) { - client.index("test", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.waitForResponses("30s"); diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportIndexAliasTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportIndexAliasTest.java index a2dfb4c..8d27231 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportIndexAliasTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportIndexAliasTest.java @@ -4,7 +4,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.index.query.QueryBuilders; -import org.junit.Ignore; import org.junit.Test; import org.xbib.elx.common.ClientBuilder; @@ -13,8 +12,8 @@ import java.util.List; import java.util.Map; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; -@Ignore public class ExtendedTransportIndexAliasTest extends NodeTestUtils { private static final Logger logger = LogManager.getLogger(ExtendedTransportIndexAliasTest.class.getSimpleName()); @@ -23,43 +22,53 @@ public class ExtendedTransportIndexAliasTest extends NodeTestUtils { public void testIndexAlias() throws Exception { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .build(); + .put(getSettings()).build(); try { client.newIndex("test1234"); for (int i = 0; i < 1; i++) { - client.index("test1234", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test1234", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.refreshIndex("test1234"); List simpleAliases = Arrays.asList("a", "b", "c"); - client.switchAliases("test", "test1234", simpleAliases); + client.switchIndex("test", "test1234", simpleAliases); client.newIndex("test5678"); for (int i = 0; i < 1; i++) { - client.index("test5678", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test5678", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.refreshIndex("test5678"); simpleAliases = Arrays.asList("d", "e", "f"); - client.switchAliases("test", "test5678", simpleAliases, (builder, index, alias) -> + client.switchIndex("test", "test5678", simpleAliases, (builder, index, alias) -> builder.addAlias(index, alias, QueryBuilders.termQuery("my_key", alias))); - Map aliases = client.getIndexFilters("test5678"); - logger.info("aliases of index test5678 = {}", aliases); + Map indexFilters = client.getIndexFilters("test5678"); + logger.info("index filters of index test5678 = {}", indexFilters); + assertTrue(indexFilters.containsKey("a")); + assertTrue(indexFilters.containsKey("b")); + assertTrue(indexFilters.containsKey("c")); + assertTrue(indexFilters.containsKey("d")); + assertTrue(indexFilters.containsKey("e")); - aliases = client.getAliasFilters("test"); + Map aliases = client.getAliasFilters("test"); logger.info("aliases of alias test = {}", aliases); + assertTrue(aliases.containsKey("a")); + assertTrue(aliases.containsKey("b")); + assertTrue(aliases.containsKey("c")); + assertTrue(aliases.containsKey("d")); + assertTrue(aliases.containsKey("e")); + client.waitForResponses("30s"); + assertFalse(client.hasThrowable()); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - client.waitForResponses("30s"); - client.shutdown(); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); } - assertFalse(client.hasThrowable()); + client.shutdown(); } } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportReplicaTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportReplicaTest.java index 6b6d6d4..be95352 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportReplicaTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportReplicaTest.java @@ -16,15 +16,13 @@ import org.elasticsearch.index.indexing.IndexingStats; import org.junit.Test; import org.xbib.elx.common.ClientBuilder; +import java.util.HashMap; import java.util.Map; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -/** - * - */ public class ExtendedTransportReplicaTest extends NodeTestUtils { private static final Logger logger = LogManager.getLogger(ExtendedTransportReplicaTest.class.getSimpleName()); @@ -53,29 +51,30 @@ public class ExtendedTransportReplicaTest extends NodeTestUtils { .build(); try { - client.newIndex("test1", settingsTest1, null) - .newIndex("test2", settingsTest2, null); + client.newIndex("test1", settingsTest1, new HashMap<>()) + .newIndex("test2", settingsTest2, new HashMap<>()); client.waitForCluster("GREEN", "30s"); for (int i = 0; i < 1234; i++) { - client.index("test1", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test1", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } for (int i = 0; i < 1234; i++) { - client.index("test2", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test2", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.waitForResponses("30s"); + client.refreshIndex("test1"); + client.refreshIndex("test2"); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - logger.info("refreshing"); - client.refreshIndex("test1"); - client.refreshIndex("test2"); SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) .setIndices("test1", "test2") .setQuery(matchAllQuery()); long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); logger.info("query total hits={}", hits); assertEquals(2468, hits); + + // TODO move to api IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.getClient(), IndicesStatsAction.INSTANCE).all(); IndicesStatsResponse response = indicesStatsRequestBuilder.execute().actionGet(); @@ -93,16 +92,15 @@ public class ExtendedTransportReplicaTest extends NodeTestUtils { } } try { - client.deleteIndex("test1") - .deleteIndex("test2"); + client.deleteIndex("test1").deleteIndex("test2"); } catch (Exception e) { logger.error("delete index failed, ignored. Reason:", e); } - client.shutdown(); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); } assertFalse(client.hasThrowable()); + client.shutdown(); } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportUpdateReplicaLevelTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportUpdateReplicaLevelTest.java index 17d71d6..1992021 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportUpdateReplicaLevelTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportUpdateReplicaLevelTest.java @@ -7,6 +7,8 @@ import org.elasticsearch.common.settings.Settings; import org.junit.Test; import org.xbib.elx.common.ClientBuilder; +import java.util.HashMap; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -37,15 +39,15 @@ public class ExtendedTransportUpdateReplicaLevelTest extends NodeTestUtils { .build(); try { - client.newIndex("replicatest", settings, null); + client.newIndex("replicatest", settings, new HashMap<>()); client.waitForCluster("GREEN", "30s"); for (int i = 0; i < 12345; i++) { - client.index("replicatest", "replicatest", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("replicatest", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.waitForResponses("30s"); - shardsAfterReplica = client.updateReplicaLevel("replicatest", replicaLevel); - assertEquals(shardsAfterReplica, numberOfShards * (replicaLevel + 1)); + client.updateReplicaLevel("replicatest", replicaLevel, "30s"); + //assertEquals(shardsAfterReplica, numberOfShards * (replicaLevel + 1)); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { diff --git a/gradle.properties b/gradle.properties index b6d2ad5..c961b52 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ group = org.xbib name = elx -version = 2.2.1.3 +version = 2.2.1.4 xbib-metrics.version = 1.1.0 xbib-guice.version = 4.0.4 diff --git a/gradle/sonarqube.gradle b/gradle/sonarqube.gradle deleted file mode 100644 index 5de408d..0000000 --- a/gradle/sonarqube.gradle +++ /dev/null @@ -1,41 +0,0 @@ -tasks.withType(FindBugs) { - ignoreFailures = true - reports { - xml.enabled = true - html.enabled = false - } -} -tasks.withType(Pmd) { - ignoreFailures = true - reports { - xml.enabled = true - html.enabled = true - } -} -tasks.withType(Checkstyle) { - ignoreFailures = true - reports { - xml.enabled = true - html.enabled = true - } -} - -jacocoTestReport { - reports { - xml.enabled true - csv.enabled false - xml.destination "${buildDir}/reports/jacoco-xml" - html.destination "${buildDir}/reports/jacoco-html" - } -} - -sonarqube { - properties { - property "sonar.projectName", "${project.group} ${project.name}" - property "sonar.sourceEncoding", "UTF-8" - property "sonar.tests", "src/integration-test/java" - property "sonar.scm.provider", "git" - property "sonar.java.coveragePlugin", "jacoco" - property "sonar.junit.reportsPath", "build/test-results/test/" - } -} From 5184b75b36d298ac6641e673282aff7a492c19a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Wed, 20 Feb 2019 18:38:28 +0100 Subject: [PATCH 3/3] refactored bulk controller, improved api, renamed und cleaned methods, result classes, removed redundant code and methods --- .../java/org/xbib/elx/api/BulkControl.java | 21 - .../java/org/xbib/elx/api/BulkController.java | 36 ++ .../java/org/xbib/elx/api/BulkMetric.java | 11 +- .../java/org/xbib/elx/api/BulkProcessor.java | 64 ++ .../java/org/xbib/elx/api/ExtendedClient.java | 145 ++--- .../org/xbib/elx/api/IndexDefinition.java | 145 +---- .../org/xbib/elx/api/IndexPruneResult.java | 16 + .../java/org/xbib/elx/api/IndexRetention.java | 24 +- .../org/xbib/elx/api/IndexShiftResult.java | 10 + .../elx/common/AbstractExtendedClient.java | 574 ++++++++---------- .../org/xbib/elx/common/ClientBuilder.java | 24 +- .../elx/common/DefaultBulkController.java | 309 ++++++++++ ...BulkMetric.java => DefaultBulkMetric.java} | 28 +- ...ocessor.java => DefaultBulkProcessor.java} | 151 ++--- .../elx/common/DefaultIndexDefinition.java | 214 +++++++ .../elx/common/DefaultIndexRetention.java | 32 + .../xbib/elx/common/MockExtendedClient.java | 28 +- .../xbib/elx/common/SimpleBulkControl.java | 66 -- .../org/xbib/elx/common/io/package-info.java | 5 +- .../xbib/elx/common/util/package-info.java | 5 +- .../org/xbib/elx/node/ExtendedNodeClient.java | 5 +- .../java/org/xbib/elx/node/package-info.java | 4 + ...dedNodeClientTest.java => ClientTest.java} | 58 +- ...erBlockTest.java => ClusterBlockTest.java} | 16 +- ...licateIDTest.java => DuplicateIDTest.java} | 19 +- .../ExtendedNodeUpdateReplicaLevelTest.java | 63 -- ...ndexAliasTest.java => IndexShiftTest.java} | 44 +- .../java/org/xbib/elx/node/NodeTestUtils.java | 15 - ...dNodeReplicaTest.java => ReplicaTest.java} | 61 +- ...endedNodeSmokeTest.java => SmokeTest.java} | 30 +- .../transport/ExtendedTransportClient.java | 8 +- .../xbib/elx/transport/TransportClient.java | 4 +- ...ansportClientTest.java => ClientTest.java} | 60 +- ...licateIDTest.java => DuplicateIDTest.java} | 22 +- ...tendedTransportUpdateReplicaLevelTest.java | 61 -- ...ndexAliasTest.java => IndexShiftTest.java} | 27 +- ...sportReplicaTest.java => ReplicaTest.java} | 63 +- ...ientSingleNodeTest.java => SmokeTest.java} | 20 +- gradle.properties | 2 +- 39 files changed, 1400 insertions(+), 1090 deletions(-) delete mode 100644 elx-api/src/main/java/org/xbib/elx/api/BulkControl.java create mode 100644 elx-api/src/main/java/org/xbib/elx/api/BulkController.java create mode 100644 elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java create mode 100644 elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java create mode 100644 elx-api/src/main/java/org/xbib/elx/api/IndexShiftResult.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java rename elx-common/src/main/java/org/xbib/elx/common/{SimpleBulkMetric.java => DefaultBulkMetric.java} (78%) rename elx-common/src/main/java/org/xbib/elx/common/{BulkProcessor.java => DefaultBulkProcessor.java} (80%) create mode 100644 elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/DefaultIndexRetention.java delete mode 100644 elx-common/src/main/java/org/xbib/elx/common/SimpleBulkControl.java create mode 100644 elx-node/src/main/java/org/xbib/elx/node/package-info.java rename elx-node/src/test/java/org/xbib/elx/node/{ExtendedNodeClientTest.java => ClientTest.java} (83%) rename elx-node/src/test/java/org/xbib/elx/node/{ExtendedNodeClusterBlockTest.java => ClusterBlockTest.java} (73%) rename elx-node/src/test/java/org/xbib/elx/node/{ExtendeNodeDuplicateIDTest.java => DuplicateIDTest.java} (78%) delete mode 100644 elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeUpdateReplicaLevelTest.java rename elx-node/src/test/java/org/xbib/elx/node/{ExtendedNodeIndexAliasTest.java => IndexShiftTest.java} (50%) rename elx-node/src/test/java/org/xbib/elx/node/{ExtendedNodeReplicaTest.java => ReplicaTest.java} (66%) rename elx-node/src/test/java/org/xbib/elx/node/{ExtendedNodeSmokeTest.java => SmokeTest.java} (70%) rename elx-transport/src/test/java/org/xbib/elx/transport/{ExtendedTransportClientTest.java => ClientTest.java} (83%) rename elx-transport/src/test/java/org/xbib/elx/transport/{ExtendedTransportDuplicateIDTest.java => DuplicateIDTest.java} (74%) delete mode 100644 elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportUpdateReplicaLevelTest.java rename elx-transport/src/test/java/org/xbib/elx/transport/{ExtendedTransportIndexAliasTest.java => IndexShiftTest.java} (72%) rename elx-transport/src/test/java/org/xbib/elx/transport/{ExtendedTransportReplicaTest.java => ReplicaTest.java} (65%) rename elx-transport/src/test/java/org/xbib/elx/transport/{ExtendedTransportClientSingleNodeTest.java => SmokeTest.java} (62%) diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkControl.java b/elx-api/src/main/java/org/xbib/elx/api/BulkControl.java deleted file mode 100644 index c43c137..0000000 --- a/elx-api/src/main/java/org/xbib/elx/api/BulkControl.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.xbib.elx.api; - -import java.util.Map; -import java.util.Set; - -public interface BulkControl { - - void startBulk(String indexName, long startRefreshInterval, long stopRefreshInterval); - - boolean isBulk(String indexName); - - void finishBulk(String indexName); - - Set indices(); - - Map getStartBulkRefreshIntervals(); - - Map getStopBulkRefreshIntervals(); - - String getMaxWaitTime(); -} diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkController.java b/elx-api/src/main/java/org/xbib/elx/api/BulkController.java new file mode 100644 index 0000000..69906ca --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkController.java @@ -0,0 +1,36 @@ +package org.xbib.elx.api; + +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.common.settings.Settings; + +import java.io.Closeable; +import java.io.Flushable; +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +public interface BulkController extends Closeable, Flushable { + + void init(Settings settings); + + Throwable getLastBulkError(); + + void startBulkMode(IndexDefinition indexDefinition) throws IOException; + + void startBulkMode(String indexName, long startRefreshIntervalInSeconds, + long stopRefreshIntervalInSeconds) throws IOException; + + void index(IndexRequest indexRequest); + + void delete(DeleteRequest deleteRequest); + + void update(UpdateRequest updateRequest); + + boolean waitForResponses(long timeout, TimeUnit timeUnit); + + void stopBulkMode(IndexDefinition indexDefinition) throws IOException; + + void stopBulkMode(String index, long timeout, TimeUnit timeUnit) throws IOException; + +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java b/elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java index 3002b8c..3a406fb 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java @@ -1,9 +1,14 @@ package org.xbib.elx.api; +import org.elasticsearch.common.settings.Settings; import org.xbib.metrics.Count; import org.xbib.metrics.Metered; -public interface BulkMetric { +import java.io.Closeable; + +public interface BulkMetric extends Closeable { + + void init(Settings settings); Metered getTotalIngest(); @@ -19,9 +24,9 @@ public interface BulkMetric { Count getFailed(); + long elapsed(); + void start(); void stop(); - - long elapsed(); } diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java b/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java new file mode 100644 index 0000000..5af92e1 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java @@ -0,0 +1,64 @@ +package org.xbib.elx.api; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; + +import java.io.Closeable; +import java.io.Flushable; +import java.util.concurrent.TimeUnit; + +public interface BulkProcessor extends Closeable, Flushable { + + BulkProcessor add(ActionRequest request); + + BulkProcessor add(ActionRequest request, Object payload); + + boolean awaitFlush(long timeout, TimeUnit unit) throws InterruptedException; + + boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException; + + interface BulkRequestHandler { + + void execute(BulkRequest bulkRequest, long executionId); + + boolean close(long timeout, TimeUnit unit) throws InterruptedException; + + } + + /** + * A listener for the execution. + */ + public interface Listener { + + /** + * Callback before the bulk is executed. + * + * @param executionId execution ID + * @param request request + */ + void beforeBulk(long executionId, BulkRequest request); + + /** + * Callback after a successful execution of bulk request. + * + * @param executionId execution ID + * @param request request + * @param response response + */ + void afterBulk(long executionId, BulkRequest request, BulkResponse response); + + /** + * Callback after a failed execution of bulk request. + * + * Note that in case an instance of InterruptedException is passed, which means that request + * processing has been + * cancelled externally, the thread's interruption status has been restored prior to calling this method. + * + * @param executionId execution ID + * @param request request + * @param failure failure + */ + void afterBulk(long executionId, BulkRequest request, Throwable failure); + } +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java index 9c2f0e1..fee17bc 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java +++ b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java @@ -6,15 +6,19 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; + +import java.io.Closeable; +import java.io.Flushable; import java.io.IOException; import java.io.InputStream; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; /** * Interface for extended managing and indexing methods of an Elasticsearch client. */ -public interface ExtendedClient { +public interface ExtendedClient extends Flushable, Closeable { /** * Set an Elasticsearch client to extend from it. May be null for TransportClient. @@ -30,40 +34,27 @@ public interface ExtendedClient { */ ElasticsearchClient getClient(); - /** - * Initiative the extended client, cerates instances and connect to cluster, if required. - * - * @param settings settings - * @return this client - * @throws IOException if init fails - */ - ExtendedClient init(Settings settings) throws IOException; - - /** - * Set bulk metric. - * @param bulkMetric the bulk metric - * @return this client - */ - ExtendedClient setBulkMetric(BulkMetric bulkMetric); - /** * Get bulk metric. * @return the bulk metric */ BulkMetric getBulkMetric(); - /** - * Set bulk control. - * @param bulkControl the bulk control - * @return this - */ - ExtendedClient setBulkControl(BulkControl bulkControl); - /** * Get buulk control. * @return the bulk control */ - BulkControl getBulkControl(); + BulkController getBulkController(); + + /** + * Initiative the extended client, the bulk metric and bulk controller, + * creates instances and connect to cluster, if required. + * + * @param settings settings + * @return this client + * @throws IOException if init fails + */ + ExtendedClient init(Settings settings) throws IOException; /** * Build index definition from settings. @@ -256,18 +247,12 @@ public interface ExtendedClient { * Stops bulk mode. * * @param index index - * @param maxWaitTime maximum wait time + * @param timeout maximum wait time + * @param timeUnit time unit for timeout * @return this * @throws IOException if bulk could not be stopped */ - ExtendedClient stopBulk(String index, String maxWaitTime) throws IOException; - - /** - * Flush bulk indexing, move all pending documents to the cluster. - * - * @return this - */ - ExtendedClient flushIngest(); + ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException; /** * Update replica level. @@ -284,10 +269,11 @@ public interface ExtendedClient { * @param index index * @param level the replica level * @param maxWaitTime maximum wait time + * @param timeUnit time unit * @return this * @throws IOException if replica setting could not be updated */ - ExtendedClient updateReplicaLevel(String index, int level, String maxWaitTime) throws IOException; + ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException; /** * Get replica level. @@ -330,43 +316,57 @@ public interface ExtendedClient { * Force segment merge of an index. * @param index the index * @param maxWaitTime maximum wait time + * @param timeUnit time unit * @return this */ - boolean forceMerge(String index, String maxWaitTime); + boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit); /** * Wait for all outstanding bulk responses. * - * @param maxWaitTime maximum wait time + * @param timeout maximum wait time + * @param timeUnit unit of timeout value * @return true if wait succeeded, false if wait timed out */ - boolean waitForResponses(String maxWaitTime); + boolean waitForResponses(long timeout, TimeUnit timeUnit); /** * Wait for cluster being healthy. * * @param healthColor cluster health color to wait for * @param maxWaitTime time value + * @param timeUnit time unit * @return true if wait succeeded, false if wait timed out */ - boolean waitForCluster(String healthColor, String maxWaitTime); + boolean waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit); /** * Get current health color. * * @param maxWaitTime maximum wait time + * @param timeUnit time unit * @return the cluster health color */ - String getHealthColor(String maxWaitTime); + String getHealthColor(long maxWaitTime, TimeUnit timeUnit); /** * Wait for index recovery (after replica change). * * @param index index * @param maxWaitTime maximum wait time + * @param timeUnit time unit * @return true if wait succeeded, false if wait timed out */ - boolean waitForRecovery(String index, String maxWaitTime); + boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit); + + /** + * Update index setting. + * @param index the index + * @param key the key of the value to be updated + * @param value the new value + * @throws IOException if update index setting failed + */ + void updateIndexSetting(String index, String key, Object value) throws IOException; /** * Resolve alias. @@ -385,14 +385,6 @@ public interface ExtendedClient { */ String resolveMostRecentIndex(String alias); - /** - * Get all alias filters. - * - * @param alias the alias - * @return map of alias filters - */ - Map getAliasFilters(String alias); - /** * Get all index filters. * @param index the index @@ -401,48 +393,49 @@ public interface ExtendedClient { Map getIndexFilters(String index); /** - * Switch from one index to another. + * Shift from one index to another. * @param indexDefinition the index definition - * @param extraAliases new aliases + * @param additionalAliases new aliases * @return this */ - ExtendedClient switchIndex(IndexDefinition indexDefinition, List extraAliases); + IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List additionalAliases); /** - * Switch from one index to another. + * Shift from one index to another. * @param indexDefinition the index definition - * @param extraAliases new aliases + * @param additionalAliases new aliases * @param indexAliasAdder method to add aliases * @return this */ - ExtendedClient switchIndex(IndexDefinition indexDefinition, List extraAliases, IndexAliasAdder indexAliasAdder); + IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List additionalAliases, + IndexAliasAdder indexAliasAdder); /** - * Switch from one index to another. - * + * Shift from one index to another. * @param index the index name * @param fullIndexName the index name with timestamp - * @param extraAliases a list of names that should be set as index aliases + * @param additionalAliases a list of names that should be set as index aliases * @return this */ - ExtendedClient switchIndex(String index, String fullIndexName, List extraAliases); + IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases); /** - * Switch from one index to another. - * + * Shift from one index to another. * @param index the index name * @param fullIndexName the index name with timestamp - * @param extraAliases a list of names that should be set as index aliases + * @param additionalAliases a list of names that should be set as index aliases * @param adder an adder method to create alias term queries * @return this */ - ExtendedClient switchIndex(String index, String fullIndexName, List extraAliases, IndexAliasAdder adder); + IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases, + IndexAliasAdder adder); /** * Prune index. * @param indexDefinition the index definition + * @return the index prune result */ - void pruneIndex(IndexDefinition indexDefinition); + IndexPruneResult pruneIndex(IndexDefinition indexDefinition); /** * Apply retention policy to prune indices. All indices before delta should be deleted, @@ -452,8 +445,10 @@ public interface ExtendedClient { * @param fullIndexName index name with timestamp * @param delta timestamp delta (for index timestamps) * @param mintokeep minimum number of indices to keep + * @param perform true if pruning should be executed, false if not + * @return the index prune result */ - void pruneIndex(String index, String fullIndexName, int delta, int mintokeep); + IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform); /** * Find the timestamp of the most recently indexed document in the index. @@ -470,24 +465,4 @@ public interface ExtendedClient { * @return the cluster name */ String getClusterName(); - - /** - * Returns true is a throwable exists. - * - * @return true if a Throwable exists - */ - boolean hasThrowable(); - - /** - * Return last throwable if exists. - * - * @return last throwable - */ - Throwable getThrowable(); - - /** - * Shutdown the client. - * @throws IOException if shutdown fails - */ - void shutdown() throws IOException; } diff --git a/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java b/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java index 7e12592..49544a7 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java @@ -2,148 +2,69 @@ package org.xbib.elx.api; import java.net.MalformedURLException; import java.net.URL; +import java.util.concurrent.TimeUnit; -public class IndexDefinition { +public interface IndexDefinition { - private String index; + IndexDefinition setIndex(String index); - private String fullIndexName; + String getIndex(); - private String dateTimePattern; + IndexDefinition setFullIndexName(String fullIndexName); - private URL settingsUrl; + String getFullIndexName(); - private URL mappingsUrl; + IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException; - private boolean enabled; + IndexDefinition setSettingsUrl(URL settingsUrl); - private boolean ignoreErrors; + URL getSettingsUrl(); - private boolean switchAliases; + IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException; - private boolean hasForceMerge; + IndexDefinition setMappingsUrl(URL mappingsUrl); - private int replicaLevel; + URL getMappingsUrl(); - private IndexRetention indexRetention; + IndexDefinition setDateTimePattern(String timeWindow); - private String maxWaitTime; + String getDateTimePattern(); - public IndexDefinition setIndex(String index) { - this.index = index; - return this; - } + IndexDefinition setEnabled(boolean enabled); - public String getIndex() { - return index; - } + boolean isEnabled(); - public IndexDefinition setFullIndexName(String fullIndexName) { - this.fullIndexName = fullIndexName; - return this; - } + IndexDefinition setIgnoreErrors(boolean ignoreErrors); - public String getFullIndexName() { - return fullIndexName; - } + boolean ignoreErrors(); - public IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException { - this.settingsUrl = settingsUrlString != null ? new URL(settingsUrlString) : null; - return this; - } + IndexDefinition setShift(boolean shift); - public IndexDefinition setSettingsUrl(URL settingsUrl) { - this.settingsUrl = settingsUrl; - return this; - } + boolean isShiftEnabled(); - public URL getSettingsUrl() { - return settingsUrl; - } + IndexDefinition setForceMerge(boolean hasForceMerge); - public IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException { - this.mappingsUrl = mappingsUrlString != null ? new URL(mappingsUrlString) : null; - return this; - } + boolean hasForceMerge(); - public IndexDefinition setMappingsUrl(URL mappingsUrl) { - this.mappingsUrl = mappingsUrl; - return this; - } + IndexDefinition setReplicaLevel(int replicaLevel); - public URL getMappingsUrl() { - return mappingsUrl; - } + int getReplicaLevel(); - public IndexDefinition setDateTimePattern(String timeWindow) { - this.dateTimePattern = timeWindow; - return this; - } + IndexDefinition setRetention(IndexRetention indexRetention); - public String getDateTimePattern() { - return dateTimePattern; - } + IndexRetention getRetention(); - public IndexDefinition setEnabled(boolean enabled) { - this.enabled = enabled; - return this; - } + IndexDefinition setMaxWaitTime(long maxWaitTime, TimeUnit timeUnit); - public boolean isEnabled() { - return enabled; - } + long getMaxWaitTime(); - public IndexDefinition setIgnoreErrors(boolean ignoreErrors) { - this.ignoreErrors = ignoreErrors; - return this; - } + TimeUnit getMaxWaitTimeUnit(); - public boolean ignoreErrors() { - return ignoreErrors; - } + IndexDefinition setStartRefreshInterval(long seconds); - public IndexDefinition setSwitchAliases(boolean switchAliases) { - this.switchAliases = switchAliases; - return this; - } + long getStartRefreshInterval(); - public boolean isSwitchAliases() { - return switchAliases; - } + IndexDefinition setStopRefreshInterval(long seconds); - public IndexDefinition setForceMerge(boolean hasForceMerge) { - this.hasForceMerge = hasForceMerge; - return this; - } - - public boolean hasForceMerge() { - return hasForceMerge; - } - - public IndexDefinition setReplicaLevel(int replicaLevel) { - this.replicaLevel = replicaLevel; - return this; - } - - public int getReplicaLevel() { - return replicaLevel; - } - - public IndexDefinition setRetention(IndexRetention indexRetention) { - this.indexRetention = indexRetention; - return this; - } - - public IndexRetention getRetention() { - return indexRetention; - } - - public IndexDefinition setMaxWaitTime(String maxWaitTime) { - this.maxWaitTime = maxWaitTime; - return this; - } - - public String getMaxWaitTime() { - return maxWaitTime; - } + long getStopRefreshInterval(); } diff --git a/elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java b/elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java new file mode 100644 index 0000000..0c118f8 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java @@ -0,0 +1,16 @@ +package org.xbib.elx.api; + +import java.util.List; + +public interface IndexPruneResult { + + enum State { NOTHING_TO_DO, SUCCESS, NONE }; + + State getState(); + + List getCandidateIndices(); + + List getDeletedIndices(); + + boolean isAcknowledged(); +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/IndexRetention.java b/elx-api/src/main/java/org/xbib/elx/api/IndexRetention.java index e1445fc..44116e2 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/IndexRetention.java +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexRetention.java @@ -1,27 +1,13 @@ package org.xbib.elx.api; -public class IndexRetention { +public interface IndexRetention { - private int timestampDiff; + IndexRetention setDelta(int delta); - private int minToKeep; + int getDelta(); - public IndexRetention setDelta(int timestampDiff) { - this.timestampDiff = timestampDiff; - return this; - } + IndexRetention setMinToKeep(int minToKeep); - public int getDelta() { - return timestampDiff; - } - - public IndexRetention setMinToKeep(int minToKeep) { - this.minToKeep = minToKeep; - return this; - } - - public int getMinToKeep() { - return minToKeep; - } + int getMinToKeep(); } diff --git a/elx-api/src/main/java/org/xbib/elx/api/IndexShiftResult.java b/elx-api/src/main/java/org/xbib/elx/api/IndexShiftResult.java new file mode 100644 index 0000000..02a2e8c --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexShiftResult.java @@ -0,0 +1,10 @@ +package org.xbib.elx.api; + +import java.util.List; + +public interface IndexShiftResult { + + List getMovedAliases(); + + List getNewAliases(); +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java index e9a2230..21f2730 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java @@ -4,7 +4,6 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; @@ -44,16 +43,12 @@ import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.elasticsearch.action.bulk.BulkItemResponse; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.Client; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.cluster.health.ClusterHealthStatus; @@ -62,7 +57,6 @@ import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.QueryBuilder; @@ -71,12 +65,14 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; -import org.xbib.elx.api.BulkControl; +import org.xbib.elx.api.BulkController; import org.xbib.elx.api.BulkMetric; import org.xbib.elx.api.ExtendedClient; import org.xbib.elx.api.IndexAliasAdder; import org.xbib.elx.api.IndexDefinition; +import org.xbib.elx.api.IndexPruneResult; import org.xbib.elx.api.IndexRetention; +import org.xbib.elx.api.IndexShiftResult; import java.io.IOException; import java.io.InputStream; @@ -99,6 +95,7 @@ import java.util.TreeSet; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.regex.Matcher; @@ -120,27 +117,58 @@ public abstract class AbstractExtendedClient implements ExtendedClient { */ private ElasticsearchClient client; - /** - * Our replacement for the buk processor. - */ - private BulkProcessor bulkProcessor; - private BulkMetric bulkMetric; - private BulkControl bulkControl; + private BulkController bulkController; - private Throwable throwable; + private AtomicBoolean closed; - private boolean closed; + private static final IndexShiftResult EMPTY_INDEX_SHIFT_RESULT = new IndexShiftResult() { + @Override + public List getMovedAliases() { + return Collections.emptyList(); + } + + @Override + public List getNewAliases() { + return Collections.emptyList(); + } + }; + + private static final IndexPruneResult EMPTY_INDEX_PRUNE_RESULT = new IndexPruneResult() { + @Override + public State getState() { + return State.NONE; + } + + @Override + public List getCandidateIndices() { + return Collections.emptyList(); + } + + @Override + public List getDeletedIndices() { + return Collections.emptyList(); + } + + @Override + public boolean isAcknowledged() { + return false; + } + }; protected abstract ElasticsearchClient createClient(Settings settings) throws IOException; protected AbstractExtendedClient() { + closed = new AtomicBoolean(false); } @Override public AbstractExtendedClient setClient(ElasticsearchClient client) { this.client = client; + this.bulkMetric = new DefaultBulkMetric(); + bulkMetric.start(); + this.bulkController = new DefaultBulkController(this, bulkMetric); return this; } @@ -149,28 +177,14 @@ public abstract class AbstractExtendedClient implements ExtendedClient { return client; } - @Override - public AbstractExtendedClient setBulkMetric(BulkMetric metric) { - this.bulkMetric = metric; - // you must start bulk metric or it will bail out at stop() - bulkMetric.start(); - return this; - } - @Override public BulkMetric getBulkMetric() { return bulkMetric; } @Override - public AbstractExtendedClient setBulkControl(BulkControl bulkControl) { - this.bulkControl = bulkControl; - return this; - } - - @Override - public BulkControl getBulkControl() { - return bulkControl; + public BulkController getBulkController() { + return bulkController; } @Override @@ -181,120 +195,33 @@ public abstract class AbstractExtendedClient implements ExtendedClient { if (bulkMetric != null) { bulkMetric.start(); } - BulkProcessor.Listener listener = new BulkProcessor.Listener() { - - private final Logger logger = LogManager.getLogger("org.xbib.elx.BulkProcessor.Listener"); - - @Override - public void beforeBulk(long executionId, BulkRequest request) { - long l = 0; - if (bulkMetric != null) { - l = bulkMetric.getCurrentIngest().getCount(); - bulkMetric.getCurrentIngest().inc(); - int n = request.numberOfActions(); - bulkMetric.getSubmitted().inc(n); - bulkMetric.getCurrentIngestNumDocs().inc(n); - bulkMetric.getTotalIngestSizeInBytes().inc(request.estimatedSizeInBytes()); - } - logger.debug("before bulk [{}] [actions={}] [bytes={}] [concurrent requests={}]", - executionId, - request.numberOfActions(), - request.estimatedSizeInBytes(), - l); - } - - @Override - public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { - long l = 0; - if (bulkMetric != null) { - l = bulkMetric.getCurrentIngest().getCount(); - bulkMetric.getCurrentIngest().dec(); - bulkMetric.getSucceeded().inc(response.getItems().length); - } - int n = 0; - for (BulkItemResponse itemResponse : response.getItems()) { - if (bulkMetric != null) { - bulkMetric.getCurrentIngest().dec(itemResponse.getIndex(), itemResponse.getType(), itemResponse.getId()); - } - if (itemResponse.isFailed()) { - n++; - if (bulkMetric != null) { - bulkMetric.getSucceeded().dec(1); - bulkMetric.getFailed().inc(1); - } - } - } - if (bulkMetric != null) { - logger.debug("after bulk [{}] [succeeded={}] [failed={}] [{}ms] {} concurrent requests", - executionId, - bulkMetric.getSucceeded().getCount(), - bulkMetric.getFailed().getCount(), - response.getTook().millis(), - l); - } - if (n > 0) { - logger.error("bulk [{}] failed with {} failed items, failure message = {}", - executionId, n, response.buildFailureMessage()); - } else { - if (bulkMetric != null) { - bulkMetric.getCurrentIngestNumDocs().dec(response.getItems().length); - } - } - } - - @Override - public void afterBulk(long executionId, BulkRequest request, Throwable failure) { - if (bulkMetric != null) { - bulkMetric.getCurrentIngest().dec(); - } - throwable = failure; - closed = true; - logger.error("after bulk [" + executionId + "] error", failure); - } - }; - if (client != null) { - int maxActionsPerRequest = settings.getAsInt(Parameters.MAX_ACTIONS_PER_REQUEST.name(), - Parameters.DEFAULT_MAX_ACTIONS_PER_REQUEST.getNum()); - int maxConcurrentRequests = settings.getAsInt(Parameters.MAX_CONCURRENT_REQUESTS.name(), - Parameters.DEFAULT_MAX_CONCURRENT_REQUESTS.getNum()); - TimeValue flushIngestInterval = settings.getAsTime(Parameters.FLUSH_INTERVAL.name(), - TimeValue.timeValueSeconds(Parameters.DEFAULT_FLUSH_INTERVAL.getNum())); - ByteSizeValue maxVolumePerRequest = settings.getAsBytesSize(Parameters.MAX_VOLUME_PER_REQUEST.name(), - ByteSizeValue.parseBytesSizeValue(Parameters.DEFAULT_MAX_VOLUME_PER_REQUEST.getString(), - "maxVolumePerRequest")); - logger.info("bulk processor up with maxActionsPerRequest = {} maxConcurrentRequests = {} " + - "flushIngestInterval = {} maxVolumePerRequest = {}", - maxActionsPerRequest, maxConcurrentRequests, flushIngestInterval, maxVolumePerRequest); - BulkProcessor.Builder builder = BulkProcessor.builder((Client) client, listener) - .setBulkActions(maxActionsPerRequest) - .setConcurrentRequests(maxConcurrentRequests) - .setFlushInterval(flushIngestInterval) - .setBulkSize(maxVolumePerRequest); - this.bulkProcessor = builder.build(); + if (bulkController != null) { + bulkController.init(settings); } - this.closed = false; - this.throwable = null; return this; } @Override - public synchronized void shutdown() throws IOException { + public void flush() throws IOException { + if (bulkController != null) { + bulkController.flush(); + } + } + + @Override + public void close() throws IOException { ensureActive(); - if (bulkProcessor != null) { - logger.info("closing bulk processor"); - bulkProcessor.close(); - } - if (bulkMetric != null) { - logger.info("stopping metric before bulk stop (for precise measurement)"); - bulkMetric.stop(); - } - if (bulkControl != null && bulkControl.indices() != null && !bulkControl.indices().isEmpty()) { - logger.info("stopping bulk mode for indices {}...", bulkControl.indices()); - for (String index : bulkControl.indices()) { - stopBulk(index, bulkControl.getMaxWaitTime()); + if (closed.compareAndSet(false, true)) { + if (bulkMetric != null) { + logger.info("closing bulk metric before bulk controller (for precise measurement)"); + bulkMetric.close(); } + if (bulkController != null) { + logger.info("closing bulk controller"); + bulkController.close(); + } + logger.info("shutdown complete"); } - logger.info("shutdown complete"); } @Override @@ -320,7 +247,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException { ensureActive(); - waitForCluster("YELLOW", "30s"); + waitForCluster("YELLOW", 30L, TimeUnit.SECONDS); URL indexSettings = indexDefinition.getSettingsUrl(); if (indexSettings == null) { logger.warn("warning while creating index '{}', no settings/mappings", @@ -417,37 +344,27 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds) throws IOException { - ensureActive(); - if (bulkControl == null) { - return this; - } - if (!bulkControl.isBulk(index) && startRefreshIntervalSeconds > 0L && stopRefreshIntervalSeconds > 0L) { - bulkControl.startBulk(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds); - updateIndexSetting(index, "refresh_interval", startRefreshIntervalSeconds + "s"); + if (bulkController != null) { + ensureActive(); + bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds); } return this; } @Override public ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException { - return stopBulk(indexDefinition.getFullIndexName(), indexDefinition.getMaxWaitTime()); + if (bulkController != null) { + ensureActive(); + bulkController.stopBulkMode(indexDefinition); + } + return this; } @Override - public ExtendedClient stopBulk(String index, String maxWaitTime) throws IOException { - ensureActive(); - if (bulkControl == null) { - return this; - } - flushIngest(); - if (waitForResponses(maxWaitTime)) { - if (bulkControl.isBulk(index)) { - long secs = bulkControl.getStopBulkRefreshIntervals().get(index); - if (secs > 0L) { - updateIndexSetting(index, "refresh_interval", secs + "s"); - } - bulkControl.finishBulk(index); - } + public ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException { + if (bulkController != null) { + ensureActive(); + bulkController.stopBulkMode(index, timeout, timeUnit); } return this; } @@ -465,16 +382,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient index(IndexRequest indexRequest) { ensureActive(); - try { - if (bulkMetric != null) { - bulkMetric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id()); - } - bulkProcessor.add(indexRequest); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of index request failed: " + e.getMessage(), e); - } + bulkController.index(indexRequest); return this; } @@ -486,16 +394,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient delete(DeleteRequest deleteRequest) { ensureActive(); - try { - if (bulkMetric != null) { - bulkMetric.getCurrentIngest().inc(deleteRequest.index(), deleteRequest.type(), deleteRequest.id()); - } - bulkProcessor.add(deleteRequest); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of delete failed: " + e.getMessage(), e); - } + bulkController.delete(deleteRequest); return this; } @@ -512,49 +411,23 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient update(UpdateRequest updateRequest) { ensureActive(); - try { - if (bulkMetric != null) { - bulkMetric.getCurrentIngest().inc(updateRequest.index(), updateRequest.type(), updateRequest.id()); - } - bulkProcessor.add(updateRequest); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of update request failed: " + e.getMessage(), e); - } + bulkController.update(updateRequest); return this; } @Override - public ExtendedClient flushIngest() { + public boolean waitForResponses(long timeout, TimeUnit timeUnit) { ensureActive(); - logger.debug("flushing bulk processor"); - bulkProcessor.flush(); - return this; + return bulkController.waitForResponses(timeout, timeUnit); } @Override - public boolean waitForResponses(String maxWaitTime) { - ensureActive(); - long millis = TimeValue.parseTimeValue(maxWaitTime, TimeValue.timeValueMinutes(1),"millis").getMillis(); - logger.debug("waiting for " + millis + " millis"); - try { - return bulkProcessor.awaitFlush(millis, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.error("interrupted"); - return false; - } - } - - @Override - public boolean waitForRecovery(String index, String maxWaitTime) { + public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) { ensureActive(); ensureIndexGiven(index); RecoveryResponse response = client.execute(RecoveryAction.INSTANCE, new RecoveryRequest(index)).actionGet(); int shards = response.getTotalShards(); - TimeValue timeout = TimeValue.parseTimeValue(maxWaitTime, TimeValue.timeValueSeconds(10), - getClass().getSimpleName() + ".timeout"); + TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest(index) .waitForActiveShards(shards).timeout(timeout)).actionGet(); @@ -566,26 +439,26 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public boolean waitForCluster(String statusString, String maxWaitTime) { + public boolean waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) { ensureActive(); ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString); - TimeValue timeout = TimeValue.parseTimeValue(maxWaitTime, TimeValue.timeValueSeconds(10), - getClass().getSimpleName() + ".timeout"); + TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet(); if (healthResponse != null && healthResponse.isTimedOut()) { - logger.error("timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name()); + if (logger.isErrorEnabled()) { + logger.error("timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name()); + } return false; } return true; } @Override - public String getHealthColor(String maxWaitTime) { + public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) { ensureActive(); try { - TimeValue timeout = TimeValue.parseTimeValue(maxWaitTime, TimeValue.timeValueSeconds(10), - getClass().getSimpleName() + ".timeout"); + TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest().timeout(timeout)).actionGet(); ClusterHealthStatus status = healthResponse.getStatus(); @@ -604,15 +477,16 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException { - return updateReplicaLevel(indexDefinition.getFullIndexName(), level, indexDefinition.getMaxWaitTime()); + return updateReplicaLevel(indexDefinition.getFullIndexName(), level, + indexDefinition.getMaxWaitTime(), indexDefinition.getMaxWaitTimeUnit()); } @Override - public ExtendedClient updateReplicaLevel(String index, int level, String maxWaitTime) throws IOException { - waitForCluster("YELLOW", maxWaitTime); // let cluster settle down from critical operations + public ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException { + waitForCluster("YELLOW", maxWaitTime, timeUnit); // let cluster settle down from critical operations if (level > 0) { updateIndexSetting(index, "number_of_replicas", level); - waitForRecovery(index, maxWaitTime); + waitForRecovery(index, maxWaitTime, timeUnit); } return this; } @@ -684,12 +558,6 @@ public abstract class AbstractExtendedClient implements ExtendedClient { return indices.isEmpty() ? alias : indices.iterator().next(); } - @Override - public Map getAliasFilters(String alias) { - GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client, GetAliasesAction.INSTANCE); - return getFilters(getAliasesRequestBuilder.setIndices(resolveAlias(alias)).execute().actionGet()); - } - @Override public Map getIndexFilters(String index) { GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client, GetAliasesAction.INSTANCE); @@ -697,50 +565,49 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public ExtendedClient switchIndex(IndexDefinition indexDefinition, List extraAliases) { - return switchIndex(indexDefinition, extraAliases, null); + public IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List additionalAliases) { + return shiftIndex(indexDefinition, additionalAliases, null); } - @Override - public ExtendedClient switchIndex(IndexDefinition indexDefinition, - List extraAliases, IndexAliasAdder indexAliasAdder) { - if (extraAliases == null) { - return this; + public IndexShiftResult shiftIndex(IndexDefinition indexDefinition, + List additionalAliases, IndexAliasAdder indexAliasAdder) { + if (additionalAliases == null) { + return EMPTY_INDEX_SHIFT_RESULT; } - if (indexDefinition.isSwitchAliases()) { - switchIndex(indexDefinition.getIndex(), - indexDefinition.getFullIndexName(), extraAliases.stream() + if (indexDefinition.isShiftEnabled()) { + return shiftIndex(indexDefinition.getIndex(), + indexDefinition.getFullIndexName(), additionalAliases.stream() .filter(a -> a != null && !a.isEmpty()) .collect(Collectors.toList()), indexAliasAdder); } - return this; + return EMPTY_INDEX_SHIFT_RESULT; } @Override - public ExtendedClient switchIndex(String index, String fullIndexName, List extraAliases) { - return switchIndex(index, fullIndexName, extraAliases, null); + public IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases) { + return shiftIndex(index, fullIndexName, additionalAliases, null); } @Override - public ExtendedClient switchIndex(String index, String fullIndexName, - List extraAliases, IndexAliasAdder adder) { + public IndexShiftResult shiftIndex(String index, String fullIndexName, + List additionalAliases, IndexAliasAdder adder) { ensureActive(); if (index.equals(fullIndexName)) { - return this; // nothing to switch to + return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to } // two situations: 1. there is a new alias 2. there is already an old index with the alias String oldIndex = resolveAlias(index); final Map oldFilterMap = oldIndex.equals(index) ? null : getIndexFilters(oldIndex); final List newAliases = new LinkedList<>(); - final List switchAliases = new LinkedList<>(); + final List moveAliases = new LinkedList<>(); IndicesAliasesRequestBuilder requestBuilder = new IndicesAliasesRequestBuilder(client, IndicesAliasesAction.INSTANCE); if (oldFilterMap == null || !oldFilterMap.containsKey(index)) { // never apply a filter for trunk index name requestBuilder.addAlias(fullIndexName, index); newAliases.add(index); } - // switch existing aliases + // move existing aliases if (oldFilterMap != null) { for (Map.Entry entry : oldFilterMap.entrySet()) { String alias = entry.getKey(); @@ -751,12 +618,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } else { requestBuilder.addAlias(fullIndexName, alias); } - switchAliases.add(alias); + moveAliases.add(alias); } } // a list of aliases that should be added, check if new or old - if (extraAliases != null) { - for (String extraAlias : extraAliases) { + if (additionalAliases != null) { + for (String extraAlias : additionalAliases) { if (oldFilterMap == null || !oldFilterMap.containsKey(extraAlias)) { // index alias adder only active on extra aliases, and if alias is new if (adder != null) { @@ -773,82 +640,72 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } else { requestBuilder.addAlias(fullIndexName, extraAlias); } - switchAliases.add(extraAlias); + moveAliases.add(extraAlias); } } } - if (!newAliases.isEmpty() || !switchAliases.isEmpty()) { - logger.info("new aliases = {}, switch aliases = {}", newAliases, switchAliases); + if (!newAliases.isEmpty() || !moveAliases.isEmpty()) { + logger.info("new aliases = {}, moved aliases = {}", newAliases, moveAliases); requestBuilder.execute().actionGet(); } - return this; + return new SuccessIndexShiftResult(moveAliases, newAliases); } @Override - public void pruneIndex(IndexDefinition indexDefinition) { - pruneIndex(indexDefinition.getIndex(), indexDefinition.getFullIndexName(), - indexDefinition.getRetention().getDelta(), indexDefinition.getRetention().getMinToKeep()); + public IndexPruneResult pruneIndex(IndexDefinition indexDefinition) { + return pruneIndex(indexDefinition.getIndex(), indexDefinition.getFullIndexName(), + indexDefinition.getRetention().getDelta(), indexDefinition.getRetention().getMinToKeep(), true); } @Override - public void pruneIndex(String index, String fullIndexName, int delta, int mintokeep) { + public IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform) { if (delta == 0 && mintokeep == 0) { - return; + return EMPTY_INDEX_PRUNE_RESULT; + } + if (index.equals(fullIndexName)) { + return EMPTY_INDEX_PRUNE_RESULT; } ensureActive(); - if (index.equals(fullIndexName)) { - return; - } GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE); GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet(); Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); - Set indices = new TreeSet<>(); logger.info("{} indices", getIndexResponse.getIndices().length); + List candidateIndices = new ArrayList<>(); for (String s : getIndexResponse.getIndices()) { Matcher m = pattern.matcher(s); if (m.matches() && index.equals(m.group(1)) && !s.equals(fullIndexName)) { - indices.add(s); + candidateIndices.add(s); } } - if (indices.isEmpty()) { - logger.info("no indices found, retention policy skipped"); - return; + if (candidateIndices.isEmpty()) { + return EMPTY_INDEX_PRUNE_RESULT; } - if (mintokeep > 0 && indices.size() <= mintokeep) { - logger.info("{} indices found, not enough for retention policy ({}), skipped", - indices.size(), mintokeep); - return; - } else { - logger.info("candidates for deletion = {}", indices); + if (mintokeep > 0 && candidateIndices.size() <= mintokeep) { + return new NothingToDoPruneResult(candidateIndices, Collections.emptyList()); } List indicesToDelete = new ArrayList<>(); - // our index Matcher m1 = pattern.matcher(fullIndexName); if (m1.matches()) { Integer i1 = Integer.parseInt(m1.group(2)); - for (String s : indices) { + for (String s : candidateIndices) { Matcher m2 = pattern.matcher(s); if (m2.matches()) { Integer i2 = Integer.parseInt(m2.group(2)); - int kept = indices.size() - indicesToDelete.size(); + int kept = candidateIndices.size() - indicesToDelete.size(); if ((delta == 0 || (delta > 0 && i1 - i2 > delta)) && mintokeep <= kept) { indicesToDelete.add(s); } } } } - logger.info("indices to delete = {}", indicesToDelete); if (indicesToDelete.isEmpty()) { - logger.info("not enough indices found to delete, retention policy complete"); - return; + return new NothingToDoPruneResult(candidateIndices, indicesToDelete); } String[] s = new String[indicesToDelete.size()]; DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest() .indices(indicesToDelete.toArray(s)); DeleteIndexResponse response = client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet(); - if (!response.isAcknowledged()) { - logger.warn("retention delete index operation was not acknowledged"); - } + return new SuccessPruneResult(candidateIndices, indicesToDelete, response); } @Override @@ -875,15 +732,15 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public boolean forceMerge(IndexDefinition indexDefinition) { if (indexDefinition.hasForceMerge()) { - return forceMerge(indexDefinition.getFullIndexName(), indexDefinition.getMaxWaitTime()); + return forceMerge(indexDefinition.getFullIndexName(), indexDefinition.getMaxWaitTime(), + indexDefinition.getMaxWaitTimeUnit()); } return false; } @Override - public boolean forceMerge(String index, String maxWaitTime) { - TimeValue timeout = TimeValue.parseTimeValue(maxWaitTime, TimeValue.timeValueSeconds(10), - getClass().getSimpleName() + ".timeout"); + public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) { + TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); ForceMergeRequestBuilder forceMergeRequestBuilder = new ForceMergeRequestBuilder(client, ForceMergeAction.INSTANCE); forceMergeRequestBuilder.setIndices(index); @@ -909,44 +766,35 @@ public abstract class AbstractExtendedClient implements ExtendedClient { String fullIndexName; String dateTimePattern = settings.get("dateTimePattern"); if (dateTimePattern != null) { - fullIndexName = resolveAlias(indexName + - DateTimeFormatter.ofPattern(dateTimePattern) + // check if index name with current date already exists, resolve to it + fullIndexName = resolveAlias(indexName + DateTimeFormatter.ofPattern(dateTimePattern) .withZone(ZoneId.systemDefault()) // not GMT .format(LocalDate.now())); - logger.info("index name {} resolved to full index name = {}", indexName, fullIndexName); } else { + // check if index name already exists, resolve to it fullIndexName = resolveMostRecentIndex(indexName); - logger.info("index name {} resolved to full index name = {}", indexName, fullIndexName); } - IndexRetention indexRetention = new IndexRetention() + IndexRetention indexRetention = new DefaultIndexRetention() .setMinToKeep(settings.getAsInt("retention.mintokeep", 0)) .setDelta(settings.getAsInt("retention.delta", 0)); - - return new IndexDefinition() + return new DefaultIndexDefinition() + .setEnabled(isEnabled) .setIndex(indexName) .setFullIndexName(fullIndexName) .setSettingsUrl(settings.get("settings")) .setMappingsUrl(settings.get("mapping")) .setDateTimePattern(dateTimePattern) - .setEnabled(isEnabled) .setIgnoreErrors(settings.getAsBoolean("skiperrors", false)) - .setSwitchAliases(settings.getAsBoolean("aliases", true)) + .setShift(settings.getAsBoolean("shift", true)) .setReplicaLevel(settings.getAsInt("replica", 0)) - .setMaxWaitTime(settings.get("timout", "30s")) - .setRetention(indexRetention); + .setMaxWaitTime(settings.getAsLong("timeout", 30L), TimeUnit.SECONDS) + .setRetention(indexRetention) + .setStartRefreshInterval(settings.getAsLong("bulk.startrefreshinterval", -1L)) + .setStopRefreshInterval(settings.getAsLong("bulk.stoprefreshinterval", -1L)); } @Override - public boolean hasThrowable() { - return throwable != null; - } - - @Override - public Throwable getThrowable() { - return throwable; - } - - private void updateIndexSetting(String index, String key, Object value) throws IOException { + public void updateIndexSetting(String index, String key, Object value) throws IOException { ensureActive(); if (index == null) { throw new IOException("no index name given"); @@ -971,9 +819,6 @@ public abstract class AbstractExtendedClient implements ExtendedClient { if (client == null) { throw new IllegalStateException("no client"); } - if (closed) { - throw new ElasticsearchException("client is closed"); - } } private void ensureIndexGiven(String index) { @@ -1096,4 +941,115 @@ public abstract class AbstractExtendedClient implements ExtendedClient { .forEachOrdered(e -> result.put(e.getKey(), e.getValue())); return result; } + + private static TimeValue toTimeValue(long timeValue, TimeUnit timeUnit) { + switch (timeUnit) { + case DAYS: + return TimeValue.timeValueHours(24 * timeValue); + case HOURS: + return TimeValue.timeValueHours(timeValue); + case MINUTES: + return TimeValue.timeValueMinutes(timeValue); + case SECONDS: + return TimeValue.timeValueSeconds(timeValue); + case MILLISECONDS: + return TimeValue.timeValueMillis(timeValue); + case MICROSECONDS: + return TimeValue.timeValueNanos(1000 * timeValue); + case NANOSECONDS: + return TimeValue.timeValueNanos(timeValue); + default: + throw new IllegalArgumentException("unknown time unit: " + timeUnit); + } + } + + private static class SuccessIndexShiftResult implements IndexShiftResult { + + List movedAliases; + + List newAliases; + + SuccessIndexShiftResult(List movedAliases, List newAliases) { + this.movedAliases = movedAliases; + this.newAliases = newAliases; + } + + @Override + public List getMovedAliases() { + return movedAliases; + } + + @Override + public List getNewAliases() { + return newAliases; + } + } + + private static class SuccessPruneResult implements IndexPruneResult { + + List candidateIndices; + + List indicesToDelete; + + DeleteIndexResponse response; + + SuccessPruneResult(List candidateIndices, List indicesToDelete, + DeleteIndexResponse response) { + this.candidateIndices = candidateIndices; + this.indicesToDelete = indicesToDelete; + this.response = response; + } + + @Override + public IndexPruneResult.State getState() { + return IndexPruneResult.State.SUCCESS; + } + + @Override + public List getCandidateIndices() { + return candidateIndices; + } + + @Override + public List getDeletedIndices() { + return indicesToDelete; + } + + @Override + public boolean isAcknowledged() { + return response.isAcknowledged(); + } + } + + private static class NothingToDoPruneResult implements IndexPruneResult { + + List candidateIndices; + + List indicesToDelete; + + NothingToDoPruneResult(List candidateIndices, List indicesToDelete) { + this.candidateIndices = candidateIndices; + this.indicesToDelete = indicesToDelete; + } + + @Override + public IndexPruneResult.State getState() { + return IndexPruneResult.State.SUCCESS; + } + + @Override + public List getCandidateIndices() { + return candidateIndices; + } + + @Override + public List getDeletedIndices() { + return indicesToDelete; + } + + @Override + public boolean isAcknowledged() { + return false; + } + } } diff --git a/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java b/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java index cc15697..ba9150f 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java +++ b/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java @@ -5,8 +5,6 @@ import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; -import org.xbib.elx.api.BulkControl; -import org.xbib.elx.api.BulkMetric; import org.xbib.elx.api.ExtendedClient; import org.xbib.elx.api.ExtendedClientProvider; @@ -26,10 +24,6 @@ public class ClientBuilder { private Class provider; - private BulkMetric metric; - - private BulkControl control; - public ClientBuilder() { this(null); } @@ -48,8 +42,6 @@ public class ClientBuilder { for (ExtendedClientProvider provider : serviceLoader) { providerMap.put(provider.getClass(), provider); } - this.metric = new SimpleBulkMetric(); - this.control = new SimpleBulkControl(); } public static ClientBuilder builder() { @@ -100,25 +92,11 @@ public class ClientBuilder { return this; } - public ClientBuilder setMetric(BulkMetric metric) { - this.metric = metric; - return this; - } - - public ClientBuilder setControl(BulkControl control) { - this.control = control; - return this; - } - @SuppressWarnings("unchecked") public C build() throws IOException { if (provider == null) { throw new IllegalArgumentException("no provider"); } - return (C) providerMap.get(provider).getExtendedClient() - .setClient(client) - .setBulkMetric(metric) - .setBulkControl(control) - .init(settingsBuilder.build()); + return (C) providerMap.get(provider).getExtendedClient().setClient(client).init(settingsBuilder.build()); } } diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java new file mode 100644 index 0000000..ca705c4 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java @@ -0,0 +1,309 @@ +package org.xbib.elx.common; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.xbib.elx.api.BulkController; +import org.xbib.elx.api.BulkMetric; +import org.xbib.elx.api.BulkProcessor; +import org.xbib.elx.api.ExtendedClient; +import org.xbib.elx.api.IndexDefinition; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +public class DefaultBulkController implements BulkController { + + private static final Logger logger = LogManager.getLogger(DefaultBulkController.class); + + private final ExtendedClient client; + + private final BulkMetric bulkMetric; + + private final List indexNames; + + private final Map startBulkRefreshIntervals; + + private final Map stopBulkRefreshIntervals; + + private long maxWaitTime; + + private TimeUnit maxWaitTimeUnit; + + private BulkProcessor bulkProcessor; + + private BulkListener bulkListener; + + private AtomicBoolean active; + + public DefaultBulkController(ExtendedClient client, BulkMetric bulkMetric) { + this.client = client; + this.bulkMetric = bulkMetric; + this.indexNames = new ArrayList<>(); + this.active = new AtomicBoolean(false); + this.startBulkRefreshIntervals = new HashMap<>(); + this.stopBulkRefreshIntervals = new HashMap<>(); + this.maxWaitTime = 30L; + this.maxWaitTimeUnit = TimeUnit.SECONDS; + } + + @Override + public Throwable getLastBulkError() { + return bulkListener.getLastBulkError(); + } + + @Override + public void init(Settings settings) { + int maxActionsPerRequest = settings.getAsInt(Parameters.MAX_ACTIONS_PER_REQUEST.name(), + Parameters.DEFAULT_MAX_ACTIONS_PER_REQUEST.getNum()); + int maxConcurrentRequests = settings.getAsInt(Parameters.MAX_CONCURRENT_REQUESTS.name(), + Parameters.DEFAULT_MAX_CONCURRENT_REQUESTS.getNum()); + TimeValue flushIngestInterval = settings.getAsTime(Parameters.FLUSH_INTERVAL.name(), + TimeValue.timeValueSeconds(Parameters.DEFAULT_FLUSH_INTERVAL.getNum())); + ByteSizeValue maxVolumePerRequest = settings.getAsBytesSize(Parameters.MAX_VOLUME_PER_REQUEST.name(), + ByteSizeValue.parseBytesSizeValue(Parameters.DEFAULT_MAX_VOLUME_PER_REQUEST.getString(), + "maxVolumePerRequest")); + if (logger.isInfoEnabled()) { + logger.info("bulk processor up with maxActionsPerRequest = {} maxConcurrentRequests = {} " + + "flushIngestInterval = {} maxVolumePerRequest = {}", + maxActionsPerRequest, maxConcurrentRequests, flushIngestInterval, maxVolumePerRequest); + } + this.bulkListener = new BulkListener(); + DefaultBulkProcessor.Builder builder = DefaultBulkProcessor.builder((Client) client.getClient(), bulkListener) + .setBulkActions(maxActionsPerRequest) + .setConcurrentRequests(maxConcurrentRequests) + .setFlushInterval(flushIngestInterval) + .setBulkSize(maxVolumePerRequest); + this.bulkProcessor = builder.build(); + this.active.set(true); + } + + @Override + public void startBulkMode(IndexDefinition indexDefinition) throws IOException { + startBulkMode(indexDefinition.getFullIndexName(), indexDefinition.getStartRefreshInterval(), + indexDefinition.getStopRefreshInterval()); + } + + @Override + public void startBulkMode(String indexName, + long startRefreshIntervalInSeconds, + long stopRefreshIntervalInSeconds) throws IOException { + if (!indexNames.contains(indexName)) { + indexNames.add(indexName); + startBulkRefreshIntervals.put(indexName, startRefreshIntervalInSeconds); + stopBulkRefreshIntervals.put(indexName, stopRefreshIntervalInSeconds); + if (startRefreshIntervalInSeconds != 0L) { + client.updateIndexSetting(indexName, "refresh_interval", startRefreshIntervalInSeconds + "s"); + } + } + } + + @Override + public void index(IndexRequest indexRequest) { + if (!active.get()) { + throw new IllegalStateException("inactive"); + } + try { + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id()); + } + bulkProcessor.add(indexRequest); + } catch (Exception e) { + bulkListener.lastBulkError = e; + active.set(false); + if (logger.isErrorEnabled()) { + logger.error("bulk add of index failed: " + e.getMessage(), e); + } + } + } + + @Override + public void delete(DeleteRequest deleteRequest) { + if (!active.get()) { + throw new IllegalStateException("inactive"); + } + try { + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().inc(deleteRequest.index(), deleteRequest.type(), deleteRequest.id()); + } + bulkProcessor.add(deleteRequest); + } catch (Exception e) { + bulkListener.lastBulkError = e; + active.set(false); + if (logger.isErrorEnabled()) { + logger.error("bulk add of delete failed: " + e.getMessage(), e); + } + } + } + + @Override + public void update(UpdateRequest updateRequest) { + if (!active.get()) { + throw new IllegalStateException("inactive"); + } + try { + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().inc(updateRequest.index(), updateRequest.type(), updateRequest.id()); + } + bulkProcessor.add(updateRequest); + } catch (Exception e) { + bulkListener.lastBulkError = e; + active.set(false); + if (logger.isErrorEnabled()) { + logger.error("bulk add of update failed: " + e.getMessage(), e); + } + } + } + + @Override + public boolean waitForResponses(long timeout, TimeUnit timeUnit) { + try { + return bulkProcessor.awaitFlush(timeout, timeUnit); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.error("interrupted"); + return false; + } + } + + @Override + public void stopBulkMode(IndexDefinition indexDefinition) throws IOException { + stopBulkMode(indexDefinition.getFullIndexName(), + indexDefinition.getMaxWaitTime(), indexDefinition.getMaxWaitTimeUnit()); + } + + @Override + public void stopBulkMode(String index, long timeout, TimeUnit timeUnit) throws IOException { + flush(); + if (waitForResponses(timeout, timeUnit)) { + if (indexNames.contains(index)) { + Long secs = stopBulkRefreshIntervals.get(index); + if (secs != null && secs != 0L) { + client.updateIndexSetting(index, "refresh_interval", secs + "s"); + } + indexNames.remove(index); + } + } + } + + @Override + public void flush() throws IOException { + if (bulkProcessor != null) { + bulkProcessor.flush(); + } + } + + @Override + public void close() throws IOException { + flush(); + if (client.waitForResponses(maxWaitTime, maxWaitTimeUnit)) { + for (String index : indexNames) { + Long secs = stopBulkRefreshIntervals.get(index); + if (secs != null && secs != 0L) + client.updateIndexSetting(index, "refresh_interval", secs + "s"); + } + indexNames.clear(); + } + if (bulkProcessor != null) { + bulkProcessor.close(); + } + } + + private class BulkListener implements DefaultBulkProcessor.Listener { + + private final Logger logger = LogManager.getLogger("org.xbib.elx.BulkProcessor.Listener"); + + private Throwable lastBulkError = null; + + @Override + public void beforeBulk(long executionId, BulkRequest request) { + long l = 0; + if (bulkMetric != null) { + l = bulkMetric.getCurrentIngest().getCount(); + bulkMetric.getCurrentIngest().inc(); + int n = request.numberOfActions(); + bulkMetric.getSubmitted().inc(n); + bulkMetric.getCurrentIngestNumDocs().inc(n); + bulkMetric.getTotalIngestSizeInBytes().inc(request.estimatedSizeInBytes()); + } + if (logger.isDebugEnabled()) { + logger.debug("before bulk [{}] [actions={}] [bytes={}] [concurrent requests={}]", + executionId, + request.numberOfActions(), + request.estimatedSizeInBytes(), + l); + } + } + + @Override + public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { + long l = 0; + if (bulkMetric != null) { + l = bulkMetric.getCurrentIngest().getCount(); + bulkMetric.getCurrentIngest().dec(); + bulkMetric.getSucceeded().inc(response.getItems().length); + } + int n = 0; + for (BulkItemResponse itemResponse : response.getItems()) { + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().dec(itemResponse.getIndex(), itemResponse.getType(), itemResponse.getId()); + } + if (itemResponse.isFailed()) { + n++; + if (bulkMetric != null) { + bulkMetric.getSucceeded().dec(1); + bulkMetric.getFailed().inc(1); + } + } + } + if (bulkMetric != null && logger.isDebugEnabled()) { + logger.debug("after bulk [{}] [succeeded={}] [failed={}] [{}ms] {} concurrent requests", + executionId, + bulkMetric.getSucceeded().getCount(), + bulkMetric.getFailed().getCount(), + response.getTook().millis(), + l); + } + if (n > 0) { + if (logger.isErrorEnabled()) { + logger.error("bulk [{}] failed with {} failed items, failure message = {}", + executionId, n, response.buildFailureMessage()); + } + } else { + if (bulkMetric != null) { + bulkMetric.getCurrentIngestNumDocs().dec(response.getItems().length); + } + } + } + + @Override + public void afterBulk(long executionId, BulkRequest request, Throwable failure) { + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().dec(); + } + lastBulkError = failure; + active.set(false); + if (logger.isErrorEnabled()) { + logger.error("after bulk [" + executionId + "] error", failure); + } + } + + Throwable getLastBulkError() { + return lastBulkError; + } + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/SimpleBulkMetric.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkMetric.java similarity index 78% rename from elx-common/src/main/java/org/xbib/elx/common/SimpleBulkMetric.java rename to elx-common/src/main/java/org/xbib/elx/common/DefaultBulkMetric.java index 1a9dd35..a956c4d 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/SimpleBulkMetric.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkMetric.java @@ -1,5 +1,6 @@ package org.xbib.elx.common; +import org.elasticsearch.common.settings.Settings; import org.xbib.elx.api.BulkMetric; import org.xbib.metrics.Count; import org.xbib.metrics.CountMetric; @@ -7,9 +8,8 @@ import org.xbib.metrics.Meter; import org.xbib.metrics.Metered; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -public class SimpleBulkMetric implements BulkMetric { +public class DefaultBulkMetric implements BulkMetric { private final Meter totalIngest; @@ -29,12 +29,8 @@ public class SimpleBulkMetric implements BulkMetric { private Long stopped; - public SimpleBulkMetric() { - this(Executors.newSingleThreadScheduledExecutor()); - } - - public SimpleBulkMetric(ScheduledExecutorService executorService) { - totalIngest = new Meter(executorService); + public DefaultBulkMetric() { + totalIngest = new Meter(Executors.newSingleThreadScheduledExecutor()); totalIngestSizeInBytes = new CountMetric(); currentIngest = new CountMetric(); currentIngestNumDocs = new CountMetric(); @@ -43,6 +39,11 @@ public class SimpleBulkMetric implements BulkMetric { failed = new CountMetric(); } + @Override + public void init(Settings settings) { + start(); + } + @Override public Metered getTotalIngest() { return totalIngest; @@ -78,6 +79,11 @@ public class SimpleBulkMetric implements BulkMetric { return failed; } + @Override + public long elapsed() { + return started != null ? ((stopped != null ? stopped : System.nanoTime()) - started) : -1L; + } + @Override public void start() { this.started = System.nanoTime(); @@ -91,8 +97,8 @@ public class SimpleBulkMetric implements BulkMetric { } @Override - public long elapsed() { - return (stopped != null ? stopped : System.nanoTime()) - started; + public void close() { + stop(); + totalIngest.shutdown(); } - } diff --git a/elx-common/src/main/java/org/xbib/elx/common/BulkProcessor.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java similarity index 80% rename from elx-common/src/main/java/org/xbib/elx/common/BulkProcessor.java rename to elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java index 1aeeed4..28dbb45 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/BulkProcessor.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java @@ -5,16 +5,14 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.bulk.BulkAction; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.client.Client; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.FutureUtils; +import org.xbib.elx.api.BulkProcessor; -import java.io.Closeable; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; @@ -28,7 +26,7 @@ import java.util.concurrent.atomic.AtomicLong; * requests allowed to be executed in parallel. * In order to create a new bulk processor, use the {@link Builder}. */ -public class BulkProcessor implements Closeable { +public class DefaultBulkProcessor implements BulkProcessor { private final int bulkActions; @@ -46,8 +44,8 @@ public class BulkProcessor implements Closeable { private volatile boolean closed; - private BulkProcessor(Client client, Listener listener, String name, int concurrentRequests, - int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval) { + private DefaultBulkProcessor(Client client, Listener listener, String name, int concurrentRequests, + int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval) { this.executionIdGen = new AtomicLong(); this.closed = false; this.bulkActions = bulkActions; @@ -77,19 +75,6 @@ public class BulkProcessor implements Closeable { return new Builder(client, listener); } - /** - * Closes the processor. If flushing by time is enabled, then it's shutdown. Any remaining bulk actions are flushed. - */ - @Override - public void close() { - try { - // 0 = immediate close - awaitClose(0, TimeUnit.NANOSECONDS); - } catch (InterruptedException exc) { - Thread.currentThread().interrupt(); - } - } - /** * Wait for bulk request handler with flush. * @param timeout the timeout value @@ -97,7 +82,8 @@ public class BulkProcessor implements Closeable { * @return true is method was successful, false if timeout * @throws InterruptedException if timeout */ - public boolean awaitFlush(long timeout, TimeUnit unit) throws InterruptedException { + @Override + public synchronized boolean awaitFlush(long timeout, TimeUnit unit) throws InterruptedException { if (closed) { return true; } @@ -124,6 +110,7 @@ public class BulkProcessor implements Closeable { * bulk requests completed * @throws InterruptedException If the current thread is interrupted */ + @Override public synchronized boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException { if (closed) { return true; @@ -139,34 +126,14 @@ public class BulkProcessor implements Closeable { return this.bulkRequestHandler.close(timeout, unit); } - /** - * Adds an {@link IndexRequest} to the list of actions to execute. Follows the same behavior of {@link IndexRequest} - * (for example, if no id is provided, one will be generated, or usage of the create flag). - * - * @param request request - * @return his bulk processor - */ - public BulkProcessor add(IndexRequest request) { - return add((ActionRequest) request); - } - - /** - * Adds an {@link DeleteRequest} to the list of actions to execute. - * - * @param request request - * @return his bulk processor - */ - public BulkProcessor add(DeleteRequest request) { - return add((ActionRequest) request); - } - /** * Adds either a delete or an index request. * * @param request request * @return his bulk processor */ - public BulkProcessor add(ActionRequest request) { + @Override + public DefaultBulkProcessor add(ActionRequest request) { return add(request, null); } @@ -177,11 +144,36 @@ public class BulkProcessor implements Closeable { * @param payload payload * @return his bulk processor */ - public BulkProcessor add(ActionRequest request, Object payload) { + @Override + public DefaultBulkProcessor add(ActionRequest request, Object payload) { internalAdd(request, payload); return this; } + /** + * Flush pending delete or index requests. + */ + @Override + public synchronized void flush() { + ensureOpen(); + if (bulkRequest.numberOfActions() > 0) { + execute(); + } + } + + /** + * Closes the processor. If flushing by time is enabled, then it's shutdown. Any remaining bulk actions are flushed. + */ + @Override + public void close() { + try { + // 0 = immediate close + awaitClose(0, TimeUnit.NANOSECONDS); + } catch (InterruptedException exc) { + Thread.currentThread().interrupt(); + } + } + private void ensureOpen() { if (closed) { throw new IllegalStateException("bulk process already closed"); @@ -213,53 +205,7 @@ public class BulkProcessor implements Closeable { return bulkActions != -1 && bulkRequest.numberOfActions() >= bulkActions || bulkSize != -1 && - bulkRequest.estimatedSizeInBytes() >= bulkSize; - } - - /** - * Flush pending delete or index requests. - */ - public synchronized void flush() { - ensureOpen(); - if (bulkRequest.numberOfActions() > 0) { - execute(); - } - } - - /** - * A listener for the execution. - */ - public interface Listener { - - /** - * Callback before the bulk is executed. - * - * @param executionId execution ID - * @param request request - */ - void beforeBulk(long executionId, BulkRequest request); - - /** - * Callback after a successful execution of bulk request. - * - * @param executionId execution ID - * @param request request - * @param response response - */ - void afterBulk(long executionId, BulkRequest request, BulkResponse response); - - /** - * Callback after a failed execution of bulk request. - * - * Note that in case an instance of InterruptedException is passed, which means that request - * processing has been - * cancelled externally, the thread's interruption status has been restored prior to calling this method. - * - * @param executionId execution ID - * @param request request - * @param failure failure - */ - void afterBulk(long executionId, BulkRequest request, Throwable failure); + bulkRequest.estimatedSizeInBytes() >= bulkSize; } /** @@ -359,8 +305,8 @@ public class BulkProcessor implements Closeable { * * @return a bulk processor */ - public BulkProcessor build() { - return new BulkProcessor(client, listener, name, concurrentRequests, bulkActions, bulkSize, flushInterval); + public DefaultBulkProcessor build() { + return new DefaultBulkProcessor(client, listener, name, concurrentRequests, bulkActions, bulkSize, flushInterval); } } @@ -368,7 +314,7 @@ public class BulkProcessor implements Closeable { @Override public void run() { - synchronized (BulkProcessor.this) { + synchronized (DefaultBulkProcessor.this) { if (closed) { return; } @@ -380,24 +326,13 @@ public class BulkProcessor implements Closeable { } } - /** - * Abstracts the low-level details of bulk request handling. - */ - interface BulkRequestHandler { - - void execute(BulkRequest bulkRequest, long executionId); - - boolean close(long timeout, TimeUnit unit) throws InterruptedException; - - } - private static class SyncBulkRequestHandler implements BulkRequestHandler { private final Client client; - private final BulkProcessor.Listener listener; + private final DefaultBulkProcessor.Listener listener; - SyncBulkRequestHandler(Client client, BulkProcessor.Listener listener) { + SyncBulkRequestHandler(Client client, DefaultBulkProcessor.Listener listener) { this.client = client; this.listener = listener; } @@ -427,13 +362,13 @@ public class BulkProcessor implements Closeable { private final Client client; - private final BulkProcessor.Listener listener; + private final DefaultBulkProcessor.Listener listener; private final Semaphore semaphore; private final int concurrentRequests; - private AsyncBulkRequestHandler(Client client, BulkProcessor.Listener listener, int concurrentRequests) { + private AsyncBulkRequestHandler(Client client, DefaultBulkProcessor.Listener listener, int concurrentRequests) { this.client = client; this.listener = listener; this.concurrentRequests = concurrentRequests; diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java new file mode 100644 index 0000000..52127e1 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java @@ -0,0 +1,214 @@ +package org.xbib.elx.common; + +import org.xbib.elx.api.IndexDefinition; +import org.xbib.elx.api.IndexRetention; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.concurrent.TimeUnit; + +public class DefaultIndexDefinition implements IndexDefinition { + + private String index; + + private String fullIndexName; + + private String dateTimePattern; + + private URL settingsUrl; + + private URL mappingsUrl; + + private boolean enabled; + + private boolean ignoreErrors; + + private boolean switchAliases; + + private boolean hasForceMerge; + + private int replicaLevel; + + private IndexRetention indexRetention; + + private long maxWaitTime; + + private TimeUnit maxWaitTimeUnit; + + private long startRefreshInterval; + + private long stopRefreshInterval; + + @Override + public IndexDefinition setIndex(String index) { + this.index = index; + return this; + } + + @Override + public String getIndex() { + return index; + } + + @Override + public IndexDefinition setFullIndexName(String fullIndexName) { + this.fullIndexName = fullIndexName; + return this; + } + + @Override + public String getFullIndexName() { + return fullIndexName; + } + + @Override + public IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException { + this.settingsUrl = settingsUrlString != null ? new URL(settingsUrlString) : null; + return this; + } + + @Override + public IndexDefinition setSettingsUrl(URL settingsUrl) { + this.settingsUrl = settingsUrl; + return this; + } + + @Override + public URL getSettingsUrl() { + return settingsUrl; + } + + @Override + public IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException { + this.mappingsUrl = mappingsUrlString != null ? new URL(mappingsUrlString) : null; + return this; + } + + @Override + public IndexDefinition setMappingsUrl(URL mappingsUrl) { + this.mappingsUrl = mappingsUrl; + return this; + } + + @Override + public URL getMappingsUrl() { + return mappingsUrl; + } + + @Override + public IndexDefinition setDateTimePattern(String timeWindow) { + this.dateTimePattern = timeWindow; + return this; + } + + @Override + public String getDateTimePattern() { + return dateTimePattern; + } + + @Override + public IndexDefinition setEnabled(boolean enabled) { + this.enabled = enabled; + return this; + } + + @Override + public boolean isEnabled() { + return enabled; + } + + @Override + public IndexDefinition setIgnoreErrors(boolean ignoreErrors) { + this.ignoreErrors = ignoreErrors; + return this; + } + + @Override + public boolean ignoreErrors() { + return ignoreErrors; + } + + @Override + public IndexDefinition setShift(boolean switchAliases) { + this.switchAliases = switchAliases; + return this; + } + + @Override + public boolean isShiftEnabled() { + return switchAliases; + } + + @Override + public IndexDefinition setForceMerge(boolean hasForceMerge) { + this.hasForceMerge = hasForceMerge; + return this; + } + + @Override + public boolean hasForceMerge() { + return hasForceMerge; + } + + @Override + public IndexDefinition setReplicaLevel(int replicaLevel) { + this.replicaLevel = replicaLevel; + return this; + } + + @Override + public int getReplicaLevel() { + return replicaLevel; + } + + @Override + public IndexDefinition setRetention(IndexRetention indexRetention) { + this.indexRetention = indexRetention; + return this; + } + + @Override + public IndexRetention getRetention() { + return indexRetention; + } + + @Override + public IndexDefinition setMaxWaitTime(long maxWaitTime, TimeUnit timeUnit) { + this.maxWaitTime = maxWaitTime; + this.maxWaitTimeUnit = timeUnit; + return this; + } + + @Override + public long getMaxWaitTime() { + return maxWaitTime; + } + + @Override + public TimeUnit getMaxWaitTimeUnit() { + return maxWaitTimeUnit; + } + + @Override + public IndexDefinition setStartRefreshInterval(long seconds) { + this.startRefreshInterval = seconds; + return this; + } + + @Override + public long getStartRefreshInterval() { + return startRefreshInterval; + } + + @Override + public IndexDefinition setStopRefreshInterval(long seconds) { + this.stopRefreshInterval = seconds; + return this; + } + + @Override + public long getStopRefreshInterval() { + return stopRefreshInterval; + } + +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexRetention.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexRetention.java new file mode 100644 index 0000000..4e49be3 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexRetention.java @@ -0,0 +1,32 @@ +package org.xbib.elx.common; + +import org.xbib.elx.api.IndexRetention; + +public class DefaultIndexRetention implements IndexRetention { + + private int delta; + + private int minToKeep; + + @Override + public IndexRetention setDelta(int delta) { + this.delta = delta; + return this; + } + + @Override + public int getDelta() { + return delta; + } + + @Override + public IndexRetention setMinToKeep(int minToKeep) { + this.minToKeep = minToKeep; + return this; + } + + @Override + public int getMinToKeep() { + return minToKeep; + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java index 7e5a1e4..58e303d 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java @@ -6,8 +6,10 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; +import java.util.concurrent.TimeUnit; + /** - * Mock client, it does not perform actions on a cluster. Useful for testing or dry runs. + * Mock client, it does not perform any actions on a cluster. Useful for testing. */ public class MockExtendedClient extends AbstractExtendedClient { @@ -56,18 +58,13 @@ public class MockExtendedClient extends AbstractExtendedClient { return this; } - @Override - public MockExtendedClient flushIngest() { - return this; - } - @Override public MockExtendedClient startBulk(String index, long startRefreshInterval, long stopRefreshIterval) { return this; } @Override - public MockExtendedClient stopBulk(String index, String maxWaitTime) { + public MockExtendedClient stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) { return this; } @@ -92,32 +89,37 @@ public class MockExtendedClient extends AbstractExtendedClient { } @Override - public boolean forceMerge(String index, String maxWaitTime) { + public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) { return true; } @Override - public boolean waitForCluster(String healthColor, String timeValue) { + public boolean waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) { return true; } @Override - public boolean waitForResponses(String maxWaitTime) { + public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) { return true; } @Override - public boolean waitForRecovery(String index, String maxWaitTime) { + public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) { return true; } @Override - public MockExtendedClient updateReplicaLevel(String index, int level, String maxWaitTime) { + public MockExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) { return this; } @Override - public void shutdown() { + public void flush() { + // nothing to do + } + + @Override + public void close() { // nothing to do } } diff --git a/elx-common/src/main/java/org/xbib/elx/common/SimpleBulkControl.java b/elx-common/src/main/java/org/xbib/elx/common/SimpleBulkControl.java deleted file mode 100644 index d606ecc..0000000 --- a/elx-common/src/main/java/org/xbib/elx/common/SimpleBulkControl.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.xbib.elx.common; - -import org.xbib.elx.api.BulkControl; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -/** - */ -public class SimpleBulkControl implements BulkControl { - - private final Set indexNames; - - private final Map startBulkRefreshIntervals; - - private final Map stopBulkRefreshIntervals; - - private String maxWaitTime; - - public SimpleBulkControl() { - indexNames = new HashSet<>(); - startBulkRefreshIntervals = new HashMap<>(); - stopBulkRefreshIntervals = new HashMap<>(); - maxWaitTime = "30s"; - } - - @Override - public void startBulk(String indexName, long startRefreshInterval, long stopRefreshInterval) { - indexNames.add(indexName); - startBulkRefreshIntervals.put(indexName, startRefreshInterval); - stopBulkRefreshIntervals.put(indexName, stopRefreshInterval); - } - - @Override - public boolean isBulk(String indexName) { - return indexNames.contains(indexName); - } - - @Override - public void finishBulk(String indexName) { - indexNames.remove(indexName); - } - - @Override - public Set indices() { - return indexNames; - } - - @Override - public Map getStartBulkRefreshIntervals() { - return startBulkRefreshIntervals; - } - - @Override - public Map getStopBulkRefreshIntervals() { - return stopBulkRefreshIntervals; - } - - @Override - public String getMaxWaitTime() { - return maxWaitTime; - } - -} diff --git a/elx-common/src/main/java/org/xbib/elx/common/io/package-info.java b/elx-common/src/main/java/org/xbib/elx/common/io/package-info.java index 62a8d78..5e86ba1 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/io/package-info.java +++ b/elx-common/src/main/java/org/xbib/elx/common/io/package-info.java @@ -1 +1,4 @@ -package org.xbib.elx.common.io; \ No newline at end of file +/** + * I/O helpers for Elasticsearch client extensions. + */ +package org.xbib.elx.common.io; diff --git a/elx-common/src/main/java/org/xbib/elx/common/util/package-info.java b/elx-common/src/main/java/org/xbib/elx/common/util/package-info.java index 3c41bfe..cd393c9 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/util/package-info.java +++ b/elx-common/src/main/java/org/xbib/elx/common/util/package-info.java @@ -1 +1,4 @@ -package org.xbib.elx.common.util; \ No newline at end of file +/** + * Utilities for Elasticsearch client extensions. + */ +package org.xbib.elx.common.util; diff --git a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java b/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java index 493a596..7eca86e 100644 --- a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java +++ b/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java @@ -47,10 +47,9 @@ public class ExtendedNodeClient extends AbstractExtendedClient { return null; } - @Override - public void shutdown() throws IOException { - super.shutdown(); + public void close() throws IOException { + super.close(); try { if (node != null) { logger.debug("closing node..."); diff --git a/elx-node/src/main/java/org/xbib/elx/node/package-info.java b/elx-node/src/main/java/org/xbib/elx/node/package-info.java new file mode 100644 index 0000000..c2a9dfb --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/package-info.java @@ -0,0 +1,4 @@ +/** + * Node client extensions. + */ +package org.xbib.elx.node; diff --git a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientTest.java b/elx-node/src/test/java/org/xbib/elx/node/ClientTest.java similarity index 83% rename from elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientTest.java rename to elx-node/src/test/java/org/xbib/elx/node/ClientTest.java index de28b4b..5aaae33 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/ClientTest.java @@ -2,7 +2,7 @@ package org.xbib.elx.node; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import org.apache.logging.log4j.LogManager; @@ -27,9 +27,9 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -public class ExtendedNodeClientTest extends NodeTestUtils { +public class ClientTest extends NodeTestUtils { - private static final Logger logger = LogManager.getLogger(ExtendedNodeClientTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(ClientTest.class.getSimpleName()); private static final Long ACTIONS = 25000L; @@ -55,17 +55,17 @@ public class ExtendedNodeClientTest extends NodeTestUtils { try { client.newIndex("test"); client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest - client.flushIngest(); - client.waitForResponses("30s"); + client.flush(); + client.waitForResponses(30L, TimeUnit.SECONDS); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); } - assertFalse(client.hasThrowable()); - client.shutdown(); + assertNull(client.getBulkController().getLastBulkError()); + client.close(); } } @@ -76,11 +76,7 @@ public class ExtendedNodeClientTest extends NodeTestUtils { .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) .build(); client.newIndex("test"); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); - } - assertFalse(client.hasThrowable()); - client.shutdown(); + client.close(); } @Test @@ -105,11 +101,7 @@ public class ExtendedNodeClientTest extends NodeTestUtils { client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); logger.info("mappings={}", getMappingsResponse.getMappings()); assertTrue(getMappingsResponse.getMappings().get("test").containsKey("doc")); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); - } - assertFalse(client.hasThrowable()); - client.shutdown(); + client.close(); } @Test @@ -125,22 +117,22 @@ public class ExtendedNodeClientTest extends NodeTestUtils { for (int i = 0; i < ACTIONS; i++) { client.index("test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } - client.flushIngest(); - client.waitForResponses("30s"); + client.flush(); + client.waitForResponses(30L, TimeUnit.SECONDS); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); } - assertFalse(client.hasThrowable()); + assertNull(client.getBulkController().getLastBulkError()); client.refreshIndex("test"); SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) .setQuery(QueryBuilders.matchAllQuery()).setSize(0); assertEquals(numactions, searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); - client.shutdown(); + client.close(); } } @@ -172,9 +164,9 @@ public class ExtendedNodeClientTest extends NodeTestUtils { } logger.info("waiting for latch..."); if (latch.await(5, TimeUnit.MINUTES)) { - logger.info("last flush..."); - client.flushIngest(); - client.waitForResponses("60s"); + logger.info("flush..."); + client.flush(); + client.waitForResponses(60L, TimeUnit.SECONDS); logger.info("got all responses, pool shutdown..."); pool.shutdown(); logger.info("pool is shut down"); @@ -184,18 +176,18 @@ public class ExtendedNodeClientTest extends NodeTestUtils { } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - client.stopBulk("test", "30s"); + client.stopBulk("test", 30L, TimeUnit.SECONDS); assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount()); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); } - assertFalse(client.hasThrowable()); + assertNull(client.getBulkController().getLastBulkError()); client.refreshIndex("test"); SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) .setQuery(QueryBuilders.matchAllQuery()).setSize(0); assertEquals(maxthreads * actions, searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); - client.shutdown(); + client.close(); } } } diff --git a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClusterBlockTest.java b/elx-node/src/test/java/org/xbib/elx/node/ClusterBlockTest.java similarity index 73% rename from elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClusterBlockTest.java rename to elx-node/src/test/java/org/xbib/elx/node/ClusterBlockTest.java index b38555a..23cbbe3 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClusterBlockTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/ClusterBlockTest.java @@ -4,17 +4,16 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; - -@Ignore -public class ExtendedNodeClusterBlockTest extends NodeTestUtils { +public class ClusterBlockTest extends NodeTestUtils { private static final Logger logger = LogManager.getLogger("test"); @@ -23,7 +22,6 @@ public class ExtendedNodeClusterBlockTest extends NodeTestUtils { try { setClusterName(); startNode("1"); - findNodeAddress(); // do not wait for green health state logger.info("ready"); } catch (Throwable t) { @@ -41,11 +39,11 @@ public class ExtendedNodeClusterBlockTest extends NodeTestUtils { @Test(expected = ClusterBlockException.class) public void testClusterBlock() throws Exception { - BulkRequestBuilder brb = client("1").prepareBulk(); - XContentBuilder builder = jsonBuilder().startObject().field("field1", "value1").endObject(); - IndexRequestBuilder irb = client("1").prepareIndex("test", "test", "1").setSource(builder); + Client client = client("1"); + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field("field1", "value1").endObject(); + IndexRequestBuilder irb = client.prepareIndex("test", "test", "1").setSource(builder); + BulkRequestBuilder brb = client.prepareBulk(); brb.add(irb); brb.execute().actionGet(); } - } diff --git a/elx-node/src/test/java/org/xbib/elx/node/ExtendeNodeDuplicateIDTest.java b/elx-node/src/test/java/org/xbib/elx/node/DuplicateIDTest.java similarity index 78% rename from elx-node/src/test/java/org/xbib/elx/node/ExtendeNodeDuplicateIDTest.java rename to elx-node/src/test/java/org/xbib/elx/node/DuplicateIDTest.java index ad75a95..d2126e5 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/ExtendeNodeDuplicateIDTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/DuplicateIDTest.java @@ -10,13 +10,14 @@ import org.junit.Test; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; +import java.util.concurrent.TimeUnit; + import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.junit.Assert.*; -@Ignore -public class ExtendeNodeDuplicateIDTest extends NodeTestUtils { +public class DuplicateIDTest extends NodeTestUtils { - private static final Logger logger = LogManager.getLogger(ExtendeNodeDuplicateIDTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getSimpleName()); private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; @@ -34,8 +35,8 @@ public class ExtendeNodeDuplicateIDTest extends NodeTestUtils { for (int i = 0; i < ACTIONS; i++) { client.index("test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); } - client.flushIngest(); - client.waitForResponses("30s"); + client.flush(); + client.waitForResponses(30L, TimeUnit.SECONDS); client.refreshIndex("test"); SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) .setIndices("test") @@ -47,12 +48,12 @@ public class ExtendeNodeDuplicateIDTest extends NodeTestUtils { } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - client.shutdown(); + client.close(); assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); } - assertFalse(client.hasThrowable()); + assertNull(client.getBulkController().getLastBulkError()); } } } diff --git a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeUpdateReplicaLevelTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeUpdateReplicaLevelTest.java deleted file mode 100644 index 01e1df4..0000000 --- a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeUpdateReplicaLevelTest.java +++ /dev/null @@ -1,63 +0,0 @@ -package org.xbib.elx.node; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.settings.Settings; -import org.junit.Ignore; -import org.junit.Test; -import org.xbib.elx.common.ClientBuilder; - -import java.util.HashMap; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; - -@Ignore -public class ExtendedNodeUpdateReplicaLevelTest extends NodeTestUtils { - - private static final Logger logger = LogManager.getLogger(ExtendedNodeUpdateReplicaLevelTest.class.getSimpleName()); - - @Test - public void testUpdateReplicaLevel() throws Exception { - - long numberOfShards = 2; - int replicaLevel = 3; - - // we need 3 nodes for replica level 3 - startNode("2"); - startNode("3"); - - long shardsAfterReplica; - - Settings settings = Settings.settingsBuilder() - .put("index.number_of_shards", numberOfShards) - .put("index.number_of_replicas", 0) - .build(); - - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) - .provider(ExtendedNodeClientProvider.class) - .build(); - - try { - client.newIndex("replicatest", settings, new HashMap<>()); - client.waitForCluster("GREEN", "30s"); - for (int i = 0; i < 12345; i++) { - client.index("replicatest",null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - client.flushIngest(); - client.waitForResponses("30s"); - client.updateReplicaLevel("replicatest", replicaLevel, "30s"); - //assertEquals(shardsAfterReplica, numberOfShards * (replicaLevel + 1)); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - client.shutdown(); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); - } - assertFalse(client.hasThrowable()); - } - } - -} diff --git a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeIndexAliasTest.java b/elx-node/src/test/java/org/xbib/elx/node/IndexShiftTest.java similarity index 50% rename from elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeIndexAliasTest.java rename to elx-node/src/test/java/org/xbib/elx/node/IndexShiftTest.java index 222b261..9fb687a 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeIndexAliasTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/IndexShiftTest.java @@ -11,16 +11,18 @@ import org.xbib.elx.common.ClientBuilder; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; @Ignore -public class ExtendedNodeIndexAliasTest extends NodeTestUtils { +public class IndexShiftTest extends NodeTestUtils { - private static final Logger logger = LogManager.getLogger(ExtendedNodeIndexAliasTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getSimpleName()); @Test - public void testIndexAlias() throws Exception { + public void testIndexShift() throws Exception { final ExtendedNodeClient client = ClientBuilder.builder(client("1")) .provider(ExtendedNodeClientProvider.class) .build(); @@ -29,37 +31,47 @@ public class ExtendedNodeIndexAliasTest extends NodeTestUtils { for (int i = 0; i < 1; i++) { client.index("test1234", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); } - client.flushIngest(); + client.flush(); client.refreshIndex("test1234"); List simpleAliases = Arrays.asList("a", "b", "c"); - client.switchIndex("test", "test1234", simpleAliases); + client.shiftIndex("test", "test1234", simpleAliases); client.newIndex("test5678"); for (int i = 0; i < 1; i++) { client.index("test5678", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); } - client.flushIngest(); + client.flush(); client.refreshIndex("test5678"); simpleAliases = Arrays.asList("d", "e", "f"); - client.switchIndex("test", "test5678", simpleAliases, (builder, index, alias) -> + client.shiftIndex("test", "test5678", simpleAliases, (builder, index, alias) -> builder.addAlias(index, alias, QueryBuilders.termQuery("my_key", alias))); - Map aliases = client.getIndexFilters("test5678"); - logger.info("aliases of index test5678 = {}", aliases); + Map indexFilters = client.getIndexFilters("test5678"); + logger.info("aliases of index test5678 = {}", indexFilters); + assertTrue(indexFilters.containsKey("a")); + assertTrue(indexFilters.containsKey("b")); + assertTrue(indexFilters.containsKey("c")); + assertTrue(indexFilters.containsKey("d")); + assertTrue(indexFilters.containsKey("e")); - aliases = client.getAliasFilters("test"); + Map aliases = client.getIndexFilters(client.resolveAlias("test")); logger.info("aliases of alias test = {}", aliases); + assertTrue(aliases.containsKey("a")); + assertTrue(aliases.containsKey("b")); + assertTrue(aliases.containsKey("c")); + assertTrue(aliases.containsKey("d")); + assertTrue(aliases.containsKey("e")); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - client.waitForResponses("30s"); - client.shutdown(); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); + client.waitForResponses(30L, TimeUnit.SECONDS); + client.close(); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); } - assertFalse(client.hasThrowable()); + assertNull(client.getBulkController().getLastBulkError()); } } } diff --git a/elx-node/src/test/java/org/xbib/elx/node/NodeTestUtils.java b/elx-node/src/test/java/org/xbib/elx/node/NodeTestUtils.java index 05bf386..7faed8d 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/NodeTestUtils.java +++ b/elx-node/src/test/java/org/xbib/elx/node/NodeTestUtils.java @@ -63,9 +63,7 @@ public class NodeTestUtils { Files.delete(dir); return FileVisitResult.CONTINUE; } - }); - } @Before @@ -74,7 +72,6 @@ public class NodeTestUtils { logger.info("starting"); setClusterName(); startNode("1"); - findNodeAddress(); try { ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN) @@ -160,18 +157,6 @@ public class NodeTestUtils { logger.info("all nodes closed"); } - protected void findNodeAddress() { - NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); - NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet(); - Object obj = response.iterator().next().getTransport().getAddress() - .publishAddress(); - if (obj instanceof InetSocketTransportAddress) { - InetSocketTransportAddress address = (InetSocketTransportAddress) obj; - String host = address.address().getHostName(); - int port = address.address().getPort(); - } - } - private Node buildNode(String id) { Settings nodeSettings = settingsBuilder() .put(getNodeSettings()) diff --git a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeReplicaTest.java b/elx-node/src/test/java/org/xbib/elx/node/ReplicaTest.java similarity index 66% rename from elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeReplicaTest.java rename to elx-node/src/test/java/org/xbib/elx/node/ReplicaTest.java index ed53cf5..762800f 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeReplicaTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/ReplicaTest.java @@ -19,15 +19,16 @@ import org.xbib.elx.common.ClientBuilder; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; @Ignore -public class ExtendedNodeReplicaTest extends NodeTestUtils { +public class ReplicaTest extends NodeTestUtils { - private static final Logger logger = LogManager.getLogger(ExtendedNodeReplicaTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getSimpleName()); @Test public void testReplicaLevel() throws Exception { @@ -54,15 +55,15 @@ public class ExtendedNodeReplicaTest extends NodeTestUtils { try { client.newIndex("test1", settingsTest1, new HashMap<>()) .newIndex("test2", settingsTest2, new HashMap<>()); - client.waitForCluster("GREEN", "30s"); + client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS); for (int i = 0; i < 1234; i++) { client.index("test1", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } for (int i = 0; i < 1234; i++) { client.index("test2", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } - client.flushIngest(); - client.waitForResponses("30s"); + client.flush(); + client.waitForResponses(30L, TimeUnit.SECONDS); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { @@ -97,11 +98,51 @@ public class ExtendedNodeReplicaTest extends NodeTestUtils { } catch (Exception e) { logger.error("delete index failed, ignored. Reason:", e); } - client.shutdown(); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); + client.close(); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); } - assertFalse(client.hasThrowable()); + assertNull(client.getBulkController().getLastBulkError()); + } + } + + @Test + public void testUpdateReplicaLevel() throws Exception { + + long numberOfShards = 2; + int replicaLevel = 3; + + // we need 3 nodes for replica level 3 + startNode("2"); + startNode("3"); + + Settings settings = Settings.settingsBuilder() + .put("index.number_of_shards", numberOfShards) + .put("index.number_of_replicas", 0) + .build(); + + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .build(); + + try { + client.newIndex("replicatest", settings, new HashMap<>()); + client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS); + for (int i = 0; i < 12345; i++) { + client.index("replicatest",null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flush(); + client.waitForResponses(30L, TimeUnit.SECONDS); + client.updateReplicaLevel("replicatest", replicaLevel, 30L, TimeUnit.SECONDS); + assertEquals(replicaLevel, client.getReplicaLevel("replicatest")); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + client.close(); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); + } + assertNull(client.getBulkController().getLastBulkError()); } } diff --git a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeSmokeTest.java b/elx-node/src/test/java/org/xbib/elx/node/SmokeTest.java similarity index 70% rename from elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeSmokeTest.java rename to elx-node/src/test/java/org/xbib/elx/node/SmokeTest.java index adac4c6..cb70fe0 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeSmokeTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/SmokeTest.java @@ -6,16 +6,16 @@ import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.junit.Test; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.common.SimpleBulkControl; -import org.xbib.elx.common.SimpleBulkMetric; import org.xbib.elx.api.IndexDefinition; +import java.util.concurrent.TimeUnit; + import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; -public class ExtendedNodeSmokeTest extends NodeTestUtils { +public class SmokeTest extends NodeTestUtils { - private static final Logger logger = LogManager.getLogger(ExtendedNodeSmokeTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(SmokeTest.class.getSimpleName()); @Test public void smokeTest() throws Exception { @@ -23,21 +23,19 @@ public class ExtendedNodeSmokeTest extends NodeTestUtils { .provider(ExtendedNodeClientProvider.class) .build(); try { - client.setBulkControl(new SimpleBulkControl()); - client.setBulkMetric(new SimpleBulkMetric()); client.newIndex("test"); client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest - client.flushIngest(); - client.waitForResponses("30s"); + client.flush(); + client.waitForResponses(30, TimeUnit.SECONDS); assertEquals(clusterName, client.getClusterName()); client.checkMapping("test"); client.update("test", "1", "{ \"name\" : \"Another name\"}"); - client.flushIngest(); + client.flush(); - client.waitForRecovery("test", "10s"); + client.waitForRecovery("test", 10L, TimeUnit.SECONDS); client.delete("test", "1"); client.deleteIndex("test"); @@ -47,7 +45,7 @@ public class ExtendedNodeSmokeTest extends NodeTestUtils { assertEquals(0, indexDefinition.getReplicaLevel()); client.newIndex(indexDefinition); client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); - client.flushIngest(); + client.flush(); client.updateReplicaLevel(indexDefinition, 2); int replica = client.getReplicaLevel(indexDefinition); @@ -59,11 +57,11 @@ public class ExtendedNodeSmokeTest extends NodeTestUtils { } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); + client.close(); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); } - assertFalse(client.hasThrowable()); - client.shutdown(); + assertNull(client.getBulkController().getLastBulkError()); } } } diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java index e883091..685b9ec 100644 --- a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java +++ b/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java @@ -74,15 +74,15 @@ public class ExtendedTransportClient extends AbstractExtendedClient { } @Override - public synchronized void shutdown() throws IOException { - super.shutdown(); - logger.info("shutting down..."); + public synchronized void close() throws IOException { + super.close(); + logger.info("closing"); if (getClient() != null) { TransportClient client = (TransportClient) getClient(); client.close(); client.threadPool().shutdown(); } - logger.info("shutting down completed"); + logger.info("close completed"); } private Collection findAddresses(Settings settings) throws IOException { diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/TransportClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportClient.java index 827f657..1a607ad 100644 --- a/elx-transport/src/main/java/org/xbib/elx/transport/TransportClient.java +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportClient.java @@ -308,7 +308,9 @@ public class TransportClient extends AbstractClient { transportService.connectToNode(node); } catch (Exception e) { it.remove(); - logger.debug("failed to connect to discovered node [" + node + "]", e); + if (logger.isDebugEnabled()) { + logger.debug("failed to connect to discovered node [" + node + "]", e); + } } } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ClientTest.java similarity index 83% rename from elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientTest.java rename to elx-transport/src/test/java/org/xbib/elx/transport/ClientTest.java index 5085df6..56da530 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ClientTest.java @@ -25,12 +25,12 @@ import java.util.concurrent.TimeUnit; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -public class ExtendedTransportClientTest extends NodeTestUtils { +public class ClientTest extends NodeTestUtils { - private static final Logger logger = LogManager.getLogger(ExtendedTransportClientTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(ClientTest.class.getSimpleName()); private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; @@ -54,10 +54,6 @@ public class ExtendedTransportClientTest extends NodeTestUtils { .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) .build(); client.newIndex("test"); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); - } - assertFalse(client.hasThrowable()); try { client.deleteIndex("test") .newIndex("test") @@ -65,11 +61,7 @@ public class ExtendedTransportClientTest extends NodeTestUtils { } catch (NoNodeAvailableException e) { logger.error("no node available"); } finally { - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); - } - assertFalse(client.hasThrowable()); - client.shutdown(); + client.close(); } } @@ -84,17 +76,17 @@ public class ExtendedTransportClientTest extends NodeTestUtils { try { client.newIndex("test"); client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); - client.flushIngest(); - client.waitForResponses("30s"); + client.flush(); + client.waitForResponses(30L, TimeUnit.SECONDS); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); } - assertFalse(client.hasThrowable()); - client.shutdown(); + assertNull(client.getBulkController().getLastBulkError()); + client.close(); } } @@ -121,11 +113,7 @@ public class ExtendedTransportClientTest extends NodeTestUtils { client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); logger.info("mappings={}", getMappingsResponse.getMappings()); assertTrue(getMappingsResponse.getMappings().get("test").containsKey("doc")); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); - } - assertFalse(client.hasThrowable()); - client.shutdown(); + client.close(); } @Test @@ -142,17 +130,17 @@ public class ExtendedTransportClientTest extends NodeTestUtils { for (int i = 0; i < ACTIONS; i++) { client.index("test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } - client.flushIngest(); - client.waitForResponses("30s"); + client.flush(); + client.waitForResponses(30L, TimeUnit.SECONDS); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); } - assertFalse(client.hasThrowable()); - client.shutdown(); + assertNull(client.getBulkController().getLastBulkError()); + client.close(); } } @@ -190,21 +178,21 @@ public class ExtendedTransportClientTest extends NodeTestUtils { logger.info("waiting for latch..."); if (latch.await(60, TimeUnit.SECONDS)) { logger.info("flush ..."); - client.flushIngest(); - client.waitForResponses("30s"); + client.flush(); + client.waitForResponses(30L, TimeUnit.SECONDS); logger.info("pool to be shut down ..."); pool.shutdown(); logger.info("poot shut down"); } - client.stopBulk("test", "30s"); + client.stopBulk("test", 30L, TimeUnit.SECONDS); assertEquals(maxthreads * maxloop, client.getBulkMetric().getSucceeded().getCount()); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); } - assertFalse(client.hasThrowable()); + assertNull(client.getBulkController().getLastBulkError()); // extra search lookup client.refreshIndex("test"); SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) @@ -214,7 +202,7 @@ public class ExtendedTransportClientTest extends NodeTestUtils { .setSize(0); assertEquals(maxthreads * maxloop, searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); - client.shutdown(); + client.close(); } } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportDuplicateIDTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/DuplicateIDTest.java similarity index 74% rename from elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportDuplicateIDTest.java rename to elx-transport/src/test/java/org/xbib/elx/transport/DuplicateIDTest.java index ea1cd1d..6f6b6bd 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportDuplicateIDTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/DuplicateIDTest.java @@ -9,12 +9,16 @@ import org.junit.Test; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; +import java.util.concurrent.TimeUnit; + import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; -public class ExtendedTransportDuplicateIDTest extends NodeTestUtils { +public class DuplicateIDTest extends NodeTestUtils { - private final static Logger logger = LogManager.getLogger(ExtendedTransportDuplicateIDTest.class.getSimpleName()); + private final static Logger logger = LogManager.getLogger(DuplicateIDTest.class.getSimpleName()); private final static Long MAX_ACTIONS_PER_REQUEST = 1000L; @@ -33,8 +37,8 @@ public class ExtendedTransportDuplicateIDTest extends NodeTestUtils { for (int i = 0; i < ACTIONS; i++) { client.index("test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); } - client.flushIngest(); - client.waitForResponses("30s"); + client.flush(); + client.waitForResponses(30L, TimeUnit.SECONDS); client.refreshIndex("test"); SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) .setIndices("test") @@ -46,12 +50,12 @@ public class ExtendedTransportDuplicateIDTest extends NodeTestUtils { } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - client.shutdown(); + client.close(); assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); } - assertFalse(client.hasThrowable()); + assertNull(client.getBulkController().getLastBulkError()); } } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportUpdateReplicaLevelTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportUpdateReplicaLevelTest.java deleted file mode 100644 index 1992021..0000000 --- a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportUpdateReplicaLevelTest.java +++ /dev/null @@ -1,61 +0,0 @@ -package org.xbib.elx.transport; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.settings.Settings; -import org.junit.Test; -import org.xbib.elx.common.ClientBuilder; - -import java.util.HashMap; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; - -public class ExtendedTransportUpdateReplicaLevelTest extends NodeTestUtils { - - private static final Logger logger = LogManager.getLogger(ExtendedTransportUpdateReplicaLevelTest.class.getSimpleName()); - - @Test - public void testUpdateReplicaLevel() throws Exception { - - long numberOfShards = 2; - int replicaLevel = 3; - - // we need 3 nodes for replica level 3 - startNode("2"); - startNode("3"); - - int shardsAfterReplica; - - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(getSettings()) - .build(); - - Settings settings = Settings.settingsBuilder() - .put("index.number_of_shards", numberOfShards) - .put("index.number_of_replicas", 0) - .build(); - - try { - client.newIndex("replicatest", settings, new HashMap<>()); - client.waitForCluster("GREEN", "30s"); - for (int i = 0; i < 12345; i++) { - client.index("replicatest", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - client.flushIngest(); - client.waitForResponses("30s"); - client.updateReplicaLevel("replicatest", replicaLevel, "30s"); - //assertEquals(shardsAfterReplica, numberOfShards * (replicaLevel + 1)); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - client.shutdown(); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); - } - assertFalse(client.hasThrowable()); - } - } -} diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportIndexAliasTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/IndexShiftTest.java similarity index 72% rename from elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportIndexAliasTest.java rename to elx-transport/src/test/java/org/xbib/elx/transport/IndexShiftTest.java index 8d27231..7c1fdff 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportIndexAliasTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/IndexShiftTest.java @@ -10,13 +10,14 @@ import org.xbib.elx.common.ClientBuilder; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -public class ExtendedTransportIndexAliasTest extends NodeTestUtils { +public class IndexShiftTest extends NodeTestUtils { - private static final Logger logger = LogManager.getLogger(ExtendedTransportIndexAliasTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getSimpleName()); @Test public void testIndexAlias() throws Exception { @@ -28,21 +29,21 @@ public class ExtendedTransportIndexAliasTest extends NodeTestUtils { for (int i = 0; i < 1; i++) { client.index("test1234", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); } - client.flushIngest(); + client.flush(); client.refreshIndex("test1234"); List simpleAliases = Arrays.asList("a", "b", "c"); - client.switchIndex("test", "test1234", simpleAliases); + client.shiftIndex("test", "test1234", simpleAliases); client.newIndex("test5678"); for (int i = 0; i < 1; i++) { client.index("test5678", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); } - client.flushIngest(); + client.flush(); client.refreshIndex("test5678"); simpleAliases = Arrays.asList("d", "e", "f"); - client.switchIndex("test", "test5678", simpleAliases, (builder, index, alias) -> + client.shiftIndex("test", "test5678", simpleAliases, (builder, index, alias) -> builder.addAlias(index, alias, QueryBuilders.termQuery("my_key", alias))); Map indexFilters = client.getIndexFilters("test5678"); logger.info("index filters of index test5678 = {}", indexFilters); @@ -52,7 +53,7 @@ public class ExtendedTransportIndexAliasTest extends NodeTestUtils { assertTrue(indexFilters.containsKey("d")); assertTrue(indexFilters.containsKey("e")); - Map aliases = client.getAliasFilters("test"); + Map aliases = client.getIndexFilters(client.resolveAlias("test")); logger.info("aliases of alias test = {}", aliases); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); @@ -60,15 +61,15 @@ public class ExtendedTransportIndexAliasTest extends NodeTestUtils { assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("e")); - client.waitForResponses("30s"); - assertFalse(client.hasThrowable()); + client.waitForResponses(30L, TimeUnit.SECONDS); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); } - client.shutdown(); + assertNull(client.getBulkController().getLastBulkError()); + client.close(); } } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportReplicaTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ReplicaTest.java similarity index 65% rename from elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportReplicaTest.java rename to elx-transport/src/test/java/org/xbib/elx/transport/ReplicaTest.java index be95352..027b034 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportReplicaTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ReplicaTest.java @@ -18,14 +18,15 @@ import org.xbib.elx.common.ClientBuilder; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; -public class ExtendedTransportReplicaTest extends NodeTestUtils { +public class ReplicaTest extends NodeTestUtils { - private static final Logger logger = LogManager.getLogger(ExtendedTransportReplicaTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getSimpleName()); @Test public void testReplicaLevel() throws Exception { @@ -53,15 +54,15 @@ public class ExtendedTransportReplicaTest extends NodeTestUtils { try { client.newIndex("test1", settingsTest1, new HashMap<>()) .newIndex("test2", settingsTest2, new HashMap<>()); - client.waitForCluster("GREEN", "30s"); + client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS); for (int i = 0; i < 1234; i++) { client.index("test1", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } for (int i = 0; i < 1234; i++) { client.index("test2", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } - client.flushIngest(); - client.waitForResponses("30s"); + client.flush(); + client.waitForResponses(30L, TimeUnit.SECONDS); client.refreshIndex("test1"); client.refreshIndex("test2"); } catch (NoNodeAvailableException e) { @@ -96,12 +97,54 @@ public class ExtendedTransportReplicaTest extends NodeTestUtils { } catch (Exception e) { logger.error("delete index failed, ignored. Reason:", e); } - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); } - assertFalse(client.hasThrowable()); - client.shutdown(); + assertNull(client.getBulkController().getLastBulkError()); + client.close(); } } + @Test + public void testUpdateReplicaLevel() throws Exception { + + long numberOfShards = 2; + int replicaLevel = 3; + + // we need 3 nodes for replica level 3 + startNode("2"); + startNode("3"); + + int shardsAfterReplica; + + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) + .put(getSettings()) + .build(); + + Settings settings = Settings.settingsBuilder() + .put("index.number_of_shards", numberOfShards) + .put("index.number_of_replicas", 0) + .build(); + + try { + client.newIndex("replicatest", settings, new HashMap<>()); + client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS); + for (int i = 0; i < 12345; i++) { + client.index("replicatest", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flush(); + client.waitForResponses(30L, TimeUnit.SECONDS); + client.updateReplicaLevel("replicatest", replicaLevel, 30L, TimeUnit.SECONDS); + assertEquals(replicaLevel, client.getReplicaLevel("replicatest")); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + client.close(); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); + } + assertNull(client.getBulkController().getLastBulkError()); + } + } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientSingleNodeTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/SmokeTest.java similarity index 62% rename from elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientSingleNodeTest.java rename to elx-transport/src/test/java/org/xbib/elx/transport/SmokeTest.java index 3422762..d745015 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientSingleNodeTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/SmokeTest.java @@ -6,12 +6,14 @@ import org.elasticsearch.client.transport.NoNodeAvailableException; import org.junit.Test; import org.xbib.elx.common.ClientBuilder; +import java.util.concurrent.TimeUnit; + import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; -public class ExtendedTransportClientSingleNodeTest extends NodeTestUtils { +public class SmokeTest extends NodeTestUtils { - private static final Logger logger = LogManager.getLogger(ExtendedTransportClientSingleNodeTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(SmokeTest.class.getSimpleName()); @Test public void testSingleDocNodeClient() throws Exception { @@ -22,17 +24,17 @@ public class ExtendedTransportClientSingleNodeTest extends NodeTestUtils { try { client.newIndex("test"); client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest - client.flushIngest(); - client.waitForResponses("30s"); + client.flush(); + client.waitForResponses(30, TimeUnit.SECONDS); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); + client.close(); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); } - assertFalse(client.hasThrowable()); - client.shutdown(); + assertNull(client.getBulkController().getLastBulkError()); } } } diff --git a/gradle.properties b/gradle.properties index c961b52..98d19b7 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ group = org.xbib name = elx -version = 2.2.1.4 +version = 2.2.1.5 xbib-metrics.version = 1.1.0 xbib-guice.version = 4.0.4