commit 7069e31fe00ae669e348533b9c6fdc310f04f025 Author: Jörg Prante Date: Tue Nov 1 17:28:05 2016 +0100 initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bf3e9b4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +/data +/work +/logs +/.idea +/target +.DS_Store +*.iml +/.settings +/.classpath +/.project +/.gradle +/build +/plugins \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..a830350 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,12 @@ +language: java +sudo: required +jdk: + - oraclejdk8 +cache: + directories: + - $HOME/.m2 +after_success: + - ./gradlew sonarqube -Dsonar.host.url=https://sonarqube.com -Dsonar.login=$SONAR_TOKEN +env: + global: + secure: n1Ai4q/yMLn/Pg5pA4lTavoJoe7mQYB1PSKnZAqwbgyla94ySzK6iyBCBiNs/foMPisB/x+DHvmUXTsjvquw9Ay48ZITCV3xhcWzD0eZM2TMoG19CpRAEe8L8LNuYiti9k89ijDdUGZ5ifsvQNTGNHksouayAuApC3PrTUejJfR6SYrp1ZsQTbsMlr+4XU3p7QknK5rGgOwATIMP28F+bVnB05WJtlJA3b0SeucCurn3wJ4FGBQXRYmdlT7bQhNE4QgZM1VzcUFD/K0TBxzzq/otb/lNRSifyoekktDmJwQnaT9uQ4R8R6KdQ2Kb38Rvgjur+TKm5i1G8qS2+6LnIxQJG1aw3JvKK6W0wWCgnAVVRrXaCLday9NuY59tuh1mfjQ10UcsMNKcTdcKEMrLow506wSETcXc7L/LEnneWQyJJeV4vhPqR7KJfsBbeqgz3yIfsCn1GZVWFlfegzYCN52YTl0Y0uRD2Z+TnzQu+Bf4DzaWXLge1rz31xkhyeNNspub4h024+XqBjcMm6M9mlMzmmK8t2DIwPy/BlQbFBUyhrxziuR/5/2NEDPyHltvWkRb4AUIa25WJqkV0gTBegbMadZ9DyOo6Ea7aoVFBae2WGR08F1kzABsWrd1S7UJmWxW35iyMEtoAIayXphIK98qO5aCutwZ+3iOQazxbAs= diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..4989a18 --- /dev/null +++ b/README.md @@ -0,0 +1,82 @@ +![Helper](https://github.com/jprante/elasticsearch-helper/raw/master/src/site/resources/helper.jpg) + +# Elasticsearch helper plugin + +This plugin offers some Java helper classes for easier use of Elasticsearch API. + +## Compatibility matrix + +| Elasticsearch | Plugin | Release date | +| ----------------- | -------------- | -------------| +| 2.4.1 | 2.4.1.0 | Oct 4, 2016 | +| 2.4.0 | 2.4.0.0 | Oct 4, 2016 | +| 2.3.5 | 2.3.5.0 | Aug 4, 2016 | +| 2.3.3 | 2.3.3.1 | Jul 6, 2016 | +| 2.3.3 | 2.3.3.0 | May 23, 2016 | +| 2.3.2 | 2.3.2.0 | May 9, 2016 | +| 2.3.1 | 2.3.1.1 | May 9, 2016 | +| 2.3.1 | 2.3.1.0 | Apr 9, 2016 | +| 2.3.0 | 2.3.0.0 | Apr 9, 2016 | +| 2.2.1 | 2.2.1.1 | Mar 30, 2016 | +| 2.2.0 | 2.2.0.5 | Mar 15, 2016 | +| 2.2.0 | 2.2.0.4 | Mar 10, 2016 | +| 2.2.0 | 2.2.0.3 | Feb 16, 2016 | +| 2.2.0 | 2.2.0.2 | Feb 12, 2016 | +| 2.2.0 | 2.2.0.0 | Feb 3, 2016 | +| 2.1.1 | 2.1.1.0 | Dec 21, 2015 | +| 2.1.0 | 2.1.0.0 | Nov 29, 2015 | +| 2.0.0 | 2.0.0.2 | Nov 3, 2015 | +| 2.0.0 | 2.0.0.1 | Oct 29, 2015 | +| 2.0.0 | 2.0.0.0 | Oct 28, 2015 | +| 1.6.0 | 1.6.0.0 | Jul 1, 2015 | +| 1.5.2 | 1.5.2.2 | May 11, 2015 | +| 1.5.2 | 1.5.2.1 | May 3, 2015 | +| 1.5.1 | 1.5.1.0 | Apr 23, 2015 | +| 1.3.1 | 1.3.0.3 | Aug 8, 2014 | +| 1.3.1 | 1.3.0.1 | Aug 4, 2014 | +| 1.3.0 | 1.3.0.0 | Jul 23, 2014 | +| 1.2.2 | 1.2.2.0 | Jul 19, 2014 | +| 1.2.1 | 1.2.1.0 | Jun 4, 2014 | +| 1.2.0 | 1.2.0.1 | May 28, 2014 | +| 1.2.0 | 1.2.0.0 | May 22, 2014 | +| 1.1.0 | 1.1.0.7 | May 11, 2014 | +| 1.0.0.RC2 | 1.0.0.RC2.1 | Feb 3, 2014 | +| 0.90.7 | 0.90.7.1 | Dec 3, 2013 | +| 0.20.6 | 0.20.6.1 | Feb 4, 2014 | +| 0.19.11.2 | 0.19.11.2 | Feb 1, 2013 | + +## Installation 2.x + + ./bin/plugin install http://xbib.org/repository/org/xbib/elasticsearch/plugin/elasticsearch-helper/2.3.3.1/elasticsearch-helper-2.3.3.1-plugin.zip + +## Installation 1.x + + ./bin/plugin -install helper -url http://xbib.org/repository/org/xbib/elasticsearch/plugin/elasticsearch-helper/1.6.0.0/elasticsearch-helper-1.6.0.0.zip + +Do not forget to restart the node after installing. + +## Project docs + +The Maven project site is available at `Github `_ + +## Issues + +All feedback is welcome! If you find issues, please post them at `Github `_ + +# License + +Elasticsearch Helper Plugin (formerly Elasticsearch Support Plugin) + +Copyright (C) 2013 Jörg Prante + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000..fd72ee1 --- /dev/null +++ b/build.gradle @@ -0,0 +1,117 @@ + +plugins { + id "org.sonarqube" version "2.2" + id "org.ajoberstar.github-pages" version "1.6.0-rc.1" + id "org.xbib.gradle.plugin.jbake" version "1.1.0" +} + +group = 'org.xbib' +version = '2.2.1.0' + +printf "Host: %s\nOS: %s %s %s\nJVM: %s %s %s %s\nGroovy: %s\nGradle: %s\n" + + "Build: group: ${project.group} name: ${project.name} version: ${project.version}\n", + InetAddress.getLocalHost(), + System.getProperty("os.name"), + System.getProperty("os.arch"), + System.getProperty("os.version"), + System.getProperty("java.version"), + System.getProperty("java.vm.version"), + System.getProperty("java.vm.vendor"), + System.getProperty("java.vm.name"), + GroovySystem.getVersion(), + gradle.gradleVersion + +apply plugin: 'java' +apply plugin: 'maven' +apply plugin: 'signing' +apply plugin: 'findbugs' +apply plugin: 'pmd' +apply plugin: 'checkstyle' +apply plugin: "jacoco" +apply plugin: 'org.ajoberstar.github-pages' + +apply from: 'gradle/ext.gradle' + +sourceSets { + integrationTest { + java { + srcDir file('src/integration-test/java') + compileClasspath += main.output + compileClasspath += test.output + } + resources { + srcDir file('src/integration-test/resources') + } + } +} + +sourceCompatibility = 1.8 +targetCompatibility = 1.8 + +configurations { + wagon + integrationTestCompile.extendsFrom testCompile + integrationTestRuntime.extendsFrom testRuntime +} + +dependencies { + compile "org.xbib:metrics:1.0.0" + compile "org.elasticsearch:elasticsearch:2.2.1" + testCompile "net.java.dev.jna:jna:4.1.0" + testCompile "junit:junit:4.12" + testCompile "org.apache.logging.log4j:log4j-core:2.7" + testCompile "org.apache.logging.log4j:log4j-slf4j-impl:2.7" + wagon 'org.apache.maven.wagon:wagon-ssh-external:2.10' +} + +tasks.withType(JavaCompile) { + options.compilerArgs << "-Xlint:all" << "-profile" << "compact3" +} + +task integrationTest(type: Test) { + include '**/MiscTestSuite.class' + include '**/BulkNodeTestSuite.class' + include '**/BulkTransportTestSuite.class' + testClassesDir = sourceSets.integrationTest.output.classesDir + classpath = configurations.integrationTestCompile + classpath += configurations.integrationTestRuntime + classpath += sourceSets.main.output + classpath += sourceSets.test.output + classpath += sourceSets.integrationTest.output + outputs.upToDateWhen { false } + systemProperty 'path.home', projectDir.absolutePath + testLogging.showStandardStreams = true +} + +integrationTest.mustRunAfter test +check.dependsOn integrationTest + +clean { + delete "plugins" + delete "logs" +} + +task javadocJar(type: Jar, dependsOn: classes) { + from javadoc + into "build/tmp" + classifier 'javadoc' +} + +task sourcesJar(type: Jar, dependsOn: classes) { + from sourceSets.main.allSource + into "build/tmp" + classifier 'sources' +} + +artifacts { + archives javadocJar, sourcesJar +} + +if (project.hasProperty('signing.keyId')) { + signing { + sign configurations.archives + } +} + +apply from: 'gradle/publish.gradle' +apply from: 'gradle/sonarqube.gradle' diff --git a/config/checkstyle/checkstyle.xml b/config/checkstyle/checkstyle.xml new file mode 100644 index 0000000..52fe33c --- /dev/null +++ b/config/checkstyle/checkstyle.xml @@ -0,0 +1,323 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/gradle/ext.gradle b/gradle/ext.gradle new file mode 100644 index 0000000..1a3a910 --- /dev/null +++ b/gradle/ext.gradle @@ -0,0 +1,8 @@ +ext { + user = 'jprante' + name = 'elasticsearch-extras-client' + description = 'Some extras implemented for using Elasticsearch clients (node and transport)' + scmUrl = 'https://github.com/' + user + '/' + name + scmConnection = 'scm:git:git://github.com/' + user + '/' + name + '.git' + scmDeveloperConnection = 'scm:git:git://github.com/' + user + '/' + name + '.git' +} diff --git a/gradle/publish.gradle b/gradle/publish.gradle new file mode 100644 index 0000000..6882a43 --- /dev/null +++ b/gradle/publish.gradle @@ -0,0 +1,63 @@ + +task xbibUpload(type: Upload) { + configuration = configurations.archives + uploadDescriptor = true + repositories { + if (project.hasProperty("xbibUsername")) { + mavenDeployer { + configuration = configurations.wagon + repository(url: 'scpexe://xbib.org/repository') { + authentication(userName: xbibUsername, privateKey: xbibPrivateKey) + } + } + } + } +} + +task sonaTypeUpload(type: Upload) { + configuration = configurations.archives + uploadDescriptor = true + repositories { + if (project.hasProperty('ossrhUsername')) { + mavenDeployer { + beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } + repository(url: 'https://oss.sonatype.org/service/local/staging/deploy/maven2') { + authentication(userName: ossrhUsername, password: ossrhPassword) + } + snapshotRepository(url: 'https://oss.sonatype.org/content/repositories/snapshots') { + authentication(userName: ossrhUsername, password: ossrhPassword) + } + pom.project { + name name + description description + packaging 'jar' + inceptionYear '2012' + url scmUrl + organization { + name 'xbib' + url 'http://xbib.org' + } + developers { + developer { + id user + name 'Jörg Prante' + email 'joergprante@gmail.com' + url 'https://github.com/jprante' + } + } + scm { + url scmUrl + connection scmConnection + developerConnection scmDeveloperConnection + } + licenses { + license { + name 'The Apache License, Version 2.0' + url 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + } + } + } + } +} diff --git a/gradle/publish.gradle~ b/gradle/publish.gradle~ new file mode 100644 index 0000000..e04b20b --- /dev/null +++ b/gradle/publish.gradle~ @@ -0,0 +1,104 @@ + +task xbibUpload(type: Upload) { + configuration = configurations.archives + uploadDescriptor = true + repositories { + if (project.hasProperty("xbibUsername")) { + mavenDeployer { + configuration = configurations.wagon + repository(url: 'scpexe://xbib.org/repository') { + authentication(userName: xbibUsername, privateKey: xbibPrivateKey) + } + } + } + } +} + +task sonaTypeUpload(type: Upload) { + configuration = configurations.archives + uploadDescriptor = true + repositories { + if (project.hasProperty('ossrhUsername')) { + mavenDeployer { + beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } + repository(url: 'https://oss.sonatype.org/service/local/staging/deploy/maven2') { + authentication(userName: ossrhUsername, password: ossrhPassword) + } + snapshotRepository(url: 'https://oss.sonatype.org/content/repositories/snapshots') { + authentication(userName: ossrhUsername, password: ossrhPassword) + } + pom.project { + name name + description description + packaging 'jar' + inceptionYear '2012' + url scmUrl + organization { + name 'xbib' + url 'http://xbib.org' + } + developers { + developer { + id user + name 'Jörg Prante' + email 'joergprante@gmail.com' + url 'https://github.com/jprante' + } + } + scm { + url scmUrl + connection scmConnection + developerConnection scmDeveloperConnection + } + licenses { + license { + name 'The Apache License, Version 2.0' + url 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + } + } + } + } +} + + +task hbzUpload(type: Upload) { + configuration = configurations.archives + uploadDescriptor = true + repositories { + if (project.hasProperty('hbzUserName')) { + mavenDeployer { + configuration = configurations.wagon + beforeDeployment { MavenDeployment deployment -> + signing.signPom(deployment) + } + repository(url: uri(hbzUrl)) { + authentication(userName: hbzUserName, privateKey: hbzPrivateKey) + } + pom.project { + developers { + developer { + id 'jprante' + name 'Jörg Prante' + email 'joergprante@gmail.com' + url 'https://github.com/jprante' + } + } + scm { + url 'https://github.com/xbib/elasticsearch-webapp-libraryservice' + connection 'scm:git:git://github.com/xbib/elasticsaerch-webapp-libraryservice.git' + developerConnection 'scm:git:git://github.com/xbib/elasticsaerch-webapp-libraryservice.git' + } + inceptionYear '2016' + licenses { + license { + name 'The Apache License, Version 2.0' + url 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + } + } + } + } +} diff --git a/gradle/sonarqube.gradle b/gradle/sonarqube.gradle new file mode 100644 index 0000000..5de408d --- /dev/null +++ b/gradle/sonarqube.gradle @@ -0,0 +1,41 @@ +tasks.withType(FindBugs) { + ignoreFailures = true + reports { + xml.enabled = true + html.enabled = false + } +} +tasks.withType(Pmd) { + ignoreFailures = true + reports { + xml.enabled = true + html.enabled = true + } +} +tasks.withType(Checkstyle) { + ignoreFailures = true + reports { + xml.enabled = true + html.enabled = true + } +} + +jacocoTestReport { + reports { + xml.enabled true + csv.enabled false + xml.destination "${buildDir}/reports/jacoco-xml" + html.destination "${buildDir}/reports/jacoco-html" + } +} + +sonarqube { + properties { + property "sonar.projectName", "${project.group} ${project.name}" + property "sonar.sourceEncoding", "UTF-8" + property "sonar.tests", "src/integration-test/java" + property "sonar.scm.provider", "git" + property "sonar.java.coveragePlugin", "jacoco" + property "sonar.junit.reportsPath", "build/test-results/test/" + } +} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..6ffa237 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..2a06e59 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Tue Nov 01 14:46:00 CET 2016 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-3.1-bin.zip diff --git a/gradlew b/gradlew new file mode 100755 index 0000000..9aa616c --- /dev/null +++ b/gradlew @@ -0,0 +1,169 @@ +#!/usr/bin/env bash + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn ( ) { + echo "$*" +} + +die ( ) { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules +function splitJvmOpts() { + JVM_OPTS=("$@") +} +eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS +JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [[ "$(uname)" == "Darwin" ]] && [[ "$HOME" == "$PWD" ]]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..e95643d --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 0000000..ef50653 --- /dev/null +++ b/settings.gradle @@ -0,0 +1 @@ +rootProject.name = 'elasticsearch-extras-client' diff --git a/src/integration-test/java/org/elasticsearch/node/MockNode.java b/src/integration-test/java/org/elasticsearch/node/MockNode.java new file mode 100644 index 0000000..b0c02eb --- /dev/null +++ b/src/integration-test/java/org/elasticsearch/node/MockNode.java @@ -0,0 +1,38 @@ +package org.elasticsearch.node; + +import org.elasticsearch.Version; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.internal.InternalSettingsPreparer; +import org.elasticsearch.plugins.Plugin; + +import java.util.ArrayList; +import java.util.Collection; + +/** + * + */ +public class MockNode extends Node { + + public MockNode() { + super(Settings.EMPTY); + } + + public MockNode(Settings settings) { + super(settings); + } + + public MockNode(Settings settings, Collection> classpathPlugins) { + super(InternalSettingsPreparer.prepareEnvironment(settings, null), Version.CURRENT, classpathPlugins); + } + + public MockNode(Settings settings, Class classpathPlugin) { + this(settings, list(classpathPlugin)); + } + + private static Collection> list(Class classpathPlugin) { + Collection> list = new ArrayList<>(); + list.add(classpathPlugin); + return list; + } + +} diff --git a/src/integration-test/java/org/elasticsearch/node/package-info.java b/src/integration-test/java/org/elasticsearch/node/package-info.java new file mode 100644 index 0000000..f299cbc --- /dev/null +++ b/src/integration-test/java/org/elasticsearch/node/package-info.java @@ -0,0 +1,4 @@ +/** + * Classes to support Elasticsearch node creation. + */ +package org.elasticsearch.node; diff --git a/src/integration-test/java/org/xbib/elasticsearch/AliasTest.java b/src/integration-test/java/org/xbib/elasticsearch/AliasTest.java new file mode 100644 index 0000000..545e9e8 --- /dev/null +++ b/src/integration-test/java/org/xbib/elasticsearch/AliasTest.java @@ -0,0 +1,92 @@ +package org.xbib.elasticsearch; + +import com.carrotsearch.hppc.cursors.ObjectCursor; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.cluster.metadata.AliasAction; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.junit.Test; + +import java.io.IOException; +import java.util.Collections; +import java.util.Iterator; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * + */ +public class AliasTest extends NodeTestUtils { + + private static final ESLogger logger = ESLoggerFactory.getLogger(AliasTest.class.getName()); + + @Test + public void testAlias() throws IOException { + CreateIndexRequest indexRequest = new CreateIndexRequest("test"); + client("1").admin().indices().create(indexRequest).actionGet(); + // put alias + IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest(); + String[] indices = new String[]{"test"}; + String[] aliases = new String[]{"test_alias"}; + IndicesAliasesRequest.AliasActions aliasAction = + new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, indices, aliases); + indicesAliasesRequest.addAliasAction(aliasAction); + client("1").admin().indices().aliases(indicesAliasesRequest).actionGet(); + // get alias + GetAliasesRequest getAliasesRequest = new GetAliasesRequest(Strings.EMPTY_ARRAY); + long t0 = System.nanoTime(); + GetAliasesResponse getAliasesResponse = client("1").admin().indices().getAliases(getAliasesRequest).actionGet(); + long t1 = (System.nanoTime() - t0) / 1000000; + logger.info("{} time(ms) = {}", getAliasesResponse.getAliases(), t1); + assertTrue(t1 >= 0); + } + + @Test + public void testMostRecentIndex() throws IOException { + String alias = "test"; + CreateIndexRequest indexRequest = new CreateIndexRequest("test20160101"); + client("1").admin().indices().create(indexRequest).actionGet(); + indexRequest = new CreateIndexRequest("test20160102"); + client("1").admin().indices().create(indexRequest).actionGet(); + indexRequest = new CreateIndexRequest("test20160103"); + client("1").admin().indices().create(indexRequest).actionGet(); + IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest(); + String[] indices = new String[]{"test20160101", "test20160102", "test20160103"}; + String[] aliases = new String[]{alias}; + IndicesAliasesRequest.AliasActions aliasAction = + new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, indices, aliases); + indicesAliasesRequest.addAliasAction(aliasAction); + client("1").admin().indices().aliases(indicesAliasesRequest).actionGet(); + + GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client("1"), + GetAliasesAction.INSTANCE); + GetAliasesResponse getAliasesResponse = getAliasesRequestBuilder.setAliases(alias).execute().actionGet(); + Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); + Set result = new TreeSet<>(Collections.reverseOrder()); + for (ObjectCursor indexName : getAliasesResponse.getAliases().keys()) { + Matcher m = pattern.matcher(indexName.value); + if (m.matches()) { + if (alias.equals(m.group(1))) { + result.add(indexName.value); + } + } + } + Iterator it = result.iterator(); + assertEquals("test20160103", it.next()); + assertEquals("test20160102", it.next()); + assertEquals("test20160101", it.next()); + logger.info("result={}", result); + } + +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/NodeTestUtils.java b/src/integration-test/java/org/xbib/elasticsearch/NodeTestUtils.java new file mode 100644 index 0000000..1815326 --- /dev/null +++ b/src/integration-test/java/org/xbib/elasticsearch/NodeTestUtils.java @@ -0,0 +1,204 @@ +package org.xbib.elasticsearch; + +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.client.support.AbstractClient; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.node.MockNode; +import org.elasticsearch.node.Node; +import org.junit.After; +import org.junit.Before; +import org.xbib.elasticsearch.extras.client.NetworkUtils; + +import java.io.IOException; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.elasticsearch.common.settings.Settings.settingsBuilder; + +/** + * + */ +public class NodeTestUtils { + + protected static final ESLogger logger = ESLoggerFactory.getLogger("test"); + private static Random random = new Random(); + private static char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); + private Map nodes = new HashMap<>(); + private Map clients = new HashMap<>(); + private AtomicInteger counter = new AtomicInteger(); + private String cluster; + private String host; + private int port; + + private static void deleteFiles() throws IOException { + Path directory = Paths.get(System.getProperty("path.home") + "/data"); + Files.walkFileTree(directory, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + + }); + + } + + @Before + public void startNodes() { + try { + logger.info("starting"); + setClusterName(); + startNode("1"); + findNodeAddress(); + try { + ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN).timeout(TimeValue.timeValueSeconds(30))).actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + throw new IOException("cluster state is " + healthResponse.getStatus().name() + + ", from here on, everything will fail!"); + } + } catch (ElasticsearchTimeoutException e) { + throw new IOException("timeout, cluster does not respond to health request, cowardly refusing to continue with operations"); + } + } catch (Throwable t) { + logger.error("startNodes failed", t); + } + } + + @After + public void stopNodes() { + try { + closeNodes(); + } catch (Exception e) { + logger.error("can not close nodes", e); + } finally { + try { + deleteFiles(); + logger.info("data files wiped"); + Thread.sleep(2000L); + } catch (IOException e) { + logger.error(e.getMessage(), e); + } catch (InterruptedException e) { + // ignore + } + } + } + + protected void setClusterName() { + this.cluster = "test-helper-cluster-" + + NetworkUtils.getLocalAddress().getHostName() + + "-" + System.getProperty("user.name") + + "-" + counter.incrementAndGet(); + } + + protected String getClusterName() { + return cluster; + } + + protected Settings getSettings() { + return settingsBuilder() + .put("host", host) + .put("port", port) + .put("cluster.name", cluster) + .put("path.home", getHome()) + .build(); + } + + protected Settings getNodeSettings() { + return settingsBuilder() + .put("cluster.name", cluster) + .put("cluster.routing.schedule", "50ms") + .put("cluster.routing.allocation.disk.threshold_enabled", false) + .put("discovery.zen.multicast.enabled", true) + .put("discovery.zen.multicast.ping_timeout", "5s") + .put("http.enabled", true) + .put("threadpool.bulk.size", Runtime.getRuntime().availableProcessors()) + .put("threadpool.bulk.queue_size", 16 * Runtime.getRuntime().availableProcessors()) // default is 50, too low + .put("index.number_of_replicas", 0) + .put("path.home", getHome()) + .build(); + } + + protected String getHome() { + return System.getProperty("path.home"); + } + + public void startNode(String id) throws IOException { + buildNode(id).start(); + } + + public AbstractClient client(String id) { + return clients.get(id); + } + + private void closeNodes() throws IOException { + logger.info("closing all clients"); + for (AbstractClient client : clients.values()) { + client.close(); + } + clients.clear(); + logger.info("closing all nodes"); + for (Node node : nodes.values()) { + if (node != null) { + node.close(); + } + } + nodes.clear(); + logger.info("all nodes closed"); + } + + protected void findNodeAddress() { + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); + NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet(); + Object obj = response.iterator().next().getTransport().getAddress() + .publishAddress(); + if (obj instanceof InetSocketTransportAddress) { + InetSocketTransportAddress address = (InetSocketTransportAddress) obj; + host = address.address().getHostName(); + port = address.address().getPort(); + } + } + + private Node buildNode(String id) throws IOException { + Settings nodeSettings = settingsBuilder() + .put(getNodeSettings()) + .put("name", id) + .build(); + logger.info("settings={}", nodeSettings.getAsMap()); + Node node = new MockNode(nodeSettings); + AbstractClient client = (AbstractClient) node.client(); + nodes.put(id, node); + clients.put(id, client); + logger.info("clients={}", clients); + return node; + } + + protected String randomString(int len) { + final char[] buf = new char[len]; + final int n = numbersAndLetters.length - 1; + for (int i = 0; i < buf.length; i++) { + buf[i] = numbersAndLetters[random.nextInt(n)]; + } + return new String(buf); + } +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/SearchTest.java b/src/integration-test/java/org/xbib/elasticsearch/SearchTest.java new file mode 100644 index 0000000..8146b19 --- /dev/null +++ b/src/integration-test/java/org/xbib/elasticsearch/SearchTest.java @@ -0,0 +1,66 @@ +package org.xbib.elasticsearch; + +import org.elasticsearch.action.bulk.BulkAction; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.sort.SortOrder; +import org.junit.Test; + +import static org.elasticsearch.client.Requests.indexRequest; +import static org.elasticsearch.client.Requests.refreshRequest; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + +/** + * + */ +public class SearchTest extends NodeTestUtils { + + @Test + public void testSearch() throws Exception { + Client client = client("1"); + long t0 = System.currentTimeMillis(); + BulkRequestBuilder builder = new BulkRequestBuilder(client, BulkAction.INSTANCE); + for (int i = 0; i < 1000; i++) { + builder.add(indexRequest() + .index("pages").type("row") + .source(jsonBuilder() + .startObject() + .field("user1", "kimchy") + .field("user2", "kimchy") + .field("user3", "kimchy") + .field("user4", "kimchy") + .field("user5", "kimchy") + .field("user6", "kimchy") + .field("user7", "kimchy") + .field("user8", "kimchy") + .field("user9", "kimchy") + .field("rowcount", i) + .field("rs", 1234))); + } + client.bulk(builder.request()).actionGet(); + + client.admin().indices().refresh(refreshRequest()).actionGet(); + + long t1 = System.currentTimeMillis(); + logger.info("t1-t0 = {}", t1 - t0); + + for (int i = 0; i < 100; i++) { + t1 = System.currentTimeMillis(); + QueryBuilder queryStringBuilder = + QueryBuilders.queryStringQuery("rs:" + 1234); + SearchRequestBuilder requestBuilder = client.prepareSearch() + .setIndices("pages") + .setTypes("row") + .setQuery(queryStringBuilder) + .addSort("rowcount", SortOrder.DESC) + .setFrom(i * 10).setSize(10); + SearchResponse response = requestBuilder.execute().actionGet(); + long t2 = System.currentTimeMillis(); + logger.info("t2-t1 = {}", t2 - t1); + } + } +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/SimpleTest.java b/src/integration-test/java/org/xbib/elasticsearch/SimpleTest.java new file mode 100644 index 0000000..7a25dce --- /dev/null +++ b/src/integration-test/java/org/xbib/elasticsearch/SimpleTest.java @@ -0,0 +1,59 @@ +package org.xbib.elasticsearch; + +import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; +import org.elasticsearch.action.index.IndexAction; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.common.settings.Settings; +import org.junit.Test; + +import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.junit.Assert.assertEquals; + +/** + * + */ +public class SimpleTest extends NodeTestUtils { + + protected Settings getNodeSettings() { + return settingsBuilder() + .put("path.home", System.getProperty("path.home")) + .put("index.analysis.analyzer.default.filter.0", "lowercase") + .put("index.analysis.analyzer.default.filter.1", "trim") + .put("index.analysis.analyzer.default.tokenizer", "keyword") + .build(); + } + + @Test + public void test() throws Exception { + try { + DeleteIndexRequestBuilder deleteIndexRequestBuilder = + new DeleteIndexRequestBuilder(client("1"), DeleteIndexAction.INSTANCE, "test"); + deleteIndexRequestBuilder.execute().actionGet(); + } catch (Exception e) { + // ignore + } + IndexRequestBuilder indexRequestBuilder = new IndexRequestBuilder(client("1"), IndexAction.INSTANCE); + indexRequestBuilder + .setIndex("test") + .setType("test") + .setId("1") + .setSource(jsonBuilder().startObject().field("field", + "1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8").endObject()) + .setRefresh(true) + .execute() + .actionGet(); + String doc = client("1").prepareSearch("test") + .setTypes("test") + .setQuery(matchQuery("field", + "1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8")) + .execute() + .actionGet() + .getHits().getAt(0).getSourceAsString(); + + assertEquals(doc, + "{\"field\":\"1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8\"}"); + } +} \ No newline at end of file diff --git a/src/integration-test/java/org/xbib/elasticsearch/WildcardTest.java b/src/integration-test/java/org/xbib/elasticsearch/WildcardTest.java new file mode 100644 index 0000000..fd9ce16 --- /dev/null +++ b/src/integration-test/java/org/xbib/elasticsearch/WildcardTest.java @@ -0,0 +1,70 @@ +package org.xbib.elasticsearch; + +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilder; +import org.junit.Test; + +import java.io.IOException; + +import static org.elasticsearch.client.Requests.indexRequest; +import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; + +/** + * + */ +public class WildcardTest extends NodeTestUtils { + + protected Settings getNodeSettings() { + return settingsBuilder() + .put("cluster.name", getClusterName()) + .put("cluster.routing.allocation.disk.threshold_enabled", false) + .put("discovery.zen.multicast.enabled", false) + .put("http.enabled", false) + .put("path.home", System.getProperty("path.home")) + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .build(); + } + + @Test + public void testWildcard() throws Exception { + index(client("1"), "1", "010"); + index(client("1"), "2", "0*0"); + // exact + validateCount(client("1"), queryStringQuery("010").defaultField("field"), 1); + validateCount(client("1"), queryStringQuery("0\\*0").defaultField("field"), 1); + // pattern + validateCount(client("1"), queryStringQuery("0*0").defaultField("field"), 1); // 2? + validateCount(client("1"), queryStringQuery("0?0").defaultField("field"), 1); // 2? + validateCount(client("1"), queryStringQuery("0**0").defaultField("field"), 1); // 2? + validateCount(client("1"), queryStringQuery("0??0").defaultField("field"), 0); + validateCount(client("1"), queryStringQuery("*10").defaultField("field"), 1); + validateCount(client("1"), queryStringQuery("*1*").defaultField("field"), 1); + validateCount(client("1"), queryStringQuery("*\\*0").defaultField("field"), 0); // 1? + validateCount(client("1"), queryStringQuery("*\\**").defaultField("field"), 0); // 1? + } + + private void index(Client client, String id, String fieldValue) throws IOException { + client.index(indexRequest() + .index("index").type("type").id(id) + .source(jsonBuilder().startObject().field("field", fieldValue).endObject()) + .refresh(true)).actionGet(); + } + + private long count(Client client, QueryBuilder queryBuilder) { + return client.prepareSearch("index").setTypes("type") + .setQuery(queryBuilder) + .execute().actionGet().getHits().getTotalHits(); + } + + private void validateCount(Client client, QueryBuilder queryBuilder, long expectedHits) { + final long actualHits = count(client, queryBuilder); + if (actualHits != expectedHits) { + throw new RuntimeException("actualHits=" + actualHits + ", expectedHits=" + expectedHits); + } + } + +} \ No newline at end of file diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/NetworkTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/NetworkTest.java new file mode 100644 index 0000000..b9e7a87 --- /dev/null +++ b/src/integration-test/java/org/xbib/elasticsearch/extras/client/NetworkTest.java @@ -0,0 +1,44 @@ +package org.xbib.elasticsearch.extras.client; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.Test; + +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.util.Collections; +import java.util.Enumeration; + +/** + * + */ +public class NetworkTest { + + private static final Logger logger = LogManager.getLogger(NetworkTest.class); + + @Test + public void testNetwork() throws Exception { + Enumeration nets = NetworkInterface.getNetworkInterfaces(); + for (NetworkInterface netint : Collections.list(nets)) { + System.out.println("checking network interface = " + netint.getName()); + Enumeration inetAddresses = netint.getInetAddresses(); + for (InetAddress addr : Collections.list(inetAddresses)) { + logger.info("found address = " + addr.getHostAddress() + + " name = " + addr.getHostName() + + " canicalhostname = " + addr.getCanonicalHostName() + + " loopback = " + addr.isLoopbackAddress() + + " sitelocal = " + addr.isSiteLocalAddress() + + " linklocal = " + addr.isLinkLocalAddress() + + " anylocal = " + addr.isAnyLocalAddress() + + " multicast = " + addr.isMulticastAddress() + + " mcglobal = " + addr.isMCGlobal() + + " mclinklocal = " + addr.isMCLinkLocal() + + " mcnodelocal = " + addr.isMCNodeLocal() + + " mcorglocal = " + addr.isMCOrgLocal() + + " mcsitelocal = " + addr.isMCSiteLocal() + + " mcsitelocal = " + addr.isReachable(1000)); + } + } + + } +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClientTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClientTest.java new file mode 100644 index 0000000..c7f7421 --- /dev/null +++ b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClientTest.java @@ -0,0 +1,208 @@ +package org.xbib.elasticsearch.extras.client.node; + +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.junit.Before; +import org.junit.Test; +import org.xbib.elasticsearch.NodeTestUtils; +import org.xbib.elasticsearch.extras.client.ClientBuilder; +import org.xbib.elasticsearch.extras.client.SimpleBulkControl; +import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * + */ +public class BulkNodeClientTest extends NodeTestUtils { + + private static final ESLogger logger = ESLoggerFactory.getLogger(BulkNodeClientTest.class.getSimpleName()); + + private static final Long MAX_ACTIONS = 1000L; + + private static final Long NUM_ACTIONS = 1234L; + + @Before + public void startNodes() { + try { + super.startNodes(); + startNode("2"); + } catch (Throwable t) { + logger.error("startNodes failed", t); + } + } + + @Test + public void testNewIndexNodeClient() throws Exception { + final BulkNodeClient client = ClientBuilder.builder() + .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5)) + .setMetric(new SimpleBulkMetric()) + .setControl(new SimpleBulkControl()) + .toBulkNodeClient(client("1")); + client.newIndex("test"); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + client.shutdown(); + } + + @Test + public void testMappingNodeClient() throws Exception { + final BulkNodeClient client = ClientBuilder.builder() + .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5)) + .setMetric(new SimpleBulkMetric()) + .setControl(new SimpleBulkControl()) + .toBulkNodeClient(client("1")); + XContentBuilder builder = jsonBuilder() + .startObject() + .startObject("test") + .startObject("properties") + .startObject("location") + .field("type", "geo_point") + .endObject() + .endObject() + .endObject() + .endObject(); + client.mapping("test", builder.string()); + client.newIndex("test"); + GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test"); + GetMappingsResponse getMappingsResponse = + client.client().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); + logger.info("mappings={}", getMappingsResponse.getMappings()); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + client.shutdown(); + } + + @Test + public void testSingleDocNodeClient() { + final BulkNodeClient client = ClientBuilder.builder() + .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS) + .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(30)) + .setMetric(new SimpleBulkMetric()) + .setControl(new SimpleBulkControl()) + .toBulkNodeClient(client("1")); + try { + client.newIndex("test"); + client.index("test", "test", "1", "{ \"name\" : \"Hello World\"}"); // single doc ingest + client.flushIngest(); + client.waitForResponses(TimeValue.timeValueSeconds(30)); + } catch (InterruptedException e) { + // ignore + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } catch (ExecutionException e) { + logger.error(e.getMessage(), e); + } finally { + assertEquals(1, client.getMetric().getSucceeded().getCount()); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + client.shutdown(); + } + } + + @Test + public void testRandomDocsNodeClient() throws Exception { + long numactions = NUM_ACTIONS; + final BulkNodeClient client = ClientBuilder.builder() + .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS) + .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60)) + .setMetric(new SimpleBulkMetric()) + .setControl(new SimpleBulkControl()) + .toBulkNodeClient(client("1")); + try { + client.newIndex("test"); + for (int i = 0; i < NUM_ACTIONS; i++) { + client.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flushIngest(); + client.waitForResponses(TimeValue.timeValueSeconds(30)); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + assertEquals(numactions, client.getMetric().getSucceeded().getCount()); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + client.shutdown(); + } + } + + @Test + public void testThreadedRandomDocsNodeClient() throws Exception { + int maxthreads = Runtime.getRuntime().availableProcessors(); + Long maxactions = MAX_ACTIONS; + final Long maxloop = NUM_ACTIONS; + logger.info("NodeClient max={} maxactions={} maxloop={}", maxthreads, maxactions, maxloop); + final BulkNodeClient client = ClientBuilder.builder() + .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, maxactions) + .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60))// disable auto flush for this test + .setMetric(new SimpleBulkMetric()) + .setControl(new SimpleBulkControl()) + .toBulkNodeClient(client("1")); + try { + client.newIndex("test") + .startBulk("test", -1, 1000); + ThreadPoolExecutor pool = EsExecutors.newFixed("bulk-nodeclient-test", maxthreads, 30, + EsExecutors.daemonThreadFactory("bulk-nodeclient-test")); + final CountDownLatch latch = new CountDownLatch(maxthreads); + for (int i = 0; i < maxthreads; i++) { + pool.execute(new Runnable() { + public void run() { + for (int i = 0; i < maxloop; i++) { + client.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + latch.countDown(); + } + }); + } + logger.info("waiting for max 30 seconds..."); + latch.await(30, TimeUnit.SECONDS); + logger.info("flush..."); + client.flushIngest(); + client.waitForResponses(TimeValue.timeValueSeconds(30)); + logger.info("got all responses, thread pool shutdown..."); + pool.shutdown(); + logger.info("pool is shut down"); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + client.stopBulk("test"); + assertEquals(maxthreads * maxloop, client.getMetric().getSucceeded().getCount()); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + client.refreshIndex("test"); + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE) + .setQuery(QueryBuilders.matchAllQuery()).setSize(0); + assertEquals(maxthreads * maxloop, + searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); + client.shutdown(); + } + } + +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClusterBlockTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClusterBlockTest.java new file mode 100644 index 0000000..58e2b8e --- /dev/null +++ b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClusterBlockTest.java @@ -0,0 +1,49 @@ +package org.xbib.elasticsearch.extras.client.node; + +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.junit.Before; +import org.junit.Test; +import org.xbib.elasticsearch.NodeTestUtils; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + +/** + * + */ +public class BulkNodeClusterBlockTest extends NodeTestUtils { + + @Before + public void startNodes() { + try { + setClusterName(); + startNode("1"); + findNodeAddress(); + // do not wait for green health state + logger.info("ready"); + } catch (Throwable t) { + logger.error("startNodes failed", t); + } + } + + protected Settings getNodeSettings() { + return Settings.settingsBuilder() + .put(super.getNodeSettings()) + .put("discovery.zen.minimum_master_nodes", 2) // block until we have two nodes + .build(); + } + + @Test(expected = ClusterBlockException.class) + public void testClusterBlock() throws Exception { + BulkRequestBuilder brb = client("1").prepareBulk(); + XContentBuilder builder = jsonBuilder().startObject().field("field1", "value1").endObject(); + String jsonString = builder.string(); + IndexRequestBuilder irb = client("1").prepareIndex("test", "test", "1").setSource(jsonString); + brb.add(irb); + brb.execute().actionGet(); + } + +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeDuplicateIDTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeDuplicateIDTest.java new file mode 100644 index 0000000..98c6a70 --- /dev/null +++ b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeDuplicateIDTest.java @@ -0,0 +1,60 @@ +package org.xbib.elasticsearch.extras.client.node; + +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.unit.TimeValue; +import org.junit.Test; +import org.xbib.elasticsearch.NodeTestUtils; +import org.xbib.elasticsearch.extras.client.ClientBuilder; +import org.xbib.elasticsearch.extras.client.SimpleBulkControl; +import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; + +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.junit.Assert.*; + +public class BulkNodeDuplicateIDTest extends NodeTestUtils { + + private final static ESLogger logger = ESLoggerFactory.getLogger(BulkNodeDuplicateIDTest.class.getSimpleName()); + + private final static Long MAX_ACTIONS = 1000L; + + private final static Long NUM_ACTIONS = 12345L; + + @Test + public void testDuplicateDocIDs() throws Exception { + long numactions = NUM_ACTIONS; + final BulkNodeClient client = ClientBuilder.builder() + .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS) + .setMetric(new SimpleBulkMetric()) + .setControl(new SimpleBulkControl()) + .toBulkNodeClient(client("1")); + try { + client.newIndex("test"); + for (int i = 0; i < NUM_ACTIONS; i++) { + client.index("test", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flushIngest(); + client.waitForResponses(TimeValue.timeValueSeconds(30)); + client.refreshIndex("test"); + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE) + .setIndices("test") + .setTypes("test") + .setQuery(matchAllQuery()); + long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); + logger.info("hits = {}", hits); + assertTrue(hits < NUM_ACTIONS); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + client.shutdown(); + assertEquals(numactions, client.getMetric().getSucceeded().getCount()); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + } + } +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeIndexAliasTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeIndexAliasTest.java new file mode 100644 index 0000000..d4b19b0 --- /dev/null +++ b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeIndexAliasTest.java @@ -0,0 +1,77 @@ +package org.xbib.elasticsearch.extras.client.node; + +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.QueryBuilders; +import org.junit.Test; +import org.xbib.elasticsearch.NodeTestUtils; +import org.xbib.elasticsearch.extras.client.ClientBuilder; +import org.xbib.elasticsearch.extras.client.IndexAliasAdder; +import org.xbib.elasticsearch.extras.client.SimpleBulkControl; +import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertFalse; + +/** + * + */ +public class BulkNodeIndexAliasTest extends NodeTestUtils { + + private static final ESLogger logger = ESLoggerFactory.getLogger(BulkNodeIndexAliasTest.class.getSimpleName()); + + @Test + public void testIndexAlias() throws Exception { + final BulkNodeClient client = ClientBuilder.builder() + .setMetric(new SimpleBulkMetric()) + .setControl(new SimpleBulkControl()) + .toBulkNodeClient(client("1")); + try { + client.newIndex("test1234"); + for (int i = 0; i < 1; i++) { + client.index("test1234", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flushIngest(); + client.refreshIndex("test1234"); + + List simpleAliases = Arrays.asList("a", "b", "c"); + client.switchAliases("test", "test1234", simpleAliases); + + client.newIndex("test5678"); + for (int i = 0; i < 1; i++) { + client.index("test5678", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flushIngest(); + client.refreshIndex("test5678"); + + simpleAliases = Arrays.asList("d", "e", "f"); + client.switchAliases("test", "test5678", simpleAliases, new IndexAliasAdder() { + @Override + public void addIndexAlias(IndicesAliasesRequestBuilder builder, String index, String alias) { + builder.addAlias(index, alias, QueryBuilders.termQuery("my_key", alias)); + } + }); + Map aliases = client.getIndexFilters("test5678"); + logger.info("aliases of index test5678 = {}", aliases); + + aliases = client.getAliasFilters("test"); + logger.info("aliases of alias test = {}", aliases); + + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + client.waitForResponses(TimeValue.timeValueSeconds(30)); + client.shutdown(); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + } + } +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeReplicaTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeReplicaTest.java new file mode 100644 index 0000000..93141e1 --- /dev/null +++ b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeReplicaTest.java @@ -0,0 +1,105 @@ +package org.xbib.elasticsearch.extras.client.node; + +import org.elasticsearch.action.admin.indices.stats.*; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.indexing.IndexingStats; +import org.junit.Test; +import org.xbib.elasticsearch.NodeTestUtils; +import org.xbib.elasticsearch.extras.client.ClientBuilder; +import org.xbib.elasticsearch.extras.client.SimpleBulkControl; +import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; + +import java.util.Map; + +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +public class BulkNodeReplicaTest extends NodeTestUtils { + + private final static ESLogger logger = ESLoggerFactory.getLogger(BulkNodeReplicaTest.class.getSimpleName()); + + @Test + public void testReplicaLevel() throws Exception { + + // we need nodes for replica levels + startNode("2"); + startNode("3"); + startNode("4"); + + Settings settingsTest1 = Settings.settingsBuilder() + .put("index.number_of_shards", 2) + .put("index.number_of_replicas", 3) + .build(); + + Settings settingsTest2 = Settings.settingsBuilder() + .put("index.number_of_shards", 2) + .put("index.number_of_replicas", 1) + .build(); + + final BulkNodeClient client = ClientBuilder.builder() + .setMetric(new SimpleBulkMetric()) + .setControl(new SimpleBulkControl()) + .toBulkNodeClient(client("1")); + + try { + client.newIndex("test1", settingsTest1, null) + .newIndex("test2", settingsTest2, null); + client.waitForCluster("GREEN", TimeValue.timeValueSeconds(30)); + for (int i = 0; i < 1234; i++) { + client.index("test1", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + for (int i = 0; i < 1234; i++) { + client.index("test2", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flushIngest(); + client.waitForResponses(TimeValue.timeValueSeconds(60)); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + logger.info("refreshing"); + client.refreshIndex("test1"); + client.refreshIndex("test2"); + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE) + .setIndices("test1", "test2") + .setQuery(matchAllQuery()); + long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); + logger.info("query total hits={}", hits); + assertEquals(2468, hits); + IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.client(), IndicesStatsAction.INSTANCE) + .all(); + IndicesStatsResponse response = indicesStatsRequestBuilder.execute().actionGet(); + for (Map.Entry m : response.getIndices().entrySet()) { + IndexStats indexStats = m.getValue(); + CommonStats commonStats = indexStats.getTotal(); + IndexingStats indexingStats = commonStats.getIndexing(); + IndexingStats.Stats stats = indexingStats.getTotal(); + logger.info("index {}: count = {}", m.getKey(), stats.getIndexCount()); + for (Map.Entry me : indexStats.getIndexShards().entrySet()) { + IndexShardStats indexShardStats = me.getValue(); + CommonStats commonShardStats = indexShardStats.getTotal(); + logger.info("shard {} count = {}", me.getKey(), + commonShardStats.getIndexing().getTotal().getIndexCount()); + } + } + try { + client.deleteIndex("test1") + .deleteIndex("test2"); + } catch (Exception e) { + logger.error("delete index failed, ignored. Reason:", e); + } + client.shutdown(); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + } + } + +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeUpdateReplicaLevelTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeUpdateReplicaLevelTest.java new file mode 100644 index 0000000..b1c88fe --- /dev/null +++ b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeUpdateReplicaLevelTest.java @@ -0,0 +1,67 @@ +package org.xbib.elasticsearch.extras.client.node; + +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.junit.Test; +import org.xbib.elasticsearch.NodeTestUtils; +import org.xbib.elasticsearch.extras.client.ClientBuilder; +import org.xbib.elasticsearch.extras.client.SimpleBulkControl; +import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * + */ +public class BulkNodeUpdateReplicaLevelTest extends NodeTestUtils { + + private static final ESLogger logger = ESLoggerFactory.getLogger(BulkNodeUpdateReplicaLevelTest.class.getSimpleName()); + + @Test + public void testUpdateReplicaLevel() throws Exception { + + int numberOfShards = 2; + int replicaLevel = 3; + + // we need 3 nodes for replica level 3 + startNode("2"); + startNode("3"); + + int shardsAfterReplica; + + Settings settings = Settings.settingsBuilder() + .put("index.number_of_shards", numberOfShards) + .put("index.number_of_replicas", 0) + .build(); + + final BulkNodeClient client = ClientBuilder.builder() + .setMetric(new SimpleBulkMetric()) + .setControl(new SimpleBulkControl()) + .toBulkNodeClient(client("1")); + + try { + client.newIndex("replicatest", settings, null); + client.waitForCluster("GREEN", TimeValue.timeValueSeconds(30)); + for (int i = 0; i < 12345; i++) { + client.index("replicatest", "replicatest", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flushIngest(); + client.waitForResponses(TimeValue.timeValueSeconds(30)); + shardsAfterReplica = client.updateReplicaLevel("replicatest", replicaLevel); + assertEquals(shardsAfterReplica, numberOfShards * (replicaLevel + 1)); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + client.shutdown(); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + } + } + +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/package-info.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/package-info.java new file mode 100644 index 0000000..873ebae --- /dev/null +++ b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/package-info.java @@ -0,0 +1,4 @@ +/** + * Classes for testing Elasticsearch node client extras. + */ +package org.xbib.elasticsearch.extras.client.node; diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/package-info.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/package-info.java new file mode 100644 index 0000000..2bfc45c --- /dev/null +++ b/src/integration-test/java/org/xbib/elasticsearch/extras/client/package-info.java @@ -0,0 +1,4 @@ +/** + * Classes to test Elasticsearch clients. + */ +package org.xbib.elasticsearch.extras.client; diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClientTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClientTest.java new file mode 100644 index 0000000..0a35742 --- /dev/null +++ b/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClientTest.java @@ -0,0 +1,201 @@ +package org.xbib.elasticsearch.extras.client.transport; + +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.index.query.QueryBuilders; +import org.junit.Before; +import org.junit.Test; +import org.xbib.elasticsearch.NodeTestUtils; +import org.xbib.elasticsearch.extras.client.ClientBuilder; +import org.xbib.elasticsearch.extras.client.SimpleBulkControl; +import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; + +import java.io.IOException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * + */ +public class BulkTransportClientTest extends NodeTestUtils { + + private static final ESLogger logger = ESLoggerFactory.getLogger(BulkTransportClientTest.class.getSimpleName()); + + private static final Long MAX_ACTIONS = 1000L; + + private static final Long NUM_ACTIONS = 1234L; + + @Before + public void startNodes() { + try { + super.startNodes(); + startNode("2"); + } catch (Throwable t) { + logger.error("startNodes failed", t); + } + } + + @Test + public void testBulkClient() throws IOException { + final BulkTransportClient client = ClientBuilder.builder() + .put(getSettings()) + .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60)) + .setMetric(new SimpleBulkMetric()) + .setControl(new SimpleBulkControl()) + .toBulkTransportClient(); + client.newIndex("test"); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + try { + client.deleteIndex("test") + .newIndex("test") + .deleteIndex("test"); + } catch (NoNodeAvailableException e) { + logger.error("no node available"); + } finally { + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + client.shutdown(); + } + } + + @Test + public void testSingleDocBulkClient() throws IOException { + final BulkTransportClient client = ClientBuilder.builder() + .put(getSettings()) + .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS) + .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60)) + .setMetric(new SimpleBulkMetric()) + .setControl(new SimpleBulkControl()) + .toBulkTransportClient(); + try { + client.newIndex("test"); + client.index("test", "test", "1", "{ \"name\" : \"Hello World\"}"); // single doc ingest + client.flushIngest(); + client.waitForResponses(TimeValue.timeValueSeconds(30)); + } catch (InterruptedException e) { + // ignore + } catch (ExecutionException e) { + logger.error(e.getMessage(), e); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + assertEquals(1, client.getMetric().getSucceeded().getCount()); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + client.shutdown(); + } + } + + @Test + public void testRandomDocsBulkClient() throws IOException { + long numactions = NUM_ACTIONS; + final BulkTransportClient client = ClientBuilder.builder() + .put(getSettings()) + .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS) + .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60)) + .setMetric(new SimpleBulkMetric()) + .setControl(new SimpleBulkControl()) + .toBulkTransportClient(); + try { + client.newIndex("test"); + for (int i = 0; i < NUM_ACTIONS; i++) { + client.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flushIngest(); + client.waitForResponses(TimeValue.timeValueSeconds(30)); + } catch (InterruptedException e) { + // ignore + } catch (ExecutionException e) { + logger.error(e.getMessage(), e); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + assertEquals(numactions, client.getMetric().getSucceeded().getCount()); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + client.shutdown(); + } + } + + @Test + public void testThreadedRandomDocsBulkClient() throws Exception { + int maxthreads = Runtime.getRuntime().availableProcessors(); + long maxactions = MAX_ACTIONS; + final long maxloop = NUM_ACTIONS; + + Settings settingsForIndex = Settings.settingsBuilder() + .put("index.number_of_shards", 2) + .put("index.number_of_replicas", 1) + .build(); + + final BulkTransportClient client = ClientBuilder.builder() + .put(getSettings()) + .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, maxactions) + .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60)) // = disable autoflush for this test + .setMetric(new SimpleBulkMetric()) + .setControl(new SimpleBulkControl()) + .toBulkTransportClient(); + try { + client.newIndex("test", settingsForIndex, null) + .startBulk("test", -1, 1000); + ThreadPoolExecutor pool = + EsExecutors.newFixed("bulkclient-test", maxthreads, 30, EsExecutors.daemonThreadFactory("bulkclient-test")); + final CountDownLatch latch = new CountDownLatch(maxthreads); + for (int i = 0; i < maxthreads; i++) { + pool.execute(() -> { + for (int i1 = 0; i1 < maxloop; i1++) { + client.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + latch.countDown(); + }); + } + logger.info("waiting for max 30 seconds..."); + latch.await(30, TimeUnit.SECONDS); + logger.info("client flush ..."); + client.flushIngest(); + client.waitForResponses(TimeValue.timeValueSeconds(30)); + logger.info("thread pool to be shut down ..."); + pool.shutdown(); + logger.info("poot shut down"); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + client.stopBulk("test"); + assertEquals(maxthreads * maxloop, client.getMetric().getSucceeded().getCount()); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + client.refreshIndex("test"); + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE) + // to avoid NPE at org.elasticsearch.action.search.SearchRequest.writeTo(SearchRequest.java:580) + .setIndices("_all") + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(0); + assertEquals(maxthreads * maxloop, + searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); + client.shutdown(); + } + } + +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportDuplicateIDTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportDuplicateIDTest.java new file mode 100644 index 0000000..00a4066 --- /dev/null +++ b/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportDuplicateIDTest.java @@ -0,0 +1,61 @@ +package org.xbib.elasticsearch.extras.client.transport; + +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.unit.TimeValue; +import org.junit.Test; +import org.xbib.elasticsearch.NodeTestUtils; +import org.xbib.elasticsearch.extras.client.ClientBuilder; +import org.xbib.elasticsearch.extras.client.SimpleBulkControl; +import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; + +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.junit.Assert.*; + +public class BulkTransportDuplicateIDTest extends NodeTestUtils { + + private final static ESLogger logger = ESLoggerFactory.getLogger(BulkTransportDuplicateIDTest.class.getSimpleName()); + + private final static Long MAX_ACTIONS = 1000L; + + private final static Long NUM_ACTIONS = 12345L; + + @Test + public void testDuplicateDocIDs() throws Exception { + long numactions = NUM_ACTIONS; + final BulkTransportClient client = ClientBuilder.builder() + .put(getSettings()) + .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS) + .setMetric(new SimpleBulkMetric()) + .setControl(new SimpleBulkControl()) + .toBulkTransportClient(); + try { + client.newIndex("test"); + for (int i = 0; i < NUM_ACTIONS; i++) { + client.index("test", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flushIngest(); + client.waitForResponses(TimeValue.timeValueSeconds(30)); + client.refreshIndex("test"); + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE) + .setIndices("test") + .setTypes("test") + .setQuery(matchAllQuery()); + long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); + logger.info("hits = {}", hits); + assertTrue(hits < NUM_ACTIONS); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + client.shutdown(); + assertEquals(numactions, client.getMetric().getSucceeded().getCount()); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + } + } +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportReplicaTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportReplicaTest.java new file mode 100644 index 0000000..119688e --- /dev/null +++ b/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportReplicaTest.java @@ -0,0 +1,108 @@ +package org.xbib.elasticsearch.extras.client.transport; + +import org.elasticsearch.action.admin.indices.stats.*; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.indexing.IndexingStats; +import org.junit.Test; +import org.xbib.elasticsearch.NodeTestUtils; +import org.xbib.elasticsearch.extras.client.ClientBuilder; +import org.xbib.elasticsearch.extras.client.SimpleBulkControl; +import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; + +import java.util.Map; + +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * + */ +public class BulkTransportReplicaTest extends NodeTestUtils { + + private static final ESLogger logger = ESLoggerFactory.getLogger(BulkTransportReplicaTest.class.getSimpleName()); + + @Test + public void testReplicaLevel() throws Exception { + + // we need nodes for replica levels + startNode("2"); + startNode("3"); + startNode("4"); + + Settings settingsTest1 = Settings.settingsBuilder() + .put("index.number_of_shards", 2) + .put("index.number_of_replicas", 3) + .build(); + + Settings settingsTest2 = Settings.settingsBuilder() + .put("index.number_of_shards", 2) + .put("index.number_of_replicas", 1) + .build(); + + final BulkTransportClient client = ClientBuilder.builder() + .put(getSettings()) + .setMetric(new SimpleBulkMetric()) + .setControl(new SimpleBulkControl()) + .toBulkTransportClient(); + try { + client.newIndex("test1", settingsTest1, null) + .newIndex("test2", settingsTest2, null); + client.waitForCluster("GREEN", TimeValue.timeValueSeconds(30)); + for (int i = 0; i < 1234; i++) { + client.index("test1", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + for (int i = 0; i < 1234; i++) { + client.index("test2", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flushIngest(); + client.waitForResponses(TimeValue.timeValueSeconds(60)); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + logger.info("refreshing"); + client.refreshIndex("test1"); + client.refreshIndex("test2"); + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE) + .setIndices("test1", "test2") + .setQuery(matchAllQuery()); + long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); + logger.info("query total hits={}", hits); + assertEquals(2468, hits); + IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.client(), + IndicesStatsAction.INSTANCE).all(); + IndicesStatsResponse response = indicesStatsRequestBuilder.execute().actionGet(); + for (Map.Entry m : response.getIndices().entrySet()) { + IndexStats indexStats = m.getValue(); + CommonStats commonStats = indexStats.getTotal(); + IndexingStats indexingStats = commonStats.getIndexing(); + IndexingStats.Stats stats = indexingStats.getTotal(); + logger.info("index {}: count = {}", m.getKey(), stats.getIndexCount()); + for (Map.Entry me : indexStats.getIndexShards().entrySet()) { + IndexShardStats indexShardStats = me.getValue(); + CommonStats commonShardStats = indexShardStats.getTotal(); + logger.info("shard {} count = {}", me.getKey(), + commonShardStats.getIndexing().getTotal().getIndexCount()); + } + } + try { + client.deleteIndex("test1") + .deleteIndex("test2"); + } catch (Exception e) { + logger.error("delete index failed, ignored. Reason:", e); + } + client.shutdown(); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + } + } + +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportUpdateReplicaLevelTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportUpdateReplicaLevelTest.java new file mode 100644 index 0000000..8ed2c4a --- /dev/null +++ b/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportUpdateReplicaLevelTest.java @@ -0,0 +1,69 @@ +package org.xbib.elasticsearch.extras.client.transport; + +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.junit.Test; +import org.xbib.elasticsearch.NodeTestUtils; +import org.xbib.elasticsearch.extras.client.ClientBuilder; +import org.xbib.elasticsearch.extras.client.SimpleBulkControl; +import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * + */ +public class BulkTransportUpdateReplicaLevelTest extends NodeTestUtils { + + private static final ESLogger logger = + ESLoggerFactory.getLogger(BulkTransportUpdateReplicaLevelTest.class.getSimpleName()); + + @Test + public void testUpdateReplicaLevel() throws Exception { + + int numberOfShards = 2; + int replicaLevel = 3; + + // we need 3 nodes for replica level 3 + startNode("2"); + startNode("3"); + + int shardsAfterReplica; + + Settings settings = Settings.settingsBuilder() + .put("index.number_of_shards", numberOfShards) + .put("index.number_of_replicas", 0) + .build(); + + final BulkTransportClient client = ClientBuilder.builder() + .put(getSettings()) + .setMetric(new SimpleBulkMetric()) + .setControl(new SimpleBulkControl()) + .toBulkTransportClient(); + + try { + client.newIndex("replicatest", settings, null); + client.waitForCluster("GREEN", TimeValue.timeValueSeconds(30)); + for (int i = 0; i < 12345; i++) { + client.index("replicatest", "replicatest", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flushIngest(); + client.waitForResponses(TimeValue.timeValueSeconds(30)); + shardsAfterReplica = client.updateReplicaLevel("replicatest", replicaLevel); + assertEquals(shardsAfterReplica, numberOfShards * (replicaLevel + 1)); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + client.shutdown(); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + } + } + +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/package-info.java b/src/integration-test/java/org/xbib/elasticsearch/package-info.java new file mode 100644 index 0000000..2958ce1 --- /dev/null +++ b/src/integration-test/java/org/xbib/elasticsearch/package-info.java @@ -0,0 +1,4 @@ +/** + * Test classes for testing Elasticsearch. + */ +package org.xbib.elasticsearch; \ No newline at end of file diff --git a/src/integration-test/java/suites/BulkNodeTestSuite.java b/src/integration-test/java/suites/BulkNodeTestSuite.java new file mode 100644 index 0000000..caac820 --- /dev/null +++ b/src/integration-test/java/suites/BulkNodeTestSuite.java @@ -0,0 +1,23 @@ +package suites; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; +import org.xbib.elasticsearch.extras.client.node.BulkNodeClientTest; +import org.xbib.elasticsearch.extras.client.node.BulkNodeDuplicateIDTest; +import org.xbib.elasticsearch.extras.client.node.BulkNodeIndexAliasTest; +import org.xbib.elasticsearch.extras.client.node.BulkNodeReplicaTest; +import org.xbib.elasticsearch.extras.client.node.BulkNodeUpdateReplicaLevelTest; + +/** + * + */ +@RunWith(ListenerSuite.class) +@Suite.SuiteClasses({ + BulkNodeClientTest.class, + BulkNodeDuplicateIDTest.class, + BulkNodeReplicaTest.class, + BulkNodeUpdateReplicaLevelTest.class, + BulkNodeIndexAliasTest.class +}) +public class BulkNodeTestSuite { +} diff --git a/src/integration-test/java/suites/BulkTransportTestSuite.java b/src/integration-test/java/suites/BulkTransportTestSuite.java new file mode 100644 index 0000000..f429dfc --- /dev/null +++ b/src/integration-test/java/suites/BulkTransportTestSuite.java @@ -0,0 +1,22 @@ +package suites; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; +import org.xbib.elasticsearch.extras.client.transport.BulkTransportClientTest; +import org.xbib.elasticsearch.extras.client.transport.BulkTransportDuplicateIDTest; +import org.xbib.elasticsearch.extras.client.transport.BulkTransportReplicaTest; +import org.xbib.elasticsearch.extras.client.transport.BulkTransportUpdateReplicaLevelTest; + +/** + * + */ +@RunWith(ListenerSuite.class) +@Suite.SuiteClasses({ + BulkTransportClientTest.class, + BulkTransportDuplicateIDTest.class, + BulkTransportReplicaTest.class, + BulkTransportUpdateReplicaLevelTest.class +}) +public class BulkTransportTestSuite { + +} diff --git a/src/integration-test/java/suites/ListenerSuite.java b/src/integration-test/java/suites/ListenerSuite.java new file mode 100644 index 0000000..c02d371 --- /dev/null +++ b/src/integration-test/java/suites/ListenerSuite.java @@ -0,0 +1,23 @@ +package suites; + +import org.junit.runner.Runner; +import org.junit.runner.notification.RunNotifier; +import org.junit.runners.Suite; +import org.junit.runners.model.InitializationError; +import org.junit.runners.model.RunnerBuilder; + +public class ListenerSuite extends Suite { + + private final TestListener listener = new TestListener(); + + public ListenerSuite(Class klass, RunnerBuilder builder) throws InitializationError { + super(klass, builder); + } + + @Override + protected void runChild(Runner runner, RunNotifier notifier) { + notifier.addListener(listener); + runner.run(notifier); + notifier.removeListener(listener); + } +} diff --git a/src/integration-test/java/suites/MiscTestSuite.java b/src/integration-test/java/suites/MiscTestSuite.java new file mode 100644 index 0000000..ea23630 --- /dev/null +++ b/src/integration-test/java/suites/MiscTestSuite.java @@ -0,0 +1,21 @@ +package suites; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; +import org.xbib.elasticsearch.AliasTest; +import org.xbib.elasticsearch.SearchTest; +import org.xbib.elasticsearch.SimpleTest; +import org.xbib.elasticsearch.WildcardTest; + +/** + * + */ +@RunWith(ListenerSuite.class) +@Suite.SuiteClasses({ + SimpleTest.class, + AliasTest.class, + SearchTest.class, + WildcardTest.class +}) +public class MiscTestSuite { +} diff --git a/src/integration-test/java/suites/TestListener.java b/src/integration-test/java/suites/TestListener.java new file mode 100644 index 0000000..7e24527 --- /dev/null +++ b/src/integration-test/java/suites/TestListener.java @@ -0,0 +1,44 @@ +package suites; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.runner.Description; +import org.junit.runner.Result; +import org.junit.runner.notification.Failure; +import org.junit.runner.notification.RunListener; + +/** + * + */ +public class TestListener extends RunListener { + + private static final Logger logger = LogManager.getLogger("test.listener"); + + public void testRunStarted(Description description) throws java.lang.Exception { + logger.info("number of tests to execute: {}", description.testCount()); + } + + public void testRunFinished(Result result) throws java.lang.Exception { + logger.info("number of tests executed: {}", result.getRunCount()); + } + + public void testStarted(Description description) throws java.lang.Exception { + logger.info("starting execution of {} {}", + description.getClassName(), description.getMethodName()); + } + + public void testFinished(Description description) throws java.lang.Exception { + logger.info("finished execution of {} {}", + description.getClassName(), description.getMethodName()); + } + + public void testFailure(Failure failure) throws java.lang.Exception { + logger.info("failed execution of tests: {}", + failure.getMessage()); + } + + public void testIgnored(Description description) throws java.lang.Exception { + logger.info("execution of test ignored: {}", + description.getClassName(), description.getMethodName()); + } +} diff --git a/src/integration-test/resources/log4j2.xml b/src/integration-test/resources/log4j2.xml new file mode 100644 index 0000000..f71aced --- /dev/null +++ b/src/integration-test/resources/log4j2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/integration-test/resources/org/xbib/elasticsearch/extras/client/settings.json b/src/integration-test/resources/org/xbib/elasticsearch/extras/client/settings.json new file mode 100644 index 0000000..86f5118 --- /dev/null +++ b/src/integration-test/resources/org/xbib/elasticsearch/extras/client/settings.json @@ -0,0 +1,3 @@ +{ + "index.analysis.analyzer.default.type" : "keyword" +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/AbstractClient.java b/src/main/java/org/xbib/elasticsearch/extras/client/AbstractClient.java new file mode 100644 index 0000000..bac6522 --- /dev/null +++ b/src/main/java/org/xbib/elasticsearch/extras/client/AbstractClient.java @@ -0,0 +1,495 @@ +package org.xbib.elasticsearch.extras.client; + +import com.carrotsearch.hppc.cursors.ObjectCursor; +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; +import org.elasticsearch.action.admin.indices.flush.FlushAction; +import org.elasticsearch.action.admin.indices.flush.FlushRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexAction; +import org.elasticsearch.action.admin.indices.get.GetIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.admin.indices.recovery.RecoveryAction; +import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest; +import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * + */ +public abstract class AbstractClient { + + private static final ESLogger logger = ESLoggerFactory.getLogger(AbstractClient.class.getName()); + + private Settings.Builder settingsBuilder; + + private Settings settings; + + private Map mappings = new HashMap<>(); + + public abstract ElasticsearchClient client(); + + protected abstract void createClient(Settings settings) throws IOException; + + public abstract void shutdown(); + + public Settings.Builder getSettingsBuilder() { + return settingsBuilder(); + } + + public void resetSettings() { + settingsBuilder = Settings.settingsBuilder(); + settings = null; + mappings = new HashMap<>(); + } + + public void setSettings(Settings settings) { + this.settings = settings; + } + + public void setting(String key, String value) { + if (settingsBuilder == null) { + settingsBuilder = Settings.settingsBuilder(); + } + settingsBuilder.put(key, value); + } + + public void setting(String key, Boolean value) { + if (settingsBuilder == null) { + settingsBuilder = Settings.settingsBuilder(); + } + settingsBuilder.put(key, value); + } + + public void setting(String key, Integer value) { + if (settingsBuilder == null) { + settingsBuilder = Settings.settingsBuilder(); + } + settingsBuilder.put(key, value); + } + + public void setting(InputStream in) throws IOException { + settingsBuilder = Settings.settingsBuilder().loadFromStream(".json", in); + } + + public Settings.Builder settingsBuilder() { + return settingsBuilder != null ? settingsBuilder : Settings.settingsBuilder(); + } + + public Settings settings() { + if (settings != null) { + return settings; + } + if (settingsBuilder == null) { + settingsBuilder = Settings.settingsBuilder(); + } + return settingsBuilder.build(); + } + + public void mapping(String type, String mapping) throws IOException { + mappings.put(type, mapping); + } + + public void mapping(String type, InputStream in) throws IOException { + if (type == null) { + return; + } + StringWriter sw = new StringWriter(); + Streams.copy(new InputStreamReader(in), sw); + mappings.put(type, sw.toString()); + } + + public Map mappings() { + return mappings.isEmpty() ? null : mappings; + } + + + public void updateIndexSetting(String index, String key, Object value) throws IOException { + if (client() == null) { + return; + } + if (index == null) { + throw new IOException("no index name given"); + } + if (key == null) { + throw new IOException("no key given"); + } + if (value == null) { + throw new IOException("no value given"); + } + Settings.Builder settingsBuilder = Settings.settingsBuilder(); + settingsBuilder.put(key, value.toString()); + UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index) + .settings(settingsBuilder); + client().execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet(); + } + + public void waitForRecovery() throws IOException { + if (client() == null) { + return; + } + client().execute(RecoveryAction.INSTANCE, new RecoveryRequest()).actionGet(); + } + + public int waitForRecovery(String index) throws IOException { + if (client() == null) { + return -1; + } + if (index == null) { + throw new IOException("unable to waitfor recovery, index not set"); + } + RecoveryResponse response = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest(index)).actionGet(); + int shards = response.getTotalShards(); + client().execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest(index) + .waitForActiveShards(shards)).actionGet(); + return shards; + } + + public void waitForCluster(String statusString, TimeValue timeout) + throws IOException, ElasticsearchTimeoutException { + if (client() == null) { + return; + } + ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString); + ClusterHealthResponse healthResponse = + client().execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest() + .waitForStatus(status).timeout(timeout)).actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + throw new IOException("cluster state is " + healthResponse.getStatus().name() + + " and not " + status.name() + + ", from here on, everything will fail!"); + } + } + + public String fetchClusterName() { + if (client() == null) { + return null; + } + try { + ClusterStateRequestBuilder clusterStateRequestBuilder = + new ClusterStateRequestBuilder(client(), ClusterStateAction.INSTANCE).all(); + ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); + String name = clusterStateResponse.getClusterName().value(); + int nodeCount = clusterStateResponse.getState().getNodes().size(); + return name + " (" + nodeCount + " nodes connected)"; + } catch (ElasticsearchTimeoutException e) { + return "TIMEOUT"; + } catch (NoNodeAvailableException e) { + return "DISCONNECTED"; + } catch (Throwable t) { + return "[" + t.getMessage() + "]"; + } + } + + public String healthColor() { + if (client() == null) { + return null; + } + try { + ClusterHealthResponse healthResponse = + client().execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().timeout(TimeValue.timeValueSeconds(30))).actionGet(); + ClusterHealthStatus status = healthResponse.getStatus(); + return status.name(); + } catch (ElasticsearchTimeoutException e) { + return "TIMEOUT"; + } catch (NoNodeAvailableException e) { + return "DISCONNECTED"; + } catch (Throwable t) { + return "[" + t.getMessage() + "]"; + } + } + + public int updateReplicaLevel(String index, int level) throws IOException { + waitForCluster("YELLOW", TimeValue.timeValueSeconds(30)); + updateIndexSetting(index, "number_of_replicas", level); + return waitForRecovery(index); + } + + public void flushIndex(String index) { + if (client() == null) { + return; + } + if (index != null) { + client().execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet(); + } + } + + public void refreshIndex(String index) { + if (client() == null) { + return; + } + if (index != null) { + client().execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet(); + } + } + + public void putMapping(String index) { + if (client() == null) { + return; + } + if (!mappings().isEmpty()) { + for (Map.Entry me : mappings().entrySet()) { + client().execute(PutMappingAction.INSTANCE, + new PutMappingRequest(index).type(me.getKey()).source(me.getValue())).actionGet(); + } + } + } + + public String resolveAlias(String alias) { + if (client() == null) { + return alias; + } + GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client(), GetAliasesAction.INSTANCE); + GetAliasesResponse getAliasesResponse = getAliasesRequestBuilder.setAliases(alias).execute().actionGet(); + if (!getAliasesResponse.getAliases().isEmpty()) { + return getAliasesResponse.getAliases().keys().iterator().next().value; + } + return alias; + } + + public String resolveMostRecentIndex(String alias) { + if (client() == null) { + return alias; + } + if (alias == null) { + return null; + } + GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client(), GetAliasesAction.INSTANCE); + GetAliasesResponse getAliasesResponse = getAliasesRequestBuilder.setAliases(alias).execute().actionGet(); + Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); + Set indices = new TreeSet<>(Collections.reverseOrder()); + for (ObjectCursor indexName : getAliasesResponse.getAliases().keys()) { + Matcher m = pattern.matcher(indexName.value); + if (m.matches()) { + if (alias.equals(m.group(1))) { + indices.add(indexName.value); + } + } + } + return indices.isEmpty() ? alias : indices.iterator().next(); + } + + public Map getAliasFilters(String alias) { + GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client(), GetAliasesAction.INSTANCE); + return getFilters(getAliasesRequestBuilder.setIndices(resolveAlias(alias)).execute().actionGet()); + } + + public Map getIndexFilters(String index) { + GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client(), GetAliasesAction.INSTANCE); + return getFilters(getAliasesRequestBuilder.setIndices(index).execute().actionGet()); + } + + private Map getFilters(GetAliasesResponse getAliasesResponse) { + Map result = new HashMap<>(); + for (ObjectObjectCursor> object : getAliasesResponse.getAliases()) { + List aliasMetaDataList = object.value; + for (AliasMetaData aliasMetaData : aliasMetaDataList) { + if (aliasMetaData.filteringRequired()) { + result.put(aliasMetaData.alias(), new String(aliasMetaData.getFilter().uncompressed())); + } else { + result.put(aliasMetaData.alias(), null); + } + } + } + return result; + } + + public void switchAliases(String index, String concreteIndex, List extraAliases) { + switchAliases(index, concreteIndex, extraAliases, null); + } + + public void switchAliases(String index, String concreteIndex, + List extraAliases, IndexAliasAdder adder) { + if (client() == null) { + return; + } + if (index.equals(concreteIndex)) { + return; + } + // two situations: 1. there is a new alias 2. there is already an old index with the alias + String oldIndex = resolveAlias(index); + final Map oldFilterMap = oldIndex.equals(index) ? null : getIndexFilters(oldIndex); + final List newAliases = new LinkedList<>(); + final List switchAliases = new LinkedList<>(); + IndicesAliasesRequestBuilder requestBuilder = new IndicesAliasesRequestBuilder(client(), IndicesAliasesAction.INSTANCE); + if (oldFilterMap == null || !oldFilterMap.containsKey(index)) { + // never apply a filter for trunk index name + requestBuilder.addAlias(concreteIndex, index); + newAliases.add(index); + } + // switch existing aliases + if (oldFilterMap != null) { + for (Map.Entry entry : oldFilterMap.entrySet()) { + String alias = entry.getKey(); + String filter = entry.getValue(); + requestBuilder.removeAlias(oldIndex, alias); + if (filter != null) { + requestBuilder.addAlias(concreteIndex, alias, filter); + } else { + requestBuilder.addAlias(concreteIndex, alias); + } + switchAliases.add(alias); + } + } + // a list of aliases that should be added, check if new or old + if (extraAliases != null) { + for (String extraAlias : extraAliases) { + if (oldFilterMap == null || !oldFilterMap.containsKey(extraAlias)) { + // index alias adder only active on extra aliases, and if alias is new + if (adder != null) { + adder.addIndexAlias(requestBuilder, concreteIndex, extraAlias); + } else { + requestBuilder.addAlias(concreteIndex, extraAlias); + } + newAliases.add(extraAlias); + } else { + String filter = oldFilterMap.get(extraAlias); + requestBuilder.removeAlias(oldIndex, extraAlias); + if (filter != null) { + requestBuilder.addAlias(concreteIndex, extraAlias, filter); + } else { + requestBuilder.addAlias(concreteIndex, extraAlias); + } + switchAliases.add(extraAlias); + } + } + } + if (!newAliases.isEmpty() || !switchAliases.isEmpty()) { + logger.info("new aliases = {}, switch aliases = {}", newAliases, switchAliases); + requestBuilder.execute().actionGet(); + } + } + + public void performRetentionPolicy(String index, String concreteIndex, int timestampdiff, int mintokeep) { + if (client() == null) { + return; + } + if (index.equals(concreteIndex)) { + return; + } + GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client(), GetIndexAction.INSTANCE); + GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet(); + Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); + Set indices = new TreeSet<>(); + logger.info("{} indices", getIndexResponse.getIndices().length); + for (String s : getIndexResponse.getIndices()) { + Matcher m = pattern.matcher(s); + if (m.matches()) { + if (index.equals(m.group(1)) && !s.equals(concreteIndex)) { + indices.add(s); + } + } + } + if (indices.isEmpty()) { + logger.info("no indices found, retention policy skipped"); + return; + } + if (mintokeep > 0 && indices.size() <= mintokeep) { + logger.info("{} indices found, not enough for retention policy ({}), skipped", + indices.size(), mintokeep); + return; + } else { + logger.info("candidates for deletion = {}", indices); + } + List indicesToDelete = new ArrayList<>(); + // our index + Matcher m1 = pattern.matcher(concreteIndex); + if (m1.matches()) { + Integer i1 = Integer.parseInt(m1.group(2)); + for (String s : indices) { + Matcher m2 = pattern.matcher(s); + if (m2.matches()) { + Integer i2 = Integer.parseInt(m2.group(2)); + int kept = indices.size() - indicesToDelete.size(); + if ((timestampdiff == 0 || (timestampdiff > 0 && i1 - i2 > timestampdiff)) && mintokeep <= kept) { + indicesToDelete.add(s); + } + } + } + } + logger.info("indices to delete = {}", indicesToDelete); + if (indicesToDelete.isEmpty()) { + logger.info("not enough indices found to delete, retention policy complete"); + return; + } + String[] s = indicesToDelete.toArray(new String[indicesToDelete.size()]); + DeleteIndexRequestBuilder requestBuilder = new DeleteIndexRequestBuilder(client(), DeleteIndexAction.INSTANCE, s); + DeleteIndexResponse response = requestBuilder.execute().actionGet(); + if (!response.isAcknowledged()) { + logger.warn("retention delete index operation was not acknowledged"); + } + } + + public Long mostRecentDocument(String index) { + if (client() == null) { + return null; + } + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client(), SearchAction.INSTANCE); + SortBuilder sort = SortBuilders.fieldSort("_timestamp").order(SortOrder.DESC); + SearchResponse searchResponse = searchRequestBuilder.setIndices(index) + .addField("_timestamp") + .setSize(1) + .addSort(sort) + .execute().actionGet(); + if (searchResponse.getHits().getHits().length == 1) { + SearchHit hit = searchResponse.getHits().getHits()[0]; + if (hit.getFields().get("_timestamp") != null) { + return hit.getFields().get("_timestamp").getValue(); + } else { + return 0L; + } + } + return null; + } + +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/BulkControl.java b/src/main/java/org/xbib/elasticsearch/extras/client/BulkControl.java new file mode 100644 index 0000000..910f2f2 --- /dev/null +++ b/src/main/java/org/xbib/elasticsearch/extras/client/BulkControl.java @@ -0,0 +1,22 @@ +package org.xbib.elasticsearch.extras.client; + +import java.util.Map; +import java.util.Set; + +/** + */ +public interface BulkControl { + + void startBulk(String indexName, long startRefreshInterval, long stopRefreshInterval); + + boolean isBulk(String indexName); + + void finishBulk(String indexName); + + Set indices(); + + Map getStartBulkRefreshIntervals(); + + Map getStopBulkRefreshIntervals(); + +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/BulkMetric.java b/src/main/java/org/xbib/elasticsearch/extras/client/BulkMetric.java new file mode 100644 index 0000000..a45e9c2 --- /dev/null +++ b/src/main/java/org/xbib/elasticsearch/extras/client/BulkMetric.java @@ -0,0 +1,31 @@ +package org.xbib.elasticsearch.extras.client; + +import org.xbib.metrics.Count; +import org.xbib.metrics.Metered; + +/** + * + */ +public interface BulkMetric { + + Metered getTotalIngest(); + + Count getTotalIngestSizeInBytes(); + + Count getCurrentIngest(); + + Count getCurrentIngestNumDocs(); + + Count getSubmitted(); + + Count getSucceeded(); + + Count getFailed(); + + void start(); + + void stop(); + + long elapsed(); + +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/BulkProcessor.java b/src/main/java/org/xbib/elasticsearch/extras/client/BulkProcessor.java new file mode 100644 index 0000000..223c27e --- /dev/null +++ b/src/main/java/org/xbib/elasticsearch/extras/client/BulkProcessor.java @@ -0,0 +1,473 @@ +package org.xbib.elasticsearch.extras.client; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.bulk.BulkAction; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.util.concurrent.FutureUtils; + +import java.io.Closeable; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A bulk processor is a thread safe bulk processing class, allowing to easily set when to "flush" a new bulk request + * (either based on number of actions, based on the size, or time), and to easily control the number of concurrent bulk + * requests allowed to be executed in parallel. + * In order to create a new bulk processor, use the {@link Builder}. + */ +public class BulkProcessor implements Closeable { + + private final int bulkActions; + + private final long bulkSize; + + private final ScheduledThreadPoolExecutor scheduler; + + private final ScheduledFuture scheduledFuture; + + private final AtomicLong executionIdGen = new AtomicLong(); + + private final BulkRequestHandler bulkRequestHandler; + + private BulkRequest bulkRequest; + + private volatile boolean closed = false; + + private BulkProcessor(Client client, Listener listener, @Nullable String name, int concurrentRequests, + int bulkActions, ByteSizeValue bulkSize, @Nullable TimeValue flushInterval) { + this.bulkActions = bulkActions; + this.bulkSize = bulkSize.bytes(); + + this.bulkRequest = new BulkRequest(); + this.bulkRequestHandler = concurrentRequests == 0 ? + new SyncBulkRequestHandler(client, listener) : + new AsyncBulkRequestHandler(client, listener, concurrentRequests); + + if (flushInterval != null) { + this.scheduler = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(1, + EsExecutors.daemonThreadFactory(client.settings(), + name != null ? "[" + name + "]" : "" + "bulk_processor")); + this.scheduler.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); + this.scheduler.setContinueExistingPeriodicTasksAfterShutdownPolicy(false); + this.scheduledFuture = this.scheduler.scheduleWithFixedDelay(new Flush(), flushInterval.millis(), + flushInterval.millis(), TimeUnit.MILLISECONDS); + } else { + this.scheduler = null; + this.scheduledFuture = null; + } + } + + public static Builder builder(Client client, Listener listener) { + if (client == null) { + throw new NullPointerException("The client you specified while building a BulkProcessor is null"); + } + return new Builder(client, listener); + } + + /** + * Closes the processor. If flushing by time is enabled, then it's shutdown. Any remaining bulk actions are flushed. + */ + @Override + public void close() { + try { + awaitClose(0, TimeUnit.NANOSECONDS); + } catch (InterruptedException exc) { + Thread.currentThread().interrupt(); + } + } + + /** + * Closes the processor. If flushing by time is enabled, then it's shutdown. Any remaining bulk actions are + * flushed. + * + * If concurrent requests are not enabled, returns {@code true} immediately. + * If concurrent requests are enabled, waits for up to the specified timeout for all bulk requests to complete then + * returns {@code true}, + * If the specified waiting time elapses before all bulk requests complete, {@code false} is returned. + * + * @param timeout The maximum time to wait for the bulk requests to complete + * @param unit The time unit of the {@code timeout} argument + * @return {@code true} if all bulk requests completed and {@code false} if the waiting time elapsed before all the + * bulk requests completed + * @throws InterruptedException If the current thread is interrupted + */ + public synchronized boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException { + if (closed) { + return true; + } + closed = true; + if (this.scheduledFuture != null) { + FutureUtils.cancel(this.scheduledFuture); + this.scheduler.shutdown(); + } + if (bulkRequest.numberOfActions() > 0) { + execute(); + } + return this.bulkRequestHandler.awaitClose(timeout, unit); + } + + /** + * Adds an {@link IndexRequest} to the list of actions to execute. Follows the same behavior of {@link IndexRequest} + * (for example, if no id is provided, one will be generated, or usage of the create flag). + * + * @param request request + * @return his bulk processor + */ + public BulkProcessor add(IndexRequest request) { + return add((ActionRequest) request); + } + + /** + * Adds an {@link DeleteRequest} to the list of actions to execute. + * + * @param request request + * @return his bulk processor + */ + public BulkProcessor add(DeleteRequest request) { + return add((ActionRequest) request); + } + + /** + * Adds either a delete or an index request. + * + * @param request request + * @return his bulk processor + */ + public BulkProcessor add(ActionRequest request) { + return add(request, null); + } + + /** + * Adds either a delete or an index request with a payload. + * + * @param request request + * @param payload payload + * @return his bulk processor + */ + public BulkProcessor add(ActionRequest request, @Nullable Object payload) { + internalAdd(request, payload); + return this; + } + + protected void ensureOpen() { + if (closed) { + throw new IllegalStateException("bulk process already closed"); + } + } + + private synchronized void internalAdd(ActionRequest request, @Nullable Object payload) { + ensureOpen(); + bulkRequest.add(request, payload); + executeIfNeeded(); + } + + public BulkProcessor add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType) + throws Exception { + return add(data, defaultIndex, defaultType, null); + } + + public synchronized BulkProcessor add(BytesReference data, @Nullable String defaultIndex, + @Nullable String defaultType, @Nullable Object payload) throws Exception { + bulkRequest.add(data, defaultIndex, defaultType, null, null, payload, true); + executeIfNeeded(); + return this; + } + + private void executeIfNeeded() { + ensureOpen(); + if (!isOverTheLimit()) { + return; + } + execute(); + } + + private void execute() { + final BulkRequest bulkRequest = this.bulkRequest; + final long executionId = executionIdGen.incrementAndGet(); + + this.bulkRequest = new BulkRequest(); + this.bulkRequestHandler.execute(bulkRequest, executionId); + } + + private boolean isOverTheLimit() { + return bulkActions != -1 && + bulkRequest.numberOfActions() >= bulkActions || + bulkSize != -1 && + bulkRequest.estimatedSizeInBytes() >= bulkSize; + } + + /** + * Flush pending delete or index requests. + */ + public synchronized void flush() { + ensureOpen(); + if (bulkRequest.numberOfActions() > 0) { + execute(); + } + } + + /** + * A listener for the execution. + */ + public interface Listener { + + /** + * Callback before the bulk is executed. + * + * @param executionId execution ID + * @param request request + */ + void beforeBulk(long executionId, BulkRequest request); + + /** + * Callback after a successful execution of bulk request. + * + * @param executionId execution ID + * @param request request + * @param response response + */ + void afterBulk(long executionId, BulkRequest request, BulkResponse response); + + /** + * Callback after a failed execution of bulk request. + * + * Note that in case an instance of InterruptedException is passed, which means that request + * processing has been + * cancelled externally, the thread's interruption status has been restored prior to calling this method. + * + * @param executionId execution ID + * @param request request + * @param failure failure + */ + void afterBulk(long executionId, BulkRequest request, Throwable failure); + } + + /** + * A builder used to create a build an instance of a bulk processor. + */ + public static class Builder { + + private final Client client; + private final Listener listener; + private String name; + private int concurrentRequests = 1; + private int bulkActions = 1000; + private ByteSizeValue bulkSize = new ByteSizeValue(5, ByteSizeUnit.MB); + private TimeValue flushInterval = null; + + /** + * Creates a builder of bulk processor with the client to use and the listener that will be used + * to be notified on the completion of bulk requests. + * + * @param client the client + * @param listener the listener + */ + Builder(Client client, Listener listener) { + this.client = client; + this.listener = listener; + } + + /** + * Sets an optional name to identify this bulk processor. + * + * @param name name + * @return this builder + */ + public Builder setName(String name) { + this.name = name; + return this; + } + + /** + * Sets the number of concurrent requests allowed to be executed. A value of 0 means that only a single + * request will be allowed to be executed. A value of 1 means 1 concurrent request is allowed to be executed + * while accumulating new bulk requests. Defaults to 1. + * + * @param concurrentRequests maximum number of concurrent requests + * @return this builder + */ + public Builder setConcurrentRequests(int concurrentRequests) { + this.concurrentRequests = concurrentRequests; + return this; + } + + /** + * Sets when to flush a new bulk request based on the number of actions currently added. Defaults to + * 1000. Can be set to -1 to disable it. + * + * @param bulkActions mbulk actions + * @return this builder + */ + public Builder setBulkActions(int bulkActions) { + this.bulkActions = bulkActions; + return this; + } + + /** + * Sets when to flush a new bulk request based on the size of actions currently added. Defaults to + * 5mb. Can be set to -1 to disable it. + * + * @param bulkSize bulk size + * @return this builder + */ + public Builder setBulkSize(ByteSizeValue bulkSize) { + this.bulkSize = bulkSize; + return this; + } + + /** + * Sets a flush interval flushing *any* bulk actions pending if the interval passes. Defaults to not set. + * Note, both {@link #setBulkActions(int)} and {@link #setBulkSize(org.elasticsearch.common.unit.ByteSizeValue)} + * can be set to -1 with the flush interval set allowing for complete async processing of bulk actions. + * + * @param flushInterval flush interval + * @return this builder + */ + public Builder setFlushInterval(TimeValue flushInterval) { + this.flushInterval = flushInterval; + return this; + } + + /** + * Builds a new bulk processor. + * + * @return a bulk processor + */ + public BulkProcessor build() { + return new BulkProcessor(client, listener, name, concurrentRequests, bulkActions, bulkSize, flushInterval); + } + } + + private class Flush implements Runnable { + + @Override + public void run() { + synchronized (BulkProcessor.this) { + if (closed) { + return; + } + if (bulkRequest.numberOfActions() == 0) { + return; + } + execute(); + } + } + } + + /** + * Abstracts the low-level details of bulk request handling. + */ + abstract class BulkRequestHandler { + + public abstract void execute(BulkRequest bulkRequest, long executionId); + + public abstract boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException; + + } + + private class SyncBulkRequestHandler extends BulkRequestHandler { + private final Client client; + private final BulkProcessor.Listener listener; + + SyncBulkRequestHandler(Client client, BulkProcessor.Listener listener) { + this.client = client; + this.listener = listener; + } + + public void execute(BulkRequest bulkRequest, long executionId) { + boolean afterCalled = false; + try { + listener.beforeBulk(executionId, bulkRequest); + BulkResponse bulkResponse = client.execute(BulkAction.INSTANCE, bulkRequest).actionGet(); + afterCalled = true; + listener.afterBulk(executionId, bulkRequest, bulkResponse); + } catch (Throwable t) { + if (!afterCalled) { + listener.afterBulk(executionId, bulkRequest, t); + } + } + } + + public boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException { + return true; + } + } + + private class AsyncBulkRequestHandler extends BulkRequestHandler { + private final Client client; + private final BulkProcessor.Listener listener; + private final Semaphore semaphore; + private final int concurrentRequests; + + private AsyncBulkRequestHandler(Client client, BulkProcessor.Listener listener, int concurrentRequests) { + this.client = client; + this.listener = listener; + this.concurrentRequests = concurrentRequests; + this.semaphore = new Semaphore(concurrentRequests); + } + + @Override + public void execute(final BulkRequest bulkRequest, final long executionId) { + boolean bulkRequestSetupSuccessful = false; + boolean acquired = false; + try { + listener.beforeBulk(executionId, bulkRequest); + semaphore.acquire(); + acquired = true; + client.execute(BulkAction.INSTANCE, bulkRequest, new ActionListener() { + @Override + public void onResponse(BulkResponse response) { + try { + listener.afterBulk(executionId, bulkRequest, response); + } finally { + semaphore.release(); + } + } + + @Override + public void onFailure(Throwable e) { + try { + listener.afterBulk(executionId, bulkRequest, e); + } finally { + semaphore.release(); + } + } + }); + bulkRequestSetupSuccessful = true; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + listener.afterBulk(executionId, bulkRequest, e); + } catch (Throwable t) { + listener.afterBulk(executionId, bulkRequest, t); + } finally { + if (!bulkRequestSetupSuccessful && acquired) { // if we fail on client.bulk() release the semaphore + semaphore.release(); + } + } + } + + @Override + public boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException { + if (semaphore.tryAcquire(this.concurrentRequests, timeout, unit)) { + semaphore.release(this.concurrentRequests); + return true; + } + return false; + } + } +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/ClientBuilder.java b/src/main/java/org/xbib/elasticsearch/extras/client/ClientBuilder.java new file mode 100644 index 0000000..4089249 --- /dev/null +++ b/src/main/java/org/xbib/elasticsearch/extras/client/ClientBuilder.java @@ -0,0 +1,105 @@ +package org.xbib.elasticsearch.extras.client; + +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.xbib.elasticsearch.extras.client.node.BulkNodeClient; +import org.xbib.elasticsearch.extras.client.transport.BulkTransportClient; +import org.xbib.elasticsearch.extras.client.transport.MockTransportClient; + +/** + * + */ +public final class ClientBuilder implements Parameters { + + private final Settings.Builder settingsBuilder; + + private BulkMetric metric; + + private BulkControl control; + + public ClientBuilder() { + settingsBuilder = Settings.builder(); + } + + public static ClientBuilder builder() { + return new ClientBuilder(); + } + + public ClientBuilder put(String key, String value) { + settingsBuilder.put(key, value); + return this; + } + + public ClientBuilder put(String key, Integer value) { + settingsBuilder.put(key, value); + return this; + } + + public ClientBuilder put(String key, Long value) { + settingsBuilder.put(key, value); + return this; + } + + public ClientBuilder put(String key, Double value) { + settingsBuilder.put(key, value); + return this; + } + + public ClientBuilder put(String key, ByteSizeValue value) { + settingsBuilder.put(key, value); + return this; + } + + public ClientBuilder put(String key, TimeValue value) { + settingsBuilder.put(key, value); + return this; + } + + public ClientBuilder put(Settings settings) { + settingsBuilder.put(settings); + return this; + } + + public ClientBuilder setMetric(BulkMetric metric) { + this.metric = metric; + return this; + } + + public ClientBuilder setControl(BulkControl control) { + this.control = control; + return this; + } + + public BulkNodeClient toBulkNodeClient(Client client) { + Settings settings = settingsBuilder.build(); + return new BulkNodeClient() + .maxActionsPerRequest(settings.getAsInt(MAX_ACTIONS_PER_REQUEST, DEFAULT_MAX_ACTIONS_PER_REQUEST)) + .maxConcurrentRequests(settings.getAsInt(MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_REQUESTS)) + .maxVolumePerRequest(settings.getAsBytesSize(MAX_VOLUME_PER_REQUEST, DEFAULT_MAX_VOLUME_PER_REQUEST)) + .flushIngestInterval(settings.getAsTime(FLUSH_INTERVAL, DEFAULT_FLUSH_INTERVAL)) + .init(client, metric, control); + } + + public BulkTransportClient toBulkTransportClient() { + Settings settings = settingsBuilder.build(); + return new BulkTransportClient() + .maxActionsPerRequest(settings.getAsInt(MAX_ACTIONS_PER_REQUEST, DEFAULT_MAX_ACTIONS_PER_REQUEST)) + .maxConcurrentRequests(settings.getAsInt(MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_REQUESTS)) + .maxVolumePerRequest(settings.getAsBytesSize(MAX_VOLUME_PER_REQUEST, DEFAULT_MAX_VOLUME_PER_REQUEST)) + .flushIngestInterval(settings.getAsTime(FLUSH_INTERVAL, DEFAULT_FLUSH_INTERVAL)) + .init(settings, metric, control); + } + + public MockTransportClient toMockTransportClient() { + Settings settings = settingsBuilder.build(); + return new MockTransportClient() + .maxActionsPerRequest(settings.getAsInt(MAX_ACTIONS_PER_REQUEST, DEFAULT_MAX_ACTIONS_PER_REQUEST)) + .maxConcurrentRequests(settings.getAsInt(MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_REQUESTS)) + .maxVolumePerRequest(settings.getAsBytesSize(MAX_VOLUME_PER_REQUEST, DEFAULT_MAX_VOLUME_PER_REQUEST)) + .flushIngestInterval(settings.getAsTime(FLUSH_INTERVAL, DEFAULT_FLUSH_INTERVAL)) + .init(settings, metric, control); + } + +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/ClientMethods.java b/src/main/java/org/xbib/elasticsearch/extras/client/ClientMethods.java new file mode 100644 index 0000000..c643924 --- /dev/null +++ b/src/main/java/org/xbib/elasticsearch/extras/client/ClientMethods.java @@ -0,0 +1,391 @@ +package org.xbib.elasticsearch.extras.client; + +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +/** + * Interface for providing convenient administrative methods for ingesting data into Elasticsearch. + */ +public interface ClientMethods extends Parameters { + + /** + * Initialize new ingest client, wrap an existing Elasticsearch client, and set up metrics. + * + * @param client the Elasticsearch client + * @param metric metric + * @param control control + * @return this ingest + * @throws IOException if client could not get created + */ + ClientMethods init(ElasticsearchClient client, BulkMetric metric, BulkControl control) throws IOException; + + /** + * Initialize, create new ingest client, and set up metrics. + * + * @param settings settings + * @param metric metric + * @param control control + * @return this ingest + * @throws IOException if client could not get created + */ + ClientMethods init(Settings settings, BulkMetric metric, BulkControl control) throws IOException; + + /** + * Return Elasticsearch client. + * + * @return Elasticsearch client + */ + ElasticsearchClient client(); + + /** + * Index document. + * + * @param index the index + * @param type the type + * @param id the id + * @param source the source + * @return this + */ + ClientMethods index(String index, String type, String id, String source); + + /** + * Delete document. + * + * @param index the index + * @param type the type + * @param id the id + * @return this ingest + */ + ClientMethods delete(String index, String type, String id); + + /** + * Update document. Use with precaution! Does not work in all cases. + * + * @param index the index + * @param type the type + * @param id the id + * @param source the source + * @return this + */ + ClientMethods update(String index, String type, String id, String source); + + /** + * Set the maximum number of actions per request. + * + * @param maxActionsPerRequest maximum number of actions per request + * @return this ingest + */ + ClientMethods maxActionsPerRequest(int maxActionsPerRequest); + + /** + * Set the maximum concurent requests. + * + * @param maxConcurentRequests maximum number of concurrent ingest requests + * @return this Ingest + */ + ClientMethods maxConcurrentRequests(int maxConcurentRequests); + + /** + * Set the maximum volume for request before flush. + * + * @param maxVolume maximum volume + * @return this ingest + */ + ClientMethods maxVolumePerRequest(ByteSizeValue maxVolume); + + /** + * Set the flush interval for automatic flushing outstanding ingest requests. + * + * @param flushInterval the flush interval, default is 30 seconds + * @return this ingest + */ + ClientMethods flushIngestInterval(TimeValue flushInterval); + + /** + * Set mapping. + * + * @param type mapping type + * @param in mapping definition as input stream + * @throws IOException if mapping could not be added + */ + void mapping(String type, InputStream in) throws IOException; + + /** + * Set mapping. + * + * @param type mapping type + * @param mapping mapping definition as input stream + * @throws IOException if mapping could not be added + */ + void mapping(String type, String mapping) throws IOException; + + /** + * Put mapping. + * + * @param index index + */ + void putMapping(String index); + + /** + * Create a new index. + * + * @param index index + * @return this ingest + */ + ClientMethods newIndex(String index); + + /** + * Create a new index. + * + * @param index index + * @param type type + * @param settings settings + * @param mappings mappings + * @return this ingest + * @throws IOException if new index creation fails + */ + ClientMethods newIndex(String index, String type, InputStream settings, InputStream mappings) throws IOException; + + /** + * Create a new index. + * + * @param index index + * @param settings settings + * @param mappings mappings + * @return this ingest + */ + ClientMethods newIndex(String index, Settings settings, Map mappings); + + /** + * Create new mapping. + * + * @param index index + * @param type index type + * @param mapping mapping + * @return this ingest + */ + ClientMethods newMapping(String index, String type, Map mapping); + + /** + * Delete index. + * + * @param index index + * @return this ingest + */ + ClientMethods deleteIndex(String index); + + /** + * Start bulk mode. + * + * @param index index + * @param startRefreshIntervalSeconds refresh interval before bulk + * @param stopRefreshIntervalSeconds refresh interval after bulk + * @return this ingest + * @throws IOException if bulk could not be started + */ + ClientMethods startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds) throws IOException; + + /** + * Stops bulk mode. + * + * @param index index + * @return this Ingest + * @throws IOException if bulk could not be stopped + */ + ClientMethods stopBulk(String index) throws IOException; + + /** + * Bulked index request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * + * @param indexRequest the index request to add + * @return this ingest + */ + ClientMethods bulkIndex(IndexRequest indexRequest); + + /** + * Bulked delete request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * + * @param deleteRequest the delete request to add + * @return this ingest + */ + ClientMethods bulkDelete(DeleteRequest deleteRequest); + + /** + * Bulked update request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * Note that updates only work correctly when all operations between nodes are synchronized! + * + * @param updateRequest the update request to add + * @return this ingest + */ + ClientMethods bulkUpdate(UpdateRequest updateRequest); + + /** + * Flush ingest, move all pending documents to the cluster. + * + * @return this + */ + ClientMethods flushIngest(); + + /** + * Wait for all outstanding responses. + * + * @param maxWait maximum wait time + * @return this ingest + * @throws InterruptedException if wait is interrupted + * @throws ExecutionException if execution failed + */ + ClientMethods waitForResponses(TimeValue maxWait) throws InterruptedException, ExecutionException; + + /** + * Refresh the index. + * + * @param index index + */ + void refreshIndex(String index); + + /** + * Flush the index. + * + * @param index index + */ + void flushIndex(String index); + + /** + * Update replica level. + * + * @param index index + * @param level the replica level + * @return number of shards after updating replica level + * @throws IOException if replica could not be updated + */ + int updateReplicaLevel(String index, int level) throws IOException; + + /** + * Wait for cluster being healthy. + * + * @param healthColor cluster health color to wait for + * @param timeValue time value + * @throws IOException if wait failed + */ + void waitForCluster(String healthColor, TimeValue timeValue) throws IOException; + + /** + * Get current health color. + * + * @return the cluster health color + */ + String healthColor(); + + /** + * Wait for index recovery (after replica change). + * + * @param index index + * @return number of shards found + * @throws IOException if wait failed + */ + int waitForRecovery(String index) throws IOException; + + /** + * Resolve alias. + * + * @param alias the alias + * @return one index name behind the alias or the alias if there is no index + */ + String resolveAlias(String alias); + + /** + * Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index + * name. + * + * @param alias the alias + * @return the most recent index name pointing to the alias + */ + String resolveMostRecentIndex(String alias); + + /** + * Get all alias filters. + * + * @param index index + * @return map of alias filters + */ + Map getAliasFilters(String index); + + /** + * Switch aliases from one index to another. + * + * @param index the index name + * @param concreteIndex the index name with timestamp + * @param extraAliases a list of names that should be set as index aliases + */ + void switchAliases(String index, String concreteIndex, List extraAliases); + + /** + * Switch aliases from one index to another. + * + * @param index the index name + * @param concreteIndex the index name with timestamp + * @param extraAliases a list of names that should be set as index aliases + * @param adder an adder method to create alias term queries + */ + void switchAliases(String index, String concreteIndex, List extraAliases, IndexAliasAdder adder); + + /** + * Retention policy for an index. All indices before timestampdiff should be deleted, + * but mintokeep indices must be kept. + * + * @param index index name + * @param concreteIndex index name with timestamp + * @param timestampdiff timestamp delta (for index timestamps) + * @param mintokeep minimum number of indices to keep + */ + void performRetentionPolicy(String index, String concreteIndex, int timestampdiff, int mintokeep); + + /** + * Log the timestamp of the most recently indexed document in the index. + * + * @param index the index name + * @return millis UTC millis of the most recent document + * @throws IOException if most rcent document can not be found + */ + Long mostRecentDocument(String index) throws IOException; + + /** + * Get metric. + * + * @return metric + */ + BulkMetric getMetric(); + + /** + * Returns true is a throwable exists. + * + * @return true if a Throwable exists + */ + boolean hasThrowable(); + + /** + * Return last throwable if exists. + * + * @return last throwable + */ + Throwable getThrowable(); + + /** + * Shutdown the ingesting. + */ + void shutdown(); +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/IndexAliasAdder.java b/src/main/java/org/xbib/elasticsearch/extras/client/IndexAliasAdder.java new file mode 100644 index 0000000..8ce2df5 --- /dev/null +++ b/src/main/java/org/xbib/elasticsearch/extras/client/IndexAliasAdder.java @@ -0,0 +1,11 @@ +package org.xbib.elasticsearch.extras.client; + +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; + +/** + * + */ +public interface IndexAliasAdder { + + void addIndexAlias(IndicesAliasesRequestBuilder builder, String index, String alias); +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/NetworkUtils.java b/src/main/java/org/xbib/elasticsearch/extras/client/NetworkUtils.java new file mode 100644 index 0000000..4dd69da --- /dev/null +++ b/src/main/java/org/xbib/elasticsearch/extras/client/NetworkUtils.java @@ -0,0 +1,264 @@ +package org.xbib.elasticsearch.extras.client; + +import java.io.IOException; +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Enumeration; +import java.util.List; +import java.util.Locale; + +/** + * + */ +public class NetworkUtils { + + private static final String IPv4_SETTING = "java.net.preferIPv4Stack"; + + private static final String IPv6_SETTING = "java.net.preferIPv6Addresses"; + + private static final InetAddress localAddress; + + static { + InetAddress address; + try { + address = InetAddress.getLocalHost(); + } catch (Throwable e) { + address = InetAddress.getLoopbackAddress(); + } + localAddress = address; + } + + private NetworkUtils() { + } + + public static InetAddress getLocalAddress() { + return localAddress; + } + + public static InetAddress getFirstNonLoopbackAddress(ProtocolVersion ipversion) throws SocketException { + InetAddress address; + for (NetworkInterface networkInterface : getNetworkInterfaces()) { + try { + if (!networkInterface.isUp() || networkInterface.isLoopback()) { + continue; + } + } catch (Exception e) { + continue; + } + address = getFirstNonLoopbackAddress(networkInterface, ipversion); + if (address != null) { + return address; + } + } + return null; + } + + public static InetAddress getFirstNonLoopbackAddress(NetworkInterface networkInterface, ProtocolVersion ipVersion) + throws SocketException { + if (networkInterface == null) { + throw new IllegalArgumentException("network interface is null"); + } + for (Enumeration addresses = networkInterface.getInetAddresses(); addresses.hasMoreElements(); ) { + InetAddress address = addresses.nextElement(); + if (!address.isLoopbackAddress()) { + if ((address instanceof Inet4Address && ipVersion == ProtocolVersion.IPv4) || + (address instanceof Inet6Address && ipVersion == ProtocolVersion.IPv6)) { + return address; + } + } + } + return null; + } + + public static InetAddress getFirstAddress(NetworkInterface networkInterface, ProtocolVersion ipVersion) + throws SocketException { + if (networkInterface == null) { + throw new IllegalArgumentException("network interface is null"); + } + for (Enumeration addresses = networkInterface.getInetAddresses(); addresses.hasMoreElements(); ) { + InetAddress address = addresses.nextElement(); + if ((address instanceof Inet4Address && ipVersion == ProtocolVersion.IPv4) || + (address instanceof Inet6Address && ipVersion == ProtocolVersion.IPv6)) { + return address; + } + } + return null; + } + + public static List getAllAvailableInterfaces() throws SocketException { + List allInterfaces = new ArrayList<>(); + for (Enumeration interfaces = NetworkInterface.getNetworkInterfaces(); + interfaces.hasMoreElements(); ) { + NetworkInterface networkInterface = interfaces.nextElement(); + allInterfaces.add(networkInterface); + Enumeration subInterfaces = networkInterface.getSubInterfaces(); + if (subInterfaces.hasMoreElements()) { + while (subInterfaces.hasMoreElements()) { + allInterfaces.add(subInterfaces.nextElement()); + } + } + } + sortInterfaces(allInterfaces); + return allInterfaces; + } + + public static List getAllAvailableAddresses() throws SocketException { + List allAddresses = new ArrayList<>(); + for (NetworkInterface networkInterface : getNetworkInterfaces()) { + Enumeration addrs = networkInterface.getInetAddresses(); + while (addrs.hasMoreElements()) { + allAddresses.add(addrs.nextElement()); + } + } + sortAddresses(allAddresses); + return allAddresses; + } + + public static ProtocolVersion getProtocolVersion() throws SocketException { + switch (findAvailableProtocols()) { + case IPv4: + return ProtocolVersion.IPv4; + case IPv6: + return ProtocolVersion.IPv6; + case IPv46: + if (Boolean.getBoolean(System.getProperty(IPv4_SETTING))) { + return ProtocolVersion.IPv4; + } + if (Boolean.getBoolean(System.getProperty(IPv6_SETTING))) { + return ProtocolVersion.IPv6; + } + return ProtocolVersion.IPv6; + } + return ProtocolVersion.NONE; + } + + public static ProtocolVersion findAvailableProtocols() throws SocketException { + boolean hasIPv4 = false; + boolean hasIPv6 = false; + for (InetAddress addr : getAllAvailableAddresses()) { + if (addr instanceof Inet4Address) { + hasIPv4 = true; + } + if (addr instanceof Inet6Address) { + hasIPv6 = true; + } + } + if (hasIPv4 && hasIPv6) { + return ProtocolVersion.IPv46; + } + if (hasIPv4) { + return ProtocolVersion.IPv4; + } + if (hasIPv6) { + return ProtocolVersion.IPv6; + } + return ProtocolVersion.NONE; + } + + public static InetAddress resolveInetAddress(String host, String defaultValue) throws IOException { + if (host == null) { + host = defaultValue; + } + String origHost = host; + int pos = host.indexOf(':'); + if (pos > 0) { + host = host.substring(0, pos - 1); + } + if ((host.startsWith("#") && host.endsWith("#")) || (host.startsWith("_") && host.endsWith("_"))) { + host = host.substring(1, host.length() - 1); + if (host.equals("local")) { + return getLocalAddress(); + } else if (host.startsWith("non_loopback")) { + if (host.toLowerCase(Locale.ROOT).endsWith(":ipv4")) { + return getFirstNonLoopbackAddress(ProtocolVersion.IPv4); + } else if (host.toLowerCase(Locale.ROOT).endsWith(":ipv6")) { + return getFirstNonLoopbackAddress(ProtocolVersion.IPv6); + } else { + return getFirstNonLoopbackAddress(getProtocolVersion()); + } + } else { + ProtocolVersion protocolVersion = getProtocolVersion(); + if (host.toLowerCase(Locale.ROOT).endsWith(":ipv4")) { + protocolVersion = ProtocolVersion.IPv4; + host = host.substring(0, host.length() - 5); + } else if (host.toLowerCase(Locale.ROOT).endsWith(":ipv6")) { + protocolVersion = ProtocolVersion.IPv6; + host = host.substring(0, host.length() - 5); + } + for (NetworkInterface ni : getAllAvailableInterfaces()) { + if (!ni.isUp()) { + continue; + } + if (host.equals(ni.getName()) || host.equals(ni.getDisplayName())) { + if (ni.isLoopback()) { + return getFirstAddress(ni, protocolVersion); + } else { + return getFirstNonLoopbackAddress(ni, protocolVersion); + } + } + } + } + throw new IOException("failed to find network interface for [" + origHost + "]"); + } + return InetAddress.getByName(host); + } + + private static List getNetworkInterfaces() throws SocketException { + List networkInterfaces = new ArrayList<>(); + Enumeration interfaces = NetworkInterface.getNetworkInterfaces(); + while (interfaces.hasMoreElements()) { + NetworkInterface networkInterface = interfaces.nextElement(); + networkInterfaces.add(networkInterface); + Enumeration subInterfaces = networkInterface.getSubInterfaces(); + if (subInterfaces.hasMoreElements()) { + while (subInterfaces.hasMoreElements()) { + networkInterfaces.add(subInterfaces.nextElement()); + } + } + } + sortInterfaces(networkInterfaces); + return networkInterfaces; + } + + private static void sortInterfaces(List interfaces) { + Collections.sort(interfaces, new Comparator() { + @Override + public int compare(NetworkInterface o1, NetworkInterface o2) { + return Integer.compare(o1.getIndex(), o2.getIndex()); + } + }); + } + + private static void sortAddresses(List addressList) { + Collections.sort(addressList, new Comparator() { + @Override + public int compare(InetAddress o1, InetAddress o2) { + return compareBytes(o1.getAddress(), o2.getAddress()); + } + }); + } + + private static int compareBytes(byte[] left, byte[] right) { + for (int i = 0, j = 0; i < left.length && j < right.length; i++, j++) { + int a = (left[i] & 0xff); + int b = (right[j] & 0xff); + if (a != b) { + return a - b; + } + } + return left.length - right.length; + } + + /** + * + */ + public enum ProtocolVersion { + IPv4, IPv6, IPv46, NONE + } +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/Parameters.java b/src/main/java/org/xbib/elasticsearch/extras/client/Parameters.java new file mode 100644 index 0000000..41cc6d2 --- /dev/null +++ b/src/main/java/org/xbib/elasticsearch/extras/client/Parameters.java @@ -0,0 +1,28 @@ +package org.xbib.elasticsearch.extras.client; + +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; + +/** + * + */ +public interface Parameters { + + int DEFAULT_MAX_ACTIONS_PER_REQUEST = 1000; + + int DEFAULT_MAX_CONCURRENT_REQUESTS = Runtime.getRuntime().availableProcessors() * 4; + + ByteSizeValue DEFAULT_MAX_VOLUME_PER_REQUEST = new ByteSizeValue(10, ByteSizeUnit.MB); + + TimeValue DEFAULT_FLUSH_INTERVAL = TimeValue.timeValueSeconds(30); + + String MAX_ACTIONS_PER_REQUEST = "max_actions_per_request"; + + String MAX_CONCURRENT_REQUESTS = "max_concurrent_requests"; + + String MAX_VOLUME_PER_REQUEST = "max_volume_per_request"; + + String FLUSH_INTERVAL = "flush_interval"; + +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkControl.java b/src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkControl.java new file mode 100644 index 0000000..b9a92d6 --- /dev/null +++ b/src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkControl.java @@ -0,0 +1,54 @@ +package org.xbib.elasticsearch.extras.client; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + */ +public class SimpleBulkControl implements BulkControl { + + private final Set indexNames = new HashSet<>(); + + private final Map startBulkRefreshIntervals = new HashMap<>(); + + private final Map stopBulkRefreshIntervals = new HashMap<>(); + + @Override + public void startBulk(String indexName, long startRefreshInterval, long stopRefreshInterval) { + synchronized (indexNames) { + indexNames.add(indexName); + startBulkRefreshIntervals.put(indexName, startRefreshInterval); + stopBulkRefreshIntervals.put(indexName, stopRefreshInterval); + } + } + + @Override + public boolean isBulk(String indexName) { + return indexNames.contains(indexName); + } + + @Override + public void finishBulk(String indexName) { + synchronized (indexNames) { + indexNames.remove(indexName); + } + } + + @Override + public Set indices() { + return indexNames; + } + + @Override + public Map getStartBulkRefreshIntervals() { + return startBulkRefreshIntervals; + } + + @Override + public Map getStopBulkRefreshIntervals() { + return stopBulkRefreshIntervals; + } + +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkMetric.java b/src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkMetric.java new file mode 100644 index 0000000..bfbde5a --- /dev/null +++ b/src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkMetric.java @@ -0,0 +1,82 @@ +package org.xbib.elasticsearch.extras.client; + +import org.xbib.metrics.Count; +import org.xbib.metrics.CountMetric; +import org.xbib.metrics.Meter; +import org.xbib.metrics.Metered; +/** + * + */ +public class SimpleBulkMetric implements BulkMetric { + + private final Meter totalIngest = new Meter(); + + private final Count totalIngestSizeInBytes = new CountMetric(); + + private final Count currentIngest = new CountMetric(); + + private final Count currentIngestNumDocs = new CountMetric(); + + private final Count submitted = new CountMetric(); + + private final Count succeeded = new CountMetric(); + + private final Count failed = new CountMetric(); + + private Long started; + + private Long stopped; + + @Override + public Metered getTotalIngest() { + return totalIngest; + } + + @Override + public Count getTotalIngestSizeInBytes() { + return totalIngestSizeInBytes; + } + + @Override + public Count getCurrentIngest() { + return currentIngest; + } + + @Override + public Count getCurrentIngestNumDocs() { + return currentIngestNumDocs; + } + + @Override + public Count getSubmitted() { + return submitted; + } + + @Override + public Count getSucceeded() { + return succeeded; + } + + @Override + public Count getFailed() { + return failed; + } + + @Override + public void start() { + this.started = System.nanoTime(); + this.totalIngest.spawn(5L); + } + + @Override + public void stop() { + this.stopped = System.nanoTime(); + totalIngest.stop(); + } + + @Override + public long elapsed() { + return (stopped != null ? stopped : System.nanoTime()) - started; + } + +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClient.java b/src/main/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClient.java new file mode 100644 index 0000000..5a0df14 --- /dev/null +++ b/src/main/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClient.java @@ -0,0 +1,502 @@ +package org.xbib.elasticsearch.extras.client.node; + +import com.google.common.collect.ImmutableSet; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.create.CreateIndexAction; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkProcessor; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.env.Environment; +import org.elasticsearch.node.Node; +import org.elasticsearch.plugins.Plugin; +import org.xbib.elasticsearch.extras.client.AbstractClient; +import org.xbib.elasticsearch.extras.client.BulkControl; +import org.xbib.elasticsearch.extras.client.BulkMetric; +import org.xbib.elasticsearch.extras.client.ClientMethods; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +/** + * + */ +public class BulkNodeClient extends AbstractClient implements ClientMethods { + + private static final ESLogger logger = ESLoggerFactory.getLogger(BulkNodeClient.class.getName()); + + private int maxActionsPerRequest = DEFAULT_MAX_ACTIONS_PER_REQUEST; + + private int maxConcurrentRequests = DEFAULT_MAX_CONCURRENT_REQUESTS; + + private ByteSizeValue maxVolume = DEFAULT_MAX_VOLUME_PER_REQUEST; + + private TimeValue flushInterval = DEFAULT_FLUSH_INTERVAL; + + private ElasticsearchClient client; + + private BulkProcessor bulkProcessor; + + private BulkMetric metric; + + private BulkControl control; + + private Throwable throwable; + + private boolean closed; + + public BulkNodeClient() { + } + + @Override + public BulkNodeClient maxActionsPerRequest(int maxActionsPerRequest) { + this.maxActionsPerRequest = maxActionsPerRequest; + return this; + } + + @Override + public BulkNodeClient maxConcurrentRequests(int maxConcurrentRequests) { + this.maxConcurrentRequests = maxConcurrentRequests; + return this; + } + + @Override + public BulkNodeClient maxVolumePerRequest(ByteSizeValue maxVolume) { + this.maxVolume = maxVolume; + return this; + } + + @Override + public BulkNodeClient flushIngestInterval(TimeValue flushInterval) { + this.flushInterval = flushInterval; + return this; + } + + @Override + public BulkNodeClient init(ElasticsearchClient client, + final BulkMetric metric, final BulkControl control) { + this.client = client; + this.metric = metric; + this.control = control; + if (metric != null) { + metric.start(); + } + BulkProcessor.Listener listener = new BulkProcessor.Listener() { + @Override + public void beforeBulk(long executionId, BulkRequest request) { + long l = -1; + if (metric != null) { + metric.getCurrentIngest().inc(); + l = metric.getCurrentIngest().getCount(); + int n = request.numberOfActions(); + metric.getSubmitted().inc(n); + metric.getCurrentIngestNumDocs().inc(n); + metric.getTotalIngestSizeInBytes().inc(request.estimatedSizeInBytes()); + } + logger.debug("before bulk [{}] [actions={}] [bytes={}] [concurrent requests={}]", + executionId, + request.numberOfActions(), + request.estimatedSizeInBytes(), + l); + } + + @Override + public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { + long l = -1; + if (metric != null) { + metric.getCurrentIngest().dec(); + l = metric.getCurrentIngest().getCount(); + metric.getSucceeded().inc(response.getItems().length); + } + int n = 0; + for (BulkItemResponse itemResponse : response.getItems()) { + if (metric != null) { + metric.getCurrentIngest().dec(itemResponse.getIndex(), itemResponse.getType(), itemResponse.getId()); + } + if (itemResponse.isFailed()) { + n++; + if (metric != null) { + metric.getSucceeded().dec(1); + metric.getFailed().inc(1); + } + } + } + if (metric != null) { + logger.debug("after bulk [{}] [succeeded={}] [failed={}] [{}ms] {} concurrent requests", + executionId, + metric.getSucceeded().getCount(), + metric.getFailed().getCount(), + response.getTook().millis(), + l); + } + if (n > 0) { + logger.error("bulk [{}] failed with {} failed items, failure message = {}", + executionId, n, response.buildFailureMessage()); + } else { + if (metric != null) { + metric.getCurrentIngestNumDocs().dec(response.getItems().length); + } + } + } + + @Override + public void afterBulk(long executionId, BulkRequest request, Throwable failure) { + if (metric != null) { + metric.getCurrentIngest().dec(); + } + throwable = failure; + closed = true; + logger.error("after bulk [" + executionId + "] error", failure); + } + }; + BulkProcessor.Builder builder = BulkProcessor.builder((Client) client, listener) + .setBulkActions(maxActionsPerRequest) + .setConcurrentRequests(maxConcurrentRequests) + .setFlushInterval(flushInterval); + if (maxVolume != null) { + builder.setBulkSize(maxVolume); + } + this.bulkProcessor = builder.build(); + this.closed = false; + return this; + } + + @Override + public BulkNodeClient init(Settings settings, BulkMetric metric, BulkControl control) throws IOException { + createClient(settings); + this.metric = metric; + this.control = control; + return this; + } + + @Override + public ElasticsearchClient client() { + return client; + } + + @Override + protected void createClient(Settings settings) throws IOException { + if (client != null) { + logger.warn("client is open, closing..."); + client.threadPool().shutdown(); + logger.warn("client is closed"); + client = null; + } + if (settings != null) { + String version = System.getProperty("os.name") + + " " + System.getProperty("java.vm.name") + + " " + System.getProperty("java.vm.vendor") + + " " + System.getProperty("java.runtime.version") + + " " + System.getProperty("java.vm.version"); + Settings effectiveSettings = Settings.builder().put(settings) + .put("node.client", true) + .put("node.master", false) + .put("node.data", false).build(); + logger.info("creating node client on {} with effective settings {}", + version, effectiveSettings.getAsMap()); + Collection> plugins = Collections.emptyList(); + Node node = new BulkNode(new Environment(effectiveSettings), plugins); + node.start(); + this.client = node.client(); + } + } + + @Override + public BulkMetric getMetric() { + return metric; + } + + @Override + public BulkNodeClient index(String index, String type, String id, String source) { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + try { + if (metric != null) { + metric.getCurrentIngest().inc(index, type, id); + } + bulkProcessor.add(new IndexRequest(index).type(type).id(id).create(false).source(source)); + } catch (Exception e) { + throwable = e; + closed = true; + logger.error("bulk add of index request failed: " + e.getMessage(), e); + } + return this; + } + + @Override + public BulkNodeClient bulkIndex(IndexRequest indexRequest) { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + try { + if (metric != null) { + metric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id()); + } + bulkProcessor.add(indexRequest); + } catch (Exception e) { + throwable = e; + closed = true; + logger.error("bulk add of index request failed: " + e.getMessage(), e); + } + return this; + } + + @Override + public BulkNodeClient delete(String index, String type, String id) { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + try { + if (metric != null) { + metric.getCurrentIngest().inc(index, type, id); + } + bulkProcessor.add(new DeleteRequest(index).type(type).id(id)); + } catch (Exception e) { + throwable = e; + closed = true; + logger.error("bulk add of delete failed: " + e.getMessage(), e); + } + return this; + } + + @Override + public BulkNodeClient bulkDelete(DeleteRequest deleteRequest) { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + try { + if (metric != null) { + metric.getCurrentIngest().inc(deleteRequest.index(), deleteRequest.type(), deleteRequest.id()); + } + bulkProcessor.add(deleteRequest); + } catch (Exception e) { + throwable = e; + closed = true; + logger.error("bulk add of delete failed: " + e.getMessage(), e); + } + return this; + } + + @Override + public BulkNodeClient update(String index, String type, String id, String source) { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + try { + if (metric != null) { + metric.getCurrentIngest().inc(index, type, id); + } + bulkProcessor.add(new UpdateRequest().index(index).type(type).id(id).upsert(source)); + } catch (Exception e) { + throwable = e; + closed = true; + logger.error("bulk add of update request failed: " + e.getMessage(), e); + } + return this; + } + + @Override + public BulkNodeClient bulkUpdate(UpdateRequest updateRequest) { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + try { + if (metric != null) { + metric.getCurrentIngest().inc(updateRequest.index(), updateRequest.type(), updateRequest.id()); + } + bulkProcessor.add(updateRequest); + } catch (Exception e) { + throwable = e; + closed = true; + logger.error("bulk add of update request failed: " + e.getMessage(), e); + } + return this; + } + + @Override + public BulkNodeClient flushIngest() { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + logger.debug("flushing bulk processor"); + bulkProcessor.flush(); + return this; + } + + @Override + public BulkNodeClient waitForResponses(TimeValue maxWaitTime) throws InterruptedException, ExecutionException { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + while (!bulkProcessor.awaitClose(maxWaitTime.getMillis(), TimeUnit.MILLISECONDS)) { + logger.warn("still waiting for responses"); + } + return this; + } + + @Override + public BulkNodeClient startBulk(String index, long startRefreshIntervalMillis, long stopRefreshItervalMillis) + throws IOException { + if (control == null) { + return this; + } + if (!control.isBulk(index)) { + control.startBulk(index, startRefreshIntervalMillis, stopRefreshItervalMillis); + updateIndexSetting(index, "refresh_interval", startRefreshIntervalMillis + "ms"); + } + return this; + } + + @Override + public BulkNodeClient stopBulk(String index) throws IOException { + if (control == null) { + return this; + } + if (control.isBulk(index)) { + updateIndexSetting(index, "refresh_interval", control.getStopBulkRefreshIntervals().get(index) + "ms"); + control.finishBulk(index); + } + return this; + } + + @Override + public synchronized void shutdown() { + try { + if (bulkProcessor != null) { + logger.debug("closing bulk processor..."); + bulkProcessor.close(); + } + if (control != null && control.indices() != null && !control.indices().isEmpty()) { + logger.debug("stopping bulk mode for indices {}...", control.indices()); + for (String index : ImmutableSet.copyOf(control.indices())) { + stopBulk(index); + } + metric.stop(); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + @Override + public BulkNodeClient newIndex(String index) { + return newIndex(index, null, null); + } + + @Override + public BulkNodeClient newIndex(String index, String type, InputStream settings, InputStream mappings) throws IOException { + resetSettings(); + setting(settings); + mapping(type, mappings); + return newIndex(index, settings(), mappings()); + } + + @Override + public BulkNodeClient newIndex(String index, Settings settings, Map mappings) { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + if (client == null) { + logger.warn("no client for create index"); + return this; + } + if (index == null) { + logger.warn("no index name given to create index"); + return this; + } + CreateIndexRequestBuilder createIndexRequestBuilder = + new CreateIndexRequestBuilder(client(), CreateIndexAction.INSTANCE).setIndex(index); + if (settings != null) { + logger.info("settings = {}", settings.getAsStructuredMap()); + createIndexRequestBuilder.setSettings(settings); + } + if (mappings != null) { + for (String type : mappings.keySet()) { + logger.info("found mapping for {}", type); + createIndexRequestBuilder.addMapping(type, mappings.get(type)); + } + } + createIndexRequestBuilder.execute().actionGet(); + logger.info("index {} created", index); + return this; + } + + @Override + public BulkNodeClient newMapping(String index, String type, Map mapping) { + PutMappingRequestBuilder putMappingRequestBuilder = + new PutMappingRequestBuilder(client(), PutMappingAction.INSTANCE) + .setIndices(index) + .setType(type) + .setSource(mapping); + putMappingRequestBuilder.execute().actionGet(); + logger.info("mapping created for index {} and type {}", index, type); + return this; + } + + @Override + public BulkNodeClient deleteIndex(String index) { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + if (client == null) { + logger.warn("no client"); + return this; + } + if (index == null) { + logger.warn("no index name given to delete index"); + return this; + } + DeleteIndexRequestBuilder deleteIndexRequestBuilder = + new DeleteIndexRequestBuilder(client(), DeleteIndexAction.INSTANCE, index); + deleteIndexRequestBuilder.execute().actionGet(); + return this; + } + + @Override + public boolean hasThrowable() { + return throwable != null; + } + + @Override + public Throwable getThrowable() { + return throwable; + } + + public Settings getSettings() { + return settings(); + } + + public Settings.Builder getSettingsBuilder() { + return settingsBuilder(); + } + + private class BulkNode extends Node { + + BulkNode(Environment env, Collection> classpathPlugins) { + super(env, Version.CURRENT, classpathPlugins); + } + } + +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/node/package-info.java b/src/main/java/org/xbib/elasticsearch/extras/client/node/package-info.java new file mode 100644 index 0000000..c5c0895 --- /dev/null +++ b/src/main/java/org/xbib/elasticsearch/extras/client/node/package-info.java @@ -0,0 +1,4 @@ +/** + * Classes for Elasticsearch node client extras. + */ +package org.xbib.elasticsearch.extras.client.node; diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/package-info.java b/src/main/java/org/xbib/elasticsearch/extras/client/package-info.java new file mode 100644 index 0000000..c231c60 --- /dev/null +++ b/src/main/java/org/xbib/elasticsearch/extras/client/package-info.java @@ -0,0 +1,4 @@ +/** + * Classes for Elasticsearch client extras. + */ +package org.xbib.elasticsearch.extras.client; diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClient.java b/src/main/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClient.java new file mode 100644 index 0000000..ac2a00e --- /dev/null +++ b/src/main/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClient.java @@ -0,0 +1,581 @@ +package org.xbib.elasticsearch.extras.client.transport; + +import com.google.common.collect.ImmutableSet; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.admin.indices.create.CreateIndexAction; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.xbib.elasticsearch.extras.client.AbstractClient; +import org.xbib.elasticsearch.extras.client.BulkProcessor; +import org.xbib.elasticsearch.extras.client.BulkMetric; +import org.xbib.elasticsearch.extras.client.BulkControl; +import org.xbib.elasticsearch.extras.client.ClientMethods; +import org.xbib.elasticsearch.extras.client.NetworkUtils; + +import java.io.IOException; +import java.io.InputStream; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +/** + * Transport client with addtitional methods using the BulkProcessor. + */ +public class BulkTransportClient extends AbstractClient implements ClientMethods { + + private static final ESLogger logger = ESLoggerFactory.getLogger(BulkTransportClient.class.getName()); + + private int maxActionsPerRequest = DEFAULT_MAX_ACTIONS_PER_REQUEST; + + private int maxConcurrentRequests = DEFAULT_MAX_CONCURRENT_REQUESTS; + + private ByteSizeValue maxVolumePerRequest = DEFAULT_MAX_VOLUME_PER_REQUEST; + + private TimeValue flushInterval = DEFAULT_FLUSH_INTERVAL; + + private BulkProcessor bulkProcessor; + + private Throwable throwable; + + private boolean closed; + + private TransportClient client; + + private BulkMetric metric; + + private BulkControl control; + + private boolean ignoreBulkErrors; + + private boolean isShutdown; + + public BulkTransportClient() { + } + + @Override + public BulkTransportClient init(ElasticsearchClient client, BulkMetric metric, BulkControl control) throws IOException { + return init(findSettings(), metric, control); + } + + @Override + public BulkTransportClient init(Settings settings, final BulkMetric metric, final BulkControl control) { + createClient(settings); + this.metric = metric; + this.control = control; + if (metric != null) { + metric.start(); + } + resetSettings(); + BulkProcessor.Listener listener = new BulkProcessor.Listener() { + @Override + public void beforeBulk(long executionId, BulkRequest request) { + long l = -1L; + if (metric != null) { + metric.getCurrentIngest().inc(); + l = metric.getCurrentIngest().getCount(); + int n = request.numberOfActions(); + metric.getSubmitted().inc(n); + metric.getCurrentIngestNumDocs().inc(n); + metric.getTotalIngestSizeInBytes().inc(request.estimatedSizeInBytes()); + } + logger.debug("before bulk [{}] [actions={}] [bytes={}] [concurrent requests={}]", + executionId, + request.numberOfActions(), + request.estimatedSizeInBytes(), + l); + } + + @Override + public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { + long l = -1L; + if (metric != null) { + metric.getCurrentIngest().dec(); + l = metric.getCurrentIngest().getCount(); + metric.getSucceeded().inc(response.getItems().length); + } + int n = 0; + for (BulkItemResponse itemResponse : response.getItems()) { + if (metric != null) { + metric.getCurrentIngest().dec(itemResponse.getIndex(), itemResponse.getType(), itemResponse.getId()); + if (itemResponse.isFailed()) { + n++; + metric.getSucceeded().dec(1); + metric.getFailed().inc(1); + } + } + } + if (metric != null) { + logger.debug("after bulk [{}] [succeeded={}] [failed={}] [{}ms] [concurrent requests={}]", + executionId, + metric.getSucceeded().getCount(), + metric.getFailed().getCount(), + response.getTook().millis(), + l); + } + if (n > 0) { + logger.error("bulk [{}] failed with {} failed items, failure message = {}", + executionId, n, response.buildFailureMessage()); + } else { + if (metric != null) { + metric.getCurrentIngestNumDocs().dec(response.getItems().length); + } + } + } + + @Override + public void afterBulk(long executionId, BulkRequest requst, Throwable failure) { + if (metric != null) { + metric.getCurrentIngest().dec(); + } + throwable = failure; + if (!ignoreBulkErrors) { + closed = true; + } + logger.error("bulk [" + executionId + "] error", failure); + } + }; + BulkProcessor.Builder builder = BulkProcessor.builder(client, listener) + .setBulkActions(maxActionsPerRequest) + .setConcurrentRequests(maxConcurrentRequests) + .setFlushInterval(flushInterval); + if (maxVolumePerRequest != null) { + builder.setBulkSize(maxVolumePerRequest); + } + this.bulkProcessor = builder.build(); + try { + Collection addrs = findAddresses(settings); + if (!connect(addrs, settings.getAsBoolean("autodiscover", false))) { + throw new NoNodeAvailableException("no cluster nodes available, check settings " + + settings.getAsMap()); + } + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + this.closed = false; + return this; + } + + @Override + public ClientMethods newMapping(String index, String type, Map mapping) { + new PutMappingRequestBuilder(client(), PutMappingAction.INSTANCE) + .setIndices(index) + .setType(type) + .setSource(mapping) + .execute().actionGet(); + logger.info("mapping created for index {} and type {}", index, type); + return this; + } + + @Override + protected void createClient(Settings settings) { + if (client != null) { + logger.warn("client is open, closing..."); + client.close(); + client.threadPool().shutdown(); + logger.warn("client is closed"); + client = null; + } + if (settings != null) { + String version = System.getProperty("os.name") + + " " + System.getProperty("java.vm.name") + + " " + System.getProperty("java.vm.vendor") + + " " + System.getProperty("java.runtime.version") + + " " + System.getProperty("java.vm.version"); + logger.info("creating transport client on {} with effective settings {}", + version, settings.getAsMap()); + this.client = TransportClient.builder() + .settings(settings) + .build(); + this.ignoreBulkErrors = settings.getAsBoolean("ignoreBulkErrors", true); + } + } + + public boolean isShutdown() { + return isShutdown; + } + + @Override + public BulkTransportClient maxActionsPerRequest(int maxActionsPerRequest) { + this.maxActionsPerRequest = maxActionsPerRequest; + return this; + } + + @Override + public BulkTransportClient maxConcurrentRequests(int maxConcurrentRequests) { + this.maxConcurrentRequests = maxConcurrentRequests; + return this; + } + + @Override + public BulkTransportClient maxVolumePerRequest(ByteSizeValue maxVolumePerRequest) { + this.maxVolumePerRequest = maxVolumePerRequest; + return this; + } + + @Override + public BulkTransportClient flushIngestInterval(TimeValue flushInterval) { + this.flushInterval = flushInterval; + return this; + } + + @Override + public ElasticsearchClient client() { + return client; + } + + @Override + public BulkMetric getMetric() { + return metric; + } + + @Override + public ClientMethods newIndex(String index) { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + return newIndex(index, null, null); + } + + @Override + public ClientMethods newIndex(String index, String type, InputStream settings, InputStream mappings) throws IOException { + resetSettings(); + setting(settings); + mapping(type, mappings); + return newIndex(index, settings(), mappings()); + } + + @Override + public ClientMethods newIndex(String index, Settings settings, Map mappings) { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + if (client == null) { + logger.warn("no client for create index"); + return this; + } + if (index == null) { + logger.warn("no index name given to create index"); + return this; + } + CreateIndexRequestBuilder createIndexRequestBuilder = + new CreateIndexRequestBuilder(client(), CreateIndexAction.INSTANCE).setIndex(index); + if (settings != null) { + logger.info("settings = {}", settings.getAsStructuredMap()); + createIndexRequestBuilder.setSettings(settings); + } + if (mappings != null) { + for (String type : mappings.keySet()) { + logger.info("found mapping for {}", type); + createIndexRequestBuilder.addMapping(type, mappings.get(type)); + } + } + createIndexRequestBuilder.execute().actionGet(); + logger.info("index {} created", index); + return this; + } + + @Override + public ClientMethods deleteIndex(String index) { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + if (client == null) { + logger.warn("no client for delete index"); + return this; + } + if (index == null) { + logger.warn("no index name given to delete index"); + return this; + } + new DeleteIndexRequestBuilder(client(), DeleteIndexAction.INSTANCE, index).execute().actionGet(); + return this; + } + + @Override + public ClientMethods startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds) + throws IOException { + if (control == null) { + return this; + } + if (!control.isBulk(index)) { + control.startBulk(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds); + updateIndexSetting(index, "refresh_interval", startRefreshIntervalSeconds + "s"); + } + return this; + } + + @Override + public ClientMethods stopBulk(String index) throws IOException { + if (control == null) { + return this; + } + if (control.isBulk(index)) { + updateIndexSetting(index, "refresh_interval", control.getStopBulkRefreshIntervals().get(index) + "s"); + control.finishBulk(index); + } + return this; + } + + @Override + public BulkTransportClient index(String index, String type, String id, String source) { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + try { + metric.getCurrentIngest().inc(index, type, id); + bulkProcessor.add(new IndexRequest().index(index).type(type).id(id).create(false).source(source)); + } catch (Exception e) { + throwable = e; + closed = true; + logger.error("bulk add of index request failed: " + e.getMessage(), e); + } + return this; + } + + @Override + public BulkTransportClient bulkIndex(IndexRequest indexRequest) { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + try { + metric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id()); + bulkProcessor.add(indexRequest); + } catch (Exception e) { + throwable = e; + closed = true; + logger.error("bulk add of index request failed: " + e.getMessage(), e); + } + return this; + } + + @Override + public BulkTransportClient delete(String index, String type, String id) { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + try { + metric.getCurrentIngest().inc(index, type, id); + bulkProcessor.add(new DeleteRequest().index(index).type(type).id(id)); + } catch (Exception e) { + throwable = e; + closed = true; + logger.error("bulk add of delete request failed: " + e.getMessage(), e); + } + return this; + } + + @Override + public BulkTransportClient bulkDelete(DeleteRequest deleteRequest) { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + try { + metric.getCurrentIngest().inc(deleteRequest.index(), deleteRequest.type(), deleteRequest.id()); + bulkProcessor.add(deleteRequest); + } catch (Exception e) { + throwable = e; + closed = true; + logger.error("bulk add of delete request failed: " + e.getMessage(), e); + } + return this; + } + + @Override + public BulkTransportClient update(String index, String type, String id, String source) { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + try { + metric.getCurrentIngest().inc(index, type, id); + bulkProcessor.add(new UpdateRequest().index(index).type(type).id(id).upsert(source)); + } catch (Exception e) { + throwable = e; + closed = true; + logger.error("bulk add of update request failed: " + e.getMessage(), e); + } + return this; + } + + @Override + public BulkTransportClient bulkUpdate(UpdateRequest updateRequest) { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + try { + metric.getCurrentIngest().inc(updateRequest.index(), updateRequest.type(), updateRequest.id()); + bulkProcessor.add(updateRequest); + } catch (Exception e) { + throwable = e; + closed = true; + logger.error("bulk add of update request failed: " + e.getMessage(), e); + } + return this; + } + + @Override + public synchronized BulkTransportClient flushIngest() { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + if (client == null) { + logger.warn("no client"); + return this; + } + logger.debug("flushing bulk processor"); + bulkProcessor.flush(); + return this; + } + + @Override + public synchronized BulkTransportClient waitForResponses(TimeValue maxWaitTime) + throws InterruptedException, ExecutionException { + if (closed) { + throw new ElasticsearchException("client is closed"); + } + if (client == null) { + logger.warn("no client"); + return this; + } + bulkProcessor.awaitClose(maxWaitTime.getMillis(), TimeUnit.MILLISECONDS); + return this; + } + + @Override + public synchronized void shutdown() { + if (closed) { + shutdownClient(); + throw new ElasticsearchException("client is closed"); + } + if (client == null) { + logger.warn("no client"); + return; + } + try { + if (bulkProcessor != null) { + logger.debug("closing bulk processor..."); + bulkProcessor.close(); + } + if (control != null && control.indices() != null && !control.indices().isEmpty()) { + logger.debug("stopping bulk mode for indices {}...", control.indices()); + for (String index : ImmutableSet.copyOf(control.indices())) { + stopBulk(index); + } + metric.stop(); + } + logger.debug("shutting down..."); + shutdownClient(); + logger.debug("shutting down completed"); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + @Override + public boolean hasThrowable() { + return throwable != null; + } + + @Override + public Throwable getThrowable() { + return throwable; + } + + private Settings findSettings() { + Settings.Builder settingsBuilder = Settings.settingsBuilder(); + settingsBuilder.put("host", "localhost"); + try { + String hostname = NetworkUtils.getLocalAddress().getHostName(); + logger.debug("the hostname is {}", hostname); + settingsBuilder.put("host", hostname) + .put("port", 9300); + } catch (Exception e) { + logger.warn(e.getMessage(), e); + } + return settingsBuilder.build(); + } + + private Collection findAddresses(Settings settings) throws IOException { + String[] hostnames = settings.getAsArray("host", new String[]{"localhost"}); + int port = settings.getAsInt("port", 9300); + Collection addresses = new ArrayList<>(); + for (String hostname : hostnames) { + String[] splitHost = hostname.split(":", 2); + if (splitHost.length == 2) { + String host = splitHost[0]; + InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null); + try { + port = Integer.parseInt(splitHost[1]); + } catch (Exception e) { + // ignore + } + addresses.add(new InetSocketTransportAddress(inetAddress, port)); + } + if (splitHost.length == 1) { + String host = splitHost[0]; + InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null); + addresses.add(new InetSocketTransportAddress(inetAddress, port)); + } + } + return addresses; + } + + private void shutdownClient() { + if (client != null) { + logger.debug("shutdown started"); + client.close(); + client.threadPool().shutdown(); + client = null; + logger.debug("shutdown complete"); + } + isShutdown = true; + } + + private boolean connect(Collection addresses, boolean autodiscover) { + logger.info("trying to connect to {}", addresses); + client.addTransportAddresses(addresses); + if (client.connectedNodes() != null) { + List nodes = client.connectedNodes(); + if (!nodes.isEmpty()) { + logger.info("connected to {}", nodes); + if (autodiscover) { + logger.info("trying to auto-discover all cluster nodes..."); + ClusterStateRequestBuilder clusterStateRequestBuilder = + new ClusterStateRequestBuilder(client, ClusterStateAction.INSTANCE); + ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); + DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes(); + client.addDiscoveryNodes(discoveryNodes); + logger.info("after auto-discovery connected to {}", client.connectedNodes()); + } + return true; + } + return false; + } + return false; + } +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/transport/MockTransportClient.java b/src/main/java/org/xbib/elasticsearch/extras/client/transport/MockTransportClient.java new file mode 100644 index 0000000..76bf69e --- /dev/null +++ b/src/main/java/org/xbib/elasticsearch/extras/client/transport/MockTransportClient.java @@ -0,0 +1,156 @@ +package org.xbib.elasticsearch.extras.client.transport; + +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.xbib.elasticsearch.extras.client.BulkControl; +import org.xbib.elasticsearch.extras.client.BulkMetric; + +import java.io.IOException; +import java.util.Map; + +/** + * Mock client, it does not perform actions on a cluster. + * Useful for testing or dry runs. + */ +public class MockTransportClient extends BulkTransportClient { + + public MockTransportClient() { + } + + @Override + public ElasticsearchClient client() { + return null; + } + + @Override + public MockTransportClient init(ElasticsearchClient client, BulkMetric metric, BulkControl control) { + return this; + } + + @Override + public MockTransportClient init(Settings settings, BulkMetric metric, BulkControl control) { + return this; + } + + @Override + public MockTransportClient maxActionsPerRequest(int maxActions) { + return this; + } + + @Override + public MockTransportClient maxConcurrentRequests(int maxConcurrentRequests) { + return this; + } + + @Override + public MockTransportClient maxVolumePerRequest(ByteSizeValue maxVolumePerRequest) { + return this; + } + + @Override + public MockTransportClient flushIngestInterval(TimeValue interval) { + return this; + } + + @Override + public MockTransportClient index(String index, String type, String id, String source) { + return this; + } + + @Override + public MockTransportClient delete(String index, String type, String id) { + return this; + } + + @Override + public MockTransportClient update(String index, String type, String id, String source) { + return this; + } + + @Override + public MockTransportClient bulkIndex(IndexRequest indexRequest) { + return this; + } + + @Override + public MockTransportClient bulkDelete(DeleteRequest deleteRequest) { + return this; + } + + @Override + public MockTransportClient bulkUpdate(UpdateRequest updateRequest) { + return this; + } + + @Override + public MockTransportClient flushIngest() { + return this; + } + + @Override + public MockTransportClient waitForResponses(TimeValue timeValue) throws InterruptedException { + return this; + } + + @Override + public MockTransportClient startBulk(String index, long startRefreshInterval, long stopRefreshIterval) { + return this; + } + + @Override + public MockTransportClient stopBulk(String index) { + return this; + } + + @Override + public MockTransportClient deleteIndex(String index) { + return this; + } + + @Override + public MockTransportClient newIndex(String index) { + return this; + } + + @Override + public MockTransportClient newMapping(String index, String type, Map mapping) { + return this; + } + + @Override + public void putMapping(String index) { + } + + @Override + public void refreshIndex(String index) { + } + + @Override + public void flushIndex(String index) { + } + + @Override + public void waitForCluster(String healthColor, TimeValue timeValue) throws IOException { + } + + @Override + public int waitForRecovery(String index) throws IOException { + return -1; + } + + @Override + public int updateReplicaLevel(String index, int level) throws IOException { + return -1; + } + + @Override + public void shutdown() { + // do nothing + } + +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/transport/TransportClient.java b/src/main/java/org/xbib/elasticsearch/extras/client/transport/TransportClient.java new file mode 100644 index 0000000..423503e --- /dev/null +++ b/src/main/java/org/xbib/elasticsearch/extras/client/transport/TransportClient.java @@ -0,0 +1,517 @@ +package org.xbib.elasticsearch.extras.client.transport; + +import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; + +import com.google.common.collect.ImmutableMap; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.Version; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionModule; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.GenericAction; +import org.elasticsearch.action.TransportActionNodeProxy; +import org.elasticsearch.action.admin.cluster.node.liveness.LivenessRequest; +import org.elasticsearch.action.admin.cluster.node.liveness.LivenessResponse; +import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction; +import org.elasticsearch.cache.recycler.PageCacheRecycler; +import org.elasticsearch.client.support.AbstractClient; +import org.elasticsearch.client.support.Headers; +import org.elasticsearch.client.transport.ClientTransportModule; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterNameModule; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.component.LifecycleComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.inject.Injector; +import org.elasticsearch.common.inject.Module; +import org.elasticsearch.common.inject.ModulesBuilder; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsModule; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.indices.breaker.CircuitBreakerModule; +import org.elasticsearch.monitor.MonitorService; +import org.elasticsearch.node.internal.InternalSettingsPreparer; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsModule; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.threadpool.ThreadPoolModule; +import org.elasticsearch.transport.ConnectTransportException; +import org.elasticsearch.transport.FutureTransportResponseHandler; +import org.elasticsearch.transport.TransportModule; +import org.elasticsearch.transport.TransportRequestOptions; +import org.elasticsearch.transport.TransportService; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Stripped-down transport client without node sampling. + * Merged together: original TransportClient, TransportClientNodesServce, TransportClientProxy + * Configurable ping interval setting added + */ +public class TransportClient extends AbstractClient { + + private static final String CLIENT_TYPE = "transport"; + + private final Injector injector; + + private final ProxyActionMap proxyActionMap; + + private final long pingTimeout; + + private final ClusterName clusterName; + + private final TransportService transportService; + + private final Version minCompatibilityVersion; + + private final Headers headers; + + private final AtomicInteger tempNodeId = new AtomicInteger(); + + private final AtomicInteger nodeCounter = new AtomicInteger(); + + private final Object mutex = new Object(); + + private volatile List listedNodes = Collections.emptyList(); + + private volatile List nodes = Collections.emptyList(); + + private volatile List filteredNodes = Collections.emptyList(); + + private volatile boolean closed; + + private TransportClient(Injector injector) { + super(injector.getInstance(Settings.class), injector.getInstance(ThreadPool.class), + injector.getInstance(Headers.class)); + this.injector = injector; + this.clusterName = injector.getInstance(ClusterName.class); + this.transportService = injector.getInstance(TransportService.class); + this.minCompatibilityVersion = injector.getInstance(Version.class).minimumCompatibilityVersion(); + this.headers = injector.getInstance(Headers.class); + this.pingTimeout = this.settings.getAsTime("client.transport.ping_timeout", timeValueSeconds(5)).millis(); + this.proxyActionMap = injector.getInstance(ProxyActionMap.class); + } + + public static Builder builder() { + return new Builder(); + } + + /** + * Returns the current registered transport addresses to use. + * + * @return list of transport addresess + */ + public List transportAddresses() { + List lstBuilder = new ArrayList<>(); + for (DiscoveryNode listedNode : listedNodes) { + lstBuilder.add(listedNode.address()); + } + return Collections.unmodifiableList(lstBuilder); + } + + /** + * Returns the current connected transport nodes that this client will use. + * The nodes include all the nodes that are currently alive based on the transport + * addresses provided. + * + * @return list of nodes + */ + public List connectedNodes() { + return this.nodes; + } + + /** + * The list of filtered nodes that were not connected to, for example, due to + * mismatch in cluster name. + * + * @return list of nodes + */ + public List filteredNodes() { + return this.filteredNodes; + } + + /** + * Returns the listed nodes in the transport client (ones added to it). + * + * @return list of nodes + */ + public List listedNodes() { + return this.listedNodes; + } + + /** + * Adds a list of transport addresses that will be used to connect to. + * The Node this transport address represents will be used if its possible to connect to it. + * If it is unavailable, it will be automatically connected to once it is up. + * In order to get the list of all the current connected nodes, please see {@link #connectedNodes()}. + * + * @param discoveryNodes nodes + * @return this transport client + */ + public TransportClient addDiscoveryNodes(DiscoveryNodes discoveryNodes) { + Collection addresses = new ArrayList<>(); + for (DiscoveryNode discoveryNode : discoveryNodes) { + addresses.add((InetSocketTransportAddress) discoveryNode.address()); + } + addTransportAddresses(addresses); + return this; + } + + public TransportClient addTransportAddresses(Collection transportAddresses) { + synchronized (mutex) { + if (closed) { + throw new IllegalStateException("transport client is closed, can't add addresses"); + } + List filtered = new ArrayList<>(transportAddresses.size()); + for (TransportAddress transportAddress : transportAddresses) { + boolean found = false; + for (DiscoveryNode otherNode : listedNodes) { + if (otherNode.address().equals(transportAddress)) { + found = true; + logger.debug("address [{}] already exists with [{}], ignoring...", transportAddress, otherNode); + break; + } + } + if (!found) { + filtered.add(transportAddress); + } + } + if (filtered.isEmpty()) { + return this; + } + List discoveryNodeList = new ArrayList<>(); + discoveryNodeList.addAll(listedNodes()); + for (TransportAddress transportAddress : filtered) { + DiscoveryNode node = new DiscoveryNode("#transport#-" + tempNodeId.incrementAndGet(), transportAddress, + minCompatibilityVersion); + logger.debug("adding address [{}]", node); + discoveryNodeList.add(node); + } + listedNodes = Collections.unmodifiableList(discoveryNodeList); + connect(); + } + return this; + } + + /** + * Removes a transport address from the list of transport addresses that are used to connect to. + * + * @param transportAddress transport address to remove + * @return this transport client + */ + public TransportClient removeTransportAddress(TransportAddress transportAddress) { + synchronized (mutex) { + if (closed) { + throw new IllegalStateException("transport client is closed, can't remove an address"); + } + List builder = new ArrayList<>(); + for (DiscoveryNode otherNode : listedNodes) { + if (!otherNode.address().equals(transportAddress)) { + builder.add(otherNode); + } else { + logger.debug("removing address [{}]", otherNode); + } + } + listedNodes = Collections.unmodifiableList(builder); + } + return this; + } + + @Override + @SuppressWarnings("rawtypes") + public void close() { + synchronized (mutex) { + if (closed) { + return; + } + closed = true; + for (DiscoveryNode node : nodes) { + transportService.disconnectFromNode(node); + } + for (DiscoveryNode listedNode : listedNodes) { + transportService.disconnectFromNode(listedNode); + } + nodes = Collections.emptyList(); + } + injector.getInstance(TransportService.class).close(); + try { + injector.getInstance(MonitorService.class).close(); + } catch (Exception e) { + // ignore, might not be bounded + } + for (Class plugin : injector.getInstance(PluginsService.class).nodeServices()) { + injector.getInstance(plugin).close(); + } + try { + ThreadPool.terminate(injector.getInstance(ThreadPool.class), 10, TimeUnit.SECONDS); + } catch (Exception e) { + // ignore + } + injector.getInstance(PageCacheRecycler.class).close(); + } + + private void connect() { + Set newNodes = new HashSet<>(); + Set newFilteredNodes = new HashSet<>(); + for (DiscoveryNode listedNode : listedNodes) { + if (!transportService.nodeConnected(listedNode)) { + try { + logger.trace("connecting to listed node (light) [{}]", listedNode); + transportService.connectToNodeLight(listedNode); + } catch (Throwable e) { + logger.debug("failed to connect to node [{}], removed from nodes list", e, listedNode); + continue; + } + } + try { + LivenessResponse livenessResponse = transportService.submitRequest(listedNode, + TransportLivenessAction.NAME, headers.applyTo(new LivenessRequest()), + TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STATE) + .withTimeout(pingTimeout).build(), + new FutureTransportResponseHandler() { + @Override + public LivenessResponse newInstance() { + return new LivenessResponse(); + } + }).txGet(); + if (!clusterName.equals(livenessResponse.getClusterName())) { + logger.warn("node {} not part of the cluster {}, ignoring...", listedNode, clusterName); + newFilteredNodes.add(listedNode); + } else if (livenessResponse.getDiscoveryNode() != null) { + DiscoveryNode nodeWithInfo = livenessResponse.getDiscoveryNode(); + newNodes.add(new DiscoveryNode(nodeWithInfo.name(), nodeWithInfo.id(), nodeWithInfo.getHostName(), + nodeWithInfo.getHostAddress(), listedNode.address(), nodeWithInfo.attributes(), + nodeWithInfo.version())); + } else { + logger.debug("node {} didn't return any discovery info, temporarily using transport discovery node", + listedNode); + newNodes.add(listedNode); + } + } catch (Throwable e) { + logger.info("failed to get node info for {}, disconnecting...", e, listedNode); + transportService.disconnectFromNode(listedNode); + } + } + for (Iterator it = newNodes.iterator(); it.hasNext(); ) { + DiscoveryNode node = it.next(); + if (!transportService.nodeConnected(node)) { + try { + logger.trace("connecting to node [{}]", node); + transportService.connectToNode(node); + } catch (Throwable e) { + it.remove(); + logger.debug("failed to connect to discovered node [" + node + "]", e); + } + } + } + this.nodes = Collections.unmodifiableList(new ArrayList<>(newNodes)); + this.filteredNodes = Collections.unmodifiableList(new ArrayList<>(newFilteredNodes)); + } + + @Override + @SuppressWarnings({"unchecked", "rawtypes"}) + protected > + void doExecute(Action action, final Request request, + ActionListener listener) { + final TransportActionNodeProxy proxyAction = proxyActionMap.getProxies().get(action); + if (proxyAction == null) { + throw new IllegalStateException("undefined action " + action); + } + NodeListenerCallback callback = new NodeListenerCallback() { + @Override + public void doWithNode(DiscoveryNode node, ActionListener listener) { + proxyAction.execute(node, request, listener); + } + }; + List nodes = this.nodes; + if (nodes.isEmpty()) { + throw new NoNodeAvailableException("none of the configured nodes are available: " + this.listedNodes); + } + int index = nodeCounter.incrementAndGet(); + if (index < 0) { + index = 0; + nodeCounter.set(0); + } + RetryListener retryListener = new RetryListener<>(callback, listener, nodes, index); + DiscoveryNode node = nodes.get((index) % nodes.size()); + try { + callback.doWithNode(node, retryListener); + } catch (Throwable t) { + listener.onFailure(t); + } + } + + /** + * + * @param + */ + interface NodeListenerCallback { + + void doWithNode(DiscoveryNode node, ActionListener listener); + } + + /** + * + */ + public static class Builder { + + private Settings settings = Settings.EMPTY; + private List> pluginClasses = new ArrayList<>(); + + public Builder settings(Settings.Builder settings) { + return settings(settings.build()); + } + + public Builder settings(Settings settings) { + this.settings = settings; + return this; + } + + public Builder addPlugin(Class pluginClass) { + pluginClasses.add(pluginClass); + return this; + } + + public TransportClient build() { + Settings settings = InternalSettingsPreparer.prepareSettings(this.settings); + settings = settingsBuilder() + .put("transport.ping.schedule", this.settings.get("ping.interval", "30s")) + .put(settings) + .put("network.server", false) + .put("node.client", true) + .put(CLIENT_TYPE_SETTING, CLIENT_TYPE) + .build(); + PluginsService pluginsService = new PluginsService(settings, null, null, pluginClasses); + this.settings = pluginsService.updatedSettings(); + Version version = Version.CURRENT; + final ThreadPool threadPool = new ThreadPool(settings); + + boolean success = false; + try { + ModulesBuilder modules = new ModulesBuilder(); + modules.add(new Version.Module(version)); + // plugin modules must be added here, before others or we can get crazy injection errors... + for (Module pluginModule : pluginsService.nodeModules()) { + modules.add(pluginModule); + } + modules.add(new PluginsModule(pluginsService)); + modules.add(new SettingsModule(this.settings)); + modules.add(new NetworkModule()); + modules.add(new ClusterNameModule(this.settings)); + modules.add(new ThreadPoolModule(threadPool)); + modules.add(new TransportModule(this.settings)); + modules.add(new SearchModule() { + @Override + protected void configure() { + // noop + } + }); + modules.add(new ActionModule(true)); + modules.add(new ClientTransportModule()); + modules.add(new CircuitBreakerModule(this.settings)); + pluginsService.processModules(modules); + Injector injector = modules.createInjector(); + injector.getInstance(TransportService.class).start(); + TransportClient transportClient = new TransportClient(injector); + success = true; + return transportClient; + } finally { + if (!success) { + ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS); + } + } + } + } + + private static class RetryListener implements ActionListener { + private final ESLogger logger = ESLoggerFactory.getLogger(RetryListener.class.getName()); + private final NodeListenerCallback callback; + private final ActionListener listener; + private final List nodes; + private final int index; + + private volatile int n; + + RetryListener(NodeListenerCallback callback, ActionListener listener, + List nodes, int index) { + this.callback = callback; + this.listener = listener; + this.nodes = nodes; + this.index = index; + } + + @Override + public void onResponse(Response response) { + listener.onResponse(response); + } + + @Override + public void onFailure(Throwable e) { + if (ExceptionsHelper.unwrapCause(e) instanceof ConnectTransportException) { + int n = ++this.n; + if (n >= nodes.size()) { + listener.onFailure(new NoNodeAvailableException("none of the configured nodes were available: " + + nodes, e)); + } else { + try { + logger.warn("retrying on another node (n={}, nodes={})", n, nodes.size()); + callback.doWithNode(nodes.get((index + n) % nodes.size()), this); + } catch (final Throwable t) { + listener.onFailure(t); + } + } + } else { + listener.onFailure(e); + } + } + } + + /** + * The {@link ProxyActionMap} must be declared public. + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + public static class ProxyActionMap { + + private final ImmutableMap proxies; + + @Inject + public ProxyActionMap(Settings settings, TransportService transportService, Map actions) { + MapBuilder actionsBuilder = new MapBuilder<>(); + for (GenericAction action : actions.values()) { + if (action instanceof Action) { + actionsBuilder.put((Action) action, new TransportActionNodeProxy(settings, action, transportService)); + } + } + this.proxies = actionsBuilder.immutableMap(); + } + + public ImmutableMap getProxies() { + return proxies; + } + } + +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/transport/package-info.java b/src/main/java/org/xbib/elasticsearch/extras/client/transport/package-info.java new file mode 100644 index 0000000..ac6a50d --- /dev/null +++ b/src/main/java/org/xbib/elasticsearch/extras/client/transport/package-info.java @@ -0,0 +1,4 @@ +/** + * Classes for Elasticsearch transport client extras. + */ +package org.xbib.elasticsearch.extras.client.transport; diff --git a/src/test/resources/log4j2.xml b/src/test/resources/log4j2.xml new file mode 100644 index 0000000..f71aced --- /dev/null +++ b/src/test/resources/log4j2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file