diff --git a/.gitignore b/.gitignore index bf3e9b4..bdd8810 100644 --- a/.gitignore +++ b/.gitignore @@ -4,10 +4,10 @@ /.idea /target .DS_Store -*.iml /.settings /.classpath /.project /.gradle -/build -/plugins \ No newline at end of file +build +*.iml +*~ diff --git a/build.gradle b/build.gradle index 2f8478a..01e2104 100644 --- a/build.gradle +++ b/build.gradle @@ -1,9 +1,11 @@ - plugins { - id "org.sonarqube" version "2.2" + id "org.sonarqube" version "2.6.1" + id "io.codearte.nexus-staging" version "0.11.0" + id "com.github.spotbugs" version "1.6.9" + id "org.xbib.gradle.plugin.asciidoctor" version "1.5.6.0.1" } -printf "Host: %s\nOS: %s %s %s\nJVM: %s %s %s %s\nGroovy: %s\nGradle: %s\n" + +printf "Host: %s\nOS: %s %s %s\nJVM: %s %s %s %s\nGradle: %s Groovy: %s Java: %s\n" + "Build: group: ${project.group} name: ${project.name} version: ${project.version}\n", InetAddress.getLocalHost(), System.getProperty("os.name"), @@ -13,102 +15,143 @@ printf "Host: %s\nOS: %s %s %s\nJVM: %s %s %s %s\nGroovy: %s\nGradle: %s\n" + System.getProperty("java.vm.version"), System.getProperty("java.vm.vendor"), System.getProperty("java.vm.name"), + gradle.gradleVersion, GroovySystem.getVersion(), - gradle.gradleVersion - -apply plugin: 'java' -apply plugin: 'maven' -apply plugin: 'signing' -apply plugin: 'findbugs' -apply plugin: 'pmd' -apply plugin: 'checkstyle' -apply plugin: "jacoco" - -apply from: 'gradle/ext.gradle' - -sourceSets { - integrationTest { - java { - srcDir file('src/integration-test/java') - compileClasspath += main.output - compileClasspath += test.output + JavaVersion.current() + +if (JavaVersion.current() < JavaVersion.VERSION_11) { + throw new GradleException("This build must be run with java 11 or higher") +} + +subprojects { + apply plugin: 'java' + apply plugin: 'maven' + apply plugin: 'signing' + apply plugin: 'com.github.spotbugs' + apply plugin: 'pmd' + apply plugin: 'checkstyle' + apply plugin: 'org.xbib.gradle.plugin.asciidoctor' + + configurations { + asciidoclet + wagon + } + + dependencies { + testCompile "junit:junit:${project.property('junit.version')}" + testCompile "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}" + testCompile "org.apache.logging.log4j:log4j-slf4j-impl:${project.property('log4j.version')}" + wagon "org.apache.maven.wagon:wagon-ssh:${project.property('wagon.version')}" + } + + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + + tasks.withType(JavaCompile) { + options.compilerArgs << "-Xlint:all" + if (!options.compilerArgs.contains("-processor")) { + options.compilerArgs << '-proc:none' } - resources { - srcDir file('src/integration-test/resources') + } + + test { + jvmArgs =[ + '--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED', + '--add-opens=java.base/java.nio=ALL-UNNAMED' + ] + systemProperty 'jna.debug_load', 'true' + testLogging { + showStandardStreams = true + exceptionFormat = 'full' } } -} + + clean { + delete "plugins" + delete "logs" + delete "out" + } -configurations { - wagon - integrationTestCompile.extendsFrom testCompile - integrationTestRuntime.extendsFrom testRuntime -} + /*javadoc { + options.docletpath = configurations.asciidoclet.files.asType(List) + options.doclet = 'org.asciidoctor.Asciidoclet' + options.overview = "src/docs/asciidoclet/overview.adoc" + options.addStringOption "-base-dir", "${projectDir}" + options.addStringOption "-attribute", + "name=${project.name},version=${project.version},title-link=https://github.com/xbib/${project.name}" + configure(options) { + noTimestamp = true + } + }*/ -dependencies { - compile "org.xbib:metrics:1.0.0" - compile("org.elasticsearch:elasticsearch:2.2.1") { - exclude module: "securesm" + task javadocJar(type: Jar, dependsOn: javadoc) { + classifier 'javadoc' } - testCompile "net.java.dev.jna:jna:4.1.0" - testCompile "junit:junit:4.12" - testCompile "org.apache.logging.log4j:log4j-core:2.7" - testCompile "org.apache.logging.log4j:log4j-slf4j-impl:2.7" - wagon 'org.apache.maven.wagon:wagon-ssh-external:2.10' -} - -sourceCompatibility = JavaVersion.VERSION_1_8 -targetCompatibility = JavaVersion.VERSION_1_8 -[compileJava, compileTestJava]*.options*.encoding = 'UTF-8' -tasks.withType(JavaCompile) { - options.compilerArgs << "-Xlint:all" -} + task sourcesJar(type: Jar, dependsOn: classes) { + from sourceSets.main.allSource + classifier 'sources' + } -task integrationTest(type: Test) { - include '**/MiscTestSuite.class' - include '**/BulkNodeTestSuite.class' - include '**/BulkTransportTestSuite.class' - testClassesDir = sourceSets.integrationTest.output.classesDir - classpath = configurations.integrationTestCompile - classpath += configurations.integrationTestRuntime - classpath += sourceSets.main.output - classpath += sourceSets.test.output - classpath += sourceSets.integrationTest.output - outputs.upToDateWhen { false } - systemProperty 'path.home', projectDir.absolutePath - testLogging.showStandardStreams = true -} + artifacts { + archives javadocJar, sourcesJar + } -integrationTest.mustRunAfter test -check.dependsOn integrationTest + if (project.hasProperty('signing.keyId')) { + signing { + sign configurations.archives + } + } -clean { - delete "plugins" - delete "logs" -} + apply from: "${rootProject.projectDir}/gradle/publish.gradle" -task javadocJar(type: Jar, dependsOn: classes) { - from javadoc - into "build/tmp" - classifier 'javadoc' -} + spotbugs { + effort = "max" + reportLevel = "low" + //includeFilter = file("findbugs-exclude.xml") + } -task sourcesJar(type: Jar, dependsOn: classes) { - from sourceSets.main.allSource - into "build/tmp" - classifier 'sources' -} + tasks.withType(com.github.spotbugs.SpotBugsTask) { + ignoreFailures = true + reports { + xml.enabled = false + html.enabled = true + } + } -artifacts { - archives javadocJar, sourcesJar -} + tasks.withType(Pmd) { + ignoreFailures = true + reports { + xml.enabled = true + html.enabled = true + } + } + tasks.withType(Checkstyle) { + ignoreFailures = true + reports { + xml.enabled = true + html.enabled = true + } + } -if (project.hasProperty('signing.keyId')) { - signing { - sign configurations.archives + pmd { + toolVersion = '6.11.0' + ruleSets = ['category/java/bestpractices.xml'] + } + + checkstyle { + configFile = rootProject.file('config/checkstyle/checkstyle.xml') + ignoreFailures = true + showViolations = true } -} -apply from: 'gradle/publish.gradle' -apply from: 'gradle/sonarqube.gradle' + sonarqube { + properties { + property "sonar.projectName", "${project.group} ${project.name}" + property "sonar.sourceEncoding", "UTF-8" + property "sonar.tests", "src/test/java" + property "sonar.scm.provider", "git" + property "sonar.junit.reportsPath", "build/test-results/test/" + } + } +} \ No newline at end of file diff --git a/config/pmd/category/java/bestpractices.xml b/config/pmd/category/java/bestpractices.xml new file mode 100644 index 0000000..816e8cc --- /dev/null +++ b/config/pmd/category/java/bestpractices.xml @@ -0,0 +1,1636 @@ + + + + + + Rules which enforce generally accepted best practices. + + + + + The abstract class does not contain any abstract methods. An abstract class suggests + an incomplete implementation, which is to be completed by subclasses implementing the + abstract methods. If the class is intended to be used as a base class only (not to be instantiated + directly) a protected constructor can be provided prevent direct instantiation. + + 3 + + + + + + + + + + + + + + + Instantiation by way of private constructors from outside of the constructor's class often causes the + generation of an accessor. A factory method, or non-privatization of the constructor can eliminate this + situation. The generated class file is actually an interface. It gives the accessing class the ability + to invoke a new hidden package scope constructor that takes the interface as a supplementary parameter. + This turns a private constructor effectively into one with package scope, and is challenging to discern. + + 3 + + + + + + + + When accessing a private field / method from another class, the Java compiler will generate a accessor methods + with package-private visibility. This adds overhead, and to the dex method count on Android. This situation can + be avoided by changing the visibility of the field / method from private to package-private. + + 3 + + + + + + + + Constructors and methods receiving arrays should clone objects and store the copy. + This prevents future changes from the user from affecting the original array. + + 3 + + + + + + + + Avoid printStackTrace(); use a logger call instead. + + 3 + + + + + + + + + + + + + + + Reassigning loop variables can lead to hard-to-find bugs. Prevent or limit how these variables can be changed. + + In foreach-loops, configured by the `foreachReassign` property: + - `deny`: Report any reassignment of the loop variable in the loop body. _This is the default._ + - `allow`: Don't check the loop variable. + - `firstOnly`: Report any reassignments of the loop variable, except as the first statement in the loop body. + _This is useful if some kind of normalization or clean-up of the value before using is permitted, but any other change of the variable is not._ + + In for-loops, configured by the `forReassign` property: + - `deny`: Report any reassignment of the control variable in the loop body. _This is the default._ + - `allow`: Don't check the control variable. + - `skip`: Report any reassignments of the control variable, except conditional increments/decrements (`++`, `--`, `+=`, `-=`). + _This prevents accidental reassignments or unconditional increments of the control variable._ + + 3 + + + + + + + + Reassigning values to incoming parameters is not recommended. Use temporary local variables instead. + + 2 + + + + + + + + StringBuffers/StringBuilders can grow considerably, and so may become a source of memory leaks + if held within objects with long lifetimes. + + 3 + + + + + + + + + + + + + + + Application with hard-coded IP addresses can become impossible to deploy in some cases. + Externalizing IP adresses is preferable. + + 3 + + + + + + + + Always check the return values of navigation methods (next, previous, first, last) of a ResultSet. + If the value return is 'false', it should be handled properly. + + 3 + + + + + + + + Avoid constants in interfaces. Interfaces should define types, constants are implementation details + better placed in classes or enums. See Effective Java, item 19. + + 3 + + + + + + + + + + + + + + + + By convention, the default label should be the last label in a switch statement. + + 3 + + + + + + + + + + + + + + + Reports loops that can be safely replaced with the foreach syntax. The rule considers loops over + lists, arrays and iterators. A loop is safe to replace if it only uses the index variable to + access an element of the list or array, only has one update statement, and loops through *every* + element of the list or array left to right. + + 3 + + l) { + for (int i = 0; i < l.size(); i++) { // pre Java 1.5 + System.out.println(l.get(i)); + } + + for (String s : l) { // post Java 1.5 + System.out.println(s); + } + } +} +]]> + + + + + + Having a lot of control variables in a 'for' loop makes it harder to see what range of values + the loop iterates over. By default this rule allows a regular 'for' loop with only one variable. + + 3 + + + + //ForInit/LocalVariableDeclaration[count(VariableDeclarator) > $maximumVariables] + + + + + + + + + + Whenever using a log level, one should check if the loglevel is actually enabled, or + otherwise skip the associate String creation and manipulation. + + 2 + + + + + + + + In JUnit 3, test suites are indicated by the suite() method. In JUnit 4, suites are indicated + through the @RunWith(Suite.class) annotation. + + 3 + + + + + + + + + + + + + + + In JUnit 3, the tearDown method was used to clean up all data entities required in running tests. + JUnit 4 skips the tearDown method and executes all methods annotated with @After after running each test. + JUnit 5 introduced @AfterEach and @AfterAll annotations to execute methods after each test or after all tests in the class, respectively. + + 3 + + + + + + + + + + + + + + + In JUnit 3, the setUp method was used to set up all data entities required in running tests. + JUnit 4 skips the setUp method and executes all methods annotated with @Before before all tests. + JUnit 5 introduced @BeforeEach and @BeforeAll annotations to execute methods before each test or before all tests in the class, respectively. + + 3 + + + + + + + + + + + + + + + In JUnit 3, the framework executed all methods which started with the word test as a unit test. + In JUnit 4, only methods annotated with the @Test annotation are executed. + In JUnit 5, one of the following annotations should be used for tests: @Test, @RepeatedTest, @TestFactory, @TestTemplate or @ParameterizedTest. + + 3 + + + + + + + + + + + + + + + + + JUnit assertions should include an informative message - i.e., use the three-argument version of + assertEquals(), not the two-argument version. + + 3 + + + + + + + + Unit tests should not contain too many asserts. Many asserts are indicative of a complex test, for which + it is harder to verify correctness. Consider breaking the test scenario into multiple, shorter test scenarios. + Customize the maximum number of assertions used by this Rule to suit your needs. + + This rule checks for JUnit4, JUnit5 and TestNG Tests, as well as methods starting with "test". + + 3 + + + + + $maximumAsserts] +]]> + + + + + + + + + + + JUnit tests should include at least one assertion. This makes the tests more robust, and using assert + with messages provide the developer a clearer idea of what the test does. + + 3 + + + + + + + + In JUnit4, use the @Test(expected) annotation to denote tests that should throw exceptions. + + 3 + + + + + + + + The use of implementation types (i.e., HashSet) as object references limits your ability to use alternate + implementations in the future as requirements change. Whenever available, referencing objects + by their interface types (i.e, Set) provides much more flexibility. + + 3 + + list = new ArrayList<>(); + + public HashSet getFoo() { + return new HashSet(); + } + + // preferred approach + private List list = new ArrayList<>(); + + public Set getFoo() { + return new HashSet(); + } +} +]]> + + + + + + Exposing internal arrays to the caller violates object encapsulation since elements can be + removed or replaced outside of the object that owns it. It is safer to return a copy of the array. + + 3 + + + + + + + + + Annotating overridden methods with @Override ensures at compile time that + the method really overrides one, which helps refactoring and clarifies intent. + + 3 + + + + + + + + Java allows the use of several variables declaration of the same type on one line. However, it + can lead to quite messy code. This rule looks for several declarations on the same line. + + 4 + + + + 1] + [$strictMode or count(distinct-values(VariableDeclarator/@BeginLine)) != count(VariableDeclarator)] +| +//FieldDeclaration + [count(VariableDeclarator) > 1] + [$strictMode or count(distinct-values(VariableDeclarator/@BeginLine)) != count(VariableDeclarator)] +]]> + + + + + + + + + + + + + Position literals first in comparisons, if the second argument is null then NullPointerExceptions + can be avoided, they will just return false. + + 3 + + + + + + + + + + + + + + + Position literals first in comparisons, if the second argument is null then NullPointerExceptions + can be avoided, they will just return false. + + 3 + + + + + + + + + + + + + + + Throwing a new exception from a catch block without passing the original exception into the + new exception will cause the original stack trace to be lost making it difficult to debug + effectively. + + 3 + + + + + + + + Consider replacing Enumeration usages with the newer java.util.Iterator + + 3 + + + + + + + + + + + + + + + Consider replacing Hashtable usage with the newer java.util.Map if thread safety is not required. + + 3 + + + //Type/ReferenceType/ClassOrInterfaceType[@Image='Hashtable'] + + + + + + + + + + Consider replacing Vector usages with the newer java.util.ArrayList if expensive thread-safe operations are not required. + + 3 + + + //Type/ReferenceType/ClassOrInterfaceType[@Image='Vector'] + + + + + + + + + + All switch statements should include a default option to catch any unspecified values. + + 3 + + + + + + + + + + + + + + References to System.(out|err).print are usually intended for debugging purposes and can remain in + the codebase even in production code. By using a logger one can enable/disable this behaviour at + will (and by priority) and avoid clogging the Standard out log. + + 2 + + + + + + + + + + + + + + + Avoid passing parameters to methods or constructors without actually referencing them in the method body. + + 3 + + + + + + + + Avoid unused import statements to prevent unwanted dependencies. + This rule will also find unused on demand imports, i.e. import com.foo.*. + + 4 + + + + + + + + Detects when a local variable is declared and/or assigned, but not used. + + 3 + + + + + + + + Detects when a private field is declared and/or assigned a value, but not used. + + 3 + + + + + + + + Unused Private Method detects when a private method is declared but is unused. + + 3 + + + + + + + + This rule detects JUnit assertions in object equality. These assertions should be made by more specific methods, like assertEquals. + + 3 + + + + + + + + + + + + + + + This rule detects JUnit assertions in object references equality. These assertions should be made by + more specific methods, like assertNull, assertNotNull. + + 3 + + + + + + + + + + + + + + + This rule detects JUnit assertions in object references equality. These assertions should be made + by more specific methods, like assertSame, assertNotSame. + + 3 + + + + + + + + + + + + + + + When asserting a value is the same as a literal or Boxed boolean, use assertTrue/assertFalse, instead of assertEquals. + + 3 + + + + + + + + + + + + + + + The isEmpty() method on java.util.Collection is provided to determine if a collection has any elements. + Comparing the value of size() to 0 does not convey intent as well as the isEmpty() method. + + 3 + + + + + + + + Java 7 introduced the try-with-resources statement. This statement ensures that each resource is closed at the end + of the statement. It avoids the need of explicitly closing the resources in a finally block. Additionally exceptions + are better handled: If an exception occurred both in the `try` block and `finally` block, then the exception from + the try block was suppressed. With the `try`-with-resources statement, the exception thrown from the try-block is + preserved. + + 3 + + + + + + + + + + + + + + + + + Java 5 introduced the varargs parameter declaration for methods and constructors. This syntactic + sugar provides flexibility for users of these methods and constructors, allowing them to avoid + having to deal with the creation of an array. + + 4 + + + + + + + + + + + + + + diff --git a/config/pmd/category/java/categories.properties b/config/pmd/category/java/categories.properties new file mode 100644 index 0000000..3189fd3 --- /dev/null +++ b/config/pmd/category/java/categories.properties @@ -0,0 +1,13 @@ +# +# BSD-style license; for more info see http://pmd.sourceforge.net/license.html +# + +rulesets.filenames=\ + category/java/bestpractices.xml,\ + category/java/codestyle.xml,\ + category/java/design.xml,\ + category/java/documentation.xml,\ + category/java/errorprone.xml,\ + category/java/multithreading.xml,\ + category/java/performance.xml,\ + category/java/security.xml diff --git a/config/pmd/category/java/codestyle.xml b/config/pmd/category/java/codestyle.xml new file mode 100644 index 0000000..ac2f0a0 --- /dev/null +++ b/config/pmd/category/java/codestyle.xml @@ -0,0 +1,2176 @@ + + + + + + Rules which enforce a specific coding style. + + + + + Abstract classes should be named 'AbstractXXX'. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by {% rule java/codestyle/ClassNamingConventions %}. + + 3 + + + + + + + + + + + + + + + + + + 3 + + + + + + + + Avoid using dollar signs in variable/method/class/interface names. + + 3 + + + + + + + Avoid using final local variables, turn them into fields. + 3 + + + + + + + + + + + + + + + Prefixing parameters by 'in' or 'out' pollutes the name of the parameters and reduces code readability. + To indicate whether or not a parameter will be modify in a method, its better to document method + behavior with Javadoc. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the more general rule {% rule java/codestyle/FormalParameterNamingConventions %}. + + 4 + + + + + + + + + + + + + + + + + + Do not use protected fields in final classes since they cannot be subclassed. + Clarify your intent by using private or package access modifiers instead. + + 3 + + + + + + + + + + + + + + + Do not use protected methods in most final classes since they cannot be subclassed. This should + only be allowed in final classes that extend other classes with protected methods (whose + visibility cannot be reduced). Clarify your intent by using private or package access modifiers instead. + + 3 + + + + + + + + + + + + + + + Unnecessary reliance on Java Native Interface (JNI) calls directly reduces application portability + and increases the maintenance burden. + + 2 + + + //Name[starts-with(@Image,'System.loadLibrary')] + + + + + + + + + + Methods that return boolean results should be named as predicate statements to denote this. + I.e, 'isReady()', 'hasValues()', 'canCommit()', 'willFail()', etc. Avoid the use of the 'get' + prefix for these methods. + + 4 + + + + + + + + + + + + + + + + It is a good practice to call super() in a constructor. If super() is not called but + another constructor (such as an overloaded constructor) is called, this rule will not report it. + + 3 + + + + 0 ] +/ClassOrInterfaceBody + /ClassOrInterfaceBodyDeclaration + /ConstructorDeclaration[ count (.//ExplicitConstructorInvocation)=0 ] +]]> + + + + + + + + + + + Configurable naming conventions for type declarations. This rule reports + type declarations which do not match the regex that applies to their + specific kind (e.g. enum or interface). Each regex can be configured through + properties. + + By default this rule uses the standard Java naming convention (Pascal case), + and reports utility class names not ending with 'Util'. + + 1 + + + + + + + + To avoid mistakes if we want that a Method, Constructor, Field or Nested class have a default access modifier + we must add a comment at the beginning of it's declaration. + By default the comment must be /* default */ or /* package */, if you want another, you have to provide a regular expression. + This rule ignores by default all cases that have a @VisibleForTesting annotation. Use the + property "ignoredAnnotations" to customize the recognized annotations. + + 3 + + + + + + + + Avoid negation within an "if" expression with an "else" clause. For example, rephrase: + `if (x != y) diff(); else same();` as: `if (x == y) same(); else diff();`. + + Most "if (x != y)" cases without an "else" are often return cases, so consistent use of this + rule makes the code easier to read. Also, this resolves trivial ordering problems, such + as "does the error case go first?" or "does the common case go first?". + + 3 + + + + + + + + Enforce a policy for braces on control statements. It is recommended to use braces on 'if ... else' + statements and loop statements, even if they are optional. This usually makes the code clearer, and + helps prepare the future when you need to add another statement. That said, this rule lets you control + which statements are required to have braces via properties. + + From 6.2.0 on, this rule supersedes WhileLoopMustUseBraces, ForLoopMustUseBraces, IfStmtMustUseBraces, + and IfElseStmtMustUseBraces. + + 3 + + + + + + + + + + + + + 1 + or (some $stmt (: in only the block statements until the next label :) + in following-sibling::BlockStatement except following-sibling::SwitchLabel[1]/following-sibling::BlockStatement + satisfies not($stmt/Statement/Block))] + ]]> + + + + + + + + + + Use explicit scoping instead of accidental usage of default package private level. + The rule allows methods and fields annotated with Guava's @VisibleForTesting. + + 3 + + + + + + + + + + + + Avoid importing anything from the package 'java.lang'. These classes are automatically imported (JLS 7.5.3). + + 4 + + + + + + + + Duplicate or overlapping import statements should be avoided. + + 4 + + + + + + + + Empty or auto-generated methods in an abstract class should be tagged as abstract. This helps to remove their inapproprate + usage by developers who should be implementing their own versions in the concrete subclasses. + + 1 + + + + + + + + + + + + + + No need to explicitly extend Object. + 4 + + + + + + + + + + + + + + + Fields should be declared at the top of the class, before any method declarations, constructors, initializers or inner classes. + + 3 + + + + + + + + + Configurable naming conventions for field declarations. This rule reports variable declarations + which do not match the regex that applies to their specific kind ---e.g. constants (static final), + enum constant, final field. Each regex can be configured through properties. + + By default this rule uses the standard Java naming convention (Camel case), and uses the ALL_UPPER + convention for constants and enum constants. + + 1 + + + + + + + + Some for loops can be simplified to while loops, this makes them more concise. + + 3 + + + + + + + + + + + + + + + Avoid using 'for' statements without using curly braces. If the code formatting or + indentation is lost then it becomes difficult to separate the code being controlled + from the rest. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the rule {% rule java/codestyle/ControlStatementBraces %}. + + 3 + + + //ForStatement[not(Statement/Block)] + + + + + + + + + + Configurable naming conventions for formal parameters of methods and lambdas. + This rule reports formal parameters which do not match the regex that applies to their + specific kind (e.g. lambda parameter, or final formal parameter). Each regex can be + configured through properties. + + By default this rule uses the standard Java naming convention (Camel case). + + 1 + + lambda1 = s_str -> { }; + + // lambda parameters with an explicit type can be configured separately + Consumer lambda1 = (String str) -> { }; + + } + + } + ]]> + + + + + + Names for references to generic values should be limited to a single uppercase letter. + + 4 + + + + 1 + or + string:upper-case(@Image) != @Image +] +]]> + + + + + extends BaseDao { + // This is ok... +} + +public interface GenericDao { + // Also this +} + +public interface GenericDao { + // 'e' should be an 'E' +} + +public interface GenericDao { + // 'EF' is not ok. +} +]]> + + + + + + + Identical `catch` branches use up vertical space and increase the complexity of code without + adding functionality. It's better style to collapse identical branches into a single multi-catch + branch. + + 3 + + + + + + + + Avoid using if..else statements without using surrounding braces. If the code formatting + or indentation is lost then it becomes difficult to separate the code being controlled + from the rest. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the rule {% rule java/codestyle/ControlStatementBraces %}. + + 3 + + + + + + + + + + + + + + + Avoid using if statements without using braces to surround the code block. If the code + formatting or indentation is lost then it becomes difficult to separate the code being + controlled from the rest. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the rule {% rule java/codestyle/ControlStatementBraces %}. + + 3 + + + + + + + + + + + + + + + This rule finds Linguistic Naming Antipatterns. It checks for fields, that are named, as if they should + be boolean but have a different type. It also checks for methods, that according to their name, should + return a boolean, but don't. Further, it checks, that getters return something and setters won't. + Finally, it checks that methods, that start with "to" - so called transform methods - actually return + something, since according to their name, they should convert or transform one object into another. + There is additionally an option, to check for methods that contain "To" in their name - which are + also transform methods. However, this is disabled by default, since this detection is prone to + false positives. + + For more information, see [Linguistic Antipatterns - What They Are and How + Developers Perceive Them](https://doi.org/10.1007/s10664-014-9350-8). + + 3 + + + + + + + + The Local Home interface of a Session EJB should be suffixed by 'LocalHome'. + + 4 + + + + + + + + + + + + + + + The Local Interface of a Session EJB should be suffixed by 'Local'. + + 4 + + + + + + + + + + + + + + + A local variable assigned only once can be declared final. + + 3 + + + + + + + + Configurable naming conventions for local variable declarations and other locally-scoped + variables. This rule reports variable declarations which do not match the regex that applies to their + specific kind (e.g. final variable, or catch-clause parameter). Each regex can be configured through + properties. + + By default this rule uses the standard Java naming convention (Camel case). + + 1 + + + + + + + + Fields, formal arguments, or local variable names that are too long can make the code difficult to follow. + + 3 + + + + + $minimum] +]]> + + + + + + + + + + + The EJB Specification states that any MessageDrivenBean or SessionBean should be suffixed by 'Bean'. + + 4 + + + + + + + + + + + + + + + A method argument that is never re-assigned within the method can be declared final. + + 3 + + + + + + + + Configurable naming conventions for method declarations. This rule reports + method declarations which do not match the regex that applies to their + specific kind (e.g. JUnit test or native method). Each regex can be + configured through properties. + + By default this rule uses the standard Java naming convention (Camel case). + + 1 + + + + + + + + Detects when a non-field has a name starting with 'm_'. This usually denotes a field and could be confusing. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the more general rule + {% rule java/codestyle/LocalVariableNamingConventions %}. + + 3 + + + + + + + + + + + + + + + Detects when a class or interface does not have a package definition. + + 3 + + + //ClassOrInterfaceDeclaration[count(preceding::PackageDeclaration) = 0] + + + + + + + + + + Since Java 1.7, numeric literals can use underscores to separate digits. This rule enforces that + numeric literals above a certain length use these underscores to increase readability. + + The rule only supports decimal (base 10) literals for now. The acceptable length under which literals + are not required to have underscores is configurable via a property. Even under that length, underscores + that are misplaced (not making groups of 3 digits) are reported. + + 3 + + + + + + + + + + + + + + + + + A method should have only one exit point, and that should be the last statement in the method. + + 3 + + 0) { + return "hey"; // first exit + } + return "hi"; // second exit + } +} +]]> + + + + + + Detects when a package definition contains uppercase characters. + + 3 + + + //PackageDeclaration/Name[lower-case(@Image)!=@Image] + + + + + + + + + + Checks for variables that are defined before they might be used. A reference is deemed to be premature if it is created right before a block of code that doesn't use it that also has the ability to return or throw an exception. + + 3 + + + + + + + + Remote Interface of a Session EJB should not have a suffix. + + 4 + + + + + + + + + + + + + + + A Remote Home interface type of a Session EJB should be suffixed by 'Home'. + + 4 + + + + + + + + + + + + + + + Short Classnames with fewer than e.g. five characters are not recommended. + + 4 + + + + + + + + + + + + + + + + Method names that are very short are not helpful to the reader. + + 3 + + + + + + + + + + + + + + + + Fields, local variables, or parameter names that are very short are not helpful to the reader. + + 3 + + + + + + + + + + + + + + + + + Field names using all uppercase characters - Sun's Java naming conventions indicating constants - should + be declared as final. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the more general rule {% rule java/codestyle/FieldNamingConventions %}. + + 3 + + + + + + + + + + + + + + + If you overuse the static import feature, it can make your program unreadable and + unmaintainable, polluting its namespace with all the static members you import. + Readers of your code (including you, a few months after you wrote it) will not know + which class a static member comes from (Sun 1.5 Language Guide). + + 3 + + + + + $maximumStaticImports] +]]> + + + + + + + + + + + + Avoid the use of value in annotations when it's the only element. + + 3 + + + + + + + + + This rule detects when a constructor is not necessary; i.e., when there is only one constructor and the + constructor is identical to the default constructor. The default constructor should has same access + modifier as the declaring class. In an enum type, the default constructor is implicitly private. + + 3 + + + + + + + + Import statements allow the use of non-fully qualified names. The use of a fully qualified name + which is covered by an import statement is redundant. Consider using the non-fully qualified name. + + 4 + + + + + + + + Avoid the creation of unnecessary local variables + + 3 + + + + + + + + Fields in interfaces and annotations are automatically `public static final`, and methods are `public abstract`. + Classes, interfaces or annotations nested in an interface or annotation are automatically `public static` + (all nested interfaces and annotations are automatically static). + Nested enums are automatically `static`. + For historical reasons, modifiers which are implied by the context are accepted by the compiler, but are superfluous. + + 3 + + + + + + + + Avoid the use of unnecessary return statements. + + 3 + + + + + + + + Use the diamond operator to let the type be inferred automatically. With the Diamond operator it is possible + to avoid duplication of the type parameters. + Instead, the compiler is now able to infer the parameter types for constructor calls, + which makes the code also more readable. + + 3 + + + + + + + + + strings = new ArrayList(); // unnecessary duplication of type parameters +List stringsWithDiamond = new ArrayList<>(); // using the diamond operator is more concise +]]> + + + + + Useless parentheses should be removed. + 4 + + + + 1] + /PrimaryPrefix/Expression + [not(./CastExpression)] + [not(./ConditionalExpression)] + [not(./AdditiveExpression)] + [not(./AssignmentOperator)] +| +//Expression[not(parent::PrimaryPrefix)]/PrimaryExpression[count(*)=1] + /PrimaryPrefix/Expression +| +//Expression/ConditionalAndExpression/PrimaryExpression/PrimaryPrefix/Expression[ + count(*)=1 and + count(./CastExpression)=0 and + count(./EqualityExpression/MultiplicativeExpression)=0 and + count(./ConditionalExpression)=0 and + count(./ConditionalOrExpression)=0] +| +//Expression/ConditionalOrExpression/PrimaryExpression/PrimaryPrefix/Expression[ + count(*)=1 and + not(./CastExpression) and + not(./ConditionalExpression) and + not(./EqualityExpression/MultiplicativeExpression)] +| +//Expression/ConditionalExpression/PrimaryExpression/PrimaryPrefix/Expression[ + count(*)=1 and + not(./CastExpression) and + not(./EqualityExpression)] +| +//Expression/AdditiveExpression[not(./PrimaryExpression/PrimaryPrefix/Literal[@StringLiteral='true'])] + /PrimaryExpression[1]/PrimaryPrefix/Expression[ + count(*)=1 and + not(./CastExpression) and + not(./AdditiveExpression[@Image = '-']) and + not(./ShiftExpression) and + not(./RelationalExpression) and + not(./InstanceOfExpression) and + not(./EqualityExpression) and + not(./AndExpression) and + not(./ExclusiveOrExpression) and + not(./InclusiveOrExpression) and + not(./ConditionalAndExpression) and + not(./ConditionalOrExpression) and + not(./ConditionalExpression)] +| +//Expression/EqualityExpression/PrimaryExpression/PrimaryPrefix/Expression[ + count(*)=1 and + not(./CastExpression) and + not(./AndExpression) and + not(./InclusiveOrExpression) and + not(./ExclusiveOrExpression) and + not(./ConditionalExpression) and + not(./ConditionalAndExpression) and + not(./ConditionalOrExpression) and + not(./EqualityExpression)] +]]> + + + + + + + + + + + Reports qualified this usages in the same class. + + 3 + + + + + + + + + + + + + + + A variable naming conventions rule - customize this to your liking. Currently, it + checks for final variables that should be fully capitalized and non-final variables + that should not include underscores. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the more general rules {% rule java/codestyle/FieldNamingConventions %}, + {% rule java/codestyle/FormalParameterNamingConventions %}, and + {% rule java/codestyle/LocalVariableNamingConventions %}. + + 1 + + + + + + + + Avoid using 'while' statements without using braces to surround the code block. If the code + formatting or indentation is lost then it becomes difficult to separate the code being + controlled from the rest. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the rule {% rule java/codestyle/ControlStatementBraces %}. + + 3 + + + //WhileStatement[not(Statement/Block)] + + + + + + + + diff --git a/config/pmd/category/java/design.xml b/config/pmd/category/java/design.xml new file mode 100644 index 0000000..ded3d80 --- /dev/null +++ b/config/pmd/category/java/design.xml @@ -0,0 +1,1657 @@ + + + + + + Rules that help you discover design issues. + + + + + If an abstract class does not provides any methods, it may be acting as a simple data container + that is not meant to be instantiated. In this case, it is probably better to use a private or + protected constructor in order to prevent instantiation than make the class misleadingly abstract. + + 1 + + + + + + + + + + + + + + + Avoid catching generic exceptions such as NullPointerException, RuntimeException, Exception in try-catch block + + 3 + + + + + + + + + + + + + + + Avoid creating deeply nested if-then statements since they are harder to read and error-prone to maintain. + + 3 + + y) { + if (y>z) { + if (z==x) { + // !! too deep + } + } + } + } +} +]]> + + + + + + Catch blocks that merely rethrow a caught exception only add to code size and runtime complexity. + + 3 + + + + + + + + + + + + + + + Catch blocks that merely rethrow a caught exception wrapped inside a new instance of the same type only add to + code size and runtime complexity. + + 3 + + + + + + + + + + + + + + + *Effective Java, 3rd Edition, Item 72: Favor the use of standard exceptions* +> +>Arguably, every erroneous method invocation boils down to an illegal argument or state, +but other exceptions are standardly used for certain kinds of illegal arguments and states. +If a caller passes null in some parameter for which null values are prohibited, convention dictates that +NullPointerException be thrown rather than IllegalArgumentException. + +To implement that, you are encouraged to use `java.util.Objects.requireNonNull()` +(introduced in Java 1.7). This method is designed primarily for doing parameter +validation in methods and constructors with multiple parameters. + +Your parameter validation could thus look like the following: +``` +public class Foo { + private String exampleValue; + + void setExampleValue(String exampleValue) { + // check, throw and assignment in a single standard call + this.exampleValue = Objects.requireNonNull(exampleValue, "exampleValue must not be null!"); + } + } +``` +]]> + + 1 + + + + + + + + + + + + + + + Avoid throwing certain exception types. Rather than throw a raw RuntimeException, Throwable, + Exception, or Error, use a subclassed exception or error instead. + + 1 + + + + + + + + + + + + + + + A class with only private constructors should be final, unless the private constructor + is invoked by a inner class. + + 1 + + + + = 1 ] +[count(./ClassOrInterfaceBody/ClassOrInterfaceBodyDeclaration/ConstructorDeclaration[(@Public = 'true') or (@Protected = 'true') or (@PackagePrivate = 'true')]) = 0 ] +[not(.//ClassOrInterfaceDeclaration)] +]]> + + + + + + + + + + + Sometimes two consecutive 'if' statements can be consolidated by separating their conditions with a boolean short-circuit operator. + + 3 + + + + + + + + + + + + + + + This rule counts the number of unique attributes, local variables, and return types within an object. + A number higher than the specified threshold can indicate a high degree of coupling. + + 3 + + + + + + + = 10. +Additionnally, classes with many methods of moderate complexity get reported as well once the total of their +methods' complexities reaches 80, even if none of the methods was directly reported. + +Reported methods should be broken down into several smaller methods. Reported classes should probably be broken down +into subcomponents.]]> + + 3 + + + + + + + + Data Classes are simple data holders, which reveal most of their state, and + without complex functionality. The lack of functionality may indicate that + their behaviour is defined elsewhere, which is a sign of poor data-behaviour + proximity. By directly exposing their internals, Data Classes break encapsulation, + and therefore reduce the system's maintainability and understandability. Moreover, + classes tend to strongly rely on their data representation, which makes for a brittle + design. + + Refactoring a Data Class should focus on restoring a good data-behaviour proximity. In + most cases, that means moving the operations defined on the data back into the class. + In some other cases it may make sense to remove entirely the class and move the data + into the former client classes. + + 3 + + + + + + + + Errors are system exceptions. Do not extend them. + + 3 + + + + + + + + + + + + + + + Using Exceptions as form of flow control is not recommended as they obscure true exceptions when debugging. + Either add the necessary validation or use an alternate control structure. + + 3 + + + + + + + + Excessive class file lengths are usually indications that the class may be burdened with excessive + responsibilities that could be provided by external classes or functions. In breaking these methods + apart the code becomes more manageable and ripe for reuse. + + 3 + + + + + + + + A high number of imports can indicate a high degree of coupling within an object. This rule + counts the number of unique imports and reports a violation if the count is above the + user-specified threshold. + + 3 + + + + + + + + When methods are excessively long this usually indicates that the method is doing more than its + name/signature might suggest. They also become challenging for others to digest since excessive + scrolling causes readers to lose focus. + Try to reduce the method length by creating helper methods and removing any copy/pasted code. + + 3 + + + + + + + + Methods with numerous parameters are a challenge to maintain, especially if most of them share the + same datatype. These situations usually denote the need for new objects to wrap the numerous parameters. + + 3 + + + + + + + + Classes with large numbers of public methods and attributes require disproportionate testing efforts + since combinational side effects grow rapidly and increase risk. Refactoring these classes into + smaller ones not only increases testability and reliability but also allows new variations to be + developed easily. + + 3 + + + + + + + + If a final field is assigned to a compile-time constant, it could be made static, thus saving overhead + in each object at runtime. + + 3 + + + + + + + + + + + + + + + The God Class rule detects the God Class design flaw using metrics. God classes do too many things, + are very big and overly complex. They should be split apart to be more object-oriented. + The rule uses the detection strategy described in "Object-Oriented Metrics in Practice". + The violations are reported against the entire class. + + See also the references: + + Michele Lanza and Radu Marinescu. Object-Oriented Metrics in Practice: + Using Software Metrics to Characterize, Evaluate, and Improve the Design + of Object-Oriented Systems. Springer, Berlin, 1 edition, October 2006. Page 80. + + 3 + + + + + Identifies private fields whose values never change once object initialization ends either in the declaration + of the field or by a constructor. This helps in converting existing classes to becoming immutable ones. + + 3 + + + + + + + + The Law of Demeter is a simple rule, that says "only talk to friends". It helps to reduce coupling between classes + or objects. + + See also the references: + + * Andrew Hunt, David Thomas, and Ward Cunningham. The Pragmatic Programmer. From Journeyman to Master. Addison-Wesley Longman, Amsterdam, October 1999.; + * K.J. Lieberherr and I.M. Holland. Assuring good style for object-oriented programs. Software, IEEE, 6(5):38–48, 1989.; + * <http://www.ccs.neu.edu/home/lieber/LoD.html> + * <http://en.wikipedia.org/wiki/Law_of_Demeter> + + 3 + + + + + + + + Use opposite operator instead of negating the whole expression with a logic complement operator. + + 3 + + + + + + + + + = + return false; + } + + return true; +} +]]> + + + + + + Avoid using classes from the configured package hierarchy outside of the package hierarchy, + except when using one of the configured allowed classes. + + 3 + + + + + + + + Complexity directly affects maintenance costs is determined by the number of decision points in a method + plus one for the method entry. The decision points include 'if', 'while', 'for', and 'case labels' calls. + Generally, numbers ranging from 1-4 denote low complexity, 5-7 denote moderate complexity, 8-10 denote + high complexity, and 11+ is very high complexity. Modified complexity treats switch statements as a single + decision point. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the rule {% rule java/design/CyclomaticComplexity %}. + + 3 + + + + + + + + This rule uses the NCSS (Non-Commenting Source Statements) algorithm to determine the number of lines + of code for a given constructor. NCSS ignores comments, and counts actual statements. Using this algorithm, + lines of code that are split are counted as one. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the rule {% rule java/design/NcssCount %}. + + 3 + + + + + + + + This rule uses the NCSS (Non-Commenting Source Statements) metric to determine the number of lines + of code in a class, method or constructor. NCSS ignores comments, blank lines, and only counts actual + statements. For more details on the calculation, see the documentation of + the [NCSS metric](/pmd_java_metrics_index.html#non-commenting-source-statements-ncss). + + 3 + + + + + + + + This rule uses the NCSS (Non-Commenting Source Statements) algorithm to determine the number of lines + of code for a given method. NCSS ignores comments, and counts actual statements. Using this algorithm, + lines of code that are split are counted as one. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the rule {% rule java/design/NcssCount %}. + + 3 + + + + + + + + This rule uses the NCSS (Non-Commenting Source Statements) algorithm to determine the number of lines + of code for a given type. NCSS ignores comments, and counts actual statements. Using this algorithm, + lines of code that are split are counted as one. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the rule {% rule java/design/NcssCount %}. + + 3 + + + + + + + + The NPath complexity of a method is the number of acyclic execution paths through that method. + While cyclomatic complexity counts the number of decision points in a method, NPath counts the number of + full paths from the beginning to the end of the block of the method. That metric grows exponentially, as + it multiplies the complexity of statements in the same block. For more details on the calculation, see the + documentation of the [NPath metric](/pmd_java_metrics_index.html#npath-complexity-npath). + + A threshold of 200 is generally considered the point where measures should be taken to reduce + complexity and increase readability. + + 3 + + + + + + + + A method/constructor shouldn't explicitly throw the generic java.lang.Exception, since it + is unclear which exceptions that can be thrown from the methods. It might be + difficult to document and understand such vague interfaces. Use either a class + derived from RuntimeException or a checked exception. + + 3 + + + + + + + + + + 3 + + + + + + + + + + + + + + + Avoid negation in an assertTrue or assertFalse test. + + For example, rephrase: + + assertTrue(!expr); + + as: + + assertFalse(expr); + + + 3 + + + + + + + + + + + + + + + Avoid unnecessary comparisons in boolean expressions, they serve no purpose and impacts readability. + + 3 + + + + + + + + + + + + + + + Avoid unnecessary if-then-else statements when returning a boolean. The result of + the conditional test can be returned instead. + + 3 + + + + + + + + No need to check for null before an instanceof; the instanceof keyword returns false when given a null argument. + + 3 + + + + + + + + + + + + + + + Fields whose scopes are limited to just single methods do not rely on the containing + object to provide them to other methods. They may be better implemented as local variables + within those methods. + + 3 + + + + + + + + Complexity directly affects maintenance costs is determined by the number of decision points in a method + plus one for the method entry. The decision points include 'if', 'while', 'for', and 'case labels' calls. + Generally, numbers ranging from 1-4 denote low complexity, 5-7 denote moderate complexity, 8-10 denote + high complexity, and 11+ is very high complexity. + + This rule is deprecated and will be removed with PMD 7.0.0. The rule is replaced + by the rule {% rule java/design/CyclomaticComplexity %}. + + 3 + + + + + + + + A high ratio of statements to labels in a switch statement implies that the switch statement + is overloaded. Consider moving the statements into new methods or creating subclasses based + on the switch variable. + + 3 + + + + + + + + Classes that have too many fields can become unwieldy and could be redesigned to have fewer fields, + possibly through grouping related fields in new objects. For example, a class with individual + city/state/zip fields could park them within a single Address field. + + 3 + + + + + + + + A class with too many methods is probably a good suspect for refactoring, in order to reduce its + complexity and find a way to have more fine grained objects. + + 3 + + + + + + $maxmethods + ] +]]> + + + + + + + + The overriding method merely calls the same method defined in a superclass. + + 3 + + + + + + + + When you write a public method, you should be thinking in terms of an API. If your method is public, it means other class + will use it, therefore, you want (or need) to offer a comprehensive and evolutive API. If you pass a lot of information + as a simple series of Strings, you may think of using an Object to represent all those information. You'll get a simpler + API (such as doWork(Workload workload), rather than a tedious series of Strings) and more importantly, if you need at some + point to pass extra data, you'll be able to do so by simply modifying or extending Workload without any modification to + your API. + + 3 + + + + 3 +] +]]> + + + + + + + + + + + For classes that only have static methods, consider making them utility classes. + Note that this doesn't apply to abstract classes, since their subclasses may + well include non-static methods. Also, if you want this class to be a utility class, + remember to add a private constructor to prevent instantiation. + (Note, that this use was known before PMD 5.1.0 as UseSingleton). + + 3 + + + + + + diff --git a/config/pmd/category/java/documentation.xml b/config/pmd/category/java/documentation.xml new file mode 100644 index 0000000..34b351a --- /dev/null +++ b/config/pmd/category/java/documentation.xml @@ -0,0 +1,144 @@ + + + + + + Rules that are related to code documentation. + + + + + A rule for the politically correct... we don't want to offend anyone. + + 3 + + + + + + + + Denotes whether comments are required (or unwanted) for specific language elements. + + 3 + + + + + + + + Determines whether the dimensions of non-header comments found are within the specified limits. + + 3 + + + + + + + + Uncommented Empty Constructor finds instances where a constructor does not + contain statements, but there is no comment. By explicitly commenting empty + constructors it is easier to distinguish between intentional (commented) + and unintentional empty constructors. + + 3 + + + + + + + + + + + + + + + + Uncommented Empty Method Body finds instances where a method body does not contain + statements, but there is no comment. By explicitly commenting empty method bodies + it is easier to distinguish between intentional (commented) and unintentional + empty methods. + + 3 + + + + + + + + + + + + + \ No newline at end of file diff --git a/config/pmd/category/java/errorprone.xml b/config/pmd/category/java/errorprone.xml new file mode 100644 index 0000000..cf289c3 --- /dev/null +++ b/config/pmd/category/java/errorprone.xml @@ -0,0 +1,3383 @@ + + + + + + Rules to detect constructs that are either broken, extremely confusing or prone to runtime errors. + + + + + Avoid assignments in operands; this can make code more complicated and harder to read. + + 3 + + + + + + + + Identifies a possible unsafe usage of a static field. + + 3 + + + + + + + + Methods such as getDeclaredConstructors(), getDeclaredConstructor(Class[]) and setAccessible(), + as the interface PrivilegedAction, allow for the runtime alteration of variable, class, or + method visibility, even if they are private. This violates the principle of encapsulation. + + 3 + + + + + + + + + + + + + + + Use of the term 'assert' will conflict with newer versions of Java since it is a reserved word. + + 2 + + + //VariableDeclaratorId[@Image='assert'] + + + + + + + + + + Using a branching statement as the last part of a loop may be a bug, and/or is confusing. + Ensure that the usage is not a bug, or consider using another approach. + + 2 + + 25) { + break; + } +} +]]> + + + + + + The method Object.finalize() is called by the garbage collector on an object when garbage collection determines + that there are no more references to the object. It should not be invoked by application logic. + + Note that Oracle has declared Object.finalize() as deprecated since JDK 9. + + 3 + + + + + + + + Code should never throw NullPointerExceptions under normal circumstances. A catch block may hide the + original error, causing other, more subtle problems later on. + + 3 + + + + + + + + + + + + + + + Catching Throwable errors is not recommended since its scope is very broad. It includes runtime issues such as + OutOfMemoryError that should be exposed and managed separately. + + 3 + + + + + + + + One might assume that the result of "new BigDecimal(0.1)" is exactly equal to 0.1, but it is actually + equal to .1000000000000000055511151231257827021181583404541015625. + This is because 0.1 cannot be represented exactly as a double (or as a binary fraction of any finite + length). Thus, the long value that is being passed in to the constructor is not exactly equal to 0.1, + appearances notwithstanding. + + The (String) constructor, on the other hand, is perfectly predictable: 'new BigDecimal("0.1")' is + exactly equal to 0.1, as one would expect. Therefore, it is generally recommended that the + (String) constructor be used in preference to this one. + + 3 + + + + + + + + + + + + + + + Code containing duplicate String literals can usually be improved by declaring the String as a constant field. + + 3 + + + + + + + + Use of the term 'enum' will conflict with newer versions of Java since it is a reserved word. + + 2 + + + //VariableDeclaratorId[@Image='enum'] + + + + + + + + + + It can be confusing to have a field name with the same name as a method. While this is permitted, + having information (field) and actions (method) is not clear naming. Developers versed in + Smalltalk often prefer this approach as the methods denote accessor methods. + + 3 + + + + + + + + It is somewhat confusing to have a field name matching the declaring class name. + This probably means that type and/or field names should be chosen more carefully. + + 3 + + + + + + + + Each caught exception type should be handled in its own catch clause. + + 3 + + + + + + + + + + + + + + + Avoid using hard-coded literals in conditional statements. By declaring them as static variables + or private members with descriptive names maintainability is enhanced. By default, the literals "-1" and "0" are ignored. + More exceptions can be defined with the property "ignoreMagicNumbers". + + 3 + + + + + + + + + + + = 0) { } // alternative approach + + if (aDouble > 0.0) {} // magic number 0.0 + if (aDouble >= Double.MIN_VALUE) {} // preferred approach +} +]]> + + + + + + Statements in a catch block that invoke accessors on the exception without using the information + only add to code size. Either remove the invocation, or use the return result. + + 2 + + + + + + + + + + + + + + + The use of multiple unary operators may be problematic, and/or confusing. + Ensure that the intended usage is not a bug, or consider simplifying the expression. + + 2 + + + + + + + + Integer literals should not start with zero since this denotes that the rest of literal will be + interpreted as an octal value. + + 3 + + + + + + + + Avoid equality comparisons with Double.NaN. Due to the implicit lack of representation + precision when comparing floating point numbers these are likely to cause logic errors. + + 3 + + + + + + + + + + + + + + + If a class is a bean, or is referenced by a bean directly or indirectly it needs to be serializable. + Member variables need to be marked as transient, static, or have accessor methods in the class. Marking + variables as transient is the safest and easiest modification. Accessor methods should follow the Java + naming conventions, i.e. for a variable named foo, getFoo() and setFoo() accessor methods should be provided. + + 3 + + + + + + + + The null check is broken since it will throw a NullPointerException itself. + It is likely that you used || instead of && or vice versa. + + 2 + + + + + + + Super should be called at the start of the method + 3 + + + + + + + + + + + + + + + Super should be called at the end of the method + + 3 + + + + + + + + + + + + + + + The skip() method may skip a smaller number of bytes than requested. Check the returned value to find out if it was the case or not. + + 3 + + + + + + + + When deriving an array of a specific class from your Collection, one should provide an array of + the same class as the parameter of the toArray() method. Doing otherwise you will will result + in a ClassCastException. + + 3 + + + + + + + + + + + + + + + The java Manual says "By convention, classes that implement this interface should override + Object.clone (which is protected) with a public method." + + 3 + + + + + + + + + + + + + + + The method clone() should only be implemented if the class implements the Cloneable interface with the exception of + a final method that only throws CloneNotSupportedException. + + The rule can also detect, if the class implements or extends a Cloneable class. + + 3 + + + + + + + + If a class implements cloneable the return type of the method clone() must be the class name. That way, the caller + of the clone method doesn't need to cast the returned clone to the correct type. + + Note: This is only possible with Java 1.5 or higher. + + 3 + + + + + + + + + + + + + + + The method clone() should throw a CloneNotSupportedException. + + 3 + + + + + + + + + + + + + + + Ensure that resources (like Connection, Statement, and ResultSet objects) are always closed after use. + + 3 + + + + + + + + Use equals() to compare object references; avoid comparing them with ==. + + 3 + + + + + + + + Calling overridable methods during construction poses a risk of invoking methods on an incompletely + constructed object and can be difficult to debug. + It may leave the sub-class unable to construct its superclass or forced to replicate the construction + process completely within itself, losing the ability to call super(). If the default constructor + contains a call to an overridable method, the subclass may be completely uninstantiable. Note that + this includes method calls throughout the control flow graph - i.e., if a constructor Foo() calls a + private method bar() that calls a public method buz(), this denotes a problem. + + 1 + + + + + + + The dataflow analysis tracks local definitions, undefinitions and references to variables on different paths on the data flow. + From those informations there can be found various problems. + + 1. UR - Anomaly: There is a reference to a variable that was not defined before. This is a bug and leads to an error. + 2. DU - Anomaly: A recently defined variable is undefined. These anomalies may appear in normal source text. + 3. DD - Anomaly: A recently defined variable is redefined. This is ominous but don't have to be a bug. + + 5 + + dd-anomaly + foo(buz); + buz = 2; +} // buz is undefined when leaving scope -> du-anomaly +]]> + + + + + + Calls to System.gc(), Runtime.getRuntime().gc(), and System.runFinalization() are not advised. Code should have the + same behavior whether the garbage collection is disabled using the option -Xdisableexplicitgc or not. + Moreover, "modern" jvms do a very good job handling garbage collections. If memory usage issues unrelated to memory + leaks develop within an application, it should be dealt with JVM options rather than within the code itself. + + 2 + + + + + + + + + + + + + + + Web applications should not call System.exit(), since only the web container or the + application server should stop the JVM. This rule also checks for the equivalent call Runtime.getRuntime().exit(). + + 3 + + + + + + + + + + + + + + + Extend Exception or RuntimeException instead of Throwable. + + 3 + + + + + + + + + + + + + + + Use Environment.getExternalStorageDirectory() instead of "/sdcard" + + 3 + + + //Literal[starts-with(@Image,'"/sdcard')] + + + + + + + + + + Throwing exceptions within a 'finally' block is confusing since they may mask other exceptions + or code defects. + Note: This is a PMD implementation of the Lint4j rule "A throw in a finally block" + + 4 + + + //FinallyStatement[descendant::ThrowStatement] + + + + + + + + + + Avoid importing anything from the 'sun.*' packages. These packages are not portable and are likely to change. + + 4 + + + + + + + + Don't use floating point for loop indices. If you must use floating point, use double + unless you're certain that float provides enough precision and you have a compelling + performance need (space or time). + + 3 + + + + + + + + + + + + + + + Empty Catch Block finds instances where an exception is caught, but nothing is done. + In most circumstances, this swallows an exception which should either be acted on + or reported. + + 3 + + + + + + + + + + + + + + + + + Empty finalize methods serve no purpose and should be removed. Note that Oracle has declared Object.finalize() as deprecated since JDK 9. + + 3 + + + + + + + + + + + + + + + Empty finally blocks serve no purpose and should be removed. + + 3 + + + + + + + + + + + + + + + Empty If Statement finds instances where a condition is checked but nothing is done about it. + + 3 + + + + + + + + + + + + + + + Empty initializers serve no purpose and should be removed. + + 3 + + + //Initializer/Block[count(*)=0] + + + + + + + + + + Empty block statements serve no purpose and should be removed. + + 3 + + + //BlockStatement/Statement/Block[count(*) = 0] + + + + + + + + + + An empty statement (or a semicolon by itself) that is not used as the sole body of a 'for' + or 'while' loop is probably a bug. It could also be a double semicolon, which has no purpose + and should be removed. + + 3 + + + + + + + + + + + + + + + Empty switch statements serve no purpose and should be removed. + + 3 + + + //SwitchStatement[count(*) = 1] + + + + + + + + + + Empty synchronized blocks serve no purpose and should be removed. + + 3 + + + //SynchronizedStatement/Block[1][count(*) = 0] + + + + + + + + + + Avoid empty try blocks - what's the point? + + 3 + + + + + + + + + + + + + + + Empty While Statement finds all instances where a while statement does nothing. + If it is a timing loop, then you should use Thread.sleep() for it; if it is + a while loop that does a lot in the exit expression, rewrite it to make it clearer. + + 3 + + + + + + + + + + + + + + + Tests for null should not use the equals() method. The '==' operator should be used instead. + + 1 + + + + + + + + + + + + + + + If the finalize() is implemented, its last action should be to call super.finalize. Note that Oracle has declared Object.finalize() as deprecated since JDK 9. + + 3 + + + + + + + + + + + + + + + + If the finalize() is implemented, it should do something besides just calling super.finalize(). Note that Oracle has declared Object.finalize() as deprecated since JDK 9. + + 3 + + + + + + + + + + + + + + + Methods named finalize() should not have parameters. It is confusing and most likely an attempt to + overload Object.finalize(). It will not be called by the VM. + + Note that Oracle has declared Object.finalize() as deprecated since JDK 9. + + 3 + + + + 0]] +]]> + + + + + + + + + + + When overriding the finalize(), the new method should be set as protected. If made public, + other classes may invoke it at inappropriate times. + + Note that Oracle has declared Object.finalize() as deprecated since JDK 9. + + 3 + + + + + + + + + + + + + + + Avoid idempotent operations - they have no effect. + + 3 + + + + + + + + There is no need to import a type that lives in the same package. + + 3 + + + + + + + + Avoid instantiating an object just to call getClass() on it; use the .class public member instead. + + 4 + + + + + + + + + + + + + + + Check for messages in slf4j loggers with non matching number of arguments and placeholders. + + 5 + + + + + + + + Avoid jumbled loop incrementers - its usually a mistake, and is confusing even if intentional. + + 3 + + + + + + + + + + + + + + Some JUnit framework methods are easy to misspell. + + 3 + + + + + + + + + + + + + + + The suite() method in a JUnit test needs to be both public and static. + + 3 + + + + + + + + + + + + + + + In most cases, the Logger reference can be declared as static and final. + + 2 + + + + + + + + + + + + + + + Non-constructor methods should not have the same name as the enclosing class. + + 3 + + + + + + + + The null check here is misplaced. If the variable is null a NullPointerException will be thrown. + Either the check is useless (the variable will never be "null") or it is incorrect. + + 3 + + + + + + + + + + + + + + + + + + Switch statements without break or return statements for each case option + may indicate problematic behaviour. Empty cases are ignored as these indicate an intentional fall-through. + + 3 + + + + + + + + + + + + + + + Serializable classes should provide a serialVersionUID field. + The serialVersionUID field is also needed for abstract base classes. Each individual class in the inheritance + chain needs an own serialVersionUID field. See also [Should an abstract class have a serialVersionUID](https://stackoverflow.com/questions/893259/should-an-abstract-class-have-a-serialversionuid). + + 3 + + + + + + + + + + + + + + + A class that has private constructors and does not have any static methods or fields cannot be used. + + 3 + + + + + + + + + + + + + + + Normally only one logger is used in each class. + + 2 + + + + + + + + A non-case label (e.g. a named break/continue label) was present in a switch statement. + This legal, but confusing. It is easy to mix up the case labels and the non-case labels. + + 3 + + + //SwitchStatement//BlockStatement/Statement/LabeledStatement + + + + + + + + + + A non-static initializer block will be called any time a constructor is invoked (just prior to + invoking the constructor). While this is a valid language construct, it is rarely used and is + confusing. + + 3 + + + + + + + + + + + + + + + Assigning a "null" to a variable (outside of its declaration) is usually bad form. Sometimes, this type + of assignment is an indication that the programmer doesn't completely understand what is going on in the code. + + NOTE: This sort of assignment may used in some cases to dereference objects and encourage garbage collection. + + 3 + + + + + + + + Override both public boolean Object.equals(Object other), and public int Object.hashCode(), or override neither. Even if you are inheriting a hashCode() from a parent class, consider implementing hashCode and explicitly delegating to your superclass. + + 3 + + + + + + + + Object clone() should be implemented with super.clone(). + + 2 + + + + 0 +] +]]> + + + + + + + + + + + A logger should normally be defined private static final and be associated with the correct class. + Private final Log log; is also allowed for rare cases where loggers need to be passed around, + with the restriction that the logger needs to be passed into the constructor. + + 3 + + + + + + + + + + + + + + + + For any method that returns an array, it is a better to return an empty array rather than a + null reference. This removes the need for null checking all results and avoids inadvertent + NullPointerExceptions. + + 1 + + + + + + + + + + + + + + + Avoid returning from a finally block, this can discard exceptions. + + 3 + + + + //FinallyStatement//ReturnStatement except //FinallyStatement//(MethodDeclaration|LambdaExpression)//ReturnStatement + + + + + + + + + + Be sure to specify a Locale when creating SimpleDateFormat instances to ensure that locale-appropriate + formatting is used. + + 3 + + + + + + + + + + + + + + + Some classes contain overloaded getInstance. The problem with overloaded getInstance methods + is that the instance created using the overloaded method is not cached and so, + for each call and new objects will be created for every invocation. + + 2 + + + + + + + + Some classes contain overloaded getInstance. The problem with overloaded getInstance methods + is that the instance created using the overloaded method is not cached and so, + for each call and new objects will be created for every invocation. + + 2 + + + + + + + + According to the J2EE specification, an EJB should not have any static fields + with write access. However, static read-only fields are allowed. This ensures proper + behavior especially when instances are distributed by the container on several JREs. + + 3 + + + + + + + + + + + + + + + Individual character values provided as initialization arguments will be converted into integers. + This can lead to internal buffer sizes that are larger than expected. Some examples: + + ``` + new StringBuffer() // 16 + new StringBuffer(6) // 6 + new StringBuffer("hello world") // 11 + 16 = 27 + new StringBuffer('A') // chr(A) = 65 + new StringBuffer("A") // 1 + 16 = 17 + + new StringBuilder() // 16 + new StringBuilder(6) // 6 + new StringBuilder("hello world") // 11 + 16 = 27 + new StringBuilder('C') // chr(C) = 67 + new StringBuilder("A") // 1 + 16 = 17 + ``` + + 4 + + + + + + + + + + + + + + + The method name and parameter number are suspiciously close to equals(Object), which can denote an + intention to override the equals(Object) method. + + 2 + + + + + + + + + + + + + + + The method name and return type are suspiciously close to hashCode(), which may denote an intention + to override the hashCode() method. + + 3 + + + + + + + + A suspicious octal escape sequence was found inside a String literal. + The Java language specification (section 3.10.6) says an octal + escape sequence inside a literal String shall consist of a backslash + followed by: + + OctalDigit | OctalDigit OctalDigit | ZeroToThree OctalDigit OctalDigit + + Any octal escape sequence followed by non-octal digits can be confusing, + e.g. "\038" is interpreted as the octal escape sequence "\03" followed by + the literal character "8". + + 3 + + + + + + + + Test classes end with the suffix Test. Having a non-test class with that name is not a good practice, + since most people will assume it is a test case. Test classes have test methods named testXXX. + + 3 + + + + + + + + Do not use "if" statements whose conditionals are always true or always false. + + 3 + + + + + + + + + + + + + + + A JUnit test assertion with a boolean literal is unnecessary since it always will evaluate to the same thing. + Consider using flow control (in case of assertTrue(false) or similar) or simply removing + statements like assertTrue(true) and assertFalse(false). If you just want a test to halt after finding + an error, use the fail() method and provide an indication message of why it did. + + 3 + + + + + + + + + + + + + + + Using equalsIgnoreCase() is faster than using toUpperCase/toLowerCase().equals() + + 3 + + + + + + + + Avoid the use temporary objects when converting primitives to Strings. Use the static conversion methods + on the wrapper classes instead. + + 3 + + + + + + + + After checking an object reference for null, you should invoke equals() on that object rather than passing it to another object's equals() method. + + 3 + + + + + + + + + + + + + + + To make sure the full stacktrace is printed out, use the logging statement with two arguments: a String and a Throwable. + + 3 + + + + + + + + + + + + + + + Using '==' or '!=' to compare strings only works if intern version is used on both sides. + Use the equals() method instead. + + 3 + + + + + + + + + + + + + + + An operation on an Immutable object (String, BigDecimal or BigInteger) won't change the object itself + since the result of the operation is a new object. Therefore, ignoring the operation result is an error. + + 3 + + + + + + + + When doing String.toLowerCase()/toUpperCase() conversions, use Locales to avoids problems with languages that + have unusual conventions, i.e. Turkish. + + 3 + + + + + + + + + + + + + + + In J2EE, the getClassLoader() method might not work as expected. Use + Thread.currentThread().getContextClassLoader() instead. + + 3 + + + //PrimarySuffix[@Image='getClassLoader'] + + + + + + + + diff --git a/config/pmd/category/java/multithreading.xml b/config/pmd/category/java/multithreading.xml new file mode 100644 index 0000000..d3e8327 --- /dev/null +++ b/config/pmd/category/java/multithreading.xml @@ -0,0 +1,393 @@ + + + + + + Rules that flag issues when dealing with multiple threads of execution. + + + + + Method-level synchronization can cause problems when new code is added to the method. + Block-level synchronization helps to ensure that only the code that needs synchronization + gets it. + + 3 + + + //MethodDeclaration[@Synchronized='true'] + + + + + + + + + + Avoid using java.lang.ThreadGroup; although it is intended to be used in a threaded environment + it contains methods that are not thread-safe. + + 3 + + + + + + + + + + + + + + + Use of the keyword 'volatile' is generally used to fine tune a Java application, and therefore, requires + a good expertise of the Java Memory Model. Moreover, its range of action is somewhat misknown. Therefore, + the volatile keyword should not be used for maintenance purpose and portability. + + 2 + + + //FieldDeclaration[contains(@Volatile,'true')] + + + + + + + + + + The J2EE specification explicitly forbids the use of threads. + + 3 + + + //ClassOrInterfaceType[@Image = 'Thread' or @Image = 'Runnable'] + + + + + + + + + + Explicitly calling Thread.run() method will execute in the caller's thread of control. Instead, call Thread.start() for the intended behavior. + + 4 + + + + + + + + + + + + + + + Partially created objects can be returned by the Double Checked Locking pattern when used in Java. + An optimizing JRE may assign a reference to the baz variable before it calls the constructor of the object the + reference points to. + + Note: With Java 5, you can make Double checked locking work, if you declare the variable to be `volatile`. + + For more details refer to: <http://www.javaworld.com/javaworld/jw-02-2001/jw-0209-double.html> + or <http://www.cs.umd.edu/~pugh/java/memoryModel/DoubleCheckedLocking.html> + + 1 + + + + + + + + Non-thread safe singletons can result in bad state changes. Eliminate + static singletons if possible by instantiating the object directly. Static + singletons are usually not needed as only a single instance exists anyway. + Other possible fixes are to synchronize the entire method or to use an + [initialize-on-demand holder class](https://en.wikipedia.org/wiki/Initialization-on-demand_holder_idiom). + + Refrain from using the double-checked locking pattern. The Java Memory Model doesn't + guarantee it to work unless the variable is declared as `volatile`, adding an uneeded + performance penalty. [Reference](http://www.cs.umd.edu/~pugh/java/memoryModel/DoubleCheckedLocking.html) + + See Effective Java, item 48. + + 3 + + + + + + + + SimpleDateFormat instances are not synchronized. Sun recommends using separate format instances + for each thread. If multiple threads must access a static formatter, the formatter must be + synchronized either on method or block level. + + This rule has been deprecated in favor of the rule {% rule UnsynchronizedStaticFormatter %}. + + 3 + + + + + + + + Instances of `java.text.Format` are generally not synchronized. + Sun recommends using separate format instances for each thread. + If multiple threads must access a static formatter, the formatter must be + synchronized either on method or block level. + + 3 + + + + + + + + Since Java5 brought a new implementation of the Map designed for multi-threaded access, you can + perform efficient map reads without blocking other threads. + + 3 + + + + + + + + + + + + + + + Thread.notify() awakens a thread monitoring the object. If more than one thread is monitoring, then only + one is chosen. The thread chosen is arbitrary; thus its usually safer to call notifyAll() instead. + + 3 + + + + + + + + + + + + + \ No newline at end of file diff --git a/config/pmd/category/java/performance.xml b/config/pmd/category/java/performance.xml new file mode 100644 index 0000000..1ce2d8d --- /dev/null +++ b/config/pmd/category/java/performance.xml @@ -0,0 +1,1006 @@ + + + + + + Rules that flag suboptimal code. + + + + + The conversion of literals to strings by concatenating them with empty strings is inefficient. + It is much better to use one of the type-specific toString() methods instead. + + 3 + + + + + + + + + + + + + + + Avoid concatenating characters as strings in StringBuffer/StringBuilder.append methods. + + 3 + + + + + + + + Instead of manually copying data between two arrays, use the efficient Arrays.copyOf or System.arraycopy method instead. + + 3 + + + + + + + + + + + + + + + The FileInputStream and FileOutputStream classes contains a finalizer method which will cause garbage + collection pauses. + See [JDK-8080225](https://bugs.openjdk.java.net/browse/JDK-8080225) for details. + + The FileReader and FileWriter constructors instantiate FileInputStream and FileOutputStream, + again causing garbage collection issues while finalizer methods are called. + + * Use `Files.newInputStream(Paths.get(fileName))` instead of `new FileInputStream(fileName)`. + * Use `Files.newOutputStream(Paths.get(fileName))` instead of `new FileOutputStream(fileName)`. + * Use `Files.newBufferedReader(Paths.get(fileName))` instead of `new FileReader(fileName)`. + * Use `Files.newBufferedWriter(Paths.get(fileName))` instead of `new FileWriter(fileName)`. + + Please note, that the `java.nio` API does not throw a `FileNotFoundException` anymore, instead + it throws a `NoSuchFileException`. If your code dealt explicitly with a `FileNotFoundException`, + then this needs to be adjusted. Both exceptions are subclasses of `IOException`, so catching + that one covers both. + + 1 + + + + + + + + + + + + + + + New objects created within loops should be checked to see if they can created outside them and reused. + + 3 + + + + + + + + Java uses the 'short' type to reduce memory usage, not to optimize calculation. In fact, the JVM does not have any + arithmetic capabilities for the short type: the JVM must convert the short into an int, do the proper calculation + and convert the int back to a short. Thus any storage gains found through use of the 'short' type may be offset by + adverse impacts on performance. + + 1 + + + + + + + + + + + + + + + Don't create instances of already existing BigInteger (BigInteger.ZERO, BigInteger.ONE) and + for Java 1.5 onwards, BigInteger.TEN and BigDecimal (BigDecimal.ZERO, BigDecimal.ONE, BigDecimal.TEN) + + 3 + + + + + + + + Avoid instantiating Boolean objects; you can reference Boolean.TRUE, Boolean.FALSE, or call Boolean.valueOf() instead. + Note that new Boolean() is deprecated since JDK 9 for that reason. + + 2 + + + + + + + + Calling new Byte() causes memory allocation that can be avoided by the static Byte.valueOf(). + It makes use of an internal cache that recycles earlier instances making it more memory efficient. + Note that new Byte() is deprecated since JDK 9 for that reason. + + 2 + + + + + + + + + + + + + + + Consecutive calls to StringBuffer/StringBuilder .append should be chained, reusing the target object. This can improve the performance + by producing a smaller bytecode, reducing overhead and improving inlining. A complete analysis can be found [here](https://github.com/pmd/pmd/issues/202#issuecomment-274349067) + + 3 + + + + + + + + Consecutively calling StringBuffer/StringBuilder.append(...) with literals should be avoided. + Since the literals are constants, they can already be combined into a single String literal and this String + can be appended in a single method call. + + 3 + + + + + + + + + + 3 + + 0) { + doSomething(); + } +} +]]> + + + + + + Avoid concatenating non-literals in a StringBuffer constructor or append() since intermediate buffers will + need to be be created and destroyed by the JVM. + + 3 + + + + + + + + Failing to pre-size a StringBuffer or StringBuilder properly could cause it to re-size many times + during runtime. This rule attempts to determine the total number the characters that are actually + passed into StringBuffer.append(), but represents a best guess "worst case" scenario. An empty + StringBuffer/StringBuilder constructor initializes the object to 16 characters. This default + is assumed if the length of the constructor can not be determined. + + 3 + + + + + + + + Calling new Integer() causes memory allocation that can be avoided by the static Integer.valueOf(). + It makes use of an internal cache that recycles earlier instances making it more memory efficient. + Note that new Integer() is deprecated since JDK 9 for that reason. + + 2 + + + + + + + + + + + + + + + Calling new Long() causes memory allocation that can be avoided by the static Long.valueOf(). + It makes use of an internal cache that recycles earlier instances making it more memory efficient. + Note that new Long() is deprecated since JDK 9 for that reason. + + 2 + + + + + + + + + + + + + + + Calls to a collection's `toArray(E[])` method should specify a target array of zero size. This allows the JVM + to optimize the memory allocation and copying as much as possible. + + Previous versions of this rule (pre PMD 6.0.0) suggested the opposite, but current JVM implementations + perform always better, when they have full control over the target array. And allocation an array via + reflection is nowadays as fast as the direct allocation. + + See also [Arrays of Wisdom of the Ancients](https://shipilev.net/blog/2016/arrays-wisdom-ancients/) + + Note: If you don't need an array of the correct type, then the simple `toArray()` method without an array + is faster, but returns only an array of type `Object[]`. + + 3 + + + + + + + + + foos = getFoos(); + +// much better; this one allows the jvm to allocate an array of the correct size and effectively skip +// the zeroing, since each array element will be overridden anyways +Foo[] fooArray = foos.toArray(new Foo[0]); + +// inefficient, the array needs to be zeroed out by the jvm before it is handed over to the toArray method +Foo[] fooArray = foos.toArray(new Foo[foos.size()]); +]]> + + + + + + Java will initialize fields with known default values so any explicit initialization of those same defaults + is redundant and results in a larger class file (approximately three additional bytecode instructions per field). + + 3 + + + + + + + + Since it passes in a literal of length 1, calls to (string).startsWith can be rewritten using (string).charAt(0) + at the expense of some readability. + + 3 + + + + + + + + + + + + + + + Calling new Short() causes memory allocation that can be avoided by the static Short.valueOf(). + It makes use of an internal cache that recycles earlier instances making it more memory efficient. + Note that new Short() is deprecated since JDK 9 for that reason. + + 2 + + + + + + + + + + + + + + + Avoid instantiating String objects; this is usually unnecessary since they are immutable and can be safely shared. + + 2 + + + + + + + + Avoid calling toString() on objects already known to be string instances; this is unnecessary. + + 3 + + + + + + + + Switch statements are intended to be used to support complex branching behaviour. Using a switch for only a few + cases is ill-advised, since switches are not as easy to understand as if-then statements. In these cases use the + if-then statement to increase code readability. + + 3 + + + + + + + + + + + + + + + + Most wrapper classes provide static conversion methods that avoid the need to create intermediate objects + just to create the primitive forms. Using these avoids the cost of creating objects that also need to be + garbage-collected later. + + 3 + + + + + + + + ArrayList is a much better Collection implementation than Vector if thread-safe operation is not required. + + 3 + + + + 0] + //AllocationExpression/ClassOrInterfaceType + [@Image='Vector' or @Image='java.util.Vector'] +]]> + + + + + + + + + + + (Arrays.asList(...)) if that is inconvenient for you (e.g. because of concurrent access). +]]> + + 3 + + + + + + + + + l= new ArrayList<>(100); + for (int i=0; i< 100; i++) { + l.add(ints[i]); + } + for (int i=0; i< 100; i++) { + l.add(a[i].toString()); // won't trigger the rule + } + } +} +]]> + + + + + + Use String.indexOf(char) when checking for the index of a single character; it executes faster. + + 3 + + + + + + + + No need to call String.valueOf to append to a string; just use the valueOf() argument directly. + + 3 + + + + + + + + The use of the '+=' operator for appending strings causes the JVM to create and use an internal StringBuffer. + If a non-trivial number of these concatenations are being used then the explicit use of a StringBuilder or + threadsafe StringBuffer is recommended to avoid this. + + 3 + + + + + + + + Use StringBuffer.length() to determine StringBuffer length rather than using StringBuffer.toString().equals("") + or StringBuffer.toString().length() == ... + + 3 + + + + + + + + + diff --git a/config/pmd/category/java/security.xml b/config/pmd/category/java/security.xml new file mode 100644 index 0000000..dbad352 --- /dev/null +++ b/config/pmd/category/java/security.xml @@ -0,0 +1,65 @@ + + + + + + Rules that flag potential security flaws. + + + + + Do not use hard coded values for cryptographic operations. Please store keys outside of source code. + + 3 + + + + + + + + Do not use hard coded initialization vector in cryptographic operations. Please use a randomly generated IV. + + 3 + + + + + + diff --git a/elx-api/build.gradle b/elx-api/build.gradle new file mode 100644 index 0000000..9e7b7be --- /dev/null +++ b/elx-api/build.gradle @@ -0,0 +1,19 @@ +dependencies { + compile "org.xbib:metrics:${project.property('xbib-metrics.version')}" + compile("org.elasticsearch:elasticsearch:${project.property('elasticsearch.version')}") { + // exclude ES jackson yaml, cbor, smile versions + exclude group: 'com.fasterxml.jackson.dataformat' + // dependencies that are not meant for client + exclude module: 'securesm' + // we use log4j2, not log4j + exclude group: 'log4j' + } + // override log4j2 of Elastic with ours + compile "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}" + // for Elasticsearch session, ES uses SMILE when encoding source for SearchRequest + compile "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${project.property('jackson-dataformat.version')}" + // CBOR ist default JSON content compression encoding in ES 2.2.1 + compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${project.property('jackson-dataformat.version')}" + // not used, but maybe in other projects + compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${project.property('jackson-dataformat.version')}" +} \ No newline at end of file diff --git a/elx-api/build.gradle~ b/elx-api/build.gradle~ new file mode 100644 index 0000000..02e43a4 --- /dev/null +++ b/elx-api/build.gradle~ @@ -0,0 +1,18 @@ + +dependencies { + compile("org.elasticsearch.client:transport:${rootProject.property('elasticsearch.version')}") { + exclude group: 'org.elasticsearch', module: 'securesm' + exclude group: 'org.elasticsearch.plugin', module: 'transport-netty3-client' + exclude group: 'org.elasticsearch.plugin', module: 'reindex-client' + exclude group: 'org.elasticsearch.plugin', module: 'percolator-client' + exclude group: 'org.elasticsearch.plugin', module: 'lang-mustache-client' + } + // we try to override the Elasticsearch netty by our netty version which might be more recent + compile "io.netty:netty-buffer:${rootProject.property('netty.version')}" + compile "io.netty:netty-codec-http:${rootProject.property('netty.version')}" + compile "io.netty:netty-handler:${rootProject.property('netty.version')}" +} + +jar { + baseName "${rootProject.name}-api" +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/BulkControl.java b/elx-api/src/main/java/org/xbib/elx/api/BulkControl.java similarity index 87% rename from src/main/java/org/xbib/elasticsearch/extras/client/BulkControl.java rename to elx-api/src/main/java/org/xbib/elx/api/BulkControl.java index 910f2f2..e0fcf84 100644 --- a/src/main/java/org/xbib/elasticsearch/extras/client/BulkControl.java +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkControl.java @@ -1,10 +1,8 @@ -package org.xbib.elasticsearch.extras.client; +package org.xbib.elx.api; import java.util.Map; import java.util.Set; -/** - */ public interface BulkControl { void startBulk(String indexName, long startRefreshInterval, long stopRefreshInterval); @@ -18,5 +16,4 @@ public interface BulkControl { Map getStartBulkRefreshIntervals(); Map getStopBulkRefreshIntervals(); - } diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/BulkMetric.java b/elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java similarity index 86% rename from src/main/java/org/xbib/elasticsearch/extras/client/BulkMetric.java rename to elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java index a45e9c2..3002b8c 100644 --- a/src/main/java/org/xbib/elasticsearch/extras/client/BulkMetric.java +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java @@ -1,11 +1,8 @@ -package org.xbib.elasticsearch.extras.client; +package org.xbib.elx.api; import org.xbib.metrics.Count; import org.xbib.metrics.Metered; -/** - * - */ public interface BulkMetric { Metered getTotalIngest(); @@ -27,5 +24,4 @@ public interface BulkMetric { void stop(); long elapsed(); - } diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/ClientMethods.java b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java similarity index 71% rename from src/main/java/org/xbib/elasticsearch/extras/client/ClientMethods.java rename to elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java index a683b63..4ba2496 100644 --- a/src/main/java/org/xbib/elasticsearch/extras/client/ClientMethods.java +++ b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java @@ -1,11 +1,11 @@ -package org.xbib.elasticsearch.extras.client; +package org.xbib.elx.api; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; - import java.io.IOException; import java.io.InputStream; import java.util.List; @@ -13,38 +13,53 @@ import java.util.Map; import java.util.concurrent.ExecutionException; /** - * Interface for providing convenient administrative methods for ingesting data into Elasticsearch. + * Interface for providing extended administrative methods for managing and ingesting Elasticsearch. */ -public interface ClientMethods extends Parameters { +public interface ExtendedClient { + + /** + * Set an Elasticsearch client to extend from it. May be null for TransportClient. + * @param client client + * @return an ELasticsearch client + */ + ExtendedClient setClient(ElasticsearchClient client); /** - * Initialize new ingest client, wrap an existing Elasticsearch client, and set up metrics. + * Return Elasticsearch client. * - * @param client the Elasticsearch client - * @param metric metric - * @param control control - * @return this ingest - * @throws IOException if client could not get created + * @return Elasticsearch client */ - ClientMethods init(ElasticsearchClient client, BulkMetric metric, BulkControl control) throws IOException; + ElasticsearchClient getClient(); + + ExtendedClient setBulkMetric(BulkMetric bulkMetric); + + BulkMetric getBulkMetric(); + + ExtendedClient setBulkControl(BulkControl bulkControl); + + BulkControl getBulkControl(); /** - * Initialize, create new ingest client, and set up metrics. + * Create new Elasticsearch client, wrap an existing Elasticsearch client. * * @param settings settings - * @param metric metric - * @param control control - * @return this ingest - * @throws IOException if client could not get created + * @return this client + * @throws IOException if init fails */ - ClientMethods init(Settings settings, BulkMetric metric, BulkControl control) throws IOException; + ExtendedClient init(Settings settings) throws IOException; /** - * Return Elasticsearch client. + * Bulked index request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. * - * @return Elasticsearch client + * @param index the index + * @param type the type + * @param id the id + * @param create true if document must be created + * @param source the source + * @return this */ - ElasticsearchClient client(); + ExtendedClient index(String index, String type, String id, boolean create, BytesReference source); /** * Index document. @@ -52,10 +67,20 @@ public interface ClientMethods extends Parameters { * @param index the index * @param type the type * @param id the id + * @param create true if document is to be created, false otherwise * @param source the source - * @return this + * @return this client methods + */ + ExtendedClient index(String index, String type, String id, boolean create, String source); + + /** + * Bulked index request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * + * @param indexRequest the index request to add + * @return this ingest */ - ClientMethods index(String index, String type, String id, String source); + ExtendedClient indexRequest(IndexRequest indexRequest); /** * Delete document. @@ -65,7 +90,29 @@ public interface ClientMethods extends Parameters { * @param id the id * @return this ingest */ - ClientMethods delete(String index, String type, String id); + ExtendedClient delete(String index, String type, String id); + + /** + * Bulked delete request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * + * @param deleteRequest the delete request to add + * @return this ingest + */ + ExtendedClient deleteRequest(DeleteRequest deleteRequest); + + /** + * Bulked update request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * Note that updates only work correctly when all operations between nodes are synchronized. + * + * @param index the index + * @param type the type + * @param id the id + * @param source the source + * @return this + */ + ExtendedClient update(String index, String type, String id, BytesReference source); /** * Update document. Use with precaution! Does not work in all cases. @@ -76,7 +123,17 @@ public interface ClientMethods extends Parameters { * @param source the source * @return this */ - ClientMethods update(String index, String type, String id, String source); + ExtendedClient update(String index, String type, String id, String source); + + /** + * Bulked update request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * Note that updates only work correctly when all operations between nodes are synchronized. + * + * @param updateRequest the update request to add + * @return this ingest + */ + ExtendedClient updateRequest(UpdateRequest updateRequest); /** * Set the maximum number of actions per request. @@ -84,7 +141,7 @@ public interface ClientMethods extends Parameters { * @param maxActionsPerRequest maximum number of actions per request * @return this ingest */ - ClientMethods maxActionsPerRequest(int maxActionsPerRequest); + ExtendedClient maxActionsPerRequest(int maxActionsPerRequest); /** * Set the maximum concurent requests. @@ -92,7 +149,7 @@ public interface ClientMethods extends Parameters { * @param maxConcurentRequests maximum number of concurrent ingest requests * @return this Ingest */ - ClientMethods maxConcurrentRequests(int maxConcurentRequests); + ExtendedClient maxConcurrentRequests(int maxConcurentRequests); /** * Set the maximum volume for request before flush. @@ -100,7 +157,7 @@ public interface ClientMethods extends Parameters { * @param maxVolume maximum volume * @return this ingest */ - ClientMethods maxVolumePerRequest(String maxVolume); + ExtendedClient maxVolumePerRequest(String maxVolume); /** * Set the flush interval for automatic flushing outstanding ingest requests. @@ -108,7 +165,7 @@ public interface ClientMethods extends Parameters { * @param flushInterval the flush interval, default is 30 seconds * @return this ingest */ - ClientMethods flushIngestInterval(String flushInterval); + ExtendedClient flushIngestInterval(String flushInterval); /** * Set mapping. @@ -141,7 +198,7 @@ public interface ClientMethods extends Parameters { * @param index index * @return this ingest */ - ClientMethods newIndex(String index); + ExtendedClient newIndex(String index); /** * Create a new index. @@ -153,7 +210,7 @@ public interface ClientMethods extends Parameters { * @return this ingest * @throws IOException if new index creation fails */ - ClientMethods newIndex(String index, String type, InputStream settings, InputStream mappings) throws IOException; + ExtendedClient newIndex(String index, String type, InputStream settings, InputStream mappings) throws IOException; /** * Create a new index. @@ -163,7 +220,7 @@ public interface ClientMethods extends Parameters { * @param mappings mappings * @return this ingest */ - ClientMethods newIndex(String index, Settings settings, Map mappings); + ExtendedClient newIndex(String index, Settings settings, Map mappings); /** * Create new mapping. @@ -173,7 +230,7 @@ public interface ClientMethods extends Parameters { * @param mapping mapping * @return this ingest */ - ClientMethods newMapping(String index, String type, Map mapping); + ExtendedClient newMapping(String index, String type, Map mapping); /** * Delete index. @@ -181,7 +238,7 @@ public interface ClientMethods extends Parameters { * @param index index * @return this ingest */ - ClientMethods deleteIndex(String index); + ExtendedClient deleteIndex(String index); /** * Start bulk mode. @@ -192,7 +249,8 @@ public interface ClientMethods extends Parameters { * @return this ingest * @throws IOException if bulk could not be started */ - ClientMethods startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds) throws IOException; + ExtendedClient startBulk(String index, long startRefreshIntervalSeconds, + long stopRefreshIntervalSeconds) throws IOException; /** * Stops bulk mode. @@ -201,42 +259,14 @@ public interface ClientMethods extends Parameters { * @return this Ingest * @throws IOException if bulk could not be stopped */ - ClientMethods stopBulk(String index) throws IOException; - - /** - * Bulked index request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when bulk limits are exceeded. - * - * @param indexRequest the index request to add - * @return this ingest - */ - ClientMethods bulkIndex(IndexRequest indexRequest); - - /** - * Bulked delete request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when bulk limits are exceeded. - * - * @param deleteRequest the delete request to add - * @return this ingest - */ - ClientMethods bulkDelete(DeleteRequest deleteRequest); - - /** - * Bulked update request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when bulk limits are exceeded. - * Note that updates only work correctly when all operations between nodes are synchronized! - * - * @param updateRequest the update request to add - * @return this ingest - */ - ClientMethods bulkUpdate(UpdateRequest updateRequest); + ExtendedClient stopBulk(String index) throws IOException; /** * Flush ingest, move all pending documents to the cluster. * * @return this */ - ClientMethods flushIngest(); + ExtendedClient flushIngest(); /** * Wait for all outstanding responses. @@ -246,7 +276,7 @@ public interface ClientMethods extends Parameters { * @throws InterruptedException if wait is interrupted * @throws ExecutionException if execution failed */ - ClientMethods waitForResponses(String maxWaitTime) throws InterruptedException, ExecutionException; + ExtendedClient waitForResponses(String maxWaitTime) throws InterruptedException, ExecutionException; /** * Refresh the index. @@ -363,11 +393,10 @@ public interface ClientMethods extends Parameters { Long mostRecentDocument(String index, String timestampfieldname) throws IOException; /** - * Get metric. - * - * @return metric + * Get cluster name. + * @return the cluster name */ - BulkMetric getMetric(); + String getClusterName(); /** * Returns true is a throwable exists. @@ -385,6 +414,7 @@ public interface ClientMethods extends Parameters { /** * Shutdown the ingesting. + * @throws IOException if shutdown fails */ - void shutdown(); + void shutdown() throws IOException; } diff --git a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java new file mode 100644 index 0000000..2a8904a --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java @@ -0,0 +1,7 @@ +package org.xbib.elx.api; + +@FunctionalInterface +public interface ExtendedClientProvider { + + C getExtendedClient(); +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/IndexAliasAdder.java b/elx-api/src/main/java/org/xbib/elx/api/IndexAliasAdder.java similarity index 80% rename from src/main/java/org/xbib/elasticsearch/extras/client/IndexAliasAdder.java rename to elx-api/src/main/java/org/xbib/elx/api/IndexAliasAdder.java index a659ab4..d92bca3 100644 --- a/src/main/java/org/xbib/elasticsearch/extras/client/IndexAliasAdder.java +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexAliasAdder.java @@ -1,10 +1,7 @@ -package org.xbib.elasticsearch.extras.client; +package org.xbib.elx.api; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; -/** - * - */ @FunctionalInterface public interface IndexAliasAdder { diff --git a/elx-api/src/main/java/org/xbib/elx/api/package-info.java b/elx-api/src/main/java/org/xbib/elx/api/package-info.java new file mode 100644 index 0000000..7991a43 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/package-info.java @@ -0,0 +1,4 @@ +/** + * The API of the Elasticsearch extensions. + */ +package org.xbib.elx.api; diff --git a/elx-common/build.gradle b/elx-common/build.gradle new file mode 100644 index 0000000..e794a8b --- /dev/null +++ b/elx-common/build.gradle @@ -0,0 +1,9 @@ +dependencies { + compile project(':elx-api') + compile "org.xbib:guice:${project.property('xbib-guice.version')}" + // add all dependencies to runtime source set, even that which are excluded by Elasticsearch jar, + // for metaprogramming. We are in Groovyland. + runtime "com.vividsolutions:jts:${project.property('jts.version')}" + runtime "com.github.spullara.mustache.java:compiler:${project.property('mustache.version')}" + runtime "net.java.dev.jna:jna:${project.property('jna.version')}" +} diff --git a/elx-common/build.gradle~ b/elx-common/build.gradle~ new file mode 100644 index 0000000..99099fb --- /dev/null +++ b/elx-common/build.gradle~ @@ -0,0 +1,65 @@ +buildscript { + repositories { + jcenter() + maven { + url 'http://xbib.org/repository' + } + } + dependencies { + classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.2.2.0" + } +} + +apply plugin: 'org.xbib.gradle.plugin.elasticsearch.build' + +configurations { + main + tests +} + +dependencies { + compile project(':api') + compile "org.xbib:metrics:${project.property('xbib-metrics.version')}" + compileOnly "org.apache.logging.log4j:log4j-api:${project.property('log4j.version')}" + testCompile "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" + testRuntime "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" +} + +jar { + baseName "${rootProject.name}-common" +} + +/* +task testJar(type: Jar, dependsOn: testClasses) { + baseName = "${project.archivesBaseName}-tests" + from sourceSets.test.output +} +*/ + +artifacts { + main jar + tests testJar + archives sourcesJar, javadocJar +} + +test { + enabled = false + jvmArgs "-javaagent:" + configurations.alpnagent.asPath + systemProperty 'path.home', project.buildDir.absolutePath + testLogging { + showStandardStreams = true + exceptionFormat = 'full' + } +} + +randomizedTest { + enabled = false +} + +esTest { + // test with the jars, not the classes, for security manager + // classpath = files(configurations.testRuntime) + configurations.main.artifacts.files + configurations.tests.artifacts.files + systemProperty 'tests.security.manager', 'true' +} +esTest.dependsOn jar, testJar + diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java new file mode 100644 index 0000000..69f2608 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java @@ -0,0 +1,1144 @@ +package org.xbib.elx.common; + +import com.carrotsearch.hppc.cursors.ObjectCursor; +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; +import org.elasticsearch.action.admin.indices.create.CreateIndexAction; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; +import org.elasticsearch.action.admin.indices.flush.FlushAction; +import org.elasticsearch.action.admin.indices.flush.FlushRequest; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequestBuilder; +import org.elasticsearch.action.admin.indices.get.GetIndexAction; +import org.elasticsearch.action.admin.indices.get.GetIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequestBuilder; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; +import org.elasticsearch.action.admin.indices.recovery.RecoveryAction; +import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest; +import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; +import org.xbib.elx.api.BulkControl; +import org.xbib.elx.api.BulkMetric; +import org.xbib.elx.api.ExtendedClient; +import org.xbib.elx.api.IndexAliasAdder; +import org.xbib.elx.common.management.IndexDefinition; +import org.xbib.elx.common.management.IndexRetention; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.io.StringWriter; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.existsQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; + +public abstract class AbstractExtendedClient implements ExtendedClient { + + private static final Logger logger = LogManager.getLogger(AbstractExtendedClient.class.getName()); + + private Map mappings; + + private ElasticsearchClient client; + + private BulkProcessor bulkProcessor; + + private BulkMetric bulkMetric; + + private BulkControl bulkControl; + + private Throwable throwable; + + private boolean closed; + + private int maxActionsPerRequest; + + private int maxConcurrentRequests; + + private String maxVolumePerRequest; + + private String flushIngestInterval; + + protected abstract ElasticsearchClient createClient(Settings settings) throws IOException; + + protected AbstractExtendedClient() { + maxActionsPerRequest = Parameters.DEFAULT_MAX_ACTIONS_PER_REQUEST.getNum(); + maxConcurrentRequests = Parameters.DEFAULT_MAX_CONCURRENT_REQUESTS.getNum(); + maxVolumePerRequest = Parameters.DEFAULT_MAX_VOLUME_PER_REQUEST.getString(); + flushIngestInterval = Parameters.DEFAULT_FLUSH_INTERVAL.getString(); + mappings = new HashMap<>(); + } + + @Override + public AbstractExtendedClient setClient(ElasticsearchClient client) { + this.client = client; + return this; + } + + @Override + public ElasticsearchClient getClient() { + return client; + } + + @Override + public AbstractExtendedClient setBulkMetric(BulkMetric metric) { + this.bulkMetric = metric; + return this; + } + + @Override + public BulkMetric getBulkMetric() { + return bulkMetric; + } + + @Override + public AbstractExtendedClient setBulkControl(BulkControl bulkControl) { + this.bulkControl = bulkControl; + return this; + } + + @Override + public BulkControl getBulkControl() { + return bulkControl; + } + + @Override + public AbstractExtendedClient init(Settings settings) throws IOException { + if (client == null) { + this.client = createClient(settings); + } + if (bulkMetric != null) { + bulkMetric.start(); + } + BulkProcessor.Listener listener = new BulkProcessor.Listener() { + + private final Logger logger = LogManager.getLogger("org.xbib.elx.BulkProcessor.Listener"); + + @Override + public void beforeBulk(long executionId, BulkRequest request) { + long l = 0; + if (bulkMetric != null) { + l = bulkMetric.getCurrentIngest().getCount(); + bulkMetric.getCurrentIngest().inc(); + int n = request.numberOfActions(); + bulkMetric.getSubmitted().inc(n); + bulkMetric.getCurrentIngestNumDocs().inc(n); + bulkMetric.getTotalIngestSizeInBytes().inc(request.estimatedSizeInBytes()); + } + logger.debug("before bulk [{}] [actions={}] [bytes={}] [concurrent requests={}]", + executionId, + request.numberOfActions(), + request.estimatedSizeInBytes(), + l); + } + + @Override + public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { + long l = 0; + if (bulkMetric != null) { + l = bulkMetric.getCurrentIngest().getCount(); + bulkMetric.getCurrentIngest().dec(); + bulkMetric.getSucceeded().inc(response.getItems().length); + } + int n = 0; + for (BulkItemResponse itemResponse : response.getItems()) { + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().dec(itemResponse.getIndex(), itemResponse.getType(), itemResponse.getId()); + } + if (itemResponse.isFailed()) { + n++; + if (bulkMetric != null) { + bulkMetric.getSucceeded().dec(1); + bulkMetric.getFailed().inc(1); + } + } + } + if (bulkMetric != null) { + logger.debug("after bulk [{}] [succeeded={}] [failed={}] [{}ms] {} concurrent requests", + executionId, + bulkMetric.getSucceeded().getCount(), + bulkMetric.getFailed().getCount(), + response.getTook().millis(), + l); + } + if (n > 0) { + logger.error("bulk [{}] failed with {} failed items, failure message = {}", + executionId, n, response.buildFailureMessage()); + } else { + if (bulkMetric != null) { + bulkMetric.getCurrentIngestNumDocs().dec(response.getItems().length); + } + } + } + + @Override + public void afterBulk(long executionId, BulkRequest request, Throwable failure) { + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().dec(); + } + throwable = failure; + closed = true; + logger.error("after bulk [" + executionId + "] error", failure); + } + }; + if (this.client != null) { + BulkProcessor.Builder builder = BulkProcessor.builder((Client)this.client, listener) + .setBulkActions(maxActionsPerRequest) + .setConcurrentRequests(maxConcurrentRequests) + .setFlushInterval(TimeValue.parseTimeValue(flushIngestInterval, null, "flushIngestInterval")); + if (maxVolumePerRequest != null) { + builder.setBulkSize(ByteSizeValue.parseBytesSizeValue(maxVolumePerRequest, "maxVolumePerRequest")); + } + this.bulkProcessor = builder.build(); + } + this.closed = false; + return this; + } + + @Override + public synchronized void shutdown() throws IOException { + ensureActive(); + if (bulkProcessor != null) { + logger.info("closing bulk processor..."); + bulkProcessor.close(); + } + if (bulkMetric != null) { + logger.info("stopping metric"); + bulkMetric.stop(); + } + if (bulkControl != null && bulkControl.indices() != null && !bulkControl.indices().isEmpty()) { + logger.info("stopping bulk mode for indices {}...", bulkControl.indices()); + for (String index : bulkControl.indices()) { + stopBulk(index); + } + } + } + + @Override + public ExtendedClient maxActionsPerRequest(int maxActionsPerRequest) { + this.maxActionsPerRequest = maxActionsPerRequest; + return this; + } + + @Override + public ExtendedClient maxConcurrentRequests(int maxConcurrentRequests) { + this.maxConcurrentRequests = maxConcurrentRequests; + return this; + } + + @Override + public ExtendedClient maxVolumePerRequest(String maxVolumePerRequest) { + this.maxVolumePerRequest = maxVolumePerRequest; + return this; + } + + @Override + public ExtendedClient flushIngestInterval(String flushIngestInterval) { + this.flushIngestInterval = flushIngestInterval; + return this; + } + + @Override + public ExtendedClient newIndex(String index) { + ensureActive(); + return newIndex(index, null, null); + } + + @Override + public ExtendedClient newIndex(String index, String type, InputStream settings, InputStream mappings) throws IOException { + mapping(type, mappings); + return newIndex(index, Settings.settingsBuilder().loadFromStream(".json", settings).build(), + this.mappings); + } + + @Override + public ExtendedClient newIndex(String index, Settings settings, Map mappings) { + ensureActive(); + if (index == null) { + logger.warn("no index name given to create index"); + return this; + } + CreateIndexRequestBuilder createIndexRequestBuilder = + new CreateIndexRequestBuilder(client, CreateIndexAction.INSTANCE).setIndex(index); + if (settings != null) { + logger.info("found settings {}", settings.getAsMap()); + createIndexRequestBuilder.setSettings(settings); + } + if (mappings != null) { + for (Map.Entry entry : mappings.entrySet()) { + String type = entry.getKey(); + String mapping = entry.getValue(); + logger.info("found mapping for {}", type); + createIndexRequestBuilder.addMapping(type, mapping, XContentType.JSON); + } + } + CreateIndexResponse createIndexResponse = createIndexRequestBuilder.execute().actionGet(); + logger.info("index {} created: {}", index, createIndexResponse); + return this; + } + + @Override + public ExtendedClient newMapping(String index, String type, Map mapping) { + PutMappingRequestBuilder putMappingRequestBuilder = + new PutMappingRequestBuilder(client, PutMappingAction.INSTANCE) + .setIndices(index) + .setType(type) + .setSource(mapping); + putMappingRequestBuilder.execute().actionGet(); + logger.info("mapping created for index {} and type {}", index, type); + return this; + } + + @Override + public ExtendedClient deleteIndex(String index) { + ensureActive(); + if (index == null) { + logger.warn("no index name given to delete index"); + return this; + } + DeleteIndexRequestBuilder deleteIndexRequestBuilder = + new DeleteIndexRequestBuilder(client, DeleteIndexAction.INSTANCE, index); + deleteIndexRequestBuilder.execute().actionGet(); + return this; + } + + @Override + public ExtendedClient startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds) + throws IOException { + ensureActive(); + if (bulkControl == null) { + return this; + } + if (!bulkControl.isBulk(index) && startRefreshIntervalSeconds > 0L && stopRefreshIntervalSeconds > 0L) { + bulkControl.startBulk(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds); + updateIndexSetting(index, "refresh_interval", startRefreshIntervalSeconds + "s"); + } + return this; + } + + @Override + public ExtendedClient stopBulk(String index) throws IOException { + ensureActive(); + if (bulkControl == null) { + return this; + } + if (bulkControl.isBulk(index)) { + long secs = bulkControl.getStopBulkRefreshIntervals().get(index); + if (secs > 0L) { + updateIndexSetting(index, "refresh_interval", secs + "s"); + } + bulkControl.finishBulk(index); + } + return this; + } + + @Override + public ExtendedClient flushIngest() { + ensureActive(); + logger.debug("flushing bulk processor"); + bulkProcessor.flush(); + return this; + } + + @Override + public ExtendedClient waitForResponses(String maxWaitTime) throws InterruptedException { + ensureActive(); + long millis = TimeValue.parseTimeValue(maxWaitTime, TimeValue.timeValueMinutes(1),"millis").getMillis(); + logger.debug("waiting for " + millis + " millis"); + while (!bulkProcessor.awaitClose(millis, TimeUnit.MILLISECONDS)) { + logger.warn("still waiting for responses"); + } + return this; + } + + @Override + public ExtendedClient index(String index, String type, String id, boolean create, BytesReference source) { + return indexRequest(new IndexRequest(index).type(type).id(id).create(create).source(source)); + } + + @Override + public ExtendedClient index(String index, String type, String id, boolean create, String source) { + return indexRequest(new IndexRequest(index).type(type).id(id).create(create).source(source.getBytes(StandardCharsets.UTF_8))); + } + + @Override + public ExtendedClient indexRequest(IndexRequest indexRequest) { + ensureActive(); + try { + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id()); + } + bulkProcessor.add(indexRequest); + } catch (Exception e) { + throwable = e; + closed = true; + logger.error("bulk add of index request failed: " + e.getMessage(), e); + } + return this; + } + + @Override + public ExtendedClient delete(String index, String type, String id) { + return deleteRequest(new DeleteRequest(index).type(type).id(id)); + } + + @Override + public ExtendedClient deleteRequest(DeleteRequest deleteRequest) { + ensureActive(); + try { + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().inc(deleteRequest.index(), deleteRequest.type(), deleteRequest.id()); + } + bulkProcessor.add(deleteRequest); + } catch (Exception e) { + throwable = e; + closed = true; + logger.error("bulk add of delete failed: " + e.getMessage(), e); + } + return this; + } + + @Override + public ExtendedClient update(String index, String type, String id, BytesReference source) { + return updateRequest(new UpdateRequest().index(index).type(type).id(id).upsert(source)); + } + + @Override + public ExtendedClient update(String index, String type, String id, String source) { + return updateRequest(new UpdateRequest().index(index).type(type).id(id).upsert(source.getBytes(StandardCharsets.UTF_8))); + } + + @Override + public ExtendedClient updateRequest(UpdateRequest updateRequest) { + ensureActive(); + try { + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().inc(updateRequest.index(), updateRequest.type(), updateRequest.id()); + } + bulkProcessor.add(updateRequest); + } catch (Exception e) { + throwable = e; + closed = true; + logger.error("bulk add of update request failed: " + e.getMessage(), e); + } + return this; + } + + @Override + public void mapping(String type, String mapping) { + mappings.put(type, mapping); + } + + @Override + public void mapping(String type, InputStream in) throws IOException { + if (type == null) { + return; + } + StringWriter sw = new StringWriter(); + Streams.copy(new InputStreamReader(in, StandardCharsets.UTF_8), sw); + mappings.put(type, sw.toString()); + } + + @Override + public int waitForRecovery(String index) throws IOException { + ensureActive(); + if (index == null) { + throw new IOException("unable to wait for recovery, no index no given"); + } + RecoveryResponse response = client.execute(RecoveryAction.INSTANCE, new RecoveryRequest(index)).actionGet(); + int shards = response.getTotalShards(); + client.execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest(index) + .waitForActiveShards(shards)).actionGet(); + return shards; + } + + @Override + public void waitForCluster(String statusString, String timeout) throws IOException { + ensureActive(); + ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString); + ClusterHealthResponse healthResponse = + client.execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest() + .waitForStatus(status).timeout(timeout)).actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + throw new IOException("cluster state is " + healthResponse.getStatus().name() + + " and not " + status.name() + + ", from here on, everything will fail!"); + } + } + + @Override + public String healthColor() { + ensureActive(); + try { + ClusterHealthResponse healthResponse = + client.execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().timeout(TimeValue.timeValueSeconds(30))).actionGet(); + ClusterHealthStatus status = healthResponse.getStatus(); + return status.name(); + } catch (ElasticsearchTimeoutException e) { + logger.warn(e.getMessage(), e); + return "TIMEOUT"; + } catch (NoNodeAvailableException e) { + logger.warn(e.getMessage(), e); + return "DISCONNECTED"; + } catch (Exception e) { + logger.warn(e.getMessage(), e); + return "[" + e.getMessage() + "]"; + } + } + + @Override + public int updateReplicaLevel(String index, int level) throws IOException { + waitForCluster("YELLOW", "30s"); + updateIndexSetting(index, "number_of_replicas", level); + return waitForRecovery(index); + } + + @Override + public void flushIndex(String index) { + ensureActive(); + if (index != null) { + client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet(); + } + } + + @Override + public void refreshIndex(String index) { + ensureActive(); + if (index != null) { + client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet(); + } + } + + @Override + public void putMapping(String index) { + ensureActive(); + if (mappings != null && !mappings.isEmpty()) { + for (Map.Entry me : mappings.entrySet()) { + client.execute(PutMappingAction.INSTANCE, + new PutMappingRequest(index).type(me.getKey()).source(me.getValue())).actionGet(); + } + } + } + + @Override + public String resolveAlias(String alias) { + ensureActive(); + GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client, GetAliasesAction.INSTANCE); + GetAliasesResponse getAliasesResponse = getAliasesRequestBuilder.setAliases(alias).execute().actionGet(); + if (!getAliasesResponse.getAliases().isEmpty()) { + return getAliasesResponse.getAliases().keys().iterator().next().value; + } + return alias; + } + + @Override + public String resolveMostRecentIndex(String alias) { + ensureActive(); + if (alias == null) { + return null; + } + GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client, GetAliasesAction.INSTANCE); + GetAliasesResponse getAliasesResponse = getAliasesRequestBuilder.setAliases(alias).execute().actionGet(); + Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); + Set indices = new TreeSet<>(Collections.reverseOrder()); + for (ObjectCursor indexName : getAliasesResponse.getAliases().keys()) { + Matcher m = pattern.matcher(indexName.value); + if (m.matches() && alias.equals(m.group(1))) { + indices.add(indexName.value); + } + } + return indices.isEmpty() ? alias : indices.iterator().next(); + } + + @Override + public Map getAliasFilters(String alias) { + GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client, GetAliasesAction.INSTANCE); + return getFilters(getAliasesRequestBuilder.setIndices(resolveAlias(alias)).execute().actionGet()); + } + + @Override + public void switchAliases(String index, String concreteIndex, List extraAliases) { + switchAliases(index, concreteIndex, extraAliases, null); + } + + @Override + public void switchAliases(String index, String concreteIndex, + List extraAliases, IndexAliasAdder adder) { + ensureActive(); + if (index.equals(concreteIndex)) { + return; + } + // two situations: 1. there is a new alias 2. there is already an old index with the alias + String oldIndex = resolveAlias(index); + final Map oldFilterMap = oldIndex.equals(index) ? null : getIndexFilters(oldIndex); + final List newAliases = new LinkedList<>(); + final List switchAliases = new LinkedList<>(); + IndicesAliasesRequestBuilder requestBuilder = new IndicesAliasesRequestBuilder(client, IndicesAliasesAction.INSTANCE); + if (oldFilterMap == null || !oldFilterMap.containsKey(index)) { + // never apply a filter for trunk index name + requestBuilder.addAlias(concreteIndex, index); + newAliases.add(index); + } + // switch existing aliases + if (oldFilterMap != null) { + for (Map.Entry entry : oldFilterMap.entrySet()) { + String alias = entry.getKey(); + String filter = entry.getValue(); + requestBuilder.removeAlias(oldIndex, alias); + if (filter != null) { + requestBuilder.addAlias(concreteIndex, alias, filter); + } else { + requestBuilder.addAlias(concreteIndex, alias); + } + switchAliases.add(alias); + } + } + // a list of aliases that should be added, check if new or old + if (extraAliases != null) { + for (String extraAlias : extraAliases) { + if (oldFilterMap == null || !oldFilterMap.containsKey(extraAlias)) { + // index alias adder only active on extra aliases, and if alias is new + if (adder != null) { + adder.addIndexAlias(requestBuilder, concreteIndex, extraAlias); + } else { + requestBuilder.addAlias(concreteIndex, extraAlias); + } + newAliases.add(extraAlias); + } else { + String filter = oldFilterMap.get(extraAlias); + requestBuilder.removeAlias(oldIndex, extraAlias); + if (filter != null) { + requestBuilder.addAlias(concreteIndex, extraAlias, filter); + } else { + requestBuilder.addAlias(concreteIndex, extraAlias); + } + switchAliases.add(extraAlias); + } + } + } + if (!newAliases.isEmpty() || !switchAliases.isEmpty()) { + logger.info("new aliases = {}, switch aliases = {}", newAliases, switchAliases); + requestBuilder.execute().actionGet(); + } + } + + @Override + public void performRetentionPolicy(String index, String concreteIndex, int timestampdiff, int mintokeep) { + if (timestampdiff == 0 && mintokeep == 0) { + return; + } + ensureActive(); + if (index.equals(concreteIndex)) { + return; + } + GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE); + GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet(); + Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); + Set indices = new TreeSet<>(); + logger.info("{} indices", getIndexResponse.getIndices().length); + for (String s : getIndexResponse.getIndices()) { + Matcher m = pattern.matcher(s); + if (m.matches() && index.equals(m.group(1)) && !s.equals(concreteIndex)) { + indices.add(s); + } + } + if (indices.isEmpty()) { + logger.info("no indices found, retention policy skipped"); + return; + } + if (mintokeep > 0 && indices.size() <= mintokeep) { + logger.info("{} indices found, not enough for retention policy ({}), skipped", + indices.size(), mintokeep); + return; + } else { + logger.info("candidates for deletion = {}", indices); + } + List indicesToDelete = new ArrayList<>(); + // our index + Matcher m1 = pattern.matcher(concreteIndex); + if (m1.matches()) { + Integer i1 = Integer.parseInt(m1.group(2)); + for (String s : indices) { + Matcher m2 = pattern.matcher(s); + if (m2.matches()) { + Integer i2 = Integer.parseInt(m2.group(2)); + int kept = indices.size() - indicesToDelete.size(); + if ((timestampdiff == 0 || (timestampdiff > 0 && i1 - i2 > timestampdiff)) && mintokeep <= kept) { + indicesToDelete.add(s); + } + } + } + } + logger.info("indices to delete = {}", indicesToDelete); + if (indicesToDelete.isEmpty()) { + logger.info("not enough indices found to delete, retention policy complete"); + return; + } + String[] s = indicesToDelete.toArray(new String[indicesToDelete.size()]); + DeleteIndexRequestBuilder requestBuilder = new DeleteIndexRequestBuilder(client, DeleteIndexAction.INSTANCE, s); + DeleteIndexResponse response = requestBuilder.execute().actionGet(); + if (!response.isAcknowledged()) { + logger.warn("retention delete index operation was not acknowledged"); + } + } + + @Override + public Long mostRecentDocument(String index, String timestampfieldname) { + ensureActive(); + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); + SortBuilder sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC); + SearchResponse searchResponse = searchRequestBuilder.setIndices(index) + .addField(timestampfieldname) + .setSize(1) + .addSort(sort) + .execute().actionGet(); + if (searchResponse.getHits().getHits().length == 1) { + SearchHit hit = searchResponse.getHits().getHits()[0]; + if (hit.getFields().get(timestampfieldname) != null) { + return hit.getFields().get(timestampfieldname).getValue(); + } else { + return 0L; + } + } + return null; + } + + @Override + public boolean hasThrowable() { + return throwable != null; + } + + @Override + public Throwable getThrowable() { + return throwable; + } + + private void updateIndexSetting(String index, String key, Object value) throws IOException { + ensureActive(); + if (index == null) { + throw new IOException("no index name given"); + } + if (key == null) { + throw new IOException("no key given"); + } + if (value == null) { + throw new IOException("no value given"); + } + Settings.Builder updateSettingsBuilder = Settings.settingsBuilder(); + updateSettingsBuilder.put(key, value.toString()); + UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index) + .settings(updateSettingsBuilder); + client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet(); + } + + private void ensureActive() { + if (this instanceof MockExtendedClient) { + return; + } + if (client == null) { + throw new IllegalStateException("no client"); + } + if (closed) { + throw new ElasticsearchException("client is closed"); + } + } + + public Map getIndexFilters(String index) { + GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client, GetAliasesAction.INSTANCE); + return getFilters(getAliasesRequestBuilder.setIndices(index).execute().actionGet()); + } + + private Map getFilters(GetAliasesResponse getAliasesResponse) { + Map result = new HashMap<>(); + for (ObjectObjectCursor> object : getAliasesResponse.getAliases()) { + List aliasMetaDataList = object.value; + for (AliasMetaData aliasMetaData : aliasMetaDataList) { + if (aliasMetaData.filteringRequired()) { + result.put(aliasMetaData.alias(), + new String(aliasMetaData.getFilter().uncompressed(), StandardCharsets.UTF_8) ); + } else { + result.put(aliasMetaData.alias(), null); + } + } + } + return result; + } + + @Override + public String getClusterName() { + ensureActive(); + try { + ClusterStateRequestBuilder clusterStateRequestBuilder = + new ClusterStateRequestBuilder(client, ClusterStateAction.INSTANCE).all(); + ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); + String name = clusterStateResponse.getClusterName().value(); + int nodeCount = clusterStateResponse.getState().getNodes().size(); + return name + " (" + nodeCount + " nodes connected)"; + } catch (ElasticsearchTimeoutException e) { + logger.warn(e.getMessage(), e); + return "TIMEOUT"; + } catch (NoNodeAvailableException e) { + logger.warn(e.getMessage(), e); + return "DISCONNECTED"; + } catch (Exception e) { + logger.warn(e.getMessage(), e); + return "[" + e.getMessage() + "]"; + } + } + + public IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) + throws MalformedURLException { + boolean isEnabled = settings.getAsBoolean("enabled", !(client instanceof MockExtendedClient)); + String indexName = settings.get("name", index); + String fullIndexName; + String dateTimePattern = settings.get("dateTimePattern"); + if (dateTimePattern != null) { + fullIndexName = resolveAlias(indexName + + DateTimeFormatter.ofPattern(dateTimePattern) + .withZone(ZoneId.systemDefault()) // not GMT + .format(LocalDate.now())); + logger.info("index name {} resolved to full index name = {}", indexName, fullIndexName); + } else { + fullIndexName = resolveMostRecentIndex(indexName); + logger.info("index name {} resolved to full index name = {}", indexName, fullIndexName); + } + IndexRetention indexRetention = new IndexRetention() + .setMinToKeep(settings.getAsInt("retention.mintokeep", 0)) + .setTimestampDiff(settings.getAsInt("retention.diff", 0)); + return new IndexDefinition() + .setIndex(indexName) + .setFullIndexName(fullIndexName) + .setType(settings.get("type")) + .setSettingsUrl(new URL(settings.get("settings"))) + .setMappingsUrl(new URL(settings.get("mapping"))) + .setDateTimePattern(dateTimePattern) + .setEnabled(isEnabled) + .setIgnoreErrors(settings.getAsBoolean("skiperrors", false)) + .setSwitchAliases(settings.getAsBoolean("aliases", true)) + .setReplicaLevel(settings.getAsInt("replica", 0)) + .setRetention(indexRetention); + } + + public void checkMapping(String index) { + ensureActive(); + GetMappingsRequestBuilder getMappingsRequestBuilder = new GetMappingsRequestBuilder(client, GetMappingsAction.INSTANCE) + .setIndices(index); + GetMappingsResponse getMappingsResponse = getMappingsRequestBuilder.execute().actionGet(); + ImmutableOpenMap> map = getMappingsResponse.getMappings(); + map.keys().forEach((Consumer>) stringObjectCursor -> { + ImmutableOpenMap mappings = map.get(stringObjectCursor.value); + for (ObjectObjectCursor cursor : mappings) { + String mappingName = cursor.key; + MappingMetaData mappingMetaData = cursor.value; + checkMapping(index, mappingName, mappingMetaData); + } + }); + } + + @SuppressWarnings("unchecked") + private void checkMapping(String index, String type, MappingMetaData mappingMetaData) { + try { + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); + SearchResponse searchResponse = searchRequestBuilder.setSize(0) + .setIndices(index) + .setTypes(type) + .setQuery(matchAllQuery()) + .execute() + .actionGet(); + long total = searchResponse.getHits().getTotalHits(); + if (total > 0L) { + Map fields = new TreeMap<>(); + Map root = mappingMetaData.getSourceAsMap(); + checkMapping(index, type, "", "", root, fields); + AtomicInteger empty = new AtomicInteger(); + Map map = sortByValue(fields); + map.forEach((key, value) -> { + logger.info("{} {} {}", + key, + value, + (double) value * 100 / total); + if (value == 0) { + empty.incrementAndGet(); + } + }); + logger.info("index={} type={} numfields={} fieldsnotused={}", + index, type, map.size(), empty.get()); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + @SuppressWarnings("unchecked") + private void checkMapping(String index, String type, + String pathDef, String fieldName, Map map, + Map fields) { + String path = pathDef; + if (!path.isEmpty() && !path.endsWith(".")) { + path = path + "."; + } + if (!"properties".equals(fieldName)) { + path = path + fieldName; + } + if (map.containsKey("index")) { + String mode = (String) map.get("index"); + if ("no".equals(mode)) { + return; + } + } + for (Map.Entry entry : map.entrySet()) { + String key = entry.getKey(); + Object o = entry.getValue(); + if (o instanceof Map) { + Map child = (Map) o; + o = map.get("type"); + String fieldType = o instanceof String ? o.toString() : null; + // do not recurse into our custom field mapper + if (!"standardnumber".equals(fieldType) && !"ref".equals(fieldType)) { + checkMapping(index, type, path, key, child, fields); + } + } else if ("type".equals(key)) { + QueryBuilder filterBuilder = existsQuery(path); + QueryBuilder queryBuilder = constantScoreQuery(filterBuilder); + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); + SearchResponse searchResponse = searchRequestBuilder.setSize(0) + .setIndices(index) + .setTypes(type) + .setQuery(queryBuilder) + .execute() + .actionGet(); + fields.put(path, searchResponse.getHits().totalHits()); + } + } + } + + private static > Map sortByValue(Map map) { + Map result = new LinkedHashMap<>(); + map.entrySet().stream().sorted(Comparator.comparing(Map.Entry::getValue)) + .forEachOrdered(e -> result.put(e.getKey(), e.getValue())); + return result; + } + + @SuppressWarnings("unchecked") + public void createIndex(IndexDefinition indexDefinition) + throws IOException { + ensureActive(); + waitForCluster("YELLOW", "30s"); + URL indexSettings = indexDefinition.getSettingsUrl(); + if (indexSettings == null) { + throw new IllegalArgumentException("no settings defined for index " + indexDefinition.getIndex()); + } + URL indexMappings = indexDefinition.getMappingsUrl(); + if (indexMappings == null) { + throw new IllegalArgumentException("no mappings defined for index " + indexDefinition.getIndex()); + } + try (InputStream indexSettingsInput = indexSettings.openStream(); + InputStream indexMappingsInput = indexMappings.openStream()) { + // multiple type? + if (indexDefinition.getType() == null) { + Map mapping = new HashMap<>(); + // get type names from input stream + Reader reader = new InputStreamReader(indexMappingsInput, StandardCharsets.UTF_8); + Map map = JsonXContent.jsonXContent.createParser(reader).mapOrdered(); + for (Map.Entry entry : map.entrySet()) { + mapping.put(entry.getKey(), JsonXContent.contentBuilder().map((Map) entry.getValue()).string()); + } + Settings settings = Settings.settingsBuilder() + .loadFromStream("", indexSettingsInput) + .build(); + newIndex(indexDefinition.getFullIndexName(), settings, mapping); + } else { + newIndex(indexDefinition.getFullIndexName(), + indexDefinition.getType(), indexSettingsInput, indexMappingsInput); + } + } catch (IOException e) { + if (indexDefinition.ignoreErrors()) { + logger.warn(e.getMessage(), e); + logger.warn("warning while creating index '{}' with settings at {} and mappings at {}", + indexDefinition.getFullIndexName(), indexSettings, indexMappings); + } else { + logger.error("error while creating index '{}' with settings at {} and mappings at {}", + indexDefinition.getFullIndexName(), indexSettings, indexMappings); + throw new IOException(e); + } + } + } + + public void startBulk(Map defs) throws IOException { + ensureActive(); + for (Map.Entry entry : defs.entrySet()) { + IndexDefinition def = entry.getValue(); + startBulk(def.getFullIndexName(), -1, 1); + } + } + + public void stopBulk(Map defs) throws IOException { + ensureActive(); + if (defs == null) { + return; + } + try { + logger.info("flush bulk"); + flushIngest(); + logger.info("waiting for all bulk responses from cluster"); + waitForResponses("120s"); + logger.info("all bulk responses received"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.error(e.getMessage(), e); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } finally { + logger.info("updating cluster settings of {}", defs.keySet()); + for (Map.Entry entry : defs.entrySet()) { + IndexDefinition def = entry.getValue(); + stopBulk(def.getFullIndexName()); + } + } + } + + public void forceMerge(Map defs) { + for (Map.Entry entry : defs.entrySet()) { + if (entry.getValue().hasForceMerge()) { + logger.info("force merge of {}", entry.getKey()); + try { + ForceMergeRequestBuilder forceMergeRequestBuilder = + new ForceMergeRequestBuilder(client, ForceMergeAction.INSTANCE); + forceMergeRequestBuilder.setIndices(entry.getValue().getFullIndexName()); + forceMergeRequestBuilder.execute().get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.error(e.getMessage(), e); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + } + } + + public void switchIndex(IndexDefinition indexDefinition, List extraAliases) { + if (extraAliases == null) { + return; + } + if (indexDefinition.isSwitchAliases()) { + // filter out null/empty values + List validAliases = extraAliases.stream() + .filter(a -> a != null && !a.isEmpty()) + .collect(Collectors.toList()); + try { + switchAliases(indexDefinition.getIndex(), + indexDefinition.getFullIndexName(), validAliases); + } catch (Exception e) { + logger.warn("switching index failed: " + e.getMessage(), e); + } + } + } + + public void switchIndex(IndexDefinition indexDefinition, + List extraAliases, IndexAliasAdder indexAliasAdder) { + if (extraAliases == null) { + return; + } + if (indexDefinition.isSwitchAliases()) { + // filter out null/empty values + List validAliases = extraAliases.stream() + .filter(a -> a != null && !a.isEmpty()) + .collect(Collectors.toList()); + try { + switchAliases(indexDefinition.getIndex(), + indexDefinition.getFullIndexName(), validAliases, indexAliasAdder); + } catch (Exception e) { + logger.warn("switching index failed: " + e.getMessage(), e); + } + } + } + + public void replica(IndexDefinition indexDefinition) { + if (indexDefinition.getReplicaLevel() > 0) { + try { + updateReplicaLevel(indexDefinition.getFullIndexName(), indexDefinition.getReplicaLevel()); + } catch (Exception e) { + logger.warn("setting replica failed: " + e.getMessage(), e); + } + } + } +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/BulkProcessor.java b/elx-common/src/main/java/org/xbib/elx/common/BulkProcessor.java similarity index 95% rename from src/main/java/org/xbib/elasticsearch/extras/client/BulkProcessor.java rename to elx-common/src/main/java/org/xbib/elx/common/BulkProcessor.java index b32637e..10f6e62 100644 --- a/src/main/java/org/xbib/elasticsearch/extras/client/BulkProcessor.java +++ b/elx-common/src/main/java/org/xbib/elx/common/BulkProcessor.java @@ -1,4 +1,4 @@ -package org.xbib.elasticsearch.extras.client; +package org.xbib.elx.common; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; @@ -248,11 +248,17 @@ public class BulkProcessor implements Closeable { public static class Builder { private final Client client; + private final Listener listener; + private String name; + private int concurrentRequests = 1; + private int bulkActions = 1000; + private ByteSizeValue bulkSize = new ByteSizeValue(5, ByteSizeUnit.MB); + private TimeValue flushInterval = null; /** @@ -281,7 +287,7 @@ public class BulkProcessor implements Closeable { /** * Sets the number of concurrent requests allowed to be executed. A value of 0 means that only a single * request will be allowed to be executed. A value of 1 means 1 concurrent request is allowed to be executed - * while accumulating new bulk requests. Defaults to 1. + * while accumulating new bulk requests. Defaults to {@code 1}. * * @param concurrentRequests maximum number of concurrent requests * @return this builder @@ -293,9 +299,9 @@ public class BulkProcessor implements Closeable { /** * Sets when to flush a new bulk request based on the number of actions currently added. Defaults to - * 1000. Can be set to -1 to disable it. + * {@code 1000}. Can be set to {@code -1} to disable it. * - * @param bulkActions mbulk actions + * @param bulkActions bulk actions * @return this builder */ public Builder setBulkActions(int bulkActions) { @@ -305,7 +311,7 @@ public class BulkProcessor implements Closeable { /** * Sets when to flush a new bulk request based on the size of actions currently added. Defaults to - * 5mb. Can be set to -1 to disable it. + * {@code 5mb}. Can be set to {@code -1} to disable it. * * @param bulkSize bulk size * @return this builder @@ -318,7 +324,7 @@ public class BulkProcessor implements Closeable { /** * Sets a flush interval flushing *any* bulk actions pending if the interval passes. Defaults to not set. * Note, both {@link #setBulkActions(int)} and {@link #setBulkSize(org.elasticsearch.common.unit.ByteSizeValue)} - * can be set to -1 with the flush interval set allowing for complete async processing of bulk actions. + * can be set to {@code -1} with the flush interval set allowing for complete async processing of bulk actions. * * @param flushInterval flush interval * @return this builder @@ -365,8 +371,10 @@ public class BulkProcessor implements Closeable { } - private class SyncBulkRequestHandler implements BulkRequestHandler { + private static class SyncBulkRequestHandler implements BulkRequestHandler { + private final Client client; + private final BulkProcessor.Listener listener; SyncBulkRequestHandler(Client client, BulkProcessor.Listener listener) { @@ -390,15 +398,19 @@ public class BulkProcessor implements Closeable { } @Override - public boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException { + public boolean awaitClose(long timeout, TimeUnit unit) { return true; } } - private class AsyncBulkRequestHandler implements BulkRequestHandler { + private static class AsyncBulkRequestHandler implements BulkRequestHandler { + private final Client client; + private final BulkProcessor.Listener listener; + private final Semaphore semaphore; + private final int concurrentRequests; private AsyncBulkRequestHandler(Client client, BulkProcessor.Listener listener, int concurrentRequests) { @@ -450,8 +462,8 @@ public class BulkProcessor implements Closeable { @Override public boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException { - if (semaphore.tryAcquire(this.concurrentRequests, timeout, unit)) { - semaphore.release(this.concurrentRequests); + if (semaphore.tryAcquire(concurrentRequests, timeout, unit)) { + semaphore.release(concurrentRequests); return true; } return false; diff --git a/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java b/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java new file mode 100644 index 0000000..cc15697 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java @@ -0,0 +1,124 @@ +package org.xbib.elx.common; + +import org.elasticsearch.Version; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.xbib.elx.api.BulkControl; +import org.xbib.elx.api.BulkMetric; +import org.xbib.elx.api.ExtendedClient; +import org.xbib.elx.api.ExtendedClientProvider; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.ServiceLoader; + +@SuppressWarnings("rawtypes") +public class ClientBuilder { + + private final ElasticsearchClient client; + + private final Settings.Builder settingsBuilder; + + private Map, ExtendedClientProvider> providerMap; + + private Class provider; + + private BulkMetric metric; + + private BulkControl control; + + public ClientBuilder() { + this(null); + } + + public ClientBuilder(ElasticsearchClient client) { + this(client, Thread.currentThread().getContextClassLoader()); + } + + public ClientBuilder(ElasticsearchClient client, ClassLoader classLoader) { + this.client = client; + this.settingsBuilder = Settings.builder(); + settingsBuilder.put("node.name", "elx-client-" + Version.CURRENT); + this.providerMap = new HashMap<>(); + ServiceLoader serviceLoader = ServiceLoader.load(ExtendedClientProvider.class, + classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader()); + for (ExtendedClientProvider provider : serviceLoader) { + providerMap.put(provider.getClass(), provider); + } + this.metric = new SimpleBulkMetric(); + this.control = new SimpleBulkControl(); + } + + public static ClientBuilder builder() { + return new ClientBuilder(); + } + + public static ClientBuilder builder(ElasticsearchClient client) { + return new ClientBuilder(client); + } + + public ClientBuilder provider(Class provider) { + this.provider = provider; + return this; + } + + public ClientBuilder put(String key, String value) { + settingsBuilder.put(key, value); + return this; + } + + public ClientBuilder put(String key, Integer value) { + settingsBuilder.put(key, value); + return this; + } + + public ClientBuilder put(String key, Long value) { + settingsBuilder.put(key, value); + return this; + } + + public ClientBuilder put(String key, Double value) { + settingsBuilder.put(key, value); + return this; + } + + public ClientBuilder put(String key, ByteSizeValue value) { + settingsBuilder.put(key, value); + return this; + } + + public ClientBuilder put(String key, TimeValue value) { + settingsBuilder.put(key, value); + return this; + } + + public ClientBuilder put(Settings settings) { + settingsBuilder.put(settings); + return this; + } + + public ClientBuilder setMetric(BulkMetric metric) { + this.metric = metric; + return this; + } + + public ClientBuilder setControl(BulkControl control) { + this.control = control; + return this; + } + + @SuppressWarnings("unchecked") + public C build() throws IOException { + if (provider == null) { + throw new IllegalArgumentException("no provider"); + } + return (C) providerMap.get(provider).getExtendedClient() + .setClient(client) + .setBulkMetric(metric) + .setBulkControl(control) + .init(settingsBuilder.build()); + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java new file mode 100644 index 0000000..dc01807 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java @@ -0,0 +1,146 @@ +package org.xbib.elx.common; + +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; + +import java.util.Map; + +/** + * Mock client, it does not perform actions on a cluster. Useful for testing or dry runs. + */ +public class MockExtendedClient extends AbstractExtendedClient { + + @Override + public ElasticsearchClient getClient() { + return null; + } + + @Override + public MockExtendedClient init(Settings settings) { + return this; + } + + @Override + protected ElasticsearchClient createClient(Settings settings) { + return null; + } + + @Override + public MockExtendedClient maxActionsPerRequest(int maxActions) { + return this; + } + + @Override + public MockExtendedClient maxConcurrentRequests(int maxConcurrentRequests) { + return this; + } + + @Override + public MockExtendedClient maxVolumePerRequest(String maxVolumePerRequest) { + return this; + } + + @Override + public MockExtendedClient flushIngestInterval(String interval) { + return this; + } + + @Override + public MockExtendedClient index(String index, String type, String id, boolean create, String source) { + return this; + } + + @Override + public MockExtendedClient delete(String index, String type, String id) { + return this; + } + + @Override + public MockExtendedClient update(String index, String type, String id, String source) { + return this; + } + + @Override + public MockExtendedClient indexRequest(IndexRequest indexRequest) { + return this; + } + + @Override + public MockExtendedClient deleteRequest(DeleteRequest deleteRequest) { + return this; + } + + @Override + public MockExtendedClient updateRequest(UpdateRequest updateRequest) { + return this; + } + + @Override + public MockExtendedClient flushIngest() { + return this; + } + + @Override + public MockExtendedClient waitForResponses(String timeValue) { + return this; + } + + @Override + public MockExtendedClient startBulk(String index, long startRefreshInterval, long stopRefreshIterval) { + return this; + } + + @Override + public MockExtendedClient stopBulk(String index) { + return this; + } + + @Override + public MockExtendedClient deleteIndex(String index) { + return this; + } + + @Override + public MockExtendedClient newIndex(String index) { + return this; + } + + @Override + public MockExtendedClient newMapping(String index, String type, Map mapping) { + return this; + } + + @Override + public void putMapping(String index) { + } + + @Override + public void refreshIndex(String index) { + } + + @Override + public void flushIndex(String index) { + } + + @Override + public void waitForCluster(String healthColor, String timeValue) { + } + + @Override + public int waitForRecovery(String index) { + return -1; + } + + @Override + public int updateReplicaLevel(String index, int level) { + return -1; + } + + @Override + public void shutdown() { + // nothing to do + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java new file mode 100644 index 0000000..87e65cc --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java @@ -0,0 +1,10 @@ +package org.xbib.elx.common; + +import org.xbib.elx.api.ExtendedClientProvider; + +public class MockExtendedClientProvider implements ExtendedClientProvider { + @Override + public MockExtendedClient getExtendedClient() { + return new MockExtendedClient(); + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/Parameters.java b/elx-common/src/main/java/org/xbib/elx/common/Parameters.java new file mode 100644 index 0000000..017c780 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/Parameters.java @@ -0,0 +1,40 @@ +package org.xbib.elx.common; + +public enum Parameters { + + DEFAULT_MAX_ACTIONS_PER_REQUEST(1000), + + DEFAULT_MAX_CONCURRENT_REQUESTS(Runtime.getRuntime().availableProcessors()), + + DEFAULT_MAX_VOLUME_PER_REQUEST("10mb"), + + DEFAULT_FLUSH_INTERVAL("30s"), + + MAX_ACTIONS_PER_REQUEST ("max_actions_per_request"), + + MAX_CONCURRENT_REQUESTS("max_concurrent_requests"), + + MAX_VOLUME_PER_REQUEST("max_volume_per_request"), + + FLUSH_INTERVAL("flush_interval"); + + int num; + + String string; + + Parameters(int num) { + this.num = num; + } + + Parameters(String string) { + this.string = string; + } + + int getNum() { + return num; + } + + String getString() { + return string; + } +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkControl.java b/elx-common/src/main/java/org/xbib/elx/common/SimpleBulkControl.java similarity index 95% rename from src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkControl.java rename to elx-common/src/main/java/org/xbib/elx/common/SimpleBulkControl.java index b9a92d6..624cec5 100644 --- a/src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkControl.java +++ b/elx-common/src/main/java/org/xbib/elx/common/SimpleBulkControl.java @@ -1,4 +1,6 @@ -package org.xbib.elasticsearch.extras.client; +package org.xbib.elx.common; + +import org.xbib.elx.api.BulkControl; import java.util.HashMap; import java.util.HashSet; diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkMetric.java b/elx-common/src/main/java/org/xbib/elx/common/SimpleBulkMetric.java similarity index 55% rename from src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkMetric.java rename to elx-common/src/main/java/org/xbib/elx/common/SimpleBulkMetric.java index e836816..1a9dd35 100644 --- a/src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkMetric.java +++ b/elx-common/src/main/java/org/xbib/elx/common/SimpleBulkMetric.java @@ -1,32 +1,48 @@ -package org.xbib.elasticsearch.extras.client; +package org.xbib.elx.common; +import org.xbib.elx.api.BulkMetric; import org.xbib.metrics.Count; import org.xbib.metrics.CountMetric; import org.xbib.metrics.Meter; import org.xbib.metrics.Metered; -/** - * - */ + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + public class SimpleBulkMetric implements BulkMetric { - private final Meter totalIngest = new Meter(); + private final Meter totalIngest; - private final Count totalIngestSizeInBytes = new CountMetric(); + private final Count totalIngestSizeInBytes; - private final Count currentIngest = new CountMetric(); + private final Count currentIngest; - private final Count currentIngestNumDocs = new CountMetric(); + private final Count currentIngestNumDocs; - private final Count submitted = new CountMetric(); + private final Count submitted; - private final Count succeeded = new CountMetric(); + private final Count succeeded; - private final Count failed = new CountMetric(); + private final Count failed; private Long started; private Long stopped; + public SimpleBulkMetric() { + this(Executors.newSingleThreadScheduledExecutor()); + } + + public SimpleBulkMetric(ScheduledExecutorService executorService) { + totalIngest = new Meter(executorService); + totalIngestSizeInBytes = new CountMetric(); + currentIngest = new CountMetric(); + currentIngestNumDocs = new CountMetric(); + submitted = new CountMetric(); + succeeded = new CountMetric(); + failed = new CountMetric(); + } + @Override public Metered getTotalIngest() { return totalIngest; @@ -65,7 +81,7 @@ public class SimpleBulkMetric implements BulkMetric { @Override public void start() { this.started = System.nanoTime(); - totalIngest.spawn(5L); + totalIngest.start(5L); } @Override diff --git a/elx-common/src/main/java/org/xbib/elx/common/io/ClasspathURLStreamHandler.java b/elx-common/src/main/java/org/xbib/elx/common/io/ClasspathURLStreamHandler.java new file mode 100644 index 0000000..e7d8727 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/io/ClasspathURLStreamHandler.java @@ -0,0 +1,25 @@ +package org.xbib.elx.common.io; + +import java.io.IOException; +import java.net.URL; +import java.net.URLConnection; +import java.net.URLStreamHandler; + +public class ClasspathURLStreamHandler extends URLStreamHandler { + + private final ClassLoader classLoader; + + public ClasspathURLStreamHandler() { + this.classLoader = getClass().getClassLoader(); + } + + public ClasspathURLStreamHandler(ClassLoader classLoader) { + this.classLoader = classLoader; + } + + @Override + protected URLConnection openConnection(URL u) throws IOException { + final URL resourceUrl = classLoader.getResource(u.getPath()); + return resourceUrl != null ? resourceUrl.openConnection() : null; + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/io/ClasspathURLStreamHandlerFactory.java b/elx-common/src/main/java/org/xbib/elx/common/io/ClasspathURLStreamHandlerFactory.java new file mode 100644 index 0000000..00c7c83 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/io/ClasspathURLStreamHandlerFactory.java @@ -0,0 +1,12 @@ +package org.xbib.elx.common.io; + +import java.net.URLStreamHandler; +import java.net.URLStreamHandlerFactory; + +public class ClasspathURLStreamHandlerFactory implements URLStreamHandlerFactory { + + @Override + public URLStreamHandler createURLStreamHandler(String protocol) { + return "classpath".equals(protocol) ? new ClasspathURLStreamHandler() : null; + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/io/package-info.java b/elx-common/src/main/java/org/xbib/elx/common/io/package-info.java new file mode 100644 index 0000000..62a8d78 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/io/package-info.java @@ -0,0 +1 @@ +package org.xbib.elx.common.io; \ No newline at end of file diff --git a/elx-common/src/main/java/org/xbib/elx/common/management/IndexDefinition.java b/elx-common/src/main/java/org/xbib/elx/common/management/IndexDefinition.java new file mode 100644 index 0000000..37dcc45 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/management/IndexDefinition.java @@ -0,0 +1,139 @@ +package org.xbib.elx.common.management; + +import java.net.URL; + +public class IndexDefinition { + + private String index; + + private String type; + + private String fullIndexName; + + private String dateTimePattern; + + private URL settingsUrl; + + private URL mappingsUrl; + + private boolean enabled; + + private boolean ignoreErrors; + + private boolean switchAliases; + + private boolean hasForceMerge; + + private int replicaLevel; + + private IndexRetention indexRetention; + + public IndexDefinition setIndex(String index) { + this.index = index; + return this; + } + + public String getIndex() { + return index; + } + + public IndexDefinition setFullIndexName(String fullIndexName) { + this.fullIndexName = fullIndexName; + return this; + } + + public String getFullIndexName() { + return fullIndexName; + } + + public IndexDefinition setType(String type) { + this.type = type; + return this; + } + + public String getType() { + return type; + } + + public IndexDefinition setSettingsUrl(URL settingsUrl) { + this.settingsUrl = settingsUrl; + return this; + } + + public URL getSettingsUrl() { + return settingsUrl; + } + + public IndexDefinition setMappingsUrl(URL mappingsUrl) { + this.mappingsUrl = mappingsUrl; + return this; + } + + public URL getMappingsUrl() { + return mappingsUrl; + } + + public IndexDefinition setDateTimePattern(String timeWindow) { + this.dateTimePattern = timeWindow; + return this; + } + + public String getDateTimePattern() { + return dateTimePattern; + } + + public IndexDefinition setEnabled(boolean enabled) { + this.enabled = enabled; + return this; + } + + public boolean isEnabled() { + return enabled; + } + + public IndexDefinition setIgnoreErrors(boolean ignoreErrors) { + this.ignoreErrors = ignoreErrors; + return this; + } + + public boolean ignoreErrors() { + return ignoreErrors; + } + + public IndexDefinition setSwitchAliases(boolean switchAliases) { + this.switchAliases = switchAliases; + return this; + } + + public boolean isSwitchAliases() { + return switchAliases; + } + + public IndexDefinition setForceMerge(boolean hasForceMerge) { + this.hasForceMerge = hasForceMerge; + return this; + } + + public boolean hasForceMerge() { + return hasForceMerge; + } + + public IndexDefinition setReplicaLevel(int replicaLevel) { + this.replicaLevel = replicaLevel; + return this; + } + + public int getReplicaLevel() { + return replicaLevel; + } + + public IndexDefinition setRetention(IndexRetention indexRetention) { + this.indexRetention = indexRetention; + return this; + } + + public IndexRetention getRetention() { + return indexRetention; + } + +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/management/IndexRetention.java b/elx-common/src/main/java/org/xbib/elx/common/management/IndexRetention.java new file mode 100644 index 0000000..8024ef4 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/management/IndexRetention.java @@ -0,0 +1,27 @@ +package org.xbib.elx.common.management; + +public class IndexRetention { + + private int timestampDiff; + + private int minToKeep; + + public IndexRetention setTimestampDiff(int timestampDiff) { + this.timestampDiff = timestampDiff; + return this; + } + + public int getTimestampDiff() { + return timestampDiff; + } + + public IndexRetention setMinToKeep(int minToKeep) { + this.minToKeep = minToKeep; + return this; + } + + public int getMinToKeep() { + return minToKeep; + } + +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/management/package-info.java b/elx-common/src/main/java/org/xbib/elx/common/management/package-info.java new file mode 100644 index 0000000..0d98623 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/management/package-info.java @@ -0,0 +1 @@ +package org.xbib.elx.common.management; \ No newline at end of file diff --git a/elx-common/src/main/java/org/xbib/elx/common/package-info.java b/elx-common/src/main/java/org/xbib/elx/common/package-info.java new file mode 100644 index 0000000..4971f08 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/package-info.java @@ -0,0 +1,4 @@ +/** + * Common classes for Elasticsearch client extensions. + */ +package org.xbib.elx.common; diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/NetworkUtils.java b/elx-common/src/main/java/org/xbib/elx/common/util/NetworkUtils.java similarity index 92% rename from src/main/java/org/xbib/elasticsearch/extras/client/NetworkUtils.java rename to elx-common/src/main/java/org/xbib/elx/common/util/NetworkUtils.java index 9c5ffc2..11dd014 100644 --- a/src/main/java/org/xbib/elasticsearch/extras/client/NetworkUtils.java +++ b/elx-common/src/main/java/org/xbib/elx/common/util/NetworkUtils.java @@ -1,7 +1,7 @@ -package org.xbib.elasticsearch.extras.client; +package org.xbib.elx.common.util; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.io.IOException; import java.net.Inet4Address; @@ -11,6 +11,7 @@ import java.net.NetworkInterface; import java.net.SocketException; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.Enumeration; import java.util.List; import java.util.Locale; @@ -20,7 +21,7 @@ import java.util.Locale; */ public class NetworkUtils { - private static final ESLogger logger = ESLoggerFactory.getLogger(NetworkUtils.class.getName()); + private static final Logger logger = LogManager.getLogger(NetworkUtils.class.getName()); private static final String IPV4_SETTING = "java.net.preferIPv4Stack"; @@ -102,10 +103,8 @@ public class NetworkUtils { NetworkInterface networkInterface = interfaces.nextElement(); allInterfaces.add(networkInterface); Enumeration subInterfaces = networkInterface.getSubInterfaces(); - if (subInterfaces.hasMoreElements()) { - while (subInterfaces.hasMoreElements()) { - allInterfaces.add(subInterfaces.nextElement()); - } + while (subInterfaces.hasMoreElements()) { + allInterfaces.add(subInterfaces.nextElement()); } } sortInterfaces(allInterfaces); @@ -223,10 +222,8 @@ public class NetworkUtils { NetworkInterface networkInterface = interfaces.nextElement(); networkInterfaces.add(networkInterface); Enumeration subInterfaces = networkInterface.getSubInterfaces(); - if (subInterfaces.hasMoreElements()) { - while (subInterfaces.hasMoreElements()) { - networkInterfaces.add(subInterfaces.nextElement()); - } + while (subInterfaces.hasMoreElements()) { + networkInterfaces.add(subInterfaces.nextElement()); } } sortInterfaces(networkInterfaces); @@ -234,7 +231,7 @@ public class NetworkUtils { } private static void sortInterfaces(List interfaces) { - Collections.sort(interfaces, (o1, o2) -> Integer.compare(o1.getIndex(), o2.getIndex())); + Collections.sort(interfaces, Comparator.comparingInt(NetworkInterface::getIndex)); } private static void sortAddresses(List addressList) { diff --git a/elx-common/src/main/java/org/xbib/elx/common/util/package-info.java b/elx-common/src/main/java/org/xbib/elx/common/util/package-info.java new file mode 100644 index 0000000..3c41bfe --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/util/package-info.java @@ -0,0 +1 @@ +package org.xbib.elx.common.util; \ No newline at end of file diff --git a/elx-common/src/main/resources/META-INF/services/java.net.URLStreamHandlerFactory b/elx-common/src/main/resources/META-INF/services/java.net.URLStreamHandlerFactory new file mode 100644 index 0000000..bb6d620 --- /dev/null +++ b/elx-common/src/main/resources/META-INF/services/java.net.URLStreamHandlerFactory @@ -0,0 +1 @@ +org.xbib.elx.common.io.ClasspathURLStreamHandlerFactory \ No newline at end of file diff --git a/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider b/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider new file mode 100644 index 0000000..9729b83 --- /dev/null +++ b/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider @@ -0,0 +1 @@ +org.xbib.elx.common.MockExtendedClientProvider \ No newline at end of file diff --git a/src/integration-test/java/org/elasticsearch/node/MockNode.java b/elx-common/src/test/java/org/elasticsearch/node/MockNode.java similarity index 98% rename from src/integration-test/java/org/elasticsearch/node/MockNode.java rename to elx-common/src/test/java/org/elasticsearch/node/MockNode.java index b0c02eb..aad8b8b 100644 --- a/src/integration-test/java/org/elasticsearch/node/MockNode.java +++ b/elx-common/src/test/java/org/elasticsearch/node/MockNode.java @@ -8,9 +8,6 @@ import org.elasticsearch.plugins.Plugin; import java.util.ArrayList; import java.util.Collection; -/** - * - */ public class MockNode extends Node { public MockNode() { @@ -34,5 +31,4 @@ public class MockNode extends Node { list.add(classpathPlugin); return list; } - } diff --git a/elx-common/src/test/java/org/elasticsearch/node/package-info.java b/elx-common/src/test/java/org/elasticsearch/node/package-info.java new file mode 100644 index 0000000..8ffed8c --- /dev/null +++ b/elx-common/src/test/java/org/elasticsearch/node/package-info.java @@ -0,0 +1 @@ +package org.elasticsearch.node; \ No newline at end of file diff --git a/src/integration-test/java/org/xbib/elasticsearch/AliasTest.java b/elx-common/src/test/java/org/xbib/elx/common/AliasTest.java similarity index 92% rename from src/integration-test/java/org/xbib/elasticsearch/AliasTest.java rename to elx-common/src/test/java/org/xbib/elx/common/AliasTest.java index 970268e..e9106d0 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/AliasTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/AliasTest.java @@ -1,9 +1,11 @@ -package org.xbib.elasticsearch; +package org.xbib.elx.common; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import com.carrotsearch.hppc.cursors.ObjectCursor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; @@ -12,8 +14,6 @@ import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.cluster.metadata.AliasAction; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.junit.Test; import java.io.IOException; @@ -29,7 +29,7 @@ import java.util.regex.Pattern; */ public class AliasTest extends NodeTestUtils { - private static final ESLogger logger = ESLoggerFactory.getLogger(AliasTest.class.getName()); + private static final Logger logger = LogManager.getLogger(AliasTest.class.getName()); @Test public void testAlias() throws IOException { @@ -53,7 +53,7 @@ public class AliasTest extends NodeTestUtils { } @Test - public void testMostRecentIndex() throws IOException { + public void testMostRecentIndex() { String alias = "test"; CreateIndexRequest indexRequest = new CreateIndexRequest("test20160101"); client("1").admin().indices().create(indexRequest).actionGet(); @@ -86,7 +86,7 @@ public class AliasTest extends NodeTestUtils { assertEquals("test20160103", it.next()); assertEquals("test20160102", it.next()); assertEquals("test20160101", it.next()); - logger.info("result={}", result); + logger.info("success: result={}", result); } } diff --git a/elx-common/src/test/java/org/xbib/elx/common/MockExtendedClientProviderTest.java b/elx-common/src/test/java/org/xbib/elx/common/MockExtendedClientProviderTest.java new file mode 100644 index 0000000..8474c1c --- /dev/null +++ b/elx-common/src/test/java/org/xbib/elx/common/MockExtendedClientProviderTest.java @@ -0,0 +1,16 @@ +package org.xbib.elx.common; + +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertNotNull; + +public class MockExtendedClientProviderTest { + + @Test + public void testMockExtendedProvider() throws IOException { + MockExtendedClient client = ClientBuilder.builder().provider(MockExtendedClientProvider.class).build(); + assertNotNull(client); + } +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/NetworkTest.java b/elx-common/src/test/java/org/xbib/elx/common/NetworkTest.java similarity index 95% rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/NetworkTest.java rename to elx-common/src/test/java/org/xbib/elx/common/NetworkTest.java index b9e7a87..248b906 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/NetworkTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/NetworkTest.java @@ -1,4 +1,4 @@ -package org.xbib.elasticsearch.extras.client; +package org.xbib.elx.common; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -9,15 +9,13 @@ import java.net.NetworkInterface; import java.util.Collections; import java.util.Enumeration; -/** - * - */ public class NetworkTest { private static final Logger logger = LogManager.getLogger(NetworkTest.class); @Test public void testNetwork() throws Exception { + // walk very slowly over all interfaces Enumeration nets = NetworkInterface.getNetworkInterfaces(); for (NetworkInterface netint : Collections.list(nets)) { System.out.println("checking network interface = " + netint.getName()); diff --git a/elx-common/src/test/java/org/xbib/elx/common/NodeTestUtils.java b/elx-common/src/test/java/org/xbib/elx/common/NodeTestUtils.java new file mode 100644 index 0000000..86e30c6 --- /dev/null +++ b/elx-common/src/test/java/org/xbib/elx/common/NodeTestUtils.java @@ -0,0 +1,213 @@ +package org.xbib.elx.common; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.client.support.AbstractClient; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.node.MockNode; +import org.elasticsearch.node.Node; +import org.junit.After; +import org.junit.Before; +import org.xbib.elx.common.util.NetworkUtils; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.elasticsearch.common.settings.Settings.settingsBuilder; + +public class NodeTestUtils { + + private static final Logger logger = LogManager.getLogger("test"); + + private static Random random = new Random(); + + private static char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); + + private Map nodes = new HashMap<>(); + + private Map clients = new HashMap<>(); + + private AtomicInteger counter = new AtomicInteger(); + + private String cluster; + + private String host; + + private int port; + + private static void deleteFiles() throws IOException { + Path directory = Paths.get(getHome() + "/data"); + Files.walkFileTree(directory, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + + }); + + } + + @Before + public void startNodes() { + try { + logger.info("starting"); + setClusterName(); + startNode("1"); + findNodeAddress(); + try { + ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN) + .timeout(TimeValue.timeValueSeconds(30))).actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + throw new IOException("cluster state is " + healthResponse.getStatus().name() + + ", from here on, everything will fail!"); + } + } catch (ElasticsearchTimeoutException e) { + throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); + } + } catch (Throwable t) { + logger.error("startNodes failed", t); + } + } + + @After + public void stopNodes() { + try { + closeNodes(); + } catch (Exception e) { + logger.error("can not close nodes", e); + } finally { + try { + deleteFiles(); + logger.info("data files wiped"); + Thread.sleep(2000L); // let OS commit changes + } catch (IOException e) { + logger.error(e.getMessage(), e); + } catch (InterruptedException e) { + // ignore + } + } + } + + protected void setClusterName() { + this.cluster = "test-helper-cluster-" + + NetworkUtils.getLocalAddress().getHostName() + + "-" + System.getProperty("user.name") + + "-" + counter.incrementAndGet(); + } + + protected String getClusterName() { + return cluster; + } + + protected Settings getSettings() { + return settingsBuilder() + .put("host", host) + .put("port", port) + .put("cluster.name", cluster) + .put("path.home", getHome()) + .build(); + } + + protected Settings getNodeSettings() { + return settingsBuilder() + .put("cluster.name", cluster) + .put("cluster.routing.schedule", "50ms") + .put("cluster.routing.allocation.disk.threshold_enabled", false) + .put("discovery.zen.multicast.enabled", true) + .put("discovery.zen.multicast.ping_timeout", "5s") + .put("http.enabled", true) + .put("threadpool.bulk.size", Runtime.getRuntime().availableProcessors()) + .put("threadpool.bulk.queue_size", 16 * Runtime.getRuntime().availableProcessors()) // default is 50, too low + .put("index.number_of_replicas", 0) + .put("path.home", getHome()) + .build(); + } + + protected static String getHome() { + return System.getProperty("path.home", System.getProperty("user.dir")); + } + + public void startNode(String id) { + buildNode(id).start(); + } + + public AbstractClient client(String id) { + return clients.get(id); + } + + private void closeNodes() { + logger.info("closing all clients"); + for (AbstractClient client : clients.values()) { + client.close(); + } + clients.clear(); + logger.info("closing all nodes"); + for (Node node : nodes.values()) { + if (node != null) { + node.close(); + } + } + nodes.clear(); + logger.info("all nodes closed"); + } + + protected void findNodeAddress() { + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); + NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet(); + Object obj = response.iterator().next().getTransport().getAddress() + .publishAddress(); + if (obj instanceof InetSocketTransportAddress) { + InetSocketTransportAddress address = (InetSocketTransportAddress) obj; + host = address.address().getHostName(); + port = address.address().getPort(); + } + } + + private Node buildNode(String id) { + Settings nodeSettings = settingsBuilder() + .put(getNodeSettings()) + .put("name", id) + .build(); + Node node = new MockNode(nodeSettings); + AbstractClient client = (AbstractClient) node.client(); + nodes.put(id, node); + clients.put(id, client); + logger.info("clients={}", clients); + return node; + } + + protected String randomString(int len) { + final char[] buf = new char[len]; + final int n = numbersAndLetters.length - 1; + for (int i = 0; i < buf.length; i++) { + buf[i] = numbersAndLetters[random.nextInt(n)]; + } + return new String(buf); + } +} diff --git a/elx-common/src/test/java/org/xbib/elx/common/SearchTest.java b/elx-common/src/test/java/org/xbib/elx/common/SearchTest.java new file mode 100644 index 0000000..63892d0 --- /dev/null +++ b/elx-common/src/test/java/org/xbib/elx/common/SearchTest.java @@ -0,0 +1,56 @@ +package org.xbib.elx.common; + +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.bulk.BulkAction; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.sort.SortOrder; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +public class SearchTest extends NodeTestUtils { + + @Test + public void testSearch() throws Exception { + Client client = client("1"); + BulkRequestBuilder builder = new BulkRequestBuilder(client, BulkAction.INSTANCE); + for (int i = 0; i < 1000; i++) { + IndexRequest indexRequest = new IndexRequest("pages", "row") + .source(XContentFactory.jsonBuilder() + .startObject() + .field("user1", "joerg") + .field("user2", "joerg") + .field("user3", "joerg") + .field("user4", "joerg") + .field("user5", "joerg") + .field("user6", "joerg") + .field("user7", "joerg") + .field("user8", "joerg") + .field("user9", "joerg") + .field("rowcount", i) + .field("rs", 1234)); + builder.add(indexRequest); + } + client.bulk(builder.request()).actionGet(); + client.admin().indices().refresh(new RefreshRequest()).actionGet(); + + for (int i = 0; i < 100; i++) { + QueryBuilder queryStringBuilder = QueryBuilders.queryStringQuery("rs:" + 1234); + SearchRequestBuilder requestBuilder = client.prepareSearch() + .setIndices("pages") + .setTypes("row") + .setQuery(queryStringBuilder) + .addSort("rowcount", SortOrder.DESC) + .setFrom(i * 10).setSize(10); + SearchResponse searchResponse = requestBuilder.execute().actionGet(); + assertTrue(searchResponse.getHits().getTotalHits() > 0); + } + } +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/SimpleTest.java b/elx-common/src/test/java/org/xbib/elx/common/SimpleTest.java similarity index 91% rename from src/integration-test/java/org/xbib/elasticsearch/SimpleTest.java rename to elx-common/src/test/java/org/xbib/elx/common/SimpleTest.java index 0af13df..75cdc29 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/SimpleTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/SimpleTest.java @@ -1,4 +1,4 @@ -package org.xbib.elasticsearch; +package org.xbib.elx.common; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; @@ -10,16 +10,14 @@ import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; import org.elasticsearch.action.index.IndexAction; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexNotFoundException; import org.junit.Test; -/** - * - */ public class SimpleTest extends NodeTestUtils { protected Settings getNodeSettings() { return settingsBuilder() - .put("path.home", System.getProperty("path.home")) + .put(super.getNodeSettings()) .put("index.analysis.analyzer.default.filter.0", "lowercase") .put("index.analysis.analyzer.default.filter.1", "trim") .put("index.analysis.analyzer.default.tokenizer", "keyword") @@ -32,8 +30,8 @@ public class SimpleTest extends NodeTestUtils { DeleteIndexRequestBuilder deleteIndexRequestBuilder = new DeleteIndexRequestBuilder(client("1"), DeleteIndexAction.INSTANCE, "test"); deleteIndexRequestBuilder.execute().actionGet(); - } catch (Exception e) { - // ignore + } catch (IndexNotFoundException e) { + // ignore if index not found } IndexRequestBuilder indexRequestBuilder = new IndexRequestBuilder(client("1"), IndexAction.INSTANCE); indexRequestBuilder diff --git a/elx-common/src/test/java/org/xbib/elx/common/WildcardTest.java b/elx-common/src/test/java/org/xbib/elx/common/WildcardTest.java new file mode 100644 index 0000000..783b440 --- /dev/null +++ b/elx-common/src/test/java/org/xbib/elx/common/WildcardTest.java @@ -0,0 +1,62 @@ +package org.xbib.elx.common; + +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.junit.Test; + +import java.io.IOException; + +public class WildcardTest extends NodeTestUtils { + + protected Settings getNodeSettings() { + return Settings.settingsBuilder() + .put(super.getNodeSettings()) + .put("cluster.routing.allocation.disk.threshold_enabled", false) + .put("discovery.zen.multicast.enabled", false) + .put("http.enabled", false) + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .build(); + } + + @Test + public void testWildcard() throws Exception { + index(client("1"), "1", "010"); + index(client("1"), "2", "0*0"); + // exact + validateCount(client("1"), QueryBuilders.queryStringQuery("010").defaultField("field"), 1); + validateCount(client("1"), QueryBuilders.queryStringQuery("0\\*0").defaultField("field"), 1); + // pattern + validateCount(client("1"), QueryBuilders.queryStringQuery("0*0").defaultField("field"), 1); // 2? + validateCount(client("1"), QueryBuilders.queryStringQuery("0?0").defaultField("field"), 1); // 2? + validateCount(client("1"), QueryBuilders.queryStringQuery("0**0").defaultField("field"), 1); // 2? + validateCount(client("1"), QueryBuilders.queryStringQuery("0??0").defaultField("field"), 0); + validateCount(client("1"), QueryBuilders.queryStringQuery("*10").defaultField("field"), 1); + validateCount(client("1"), QueryBuilders.queryStringQuery("*1*").defaultField("field"), 1); + validateCount(client("1"), QueryBuilders.queryStringQuery("*\\*0").defaultField("field"), 0); // 1? + validateCount(client("1"), QueryBuilders.queryStringQuery("*\\**").defaultField("field"), 0); // 1? + } + + private void index(Client client, String id, String fieldValue) throws IOException { + client.index(new IndexRequest("index", "type", id) + .source(XContentFactory.jsonBuilder().startObject().field("field", fieldValue).endObject()) + .refresh(true)).actionGet(); + } + + private long count(Client client, QueryBuilder queryBuilder) { + return client.prepareSearch("index").setTypes("type") + .setQuery(queryBuilder) + .execute().actionGet().getHits().getTotalHits(); + } + + private void validateCount(Client client, QueryBuilder queryBuilder, long expectedHits) { + final long actualHits = count(client, queryBuilder); + if (actualHits != expectedHits) { + throw new RuntimeException("actualHits=" + actualHits + ", expectedHits=" + expectedHits); + } + } +} diff --git a/elx-common/src/test/java/org/xbib/elx/common/package-info.java b/elx-common/src/test/java/org/xbib/elx/common/package-info.java new file mode 100644 index 0000000..9a9e4ce --- /dev/null +++ b/elx-common/src/test/java/org/xbib/elx/common/package-info.java @@ -0,0 +1 @@ +package org.xbib.elx.common; \ No newline at end of file diff --git a/elx-common/src/test/resources/log4j2.xml b/elx-common/src/test/resources/log4j2.xml new file mode 100644 index 0000000..6c323f8 --- /dev/null +++ b/elx-common/src/test/resources/log4j2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/elx-http/build.gradle~ b/elx-http/build.gradle~ new file mode 100644 index 0000000..da70162 --- /dev/null +++ b/elx-http/build.gradle~ @@ -0,0 +1,65 @@ +buildscript { + repositories { + jcenter() + maven { + url 'http://xbib.org/repository' + } + } + dependencies { + classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.2.2.0" + } +} + +apply plugin: 'org.xbib.gradle.plugin.elasticsearch.build' + +configurations { + main + tests +} + +dependencies { + compile project(':common') + compile "org.xbib:netty-http-client:${project.property('xbib-netty-http-client.version')}" + testCompile "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" + testRuntime "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" +} + +jar { + baseName "${rootProject.name}-common" +} + +/* +task testJar(type: Jar, dependsOn: testClasses) { + baseName = "${project.archivesBaseName}-tests" + from sourceSets.test.output +} +*/ + +artifacts { + main jar + tests testJar + archives sourcesJar, javadocJar +} + +test { + enabled = true + include '**/SimpleTest.*' + testLogging { + showStandardStreams = true + exceptionFormat = 'full' + } +} + +randomizedTest { + enabled = false +} + +esTest { + enabled = true + // test with the jars, not the classes, for security manager + // classpath = files(configurations.testRuntime) + configurations.main.artifacts.files + configurations.tests.artifacts.files + systemProperty 'tests.security.manager', 'true' + // maybe we like some extra security policy for our code + systemProperty 'tests.security.policy', '/extra-security.policy' +} +esTest.dependsOn jar, testJar diff --git a/elx-node/build.gradle b/elx-node/build.gradle new file mode 100644 index 0000000..bc5e01e --- /dev/null +++ b/elx-node/build.gradle @@ -0,0 +1,3 @@ +dependencies { + compile project(':elx-common') +} \ No newline at end of file diff --git a/elx-node/build.gradle~ b/elx-node/build.gradle~ new file mode 100644 index 0000000..0da2929 --- /dev/null +++ b/elx-node/build.gradle~ @@ -0,0 +1,65 @@ +buildscript { + repositories { + jcenter() + maven { + url 'http://xbib.org/repository' + } + } + dependencies { + classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.2.3.4" + } +} + +apply plugin: 'org.xbib.gradle.plugin.elasticsearch.build' + +configurations { + main + tests +} + +dependencies { + compile project(':common') + testCompile "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" + testRuntime "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" +} + +jar { + baseName "${rootProject.name}-node" +} + +/* +task testJar(type: Jar, dependsOn: testClasses) { + baseName = "${project.archivesBaseName}-tests" + from sourceSets.test.output +} +*/ + +artifacts { + main jar + tests testJar + archives sourcesJar, javadocJar +} + +test { + enabled = false + jvmArgs "-javaagent:" + configurations.alpnagent.asPath + systemProperty 'path.home', projectDir.absolutePath + testLogging { + showStandardStreams = true + exceptionFormat = 'full' + } +} + +randomizedTest { + enabled = false +} + + +esTest { + // test with the jars, not the classes, for security manager + // classpath = files(configurations.testRuntime) + configurations.main.artifacts.files + configurations.tests.artifacts.files + systemProperty 'tests.security.manager', 'true' + // maybe we like some extra security policy for our code + systemProperty 'tests.security.policy', '/extra-security.policy' +} +esTest.dependsOn jar, testJar diff --git a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java b/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java new file mode 100644 index 0000000..493a596 --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java @@ -0,0 +1,70 @@ +package org.xbib.elx.node; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.node.Node; +import org.elasticsearch.plugins.Plugin; +import org.xbib.elx.common.AbstractExtendedClient; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; + +public class ExtendedNodeClient extends AbstractExtendedClient { + + private static final Logger logger = LogManager.getLogger(ExtendedNodeClient.class.getName()); + + private Node node; + + @Override + protected ElasticsearchClient createClient(Settings settings) throws IOException { + if (settings != null) { + String version = System.getProperty("os.name") + + " " + System.getProperty("java.vm.name") + + " " + System.getProperty("java.vm.vendor") + + " " + System.getProperty("java.runtime.version") + + " " + System.getProperty("java.vm.version"); + Settings effectiveSettings = Settings.builder().put(settings) + .put("node.client", true) + .put("node.master", false) + .put("node.data", false) + .build(); + logger.info("creating node client on {} with effective settings {}", + version, effectiveSettings.toString()); + Collection> plugins = Collections.emptyList(); + this.node = new BulkNode(new Environment(effectiveSettings), plugins); + try { + node.start(); + } catch (Exception e) { + throw new IOException(e); + } + return node.client(); + } + return null; + } + + + @Override + public void shutdown() throws IOException { + super.shutdown(); + try { + if (node != null) { + logger.debug("closing node..."); + node.close(); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + private static class BulkNode extends Node { + + BulkNode(Environment env, Collection> classpathPlugins) { + super(env, Version.CURRENT, classpathPlugins); + } + } +} diff --git a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java b/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java new file mode 100644 index 0000000..46a4e9a --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java @@ -0,0 +1,10 @@ +package org.xbib.elx.node; + +import org.xbib.elx.api.ExtendedClientProvider; + +public class ExtendedNodeClientProvider implements ExtendedClientProvider { + @Override + public ExtendedNodeClient getExtendedClient() { + return new ExtendedNodeClient(); + } +} diff --git a/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider new file mode 100644 index 0000000..372aaad --- /dev/null +++ b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider @@ -0,0 +1 @@ +org.xbib.elx.node.ExtendedNodeClientProvider \ No newline at end of file diff --git a/elx-node/src/test/java/org/elasticsearch/node/MockNode.java b/elx-node/src/test/java/org/elasticsearch/node/MockNode.java new file mode 100644 index 0000000..aad8b8b --- /dev/null +++ b/elx-node/src/test/java/org/elasticsearch/node/MockNode.java @@ -0,0 +1,34 @@ +package org.elasticsearch.node; + +import org.elasticsearch.Version; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.internal.InternalSettingsPreparer; +import org.elasticsearch.plugins.Plugin; + +import java.util.ArrayList; +import java.util.Collection; + +public class MockNode extends Node { + + public MockNode() { + super(Settings.EMPTY); + } + + public MockNode(Settings settings) { + super(settings); + } + + public MockNode(Settings settings, Collection> classpathPlugins) { + super(InternalSettingsPreparer.prepareEnvironment(settings, null), Version.CURRENT, classpathPlugins); + } + + public MockNode(Settings settings, Class classpathPlugin) { + this(settings, list(classpathPlugin)); + } + + private static Collection> list(Class classpathPlugin) { + Collection> list = new ArrayList<>(); + list.add(classpathPlugin); + return list; + } +} diff --git a/elx-node/src/test/java/org/xbib/elx/node/ExtendeNodeDuplicateIDTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendeNodeDuplicateIDTest.java new file mode 100644 index 0000000..97cb185 --- /dev/null +++ b/elx-node/src/test/java/org/xbib/elx/node/ExtendeNodeDuplicateIDTest.java @@ -0,0 +1,58 @@ +package org.xbib.elx.node; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.junit.Ignore; +import org.junit.Test; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.Parameters; + +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.junit.Assert.*; + +@Ignore +public class ExtendeNodeDuplicateIDTest extends NodeTestUtils { + + private static final Logger logger = LogManager.getLogger(ExtendeNodeDuplicateIDTest.class.getSimpleName()); + + private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; + + private static final Long ACTIONS = 12345L; + + @Test + public void testDuplicateDocIDs() throws Exception { + long numactions = ACTIONS; + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .build(); + try { + client.newIndex("test"); + for (int i = 0; i < ACTIONS; i++) { + client.index("test", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flushIngest(); + client.waitForResponses("30s"); + client.refreshIndex("test"); + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) + .setIndices("test") + .setTypes("test") + .setQuery(matchAllQuery()); + long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); + logger.info("hits = {}", hits); + assertTrue(hits < ACTIONS); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + client.shutdown(); + assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + } + } +} diff --git a/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientSingleNodeTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientSingleNodeTest.java new file mode 100644 index 0000000..0d7335d --- /dev/null +++ b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientSingleNodeTest.java @@ -0,0 +1,39 @@ +package org.xbib.elx.node; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.junit.Test; +import org.xbib.elx.common.ClientBuilder; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +public class ExtendedNodeClientSingleNodeTest extends NodeTestUtils { + + private static final Logger logger = LogManager.getLogger(ExtendedNodeClientSingleNodeTest.class.getSimpleName()); + + @Test + public void testSingleDocNodeClient() throws Exception { + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .build(); + try { + client.newIndex("test"); + client.index("test", "test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + client.flushIngest(); + client.waitForResponses("30s"); + } catch (InterruptedException e) { + // ignore + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + client.shutdown(); + } + } +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClientTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientTest.java similarity index 53% rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClientTest.java rename to elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientTest.java index 77b004f..957972d 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClientTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClientTest.java @@ -1,43 +1,37 @@ -package org.xbib.elasticsearch.extras.client.node; +package org.xbib.elx.node; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.junit.Before; import org.junit.Test; -import org.xbib.elasticsearch.NodeTestUtils; -import org.xbib.elasticsearch.extras.client.Clients; -import org.xbib.elasticsearch.extras.client.SimpleBulkControl; -import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.Parameters; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -/** - * - */ -public class BulkNodeClientTest extends NodeTestUtils { +public class ExtendedNodeClientTest extends NodeTestUtils { - private static final ESLogger logger = ESLoggerFactory.getLogger(BulkNodeClientTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(ExtendedNodeClientTest.class.getSimpleName()); - private static final Long MAX_ACTIONS = 1000L; + private static final Long ACTIONS = 25000L; - private static final Long NUM_ACTIONS = 1234L; + private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; @Before public void startNodes() { @@ -49,13 +43,38 @@ public class BulkNodeClientTest extends NodeTestUtils { } } + @Test + public void testSingleDocNodeClient() throws Exception { + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) + .build(); + try { + client.newIndex("test"); + client.index("test", "test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + client.flushIngest(); + client.waitForResponses("30s"); + } catch (InterruptedException e) { + // ignore + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + client.shutdown(); + } + } + @Test public void testNewIndexNodeClient() throws Exception { - final BulkNodeClient client = Clients.builder() - .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5)) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkNodeClient(client("1")); + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) + .build(); client.newIndex("test"); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); @@ -66,11 +85,10 @@ public class BulkNodeClientTest extends NodeTestUtils { @Test public void testMappingNodeClient() throws Exception { - final BulkNodeClient client = Clients.builder() - .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5)) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkNodeClient(client("1")); + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) + .build(); XContentBuilder builder = jsonBuilder() .startObject() .startObject("test") @@ -85,7 +103,7 @@ public class BulkNodeClientTest extends NodeTestUtils { client.newIndex("test"); GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test"); GetMappingsResponse getMappingsResponse = - client.client().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); + client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); logger.info("mappings={}", getMappingsResponse.getMappings()); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); @@ -94,59 +112,34 @@ public class BulkNodeClientTest extends NodeTestUtils { client.shutdown(); } - @Test - public void testSingleDocNodeClient() { - final BulkNodeClient client = Clients.builder() - .put(Clients.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS) - .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(30)) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkNodeClient(client("1")); - try { - client.newIndex("test"); - client.index("test", "test", "1", "{ \"name\" : \"Hello World\"}"); // single doc ingest - client.flushIngest(); - client.waitForResponses("30s"); - } catch (InterruptedException e) { - // ignore - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } catch (ExecutionException e) { - logger.error(e.getMessage(), e); - } finally { - assertEquals(1, client.getMetric().getSucceeded().getCount()); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); - } - assertFalse(client.hasThrowable()); - client.shutdown(); - } - } - @Test public void testRandomDocsNodeClient() throws Exception { - long numactions = NUM_ACTIONS; - final BulkNodeClient client = Clients.builder() - .put(Clients.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS) - .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60)) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkNodeClient(client("1")); + long numactions = ACTIONS; + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); try { client.newIndex("test"); - for (int i = 0; i < NUM_ACTIONS; i++) { - client.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + for (int i = 0; i < ACTIONS; i++) { + client.index("test", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.waitForResponses("30s"); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - assertEquals(numactions, client.getMetric().getSucceeded().getCount()); + assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); } assertFalse(client.hasThrowable()); + client.refreshIndex("test"); + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) + .setQuery(QueryBuilders.matchAllQuery()).setSize(0); + assertEquals(numactions, + searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); client.shutdown(); } } @@ -154,15 +147,15 @@ public class BulkNodeClientTest extends NodeTestUtils { @Test public void testThreadedRandomDocsNodeClient() throws Exception { int maxthreads = Runtime.getRuntime().availableProcessors(); - Long maxactions = MAX_ACTIONS; - final Long maxloop = NUM_ACTIONS; - logger.info("NodeClient max={} maxactions={} maxloop={}", maxthreads, maxactions, maxloop); - final BulkNodeClient client = Clients.builder() - .put(Clients.MAX_ACTIONS_PER_REQUEST, maxactions) - .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60))// disable auto flush for this test - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkNodeClient(client("1")); + Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; + final Long actions = ACTIONS; + logger.info("NodeClient max={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); try { client.newIndex("test") .startBulk("test", -1, 1000); @@ -170,39 +163,39 @@ public class BulkNodeClientTest extends NodeTestUtils { EsExecutors.daemonThreadFactory("bulk-nodeclient-test")); final CountDownLatch latch = new CountDownLatch(maxthreads); for (int i = 0; i < maxthreads; i++) { - pool.execute(new Runnable() { - public void run() { - for (int i = 0; i < maxloop; i++) { - client.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - latch.countDown(); + pool.execute(() -> { + for (int i1 = 0; i1 < actions; i1++) { + client.index("test", "test", null, false,"{ \"name\" : \"" + randomString(32) + "\"}"); } + latch.countDown(); }); } - logger.info("waiting for max 30 seconds..."); - latch.await(30, TimeUnit.SECONDS); - logger.info("flush..."); - client.flushIngest(); - client.waitForResponses("30s"); - logger.info("got all responses, thread pool shutdown..."); - pool.shutdown(); - logger.info("pool is shut down"); + logger.info("waiting for latch..."); + if (latch.await(5, TimeUnit.MINUTES)) { + logger.info("last flush..."); + client.flushIngest(); + client.waitForResponses("60s"); + logger.info("got all responses, pool shutdown..."); + pool.shutdown(); + logger.info("pool is shut down"); + } else { + logger.warn("latch timeout"); + } } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { client.stopBulk("test"); - assertEquals(maxthreads * maxloop, client.getMetric().getSucceeded().getCount()); + assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount()); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); } assertFalse(client.hasThrowable()); client.refreshIndex("test"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE) + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) .setQuery(QueryBuilders.matchAllQuery()).setSize(0); - assertEquals(maxthreads * maxloop, + assertEquals(maxthreads * actions, searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); client.shutdown(); } } - } diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClusterBlockTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClusterBlockTest.java similarity index 76% rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClusterBlockTest.java rename to elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClusterBlockTest.java index 09c628d..b38555a 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClusterBlockTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeClusterBlockTest.java @@ -1,24 +1,22 @@ -package org.xbib.elasticsearch.extras.client.node; +package org.xbib.elx.node; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; -import org.xbib.elasticsearch.NodeTestUtils; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -/** - * - */ -public class BulkNodeClusterBlockTest extends NodeTestUtils { +@Ignore +public class ExtendedNodeClusterBlockTest extends NodeTestUtils { - private static final ESLogger logger = ESLoggerFactory.getLogger("test"); + private static final Logger logger = LogManager.getLogger("test"); @Before public void startNodes() { @@ -33,6 +31,7 @@ public class BulkNodeClusterBlockTest extends NodeTestUtils { } } + @Override protected Settings getNodeSettings() { return Settings.settingsBuilder() .put(super.getNodeSettings()) @@ -44,8 +43,7 @@ public class BulkNodeClusterBlockTest extends NodeTestUtils { public void testClusterBlock() throws Exception { BulkRequestBuilder brb = client("1").prepareBulk(); XContentBuilder builder = jsonBuilder().startObject().field("field1", "value1").endObject(); - String jsonString = builder.string(); - IndexRequestBuilder irb = client("1").prepareIndex("test", "test", "1").setSource(jsonString); + IndexRequestBuilder irb = client("1").prepareIndex("test", "test", "1").setSource(builder); brb.add(irb); brb.execute().actionGet(); } diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeIndexAliasTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeIndexAliasTest.java similarity index 58% rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeIndexAliasTest.java rename to elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeIndexAliasTest.java index eb5256c..1503fee 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeIndexAliasTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeIndexAliasTest.java @@ -1,17 +1,12 @@ -package org.xbib.elasticsearch.extras.client.node; +package org.xbib.elx.node; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilders; +import org.junit.Ignore; import org.junit.Test; -import org.xbib.elasticsearch.NodeTestUtils; -import org.xbib.elasticsearch.extras.client.Clients; -import org.xbib.elasticsearch.extras.client.IndexAliasAdder; -import org.xbib.elasticsearch.extras.client.SimpleBulkControl; -import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; +import org.xbib.elx.common.ClientBuilder; import java.util.Arrays; import java.util.List; @@ -19,23 +14,20 @@ import java.util.Map; import static org.junit.Assert.assertFalse; -/** - * - */ -public class BulkNodeIndexAliasTest extends NodeTestUtils { +@Ignore +public class ExtendedNodeIndexAliasTest extends NodeTestUtils { - private static final ESLogger logger = ESLoggerFactory.getLogger(BulkNodeIndexAliasTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(ExtendedNodeIndexAliasTest.class.getSimpleName()); @Test public void testIndexAlias() throws Exception { - final BulkNodeClient client = Clients.builder() - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkNodeClient(client("1")); + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .build(); try { client.newIndex("test1234"); for (int i = 0; i < 1; i++) { - client.index("test1234", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test1234", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.refreshIndex("test1234"); @@ -45,18 +37,14 @@ public class BulkNodeIndexAliasTest extends NodeTestUtils { client.newIndex("test5678"); for (int i = 0; i < 1; i++) { - client.index("test5678", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test5678", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.refreshIndex("test5678"); simpleAliases = Arrays.asList("d", "e", "f"); - client.switchAliases("test", "test5678", simpleAliases, new IndexAliasAdder() { - @Override - public void addIndexAlias(IndicesAliasesRequestBuilder builder, String index, String alias) { - builder.addAlias(index, alias, QueryBuilders.termQuery("my_key", alias)); - } - }); + client.switchAliases("test", "test5678", simpleAliases, (builder, index, alias) -> + builder.addAlias(index, alias, QueryBuilders.termQuery("my_key", alias))); Map aliases = client.getIndexFilters("test5678"); logger.info("aliases of index test5678 = {}", aliases); diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeReplicaTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeReplicaTest.java similarity index 78% rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeReplicaTest.java rename to elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeReplicaTest.java index b4fec6b..89de2df 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeReplicaTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeReplicaTest.java @@ -1,5 +1,7 @@ -package org.xbib.elasticsearch.extras.client.node; +package org.xbib.elx.node; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.admin.indices.stats.IndexShardStats; import org.elasticsearch.action.admin.indices.stats.IndexStats; @@ -9,15 +11,11 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.indexing.IndexingStats; +import org.junit.Ignore; import org.junit.Test; -import org.xbib.elasticsearch.NodeTestUtils; -import org.xbib.elasticsearch.extras.client.Clients; -import org.xbib.elasticsearch.extras.client.SimpleBulkControl; -import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; +import org.xbib.elx.common.ClientBuilder; import java.util.Map; @@ -25,9 +23,10 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -public class BulkNodeReplicaTest extends NodeTestUtils { +@Ignore +public class ExtendedNodeReplicaTest extends NodeTestUtils { - private final static ESLogger logger = ESLoggerFactory.getLogger(BulkNodeReplicaTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(ExtendedNodeReplicaTest.class.getSimpleName()); @Test public void testReplicaLevel() throws Exception { @@ -47,20 +46,19 @@ public class BulkNodeReplicaTest extends NodeTestUtils { .put("index.number_of_replicas", 1) .build(); - final BulkNodeClient client = Clients.builder() - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkNodeClient(client("1")); + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .build(); try { client.newIndex("test1", settingsTest1, null) .newIndex("test2", settingsTest2, null); client.waitForCluster("GREEN", "30s"); for (int i = 0; i < 1234; i++) { - client.index("test1", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test1", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } for (int i = 0; i < 1234; i++) { - client.index("test2", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test2", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.waitForResponses("30s"); @@ -70,13 +68,13 @@ public class BulkNodeReplicaTest extends NodeTestUtils { logger.info("refreshing"); client.refreshIndex("test1"); client.refreshIndex("test2"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE) + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) .setIndices("test1", "test2") .setQuery(matchAllQuery()); long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); logger.info("query total hits={}", hits); assertEquals(2468, hits); - IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.client(), IndicesStatsAction.INSTANCE) + IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.getClient(), IndicesStatsAction.INSTANCE) .all(); IndicesStatsResponse response = indicesStatsRequestBuilder.execute().actionGet(); for (Map.Entry m : response.getIndices().entrySet()) { diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportUpdateReplicaLevelTest.java b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeUpdateReplicaLevelTest.java similarity index 60% rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportUpdateReplicaLevelTest.java rename to elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeUpdateReplicaLevelTest.java index 1f56df8..a1b29a4 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportUpdateReplicaLevelTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/ExtendedNodeUpdateReplicaLevelTest.java @@ -1,54 +1,47 @@ -package org.xbib.elasticsearch.extras.client.transport; +package org.xbib.elx.node; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; +import org.junit.Ignore; import org.junit.Test; -import org.xbib.elasticsearch.NodeTestUtils; -import org.xbib.elasticsearch.extras.client.Clients; -import org.xbib.elasticsearch.extras.client.SimpleBulkControl; -import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; +import org.xbib.elx.common.ClientBuilder; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -/** - * - */ -public class BulkTransportUpdateReplicaLevelTest extends NodeTestUtils { +@Ignore +public class ExtendedNodeUpdateReplicaLevelTest extends NodeTestUtils { - private static final ESLogger logger = - ESLoggerFactory.getLogger(BulkTransportUpdateReplicaLevelTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(ExtendedNodeUpdateReplicaLevelTest.class.getSimpleName()); @Test public void testUpdateReplicaLevel() throws Exception { - int numberOfShards = 2; + long numberOfShards = 2; int replicaLevel = 3; // we need 3 nodes for replica level 3 startNode("2"); startNode("3"); - int shardsAfterReplica; + long shardsAfterReplica; Settings settings = Settings.settingsBuilder() .put("index.number_of_shards", numberOfShards) .put("index.number_of_replicas", 0) .build(); - final BulkTransportClient client = Clients.builder() - .put(getSettings()) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkTransportClient(); + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .build(); try { client.newIndex("replicatest", settings, null); client.waitForCluster("GREEN", "30s"); for (int i = 0; i < 12345; i++) { - client.index("replicatest", "replicatest", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("replicatest", "replicatest", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.waitForResponses("30s"); diff --git a/elx-node/src/test/java/org/xbib/elx/node/NodeTestUtils.java b/elx-node/src/test/java/org/xbib/elx/node/NodeTestUtils.java new file mode 100644 index 0000000..9a4750e --- /dev/null +++ b/elx-node/src/test/java/org/xbib/elx/node/NodeTestUtils.java @@ -0,0 +1,201 @@ +package org.xbib.elx.node; + +import static org.elasticsearch.common.settings.Settings.settingsBuilder; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.client.support.AbstractClient; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.node.MockNode; +import org.elasticsearch.node.Node; +import org.junit.After; +import org.junit.Before; +import org.xbib.elx.common.util.NetworkUtils; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +public class NodeTestUtils { + + private static final Logger logger = LogManager.getLogger("test"); + + private static Random random = new Random(); + + private static char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); + + private Map nodes = new HashMap<>(); + + private Map clients = new HashMap<>(); + + private AtomicInteger counter = new AtomicInteger(); + + private String cluster; + + private String host; + + private int port; + + private static void deleteFiles() throws IOException { + Path directory = Paths.get(getHome() + "/data"); + Files.walkFileTree(directory, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + + }); + + } + + @Before + public void startNodes() { + try { + logger.info("starting"); + setClusterName(); + startNode("1"); + findNodeAddress(); + try { + ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN) + .timeout(TimeValue.timeValueSeconds(30))).actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + throw new IOException("cluster state is " + healthResponse.getStatus().name() + + ", from here on, everything will fail!"); + } + } catch (ElasticsearchTimeoutException e) { + throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); + } + } catch (Throwable t) { + logger.error("startNodes failed", t); + } + } + + @After + public void stopNodes() { + try { + closeNodes(); + } catch (Exception e) { + logger.error("can not close nodes", e); + } finally { + try { + deleteFiles(); + logger.info("data files wiped"); + Thread.sleep(2000L); // let OS commit changes + } catch (IOException e) { + logger.error(e.getMessage(), e); + } catch (InterruptedException e) { + // ignore + } + } + } + + protected void setClusterName() { + this.cluster = "test-helper-cluster-" + + NetworkUtils.getLocalAddress().getHostName() + + "-" + System.getProperty("user.name") + + "-" + counter.incrementAndGet(); + } + + protected Settings getNodeSettings() { + return settingsBuilder() + .put("cluster.name", cluster) + .put("cluster.routing.schedule", "50ms") + .put("cluster.routing.allocation.disk.threshold_enabled", false) + .put("discovery.zen.multicast.enabled", true) + .put("discovery.zen.multicast.ping_timeout", "5s") + .put("http.enabled", true) + .put("threadpool.bulk.size", Runtime.getRuntime().availableProcessors()) + .put("threadpool.bulk.queue_size", 16 * Runtime.getRuntime().availableProcessors()) // default is 50, too low + .put("index.number_of_replicas", 0) + .put("path.home", getHome()) + .build(); + } + + protected static String getHome() { + return System.getProperty("path.home", System.getProperty("user.dir")); + } + + public void startNode(String id) { + buildNode(id).start(); + } + + public AbstractClient client(String id) { + return clients.get(id); + } + + private void closeNodes() { + logger.info("closing all clients"); + for (AbstractClient client : clients.values()) { + client.close(); + } + clients.clear(); + logger.info("closing all nodes"); + for (Node node : nodes.values()) { + if (node != null) { + node.close(); + } + } + nodes.clear(); + logger.info("all nodes closed"); + } + + protected void findNodeAddress() { + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); + NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet(); + Object obj = response.iterator().next().getTransport().getAddress() + .publishAddress(); + if (obj instanceof InetSocketTransportAddress) { + InetSocketTransportAddress address = (InetSocketTransportAddress) obj; + host = address.address().getHostName(); + port = address.address().getPort(); + } + } + + private Node buildNode(String id) { + Settings nodeSettings = settingsBuilder() + .put(getNodeSettings()) + .put("name", id) + .build(); + logger.info("settings={}", nodeSettings.getAsMap()); + Node node = new MockNode(nodeSettings); + AbstractClient client = (AbstractClient) node.client(); + nodes.put(id, node); + clients.put(id, client); + logger.info("clients={}", clients); + return node; + } + + protected String randomString(int len) { + final char[] buf = new char[len]; + final int n = numbersAndLetters.length - 1; + for (int i = 0; i < buf.length; i++) { + buf[i] = numbersAndLetters[random.nextInt(n)]; + } + return new String(buf); + } +} diff --git a/src/integration-test/resources/log4j2.xml b/elx-node/src/test/resources/log4j2.xml similarity index 75% rename from src/integration-test/resources/log4j2.xml rename to elx-node/src/test/resources/log4j2.xml index b175dfc..1258d7f 100644 --- a/src/integration-test/resources/log4j2.xml +++ b/elx-node/src/test/resources/log4j2.xml @@ -2,7 +2,7 @@ - + diff --git a/elx-transport/build.gradle b/elx-transport/build.gradle new file mode 100644 index 0000000..bc5e01e --- /dev/null +++ b/elx-transport/build.gradle @@ -0,0 +1,3 @@ +dependencies { + compile project(':elx-common') +} \ No newline at end of file diff --git a/elx-transport/build.gradle~ b/elx-transport/build.gradle~ new file mode 100644 index 0000000..b47f835 --- /dev/null +++ b/elx-transport/build.gradle~ @@ -0,0 +1,63 @@ +buildscript { + repositories { + jcenter() + maven { + url 'http://xbib.org/repository' + } + } + dependencies { + classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.2.2.0" + } +} + +apply plugin: 'org.xbib.gradle.plugin.elasticsearch.build' + +configurations { + main + tests +} + +dependencies { + compile project(':common') + testCompile "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" + testRuntime "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" +} + +jar { + baseName "${rootProject.name}-transport" +} + +task testJar(type: Jar, dependsOn: testClasses) { + baseName = "${project.archivesBaseName}-tests" + from sourceSets.test.output +} + +artifacts { + main jar + tests testJar + archives sourcesJar, javadocJar +} + +esTest { + enabled = true + // test with the jars, not the classes, for security manager + classpath = files(configurations.testRuntime) + configurations.main.artifacts.files + configurations.tests.artifacts.files + systemProperty 'tests.security.manager', 'true' + // maybe we like some extra security policy for our code + systemProperty 'tests.security.policy', '/extra-security.policy' +} +esTest.dependsOn jar, testJar + +randomizedTest { + enabled = false +} + +test { + enabled = false + jvmArgs "-javaagent:" + configurations.alpnagent.asPath + systemProperty 'path.home', projectDir.absolutePath + testLogging { + showStandardStreams = true + exceptionFormat = 'full' + } +} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java new file mode 100644 index 0000000..f66ac58 --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java @@ -0,0 +1,129 @@ +package org.xbib.elx.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.xbib.elx.common.AbstractExtendedClient; +import org.xbib.elx.common.util.NetworkUtils; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Transport client with additional methods using the BulkProcessor. + */ +public class ExtendedTransportClient extends AbstractExtendedClient { + + private static final Logger logger = LogManager.getLogger(ExtendedTransportClient.class.getName()); + + @Override + protected ElasticsearchClient createClient(Settings settings) { + if (settings != null) { + String systemIdentifier = System.getProperty("os.name") + + " " + System.getProperty("java.vm.name") + + " " + System.getProperty("java.vm.vendor") + + " " + System.getProperty("java.vm.version") + + " Elasticsearch " + Version.CURRENT.toString(); + logger.info("creating transport client on {} with effective settings {}", + systemIdentifier, settings.getAsMap()); + TransportClient.Builder builder = TransportClient.builder() + .settings(Settings.builder() + .put("cluster.name", settings.get("cluster.name")) + .put("processors", settings.getAsInt("processors", Runtime.getRuntime().availableProcessors())) + .put("client.transport.ignore_cluster_name", true) + .build()); + return builder.build(); + } + return null; + } + + @Override + public ExtendedTransportClient init(Settings settings) throws IOException { + super.init(settings); + // additional auto-connect + try { + Collection addrs = findAddresses(settings); + if (!connect(addrs, settings.getAsBoolean("autodiscover", false))) { + throw new NoNodeAvailableException("no cluster nodes available, check settings " + + settings.toString()); + } + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + return this; + } + + @Override + public synchronized void shutdown() throws IOException { + super.shutdown(); + logger.info("shutting down..."); + if (getClient() != null) { + TransportClient client = (TransportClient) getClient(); + client.close(); + client.threadPool().shutdown(); + } + logger.info("shutting down completed"); + } + + private Collection findAddresses(Settings settings) throws IOException { + final int defaultPort = settings.getAsInt("port", 9300); + Collection addresses = new ArrayList<>(); + for (String hostname : settings.getAsArray("host")) { + String[] splitHost = hostname.split(":", 2); + if (splitHost.length == 2) { + try { + String host = splitHost[0]; + InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null); + int port = Integer.parseInt(splitHost[1]); + InetSocketTransportAddress address = new InetSocketTransportAddress(inetAddress, port); + addresses.add(address); + } catch (NumberFormatException e) { + logger.warn(e.getMessage(), e); + } + } + if (splitHost.length == 1) { + String host = splitHost[0]; + InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null); + InetSocketTransportAddress address = new InetSocketTransportAddress(inetAddress, defaultPort); + addresses.add(address); + } + } + return addresses; + } + + private boolean connect(Collection addresses, boolean autodiscover) { + if (getClient() == null) { + throw new IllegalStateException("no client present"); + } + logger.debug("trying to connect to {}", addresses); + TransportClient transportClient = (TransportClient) getClient(); + transportClient.addTransportAddresses(addresses); + List nodes = transportClient.connectedNodes(); + logger.info("connected to nodes = {}", nodes); + if (nodes != null && !nodes.isEmpty()) { + if (autodiscover) { + logger.debug("trying to auto-discover all nodes..."); + ClusterStateRequestBuilder clusterStateRequestBuilder = + new ClusterStateRequestBuilder(getClient(), ClusterStateAction.INSTANCE); + ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); + DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes(); + transportClient.addDiscoveryNodes(discoveryNodes); + logger.info("after auto-discovery: connected to {}", transportClient.connectedNodes()); + } + return true; + } + return false; + } +} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java b/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java new file mode 100644 index 0000000..669d21a --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.transport; + +import org.xbib.elx.api.ExtendedClientProvider; + +public class ExtendedTransportClientProvider implements ExtendedClientProvider { + + @Override + public ExtendedTransportClient getExtendedClient() { + return new ExtendedTransportClient(); + } +} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/transport/TransportClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportClient.java similarity index 93% rename from src/main/java/org/xbib/elasticsearch/extras/client/transport/TransportClient.java rename to elx-transport/src/main/java/org/xbib/elx/transport/TransportClient.java index 3912ce7..827f657 100644 --- a/src/main/java/org/xbib/elasticsearch/extras/client/transport/TransportClient.java +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportClient.java @@ -1,9 +1,8 @@ -package org.xbib.elasticsearch.extras.client.transport; +package org.xbib.elx.transport; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; -import com.google.common.collect.ImmutableMap; import org.elasticsearch.Version; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionListener; @@ -25,7 +24,6 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterNameModule; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Injector; @@ -54,6 +52,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -278,16 +277,11 @@ public class TransportClient extends AbstractClient { } } try { + FutureTransportResponseHandler responseHandler = new LivenessResponseHandler(); LivenessResponse livenessResponse = transportService.submitRequest(listedNode, TransportLivenessAction.NAME, headers.applyTo(new LivenessRequest()), TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STATE) - .withTimeout(pingTimeout).build(), - new FutureTransportResponseHandler() { - @Override - public LivenessResponse newInstance() { - return new LivenessResponse(); - } - }).txGet(); + .withTimeout(pingTimeout).build(),responseHandler).txGet(); if (!clusterName.equals(livenessResponse.getClusterName())) { logger.warn("node {} not part of the cluster {}, ignoring...", listedNode, clusterName); newFilteredNodes.add(listedNode); @@ -347,12 +341,10 @@ public class TransportClient extends AbstractClient { } } - /** - * - */ public static class Builder { private Settings settings = Settings.EMPTY; + private List> pluginClasses = new ArrayList<>(); public Builder settings(Settings.Builder settings) { @@ -395,12 +387,7 @@ public class TransportClient extends AbstractClient { modules.add(new ClusterNameModule(this.settings)); modules.add(new ThreadPoolModule(threadPool)); modules.add(new TransportModule(this.settings)); - modules.add(new SearchModule() { - @Override - protected void configure() { - // noop - } - }); + modules.add(new TransportSearchModule()); modules.add(new ActionModule(true)); modules.add(new ClientTransportModule()); modules.add(new CircuitBreakerModule(this.settings)); @@ -419,27 +406,39 @@ public class TransportClient extends AbstractClient { } /** - * The {@link ProxyActionMap} must be declared public. + * The {@link ProxyActionMap} must be declared public for injection. */ @SuppressWarnings({"unchecked", "rawtypes"}) public static class ProxyActionMap { - private final ImmutableMap proxies; + private final Map proxies; @Inject public ProxyActionMap(Settings settings, TransportService transportService, Map actions) { - MapBuilder actionsBuilder = new MapBuilder<>(); + this.proxies = new LinkedHashMap<>(); for (GenericAction action : actions.values()) { if (action instanceof Action) { - actionsBuilder.put((Action) action, new TransportActionNodeProxy(settings, action, transportService)); + this.proxies.put((Action) action, new TransportActionNodeProxy(settings, action, transportService)); } } - this.proxies = actionsBuilder.immutableMap(); } - public ImmutableMap getProxies() { + public Map getProxies() { return proxies; } } + private static class LivenessResponseHandler extends FutureTransportResponseHandler { + @Override + public LivenessResponse newInstance() { + return new LivenessResponse(); + } + } + + private static class TransportSearchModule extends SearchModule { + @Override + protected void configure() { + // noop + } + } } diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/package-info.java b/elx-transport/src/main/java/org/xbib/elx/transport/package-info.java new file mode 100644 index 0000000..3697854 --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/package-info.java @@ -0,0 +1,4 @@ +/** + * Classes for Elasticsearch transport client extensions. + */ +package org.xbib.elx.transport; diff --git a/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider new file mode 100644 index 0000000..640e2f9 --- /dev/null +++ b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider @@ -0,0 +1 @@ +org.xbib.elx.transport.ExtendedTransportClientProvider \ No newline at end of file diff --git a/elx-transport/src/test/java/org/elasticsearch/node/MockNode.java b/elx-transport/src/test/java/org/elasticsearch/node/MockNode.java new file mode 100644 index 0000000..aad8b8b --- /dev/null +++ b/elx-transport/src/test/java/org/elasticsearch/node/MockNode.java @@ -0,0 +1,34 @@ +package org.elasticsearch.node; + +import org.elasticsearch.Version; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.internal.InternalSettingsPreparer; +import org.elasticsearch.plugins.Plugin; + +import java.util.ArrayList; +import java.util.Collection; + +public class MockNode extends Node { + + public MockNode() { + super(Settings.EMPTY); + } + + public MockNode(Settings settings) { + super(settings); + } + + public MockNode(Settings settings, Collection> classpathPlugins) { + super(InternalSettingsPreparer.prepareEnvironment(settings, null), Version.CURRENT, classpathPlugins); + } + + public MockNode(Settings settings, Class classpathPlugin) { + this(settings, list(classpathPlugin)); + } + + private static Collection> list(Class classpathPlugin) { + Collection> list = new ArrayList<>(); + list.add(classpathPlugin); + return list; + } +} diff --git a/elx-transport/src/test/java/org/elasticsearch/node/package-info.java b/elx-transport/src/test/java/org/elasticsearch/node/package-info.java new file mode 100644 index 0000000..8ffed8c --- /dev/null +++ b/elx-transport/src/test/java/org/elasticsearch/node/package-info.java @@ -0,0 +1 @@ +package org.elasticsearch.node; \ No newline at end of file diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientSingleNodeTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientSingleNodeTest.java new file mode 100644 index 0000000..cb9dba9 --- /dev/null +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientSingleNodeTest.java @@ -0,0 +1,40 @@ +package org.xbib.elx.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.junit.Test; +import org.xbib.elx.common.ClientBuilder; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +public class ExtendedTransportClientSingleNodeTest extends NodeTestUtils { + + private static final Logger logger = LogManager.getLogger(ExtendedTransportClientSingleNodeTest.class.getSimpleName()); + + @Test + public void testSingleDocNodeClient() throws Exception { + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) + .put(getSettings()) + .build(); + try { + client.newIndex("test"); + client.index("test", "test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + client.flushIngest(); + client.waitForResponses("30s"); + } catch (InterruptedException e) { + // ignore + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + client.shutdown(); + } + } +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClientTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientTest.java similarity index 53% rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClientTest.java rename to elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientTest.java index c7c82e0..cfca3da 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClientTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportClientTest.java @@ -1,40 +1,34 @@ -package org.xbib.elasticsearch.extras.client.transport; +package org.xbib.elx.transport; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.query.QueryBuilders; import org.junit.Before; import org.junit.Test; -import org.xbib.elasticsearch.NodeTestUtils; -import org.xbib.elasticsearch.extras.client.Clients; -import org.xbib.elasticsearch.extras.client.SimpleBulkControl; -import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.Parameters; -import java.io.IOException; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -/** - * - */ -public class BulkTransportClientTest extends NodeTestUtils { - private static final ESLogger logger = ESLoggerFactory.getLogger(BulkTransportClientTest.class.getSimpleName()); +public class ExtendedTransportClientTest extends NodeTestUtils { - private static final Long MAX_ACTIONS = 1000L; + private static final Logger logger = LogManager.getLogger(ExtendedTransportClientTest.class.getSimpleName()); - private static final Long NUM_ACTIONS = 1234L; + private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; + + private static final Long ACTIONS = 1234L; @Before public void startNodes() { @@ -47,13 +41,12 @@ public class BulkTransportClientTest extends NodeTestUtils { } @Test - public void testBulkClient() throws IOException { - final BulkTransportClient client = Clients.builder() + public void testBulkClient() throws Exception { + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) .put(getSettings()) - .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60)) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkTransportClient(); + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); client.newIndex("test"); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); @@ -75,27 +68,24 @@ public class BulkTransportClientTest extends NodeTestUtils { } @Test - public void testSingleDocBulkClient() throws IOException { - final BulkTransportClient client = Clients.builder() + public void testSingleDocBulkClient() throws Exception { + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) .put(getSettings()) - .put(Clients.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS) - .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60)) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkTransportClient(); + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); try { client.newIndex("test"); - client.index("test", "test", "1", "{ \"name\" : \"Hello World\"}"); // single doc ingest + client.index("test", "test", "1", true, "{ \"name\" : \"Hello World\"}"); client.flushIngest(); client.waitForResponses("30s"); } catch (InterruptedException e) { // ignore - } catch (ExecutionException e) { - logger.error(e.getMessage(), e); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - assertEquals(1, client.getMetric().getSucceeded().getCount()); + assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); } @@ -105,30 +95,27 @@ public class BulkTransportClientTest extends NodeTestUtils { } @Test - public void testRandomDocsBulkClient() throws IOException { - long numactions = NUM_ACTIONS; - final BulkTransportClient client = Clients.builder() + public void testRandomDocsBulkClient() throws Exception { + long numactions = ACTIONS; + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) .put(getSettings()) - .put(Clients.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS) - .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60)) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkTransportClient(); + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); try { client.newIndex("test"); - for (int i = 0; i < NUM_ACTIONS; i++) { - client.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + for (int i = 0; i < ACTIONS; i++) { + client.index("test", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.waitForResponses("30s"); } catch (InterruptedException e) { // ignore - } catch (ExecutionException e) { - logger.error(e.getMessage(), e); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - assertEquals(numactions, client.getMetric().getSucceeded().getCount()); + assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); } @@ -140,54 +127,54 @@ public class BulkTransportClientTest extends NodeTestUtils { @Test public void testThreadedRandomDocsBulkClient() throws Exception { int maxthreads = Runtime.getRuntime().availableProcessors(); - long maxactions = MAX_ACTIONS; - final long maxloop = NUM_ACTIONS; + long maxactions = MAX_ACTIONS_PER_REQUEST; + final long maxloop = ACTIONS; Settings settingsForIndex = Settings.settingsBuilder() .put("index.number_of_shards", 2) .put("index.number_of_replicas", 1) .build(); - final BulkTransportClient client = Clients.builder() + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) .put(getSettings()) - .put(Clients.MAX_ACTIONS_PER_REQUEST, maxactions) - .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60)) // = disable autoflush for this test - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkTransportClient(); + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxactions) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); try { client.newIndex("test", settingsForIndex, null) .startBulk("test", -1, 1000); - ThreadPoolExecutor pool = - EsExecutors.newFixed("bulkclient-test", maxthreads, 30, EsExecutors.daemonThreadFactory("bulkclient-test")); + ThreadPoolExecutor pool = EsExecutors.newFixed("bulkclient-test", maxthreads, 30, + EsExecutors.daemonThreadFactory("bulkclient-test")); final CountDownLatch latch = new CountDownLatch(maxthreads); for (int i = 0; i < maxthreads; i++) { pool.execute(() -> { for (int i1 = 0; i1 < maxloop; i1++) { - client.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } latch.countDown(); }); } - logger.info("waiting for max 30 seconds..."); - latch.await(30, TimeUnit.SECONDS); - logger.info("client flush ..."); - client.flushIngest(); - client.waitForResponses("30s"); - logger.info("thread pool to be shut down ..."); - pool.shutdown(); - logger.info("poot shut down"); + logger.info("waiting for latch..."); + if (latch.await(60, TimeUnit.SECONDS)) { + logger.info("flush ..."); + client.flushIngest(); + client.waitForResponses("30s"); + logger.info("pool to be shut down ..."); + pool.shutdown(); + logger.info("poot shut down"); + } } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { client.stopBulk("test"); - assertEquals(maxthreads * maxloop, client.getMetric().getSucceeded().getCount()); + assertEquals(maxthreads * maxloop, client.getBulkMetric().getSucceeded().getCount()); if (client.hasThrowable()) { logger.error("error", client.getThrowable()); } assertFalse(client.hasThrowable()); client.refreshIndex("test"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE) + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) // to avoid NPE at org.elasticsearch.action.search.SearchRequest.writeTo(SearchRequest.java:580) .setIndices("_all") .setQuery(QueryBuilders.matchAllQuery()) @@ -197,5 +184,4 @@ public class BulkTransportClientTest extends NodeTestUtils { client.shutdown(); } } - } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportDuplicateIDTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportDuplicateIDTest.java new file mode 100644 index 0000000..1fa73c9 --- /dev/null +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportDuplicateIDTest.java @@ -0,0 +1,57 @@ +package org.xbib.elx.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.junit.Test; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.Parameters; + +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.junit.Assert.*; + +public class ExtendedTransportDuplicateIDTest extends NodeTestUtils { + + private final static Logger logger = LogManager.getLogger(ExtendedTransportDuplicateIDTest.class.getSimpleName()); + + private final static Long MAX_ACTIONS_PER_REQUEST = 1000L; + + private final static Long ACTIONS = 12345L; + + @Test + public void testDuplicateDocIDs() throws Exception { + long numactions = ACTIONS; + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) + .put(getSettings()) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .build(); + try { + client.newIndex("test"); + for (int i = 0; i < ACTIONS; i++) { + client.index("test", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flushIngest(); + client.waitForResponses("30s"); + client.refreshIndex("test"); + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) + .setIndices("test") + .setTypes("test") + .setQuery(matchAllQuery()); + long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); + logger.info("hits = {}", hits); + assertTrue(hits < ACTIONS); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + client.shutdown(); + assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + } + } +} diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportIndexAliasTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportIndexAliasTest.java new file mode 100644 index 0000000..a2dfb4c --- /dev/null +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportIndexAliasTest.java @@ -0,0 +1,65 @@ +package org.xbib.elx.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.index.query.QueryBuilders; +import org.junit.Ignore; +import org.junit.Test; +import org.xbib.elx.common.ClientBuilder; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertFalse; + +@Ignore +public class ExtendedTransportIndexAliasTest extends NodeTestUtils { + + private static final Logger logger = LogManager.getLogger(ExtendedTransportIndexAliasTest.class.getSimpleName()); + + @Test + public void testIndexAlias() throws Exception { + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) + .build(); + try { + client.newIndex("test1234"); + for (int i = 0; i < 1; i++) { + client.index("test1234", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flushIngest(); + client.refreshIndex("test1234"); + + List simpleAliases = Arrays.asList("a", "b", "c"); + client.switchAliases("test", "test1234", simpleAliases); + + client.newIndex("test5678"); + for (int i = 0; i < 1; i++) { + client.index("test5678", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flushIngest(); + client.refreshIndex("test5678"); + + simpleAliases = Arrays.asList("d", "e", "f"); + client.switchAliases("test", "test5678", simpleAliases, (builder, index, alias) -> + builder.addAlias(index, alias, QueryBuilders.termQuery("my_key", alias))); + Map aliases = client.getIndexFilters("test5678"); + logger.info("aliases of index test5678 = {}", aliases); + + aliases = client.getAliasFilters("test"); + logger.info("aliases of alias test = {}", aliases); + + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + client.waitForResponses("30s"); + client.shutdown(); + if (client.hasThrowable()) { + logger.error("error", client.getThrowable()); + } + assertFalse(client.hasThrowable()); + } + } +} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportReplicaTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportReplicaTest.java similarity index 78% rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportReplicaTest.java rename to elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportReplicaTest.java index bc8f449..6b6d6d4 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportReplicaTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportReplicaTest.java @@ -1,5 +1,7 @@ -package org.xbib.elasticsearch.extras.client.transport; +package org.xbib.elx.transport; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.admin.indices.stats.IndexShardStats; import org.elasticsearch.action.admin.indices.stats.IndexStats; @@ -9,15 +11,10 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.indexing.IndexingStats; import org.junit.Test; -import org.xbib.elasticsearch.NodeTestUtils; -import org.xbib.elasticsearch.extras.client.Clients; -import org.xbib.elasticsearch.extras.client.SimpleBulkControl; -import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; +import org.xbib.elx.common.ClientBuilder; import java.util.Map; @@ -28,9 +25,9 @@ import static org.junit.Assert.assertFalse; /** * */ -public class BulkTransportReplicaTest extends NodeTestUtils { +public class ExtendedTransportReplicaTest extends NodeTestUtils { - private static final ESLogger logger = ESLoggerFactory.getLogger(BulkTransportReplicaTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(ExtendedTransportReplicaTest.class.getSimpleName()); @Test public void testReplicaLevel() throws Exception { @@ -50,20 +47,20 @@ public class BulkTransportReplicaTest extends NodeTestUtils { .put("index.number_of_replicas", 1) .build(); - final BulkTransportClient client = Clients.builder() + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) .put(getSettings()) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkTransportClient(); + .build(); + try { client.newIndex("test1", settingsTest1, null) .newIndex("test2", settingsTest2, null); client.waitForCluster("GREEN", "30s"); for (int i = 0; i < 1234; i++) { - client.index("test1", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test1", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } for (int i = 0; i < 1234; i++) { - client.index("test2", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test2", "test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.waitForResponses("30s"); @@ -73,13 +70,13 @@ public class BulkTransportReplicaTest extends NodeTestUtils { logger.info("refreshing"); client.refreshIndex("test1"); client.refreshIndex("test2"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE) + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) .setIndices("test1", "test2") .setQuery(matchAllQuery()); long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); logger.info("query total hits={}", hits); assertEquals(2468, hits); - IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.client(), + IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.getClient(), IndicesStatsAction.INSTANCE).all(); IndicesStatsResponse response = indicesStatsRequestBuilder.execute().actionGet(); for (Map.Entry m : response.getIndices().entrySet()) { diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeUpdateReplicaLevelTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportUpdateReplicaLevelTest.java similarity index 63% rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeUpdateReplicaLevelTest.java rename to elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportUpdateReplicaLevelTest.java index 5dc9202..17d71d6 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeUpdateReplicaLevelTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ExtendedTransportUpdateReplicaLevelTest.java @@ -1,29 +1,23 @@ -package org.xbib.elasticsearch.extras.client.node; +package org.xbib.elx.transport; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; import org.junit.Test; -import org.xbib.elasticsearch.NodeTestUtils; -import org.xbib.elasticsearch.extras.client.Clients; -import org.xbib.elasticsearch.extras.client.SimpleBulkControl; -import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; +import org.xbib.elx.common.ClientBuilder; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -/** - * - */ -public class BulkNodeUpdateReplicaLevelTest extends NodeTestUtils { +public class ExtendedTransportUpdateReplicaLevelTest extends NodeTestUtils { - private static final ESLogger logger = ESLoggerFactory.getLogger(BulkNodeUpdateReplicaLevelTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(ExtendedTransportUpdateReplicaLevelTest.class.getSimpleName()); @Test public void testUpdateReplicaLevel() throws Exception { - int numberOfShards = 2; + long numberOfShards = 2; int replicaLevel = 3; // we need 3 nodes for replica level 3 @@ -32,21 +26,21 @@ public class BulkNodeUpdateReplicaLevelTest extends NodeTestUtils { int shardsAfterReplica; + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) + .put(getSettings()) + .build(); + Settings settings = Settings.settingsBuilder() .put("index.number_of_shards", numberOfShards) .put("index.number_of_replicas", 0) .build(); - final BulkNodeClient client = Clients.builder() - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkNodeClient(client("1")); - try { client.newIndex("replicatest", settings, null); client.waitForCluster("GREEN", "30s"); for (int i = 0; i < 12345; i++) { - client.index("replicatest", "replicatest", null, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("replicatest", "replicatest", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flushIngest(); client.waitForResponses("30s"); @@ -62,5 +56,4 @@ public class BulkNodeUpdateReplicaLevelTest extends NodeTestUtils { assertFalse(client.hasThrowable()); } } - } diff --git a/src/integration-test/java/org/xbib/elasticsearch/NodeTestUtils.java b/elx-transport/src/test/java/org/xbib/elx/transport/NodeTestUtils.java similarity index 91% rename from src/integration-test/java/org/xbib/elasticsearch/NodeTestUtils.java rename to elx-transport/src/test/java/org/xbib/elx/transport/NodeTestUtils.java index d098332..736f87a 100644 --- a/src/integration-test/java/org/xbib/elasticsearch/NodeTestUtils.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/NodeTestUtils.java @@ -1,7 +1,7 @@ -package org.xbib.elasticsearch; - -import static org.elasticsearch.common.settings.Settings.settingsBuilder; +package org.xbib.elx.transport; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; @@ -10,8 +10,6 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.unit.TimeValue; @@ -19,22 +17,25 @@ import org.elasticsearch.node.MockNode; import org.elasticsearch.node.Node; import org.junit.After; import org.junit.Before; -import org.xbib.elasticsearch.extras.client.NetworkUtils; +import org.xbib.elx.common.util.NetworkUtils; import java.io.IOException; -import java.nio.file.*; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.util.HashMap; import java.util.Map; import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; -/** - * - */ +import static org.elasticsearch.common.settings.Settings.settingsBuilder; + public class NodeTestUtils { - private static final ESLogger logger = ESLoggerFactory.getLogger("test"); + private static final Logger logger = LogManager.getLogger("test"); private static Random random = new Random(); @@ -53,7 +54,7 @@ public class NodeTestUtils { private int port; private static void deleteFiles() throws IOException { - Path directory = Paths.get(System.getProperty("path.home") + "/data"); + Path directory = Paths.get(getHome() + "/data"); Files.walkFileTree(directory, new SimpleFileVisitor() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { @@ -148,11 +149,11 @@ public class NodeTestUtils { .build(); } - protected String getHome() { - return System.getProperty("path.home"); + protected static String getHome() { + return System.getProperty("path.home", System.getProperty("user.dir")); } - public void startNode(String id) throws IOException { + public void startNode(String id) { buildNode(id).start(); } @@ -160,7 +161,7 @@ public class NodeTestUtils { return clients.get(id); } - private void closeNodes() throws IOException { + private void closeNodes() { logger.info("closing all clients"); for (AbstractClient client : clients.values()) { client.close(); @@ -188,7 +189,7 @@ public class NodeTestUtils { } } - private Node buildNode(String id) throws IOException { + private Node buildNode(String id) { Settings nodeSettings = settingsBuilder() .put(getNodeSettings()) .put("name", id) diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/package-info.java b/elx-transport/src/test/java/org/xbib/elx/transport/package-info.java new file mode 100644 index 0000000..7abcc5a --- /dev/null +++ b/elx-transport/src/test/java/org/xbib/elx/transport/package-info.java @@ -0,0 +1 @@ +package org.xbib.elx.transport; \ No newline at end of file diff --git a/elx-transport/src/test/resources/log4j2.xml b/elx-transport/src/test/resources/log4j2.xml new file mode 100644 index 0000000..6c323f8 --- /dev/null +++ b/elx-transport/src/test/resources/log4j2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index 3f32f32..b6d2ad5 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,19 @@ group = org.xbib -name = elasticsearch-extras-client -version = 2.2.1.2 +name = elx +version = 2.2.1.3 + +xbib-metrics.version = 1.1.0 +xbib-guice.version = 4.0.4 + +elasticsearch.version = 2.2.1 +jna.version = 4.5.2 +log4j.version = 2.11.1 +mustache.version = 0.9.5 +jts.version = 1.13 +jackson-dataformat.version = 2.8.11 + +junit.version = 4.12 +wagon.version = 3.0.0 +asciidoclet.version = 1.5.4 + +org.gradle.warning.mode = all diff --git a/gradle/ext.gradle b/gradle/ext.gradle deleted file mode 100644 index 7bb7c73..0000000 --- a/gradle/ext.gradle +++ /dev/null @@ -1,8 +0,0 @@ -ext { - user = 'xbib' - name = 'elasticsearch-extras-client' - description = 'Some extras implemented for using Elasticsearch clients (node and transport)' - scmUrl = 'https://github.com/' + user + '/' + name - scmConnection = 'scm:git:git://github.com/' + user + '/' + name + '.git' - scmDeveloperConnection = 'scm:git:git://github.com/' + user + '/' + name + '.git' -} diff --git a/gradle/publish.gradle b/gradle/publish.gradle index 0337849..8675487 100644 --- a/gradle/publish.gradle +++ b/gradle/publish.gradle @@ -1,12 +1,19 @@ +ext { + description = 'Extensions for Elasticsearch clients (node and transport)' + scmUrl = 'https://github.com/jprante/elx' + scmConnection = 'scm:git:git://github.com/jprante/elx.git' + scmDeveloperConnection = 'scm:git:git://github.com/jprante/elx.git' +} -task xbibUpload(type: Upload) { +task xbibUpload(type: Upload, dependsOn: build) { + group = 'publish' configuration = configurations.archives uploadDescriptor = true repositories { if (project.hasProperty("xbibUsername")) { mavenDeployer { configuration = configurations.wagon - repository(url: 'scpexe://xbib.org/repository') { + repository(url: uri(project.property('xbibUrl'))) { authentication(userName: xbibUsername, privateKey: xbibPrivateKey) } } @@ -14,7 +21,8 @@ task xbibUpload(type: Upload) { } } -task sonaTypeUpload(type: Upload) { +task sonaTypeUpload(type: Upload, dependsOn: build) { + group = 'publish' configuration = configurations.archives uploadDescriptor = true repositories { @@ -34,7 +42,7 @@ task sonaTypeUpload(type: Upload) { name project.name description description packaging 'jar' - inceptionYear '2012' + inceptionYear '2019' url scmUrl organization { name 'xbib' @@ -42,7 +50,7 @@ task sonaTypeUpload(type: Upload) { } developers { developer { - id user + id 'xbib' name 'Jörg Prante' email 'joergprante@gmail.com' url 'https://github.com/jprante' @@ -64,3 +72,7 @@ task sonaTypeUpload(type: Upload) { } } } + +nexusStaging { + packageGroup = "org.xbib" +} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 51288f9..87b738c 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 27b5466..710ced3 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Tue Jan 03 14:13:22 CET 2017 +#Fri Feb 15 11:59:10 CET 2019 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-5.2.1-all.zip diff --git a/gradlew b/gradlew index 4453cce..af6708f 100755 --- a/gradlew +++ b/gradlew @@ -28,16 +28,16 @@ APP_NAME="Gradle" APP_BASE_NAME=`basename "$0"` # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" +DEFAULT_JVM_OPTS='"-Xmx64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" -warn ( ) { +warn () { echo "$*" } -die ( ) { +die () { echo echo "$*" echo @@ -155,7 +155,7 @@ if $cygwin ; then fi # Escape application args -save ( ) { +save () { for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done echo " " } diff --git a/gradlew.bat b/gradlew.bat index e95643d..0f8d593 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -14,7 +14,7 @@ set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= +set DEFAULT_JVM_OPTS="-Xmx64m" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome diff --git a/settings.gradle b/settings.gradle index ef50653..57d6828 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1 +1,5 @@ -rootProject.name = 'elasticsearch-extras-client' +include 'elx-api' +include 'elx-common' +include 'elx-node' +include 'elx-transport' +include 'elx-http' diff --git a/src/integration-test/java/org/elasticsearch/node/package-info.java b/src/integration-test/java/org/elasticsearch/node/package-info.java deleted file mode 100644 index f299cbc..0000000 --- a/src/integration-test/java/org/elasticsearch/node/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Classes to support Elasticsearch node creation. - */ -package org.elasticsearch.node; diff --git a/src/integration-test/java/org/xbib/elasticsearch/SearchTest.java b/src/integration-test/java/org/xbib/elasticsearch/SearchTest.java deleted file mode 100644 index 8d1276a..0000000 --- a/src/integration-test/java/org/xbib/elasticsearch/SearchTest.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.xbib.elasticsearch; - -import static org.elasticsearch.client.Requests.indexRequest; -import static org.elasticsearch.client.Requests.refreshRequest; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; - -import org.elasticsearch.action.bulk.BulkAction; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.sort.SortOrder; -import org.junit.Test; - -/** - * - */ -public class SearchTest extends NodeTestUtils { - - private static final ESLogger logger = ESLoggerFactory.getLogger("test"); - - @Test - public void testSearch() throws Exception { - Client client = client("1"); - long t0 = System.currentTimeMillis(); - BulkRequestBuilder builder = new BulkRequestBuilder(client, BulkAction.INSTANCE); - for (int i = 0; i < 1000; i++) { - builder.add(indexRequest() - .index("pages").type("row") - .source(jsonBuilder() - .startObject() - .field("user1", "kimchy") - .field("user2", "kimchy") - .field("user3", "kimchy") - .field("user4", "kimchy") - .field("user5", "kimchy") - .field("user6", "kimchy") - .field("user7", "kimchy") - .field("user8", "kimchy") - .field("user9", "kimchy") - .field("rowcount", i) - .field("rs", 1234))); - } - client.bulk(builder.request()).actionGet(); - - client.admin().indices().refresh(refreshRequest()).actionGet(); - - long t1 = System.currentTimeMillis(); - logger.info("t1-t0 = {}", t1 - t0); - - for (int i = 0; i < 100; i++) { - t1 = System.currentTimeMillis(); - QueryBuilder queryStringBuilder = - QueryBuilders.queryStringQuery("rs:" + 1234); - SearchRequestBuilder requestBuilder = client.prepareSearch() - .setIndices("pages") - .setTypes("row") - .setQuery(queryStringBuilder) - .addSort("rowcount", SortOrder.DESC) - .setFrom(i * 10).setSize(10); - SearchResponse response = requestBuilder.execute().actionGet(); - long t2 = System.currentTimeMillis(); - logger.info("t2-t1 = {}", t2 - t1); - } - } -} diff --git a/src/integration-test/java/org/xbib/elasticsearch/WildcardTest.java b/src/integration-test/java/org/xbib/elasticsearch/WildcardTest.java deleted file mode 100644 index 6e252d1..0000000 --- a/src/integration-test/java/org/xbib/elasticsearch/WildcardTest.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.xbib.elasticsearch; - -import static org.elasticsearch.client.Requests.indexRequest; -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; - -import org.elasticsearch.client.Client; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.query.QueryBuilder; -import org.junit.Test; - -import java.io.IOException; - -/** - * - */ -public class WildcardTest extends NodeTestUtils { - - protected Settings getNodeSettings() { - return settingsBuilder() - .put("cluster.name", getClusterName()) - .put("cluster.routing.allocation.disk.threshold_enabled", false) - .put("discovery.zen.multicast.enabled", false) - .put("http.enabled", false) - .put("path.home", System.getProperty("path.home")) - .put("index.number_of_shards", 1) - .put("index.number_of_replicas", 0) - .build(); - } - - @Test - public void testWildcard() throws Exception { - index(client("1"), "1", "010"); - index(client("1"), "2", "0*0"); - // exact - validateCount(client("1"), queryStringQuery("010").defaultField("field"), 1); - validateCount(client("1"), queryStringQuery("0\\*0").defaultField("field"), 1); - // pattern - validateCount(client("1"), queryStringQuery("0*0").defaultField("field"), 1); // 2? - validateCount(client("1"), queryStringQuery("0?0").defaultField("field"), 1); // 2? - validateCount(client("1"), queryStringQuery("0**0").defaultField("field"), 1); // 2? - validateCount(client("1"), queryStringQuery("0??0").defaultField("field"), 0); - validateCount(client("1"), queryStringQuery("*10").defaultField("field"), 1); - validateCount(client("1"), queryStringQuery("*1*").defaultField("field"), 1); - validateCount(client("1"), queryStringQuery("*\\*0").defaultField("field"), 0); // 1? - validateCount(client("1"), queryStringQuery("*\\**").defaultField("field"), 0); // 1? - } - - private void index(Client client, String id, String fieldValue) throws IOException { - client.index(indexRequest() - .index("index").type("type").id(id) - .source(jsonBuilder().startObject().field("field", fieldValue).endObject()) - .refresh(true)).actionGet(); - } - - private long count(Client client, QueryBuilder queryBuilder) { - return client.prepareSearch("index").setTypes("type") - .setQuery(queryBuilder) - .execute().actionGet().getHits().getTotalHits(); - } - - private void validateCount(Client client, QueryBuilder queryBuilder, long expectedHits) { - final long actualHits = count(client, queryBuilder); - if (actualHits != expectedHits) { - throw new RuntimeException("actualHits=" + actualHits + ", expectedHits=" + expectedHits); - } - } -} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeDuplicateIDTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeDuplicateIDTest.java deleted file mode 100644 index 7c11526..0000000 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeDuplicateIDTest.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.xbib.elasticsearch.extras.client.node; - -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.junit.Test; -import org.xbib.elasticsearch.NodeTestUtils; -import org.xbib.elasticsearch.extras.client.Clients; -import org.xbib.elasticsearch.extras.client.SimpleBulkControl; -import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; - -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.junit.Assert.*; - -/** - * - */ -public class BulkNodeDuplicateIDTest extends NodeTestUtils { - - private static final ESLogger logger = ESLoggerFactory.getLogger(BulkNodeDuplicateIDTest.class.getSimpleName()); - - private static final Long MAX_ACTIONS = 1000L; - - private static final Long NUM_ACTIONS = 12345L; - - @Test - public void testDuplicateDocIDs() throws Exception { - long numactions = NUM_ACTIONS; - final BulkNodeClient client = Clients.builder() - .put(Clients.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkNodeClient(client("1")); - try { - client.newIndex("test"); - for (int i = 0; i < NUM_ACTIONS; i++) { - client.index("test", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}"); - } - client.flushIngest(); - client.waitForResponses("30s"); - client.refreshIndex("test"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE) - .setIndices("test") - .setTypes("test") - .setQuery(matchAllQuery()); - long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); - logger.info("hits = {}", hits); - assertTrue(hits < NUM_ACTIONS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - client.shutdown(); - assertEquals(numactions, client.getMetric().getSucceeded().getCount()); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); - } - assertFalse(client.hasThrowable()); - } - } -} diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/package-info.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/package-info.java deleted file mode 100644 index 873ebae..0000000 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Classes for testing Elasticsearch node client extras. - */ -package org.xbib.elasticsearch.extras.client.node; diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/package-info.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/package-info.java deleted file mode 100644 index 2bfc45c..0000000 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Classes to test Elasticsearch clients. - */ -package org.xbib.elasticsearch.extras.client; diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportDuplicateIDTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportDuplicateIDTest.java deleted file mode 100644 index c087601..0000000 --- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportDuplicateIDTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.xbib.elasticsearch.extras.client.transport; - -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.junit.Test; -import org.xbib.elasticsearch.NodeTestUtils; -import org.xbib.elasticsearch.extras.client.Clients; -import org.xbib.elasticsearch.extras.client.SimpleBulkControl; -import org.xbib.elasticsearch.extras.client.SimpleBulkMetric; - -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.junit.Assert.*; - -public class BulkTransportDuplicateIDTest extends NodeTestUtils { - - private final static ESLogger logger = ESLoggerFactory.getLogger(BulkTransportDuplicateIDTest.class.getSimpleName()); - - private final static Long MAX_ACTIONS = 1000L; - - private final static Long NUM_ACTIONS = 12345L; - - @Test - public void testDuplicateDocIDs() throws Exception { - long numactions = NUM_ACTIONS; - final BulkTransportClient client = Clients.builder() - .put(getSettings()) - .put(Clients.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS) - .setMetric(new SimpleBulkMetric()) - .setControl(new SimpleBulkControl()) - .toBulkTransportClient(); - try { - client.newIndex("test"); - for (int i = 0; i < NUM_ACTIONS; i++) { - client.index("test", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}"); - } - client.flushIngest(); - client.waitForResponses("30s"); - client.refreshIndex("test"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE) - .setIndices("test") - .setTypes("test") - .setQuery(matchAllQuery()); - long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); - logger.info("hits = {}", hits); - assertTrue(hits < NUM_ACTIONS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - client.shutdown(); - assertEquals(numactions, client.getMetric().getSucceeded().getCount()); - if (client.hasThrowable()) { - logger.error("error", client.getThrowable()); - } - assertFalse(client.hasThrowable()); - } - } -} diff --git a/src/integration-test/java/org/xbib/elasticsearch/package-info.java b/src/integration-test/java/org/xbib/elasticsearch/package-info.java deleted file mode 100644 index 2958ce1..0000000 --- a/src/integration-test/java/org/xbib/elasticsearch/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Test classes for testing Elasticsearch. - */ -package org.xbib.elasticsearch; \ No newline at end of file diff --git a/src/integration-test/java/suites/BulkNodeTestSuite.java b/src/integration-test/java/suites/BulkNodeTestSuite.java deleted file mode 100644 index caac820..0000000 --- a/src/integration-test/java/suites/BulkNodeTestSuite.java +++ /dev/null @@ -1,23 +0,0 @@ -package suites; - -import org.junit.runner.RunWith; -import org.junit.runners.Suite; -import org.xbib.elasticsearch.extras.client.node.BulkNodeClientTest; -import org.xbib.elasticsearch.extras.client.node.BulkNodeDuplicateIDTest; -import org.xbib.elasticsearch.extras.client.node.BulkNodeIndexAliasTest; -import org.xbib.elasticsearch.extras.client.node.BulkNodeReplicaTest; -import org.xbib.elasticsearch.extras.client.node.BulkNodeUpdateReplicaLevelTest; - -/** - * - */ -@RunWith(ListenerSuite.class) -@Suite.SuiteClasses({ - BulkNodeClientTest.class, - BulkNodeDuplicateIDTest.class, - BulkNodeReplicaTest.class, - BulkNodeUpdateReplicaLevelTest.class, - BulkNodeIndexAliasTest.class -}) -public class BulkNodeTestSuite { -} diff --git a/src/integration-test/java/suites/BulkTransportTestSuite.java b/src/integration-test/java/suites/BulkTransportTestSuite.java deleted file mode 100644 index f429dfc..0000000 --- a/src/integration-test/java/suites/BulkTransportTestSuite.java +++ /dev/null @@ -1,22 +0,0 @@ -package suites; - -import org.junit.runner.RunWith; -import org.junit.runners.Suite; -import org.xbib.elasticsearch.extras.client.transport.BulkTransportClientTest; -import org.xbib.elasticsearch.extras.client.transport.BulkTransportDuplicateIDTest; -import org.xbib.elasticsearch.extras.client.transport.BulkTransportReplicaTest; -import org.xbib.elasticsearch.extras.client.transport.BulkTransportUpdateReplicaLevelTest; - -/** - * - */ -@RunWith(ListenerSuite.class) -@Suite.SuiteClasses({ - BulkTransportClientTest.class, - BulkTransportDuplicateIDTest.class, - BulkTransportReplicaTest.class, - BulkTransportUpdateReplicaLevelTest.class -}) -public class BulkTransportTestSuite { - -} diff --git a/src/integration-test/java/suites/ListenerSuite.java b/src/integration-test/java/suites/ListenerSuite.java deleted file mode 100644 index c02d371..0000000 --- a/src/integration-test/java/suites/ListenerSuite.java +++ /dev/null @@ -1,23 +0,0 @@ -package suites; - -import org.junit.runner.Runner; -import org.junit.runner.notification.RunNotifier; -import org.junit.runners.Suite; -import org.junit.runners.model.InitializationError; -import org.junit.runners.model.RunnerBuilder; - -public class ListenerSuite extends Suite { - - private final TestListener listener = new TestListener(); - - public ListenerSuite(Class klass, RunnerBuilder builder) throws InitializationError { - super(klass, builder); - } - - @Override - protected void runChild(Runner runner, RunNotifier notifier) { - notifier.addListener(listener); - runner.run(notifier); - notifier.removeListener(listener); - } -} diff --git a/src/integration-test/java/suites/MiscTestSuite.java b/src/integration-test/java/suites/MiscTestSuite.java deleted file mode 100644 index ea23630..0000000 --- a/src/integration-test/java/suites/MiscTestSuite.java +++ /dev/null @@ -1,21 +0,0 @@ -package suites; - -import org.junit.runner.RunWith; -import org.junit.runners.Suite; -import org.xbib.elasticsearch.AliasTest; -import org.xbib.elasticsearch.SearchTest; -import org.xbib.elasticsearch.SimpleTest; -import org.xbib.elasticsearch.WildcardTest; - -/** - * - */ -@RunWith(ListenerSuite.class) -@Suite.SuiteClasses({ - SimpleTest.class, - AliasTest.class, - SearchTest.class, - WildcardTest.class -}) -public class MiscTestSuite { -} diff --git a/src/integration-test/java/suites/TestListener.java b/src/integration-test/java/suites/TestListener.java deleted file mode 100644 index 7e24527..0000000 --- a/src/integration-test/java/suites/TestListener.java +++ /dev/null @@ -1,44 +0,0 @@ -package suites; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.junit.runner.Description; -import org.junit.runner.Result; -import org.junit.runner.notification.Failure; -import org.junit.runner.notification.RunListener; - -/** - * - */ -public class TestListener extends RunListener { - - private static final Logger logger = LogManager.getLogger("test.listener"); - - public void testRunStarted(Description description) throws java.lang.Exception { - logger.info("number of tests to execute: {}", description.testCount()); - } - - public void testRunFinished(Result result) throws java.lang.Exception { - logger.info("number of tests executed: {}", result.getRunCount()); - } - - public void testStarted(Description description) throws java.lang.Exception { - logger.info("starting execution of {} {}", - description.getClassName(), description.getMethodName()); - } - - public void testFinished(Description description) throws java.lang.Exception { - logger.info("finished execution of {} {}", - description.getClassName(), description.getMethodName()); - } - - public void testFailure(Failure failure) throws java.lang.Exception { - logger.info("failed execution of tests: {}", - failure.getMessage()); - } - - public void testIgnored(Description description) throws java.lang.Exception { - logger.info("execution of test ignored: {}", - description.getClassName(), description.getMethodName()); - } -} diff --git a/src/integration-test/resources/org/xbib/elasticsearch/extras/client/settings.json b/src/integration-test/resources/org/xbib/elasticsearch/extras/client/settings.json deleted file mode 100644 index 86f5118..0000000 --- a/src/integration-test/resources/org/xbib/elasticsearch/extras/client/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "index.analysis.analyzer.default.type" : "keyword" -} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/AbstractClient.java b/src/main/java/org/xbib/elasticsearch/extras/client/AbstractClient.java deleted file mode 100644 index aed7be0..0000000 --- a/src/main/java/org/xbib/elasticsearch/extras/client/AbstractClient.java +++ /dev/null @@ -1,496 +0,0 @@ -package org.xbib.elasticsearch.extras.client; - -import com.carrotsearch.hppc.cursors.ObjectCursor; -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import org.elasticsearch.ElasticsearchTimeoutException; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; -import org.elasticsearch.action.admin.indices.flush.FlushAction; -import org.elasticsearch.action.admin.indices.flush.FlushRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexAction; -import org.elasticsearch.action.admin.indices.get.GetIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.get.GetIndexResponse; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.action.admin.indices.recovery.RecoveryAction; -import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest; -import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; -import org.elasticsearch.action.admin.indices.refresh.RefreshAction; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.cluster.metadata.AliasMetaData; -import org.elasticsearch.common.io.Streams; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.sort.SortBuilder; -import org.elasticsearch.search.sort.SortBuilders; -import org.elasticsearch.search.sort.SortOrder; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * - */ -public abstract class AbstractClient { - - private static final ESLogger logger = ESLoggerFactory.getLogger(AbstractClient.class.getName()); - - private Settings.Builder settingsBuilder; - - private Settings settings; - - private Map mappings = new HashMap<>(); - - public abstract ElasticsearchClient client(); - - protected abstract void createClient(Settings settings) throws IOException; - - public abstract void shutdown(); - - public Settings.Builder getSettingsBuilder() { - return settingsBuilder(); - } - - public void resetSettings() { - this.settingsBuilder = Settings.settingsBuilder(); - settings = null; - mappings = new HashMap<>(); - } - - public void setSettings(Settings settings) { - this.settings = settings; - } - - public void setting(String key, String value) { - if (settingsBuilder == null) { - settingsBuilder = Settings.settingsBuilder(); - } - settingsBuilder.put(key, value); - } - - public void setting(String key, Boolean value) { - if (settingsBuilder == null) { - settingsBuilder = Settings.settingsBuilder(); - } - settingsBuilder.put(key, value); - } - - public void setting(String key, Integer value) { - if (settingsBuilder == null) { - settingsBuilder = Settings.settingsBuilder(); - } - settingsBuilder.put(key, value); - } - - public void setting(InputStream in) throws IOException { - settingsBuilder = Settings.settingsBuilder().loadFromStream(".json", in); - } - - public Settings.Builder settingsBuilder() { - return settingsBuilder != null ? settingsBuilder : Settings.settingsBuilder(); - } - - public Settings settings() { - if (settings != null) { - return settings; - } - if (settingsBuilder == null) { - settingsBuilder = Settings.settingsBuilder(); - } - return settingsBuilder.build(); - } - - public void mapping(String type, String mapping) throws IOException { - mappings.put(type, mapping); - } - - public void mapping(String type, InputStream in) throws IOException { - if (type == null) { - return; - } - StringWriter sw = new StringWriter(); - Streams.copy(new InputStreamReader(in), sw); - mappings.put(type, sw.toString()); - } - - public Map mappings() { - return mappings.isEmpty() ? null : mappings; - } - - - public void updateIndexSetting(String index, String key, Object value) throws IOException { - if (client() == null) { - return; - } - if (index == null) { - throw new IOException("no index name given"); - } - if (key == null) { - throw new IOException("no key given"); - } - if (value == null) { - throw new IOException("no value given"); - } - Settings.Builder updateSettingsBuilder = Settings.settingsBuilder(); - updateSettingsBuilder.put(key, value.toString()); - UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index) - .settings(updateSettingsBuilder); - client().execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet(); - } - - public void waitForRecovery() throws IOException { - if (client() == null) { - return; - } - client().execute(RecoveryAction.INSTANCE, new RecoveryRequest()).actionGet(); - } - - public int waitForRecovery(String index) throws IOException { - if (client() == null) { - return -1; - } - if (index == null) { - throw new IOException("unable to waitfor recovery, index not set"); - } - RecoveryResponse response = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest(index)).actionGet(); - int shards = response.getTotalShards(); - client().execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest(index) - .waitForActiveShards(shards)).actionGet(); - return shards; - } - - public void waitForCluster(String statusString, String timeout) throws IOException { - if (client() == null) { - return; - } - ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString); - ClusterHealthResponse healthResponse = - client().execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest() - .waitForStatus(status).timeout(timeout)).actionGet(); - if (healthResponse != null && healthResponse.isTimedOut()) { - throw new IOException("cluster state is " + healthResponse.getStatus().name() - + " and not " + status.name() - + ", from here on, everything will fail!"); - } - } - - public String fetchClusterName() { - if (client() == null) { - return null; - } - try { - ClusterStateRequestBuilder clusterStateRequestBuilder = - new ClusterStateRequestBuilder(client(), ClusterStateAction.INSTANCE).all(); - ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); - String name = clusterStateResponse.getClusterName().value(); - int nodeCount = clusterStateResponse.getState().getNodes().size(); - return name + " (" + nodeCount + " nodes connected)"; - } catch (ElasticsearchTimeoutException e) { - logger.warn(e.getMessage(), e); - return "TIMEOUT"; - } catch (NoNodeAvailableException e) { - logger.warn(e.getMessage(), e); - return "DISCONNECTED"; - } catch (Exception e) { - logger.warn(e.getMessage(), e); - return "[" + e.getMessage() + "]"; - } - } - - public String healthColor() { - if (client() == null) { - return null; - } - try { - ClusterHealthResponse healthResponse = - client().execute(ClusterHealthAction.INSTANCE, - new ClusterHealthRequest().timeout(TimeValue.timeValueSeconds(30))).actionGet(); - ClusterHealthStatus status = healthResponse.getStatus(); - return status.name(); - } catch (ElasticsearchTimeoutException e) { - logger.warn(e.getMessage(), e); - return "TIMEOUT"; - } catch (NoNodeAvailableException e) { - logger.warn(e.getMessage(), e); - return "DISCONNECTED"; - } catch (Exception e) { - logger.warn(e.getMessage(), e); - return "[" + e.getMessage() + "]"; - } - } - - public int updateReplicaLevel(String index, int level) throws IOException { - waitForCluster("YELLOW", "30s"); - updateIndexSetting(index, "number_of_replicas", level); - return waitForRecovery(index); - } - - public void flushIndex(String index) { - if (client() == null) { - return; - } - if (index != null) { - client().execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet(); - } - } - - public void refreshIndex(String index) { - if (client() == null) { - return; - } - if (index != null) { - client().execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet(); - } - } - - public void putMapping(String index) { - if (client() == null) { - return; - } - if (!mappings().isEmpty()) { - for (Map.Entry me : mappings().entrySet()) { - client().execute(PutMappingAction.INSTANCE, - new PutMappingRequest(index).type(me.getKey()).source(me.getValue())).actionGet(); - } - } - } - - public String resolveAlias(String alias) { - if (client() == null) { - return alias; - } - GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client(), GetAliasesAction.INSTANCE); - GetAliasesResponse getAliasesResponse = getAliasesRequestBuilder.setAliases(alias).execute().actionGet(); - if (!getAliasesResponse.getAliases().isEmpty()) { - return getAliasesResponse.getAliases().keys().iterator().next().value; - } - return alias; - } - - public String resolveMostRecentIndex(String alias) { - if (client() == null) { - return alias; - } - if (alias == null) { - return null; - } - GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client(), GetAliasesAction.INSTANCE); - GetAliasesResponse getAliasesResponse = getAliasesRequestBuilder.setAliases(alias).execute().actionGet(); - Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); - Set indices = new TreeSet<>(Collections.reverseOrder()); - for (ObjectCursor indexName : getAliasesResponse.getAliases().keys()) { - Matcher m = pattern.matcher(indexName.value); - if (m.matches() && alias.equals(m.group(1))) { - indices.add(indexName.value); - } - } - return indices.isEmpty() ? alias : indices.iterator().next(); - } - - public Map getAliasFilters(String alias) { - GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client(), GetAliasesAction.INSTANCE); - return getFilters(getAliasesRequestBuilder.setIndices(resolveAlias(alias)).execute().actionGet()); - } - - public Map getIndexFilters(String index) { - GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client(), GetAliasesAction.INSTANCE); - return getFilters(getAliasesRequestBuilder.setIndices(index).execute().actionGet()); - } - - private Map getFilters(GetAliasesResponse getAliasesResponse) { - Map result = new HashMap<>(); - for (ObjectObjectCursor> object : getAliasesResponse.getAliases()) { - List aliasMetaDataList = object.value; - for (AliasMetaData aliasMetaData : aliasMetaDataList) { - if (aliasMetaData.filteringRequired()) { - result.put(aliasMetaData.alias(), new String(aliasMetaData.getFilter().uncompressed())); - } else { - result.put(aliasMetaData.alias(), null); - } - } - } - return result; - } - - public void switchAliases(String index, String concreteIndex, List extraAliases) { - switchAliases(index, concreteIndex, extraAliases, null); - } - - public void switchAliases(String index, String concreteIndex, - List extraAliases, IndexAliasAdder adder) { - if (client() == null) { - return; - } - if (index.equals(concreteIndex)) { - return; - } - // two situations: 1. there is a new alias 2. there is already an old index with the alias - String oldIndex = resolveAlias(index); - final Map oldFilterMap = oldIndex.equals(index) ? null : getIndexFilters(oldIndex); - final List newAliases = new LinkedList<>(); - final List switchAliases = new LinkedList<>(); - IndicesAliasesRequestBuilder requestBuilder = new IndicesAliasesRequestBuilder(client(), IndicesAliasesAction.INSTANCE); - if (oldFilterMap == null || !oldFilterMap.containsKey(index)) { - // never apply a filter for trunk index name - requestBuilder.addAlias(concreteIndex, index); - newAliases.add(index); - } - // switch existing aliases - if (oldFilterMap != null) { - for (Map.Entry entry : oldFilterMap.entrySet()) { - String alias = entry.getKey(); - String filter = entry.getValue(); - requestBuilder.removeAlias(oldIndex, alias); - if (filter != null) { - requestBuilder.addAlias(concreteIndex, alias, filter); - } else { - requestBuilder.addAlias(concreteIndex, alias); - } - switchAliases.add(alias); - } - } - // a list of aliases that should be added, check if new or old - if (extraAliases != null) { - for (String extraAlias : extraAliases) { - if (oldFilterMap == null || !oldFilterMap.containsKey(extraAlias)) { - // index alias adder only active on extra aliases, and if alias is new - if (adder != null) { - adder.addIndexAlias(requestBuilder, concreteIndex, extraAlias); - } else { - requestBuilder.addAlias(concreteIndex, extraAlias); - } - newAliases.add(extraAlias); - } else { - String filter = oldFilterMap.get(extraAlias); - requestBuilder.removeAlias(oldIndex, extraAlias); - if (filter != null) { - requestBuilder.addAlias(concreteIndex, extraAlias, filter); - } else { - requestBuilder.addAlias(concreteIndex, extraAlias); - } - switchAliases.add(extraAlias); - } - } - } - if (!newAliases.isEmpty() || !switchAliases.isEmpty()) { - logger.info("new aliases = {}, switch aliases = {}", newAliases, switchAliases); - requestBuilder.execute().actionGet(); - } - } - - public void performRetentionPolicy(String index, String concreteIndex, int timestampdiff, int mintokeep) { - if (client() == null) { - return; - } - if (index.equals(concreteIndex)) { - return; - } - GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client(), GetIndexAction.INSTANCE); - GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet(); - Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); - Set indices = new TreeSet<>(); - logger.info("{} indices", getIndexResponse.getIndices().length); - for (String s : getIndexResponse.getIndices()) { - Matcher m = pattern.matcher(s); - if (m.matches() && index.equals(m.group(1)) && !s.equals(concreteIndex)) { - indices.add(s); - } - } - if (indices.isEmpty()) { - logger.info("no indices found, retention policy skipped"); - return; - } - if (mintokeep > 0 && indices.size() <= mintokeep) { - logger.info("{} indices found, not enough for retention policy ({}), skipped", - indices.size(), mintokeep); - return; - } else { - logger.info("candidates for deletion = {}", indices); - } - List indicesToDelete = new ArrayList<>(); - // our index - Matcher m1 = pattern.matcher(concreteIndex); - if (m1.matches()) { - Integer i1 = Integer.parseInt(m1.group(2)); - for (String s : indices) { - Matcher m2 = pattern.matcher(s); - if (m2.matches()) { - Integer i2 = Integer.parseInt(m2.group(2)); - int kept = indices.size() - indicesToDelete.size(); - if ((timestampdiff == 0 || (timestampdiff > 0 && i1 - i2 > timestampdiff)) && mintokeep <= kept) { - indicesToDelete.add(s); - } - } - } - } - logger.info("indices to delete = {}", indicesToDelete); - if (indicesToDelete.isEmpty()) { - logger.info("not enough indices found to delete, retention policy complete"); - return; - } - String[] s = indicesToDelete.toArray(new String[indicesToDelete.size()]); - DeleteIndexRequestBuilder requestBuilder = new DeleteIndexRequestBuilder(client(), DeleteIndexAction.INSTANCE, s); - DeleteIndexResponse response = requestBuilder.execute().actionGet(); - if (!response.isAcknowledged()) { - logger.warn("retention delete index operation was not acknowledged"); - } - } - - public Long mostRecentDocument(String index, String timestampfieldname) { - if (client() == null) { - return null; - } - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client(), SearchAction.INSTANCE); - SortBuilder sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC); - SearchResponse searchResponse = searchRequestBuilder.setIndices(index) - .addField(timestampfieldname) - .setSize(1) - .addSort(sort) - .execute().actionGet(); - if (searchResponse.getHits().getHits().length == 1) { - SearchHit hit = searchResponse.getHits().getHits()[0]; - if (hit.getFields().get(timestampfieldname) != null) { - return hit.getFields().get(timestampfieldname).getValue(); - } else { - return 0L; - } - } - return null; - } - -} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/Clients.java b/src/main/java/org/xbib/elasticsearch/extras/client/Clients.java deleted file mode 100644 index daa4981..0000000 --- a/src/main/java/org/xbib/elasticsearch/extras/client/Clients.java +++ /dev/null @@ -1,105 +0,0 @@ -package org.xbib.elasticsearch.extras.client; - -import org.elasticsearch.client.Client; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.xbib.elasticsearch.extras.client.node.BulkNodeClient; -import org.xbib.elasticsearch.extras.client.transport.BulkTransportClient; -import org.xbib.elasticsearch.extras.client.transport.MockTransportClient; - -/** - * - */ -public final class Clients implements Parameters { - - private final Settings.Builder settingsBuilder; - - private BulkMetric metric; - - private BulkControl control; - - public Clients() { - settingsBuilder = Settings.builder(); - } - - public static Clients builder() { - return new Clients(); - } - - public Clients put(String key, String value) { - settingsBuilder.put(key, value); - return this; - } - - public Clients put(String key, Integer value) { - settingsBuilder.put(key, value); - return this; - } - - public Clients put(String key, Long value) { - settingsBuilder.put(key, value); - return this; - } - - public Clients put(String key, Double value) { - settingsBuilder.put(key, value); - return this; - } - - public Clients put(String key, ByteSizeValue value) { - settingsBuilder.put(key, value); - return this; - } - - public Clients put(String key, TimeValue value) { - settingsBuilder.put(key, value); - return this; - } - - public Clients put(Settings settings) { - settingsBuilder.put(settings); - return this; - } - - public Clients setMetric(BulkMetric metric) { - this.metric = metric; - return this; - } - - public Clients setControl(BulkControl control) { - this.control = control; - return this; - } - - public BulkNodeClient toBulkNodeClient(Client client) { - Settings settings = settingsBuilder.build(); - return new BulkNodeClient() - .maxActionsPerRequest(settings.getAsInt(MAX_ACTIONS_PER_REQUEST, DEFAULT_MAX_ACTIONS_PER_REQUEST)) - .maxConcurrentRequests(settings.getAsInt(MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_REQUESTS)) - .maxVolumePerRequest(settings.get(MAX_VOLUME_PER_REQUEST, DEFAULT_MAX_VOLUME_PER_REQUEST)) - .flushIngestInterval(settings.get(FLUSH_INTERVAL, DEFAULT_FLUSH_INTERVAL)) - .init(client, metric, control); - } - - public BulkTransportClient toBulkTransportClient() { - Settings settings = settingsBuilder.build(); - return new BulkTransportClient() - .maxActionsPerRequest(settings.getAsInt(MAX_ACTIONS_PER_REQUEST, DEFAULT_MAX_ACTIONS_PER_REQUEST)) - .maxConcurrentRequests(settings.getAsInt(MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_REQUESTS)) - .maxVolumePerRequest(settings.get(MAX_VOLUME_PER_REQUEST, DEFAULT_MAX_VOLUME_PER_REQUEST)) - .flushIngestInterval(settings.get(FLUSH_INTERVAL, DEFAULT_FLUSH_INTERVAL)) - .init(settings, metric, control); - } - - public MockTransportClient toMockTransportClient() { - Settings settings = settingsBuilder.build(); - return new MockTransportClient() - .maxActionsPerRequest(settings.getAsInt(MAX_ACTIONS_PER_REQUEST, DEFAULT_MAX_ACTIONS_PER_REQUEST)) - .maxConcurrentRequests(settings.getAsInt(MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_REQUESTS)) - .maxVolumePerRequest(settings.get(MAX_VOLUME_PER_REQUEST, DEFAULT_MAX_VOLUME_PER_REQUEST)) - .flushIngestInterval(settings.get(FLUSH_INTERVAL, DEFAULT_FLUSH_INTERVAL)) - .init(settings, metric, control); - } - -} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/Parameters.java b/src/main/java/org/xbib/elasticsearch/extras/client/Parameters.java deleted file mode 100644 index d77ce24..0000000 --- a/src/main/java/org/xbib/elasticsearch/extras/client/Parameters.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.xbib.elasticsearch.extras.client; - -/** - * - */ -public interface Parameters { - - int DEFAULT_MAX_ACTIONS_PER_REQUEST = 1000; - - int DEFAULT_MAX_CONCURRENT_REQUESTS = Runtime.getRuntime().availableProcessors() * 4; - - String DEFAULT_MAX_VOLUME_PER_REQUEST = "10mb"; - - String DEFAULT_FLUSH_INTERVAL = "30s"; - - String MAX_ACTIONS_PER_REQUEST = "max_actions_per_request"; - - String MAX_CONCURRENT_REQUESTS = "max_concurrent_requests"; - - String MAX_VOLUME_PER_REQUEST = "max_volume_per_request"; - - String FLUSH_INTERVAL = "flush_interval"; - -} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClient.java b/src/main/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClient.java deleted file mode 100644 index 0f387b6..0000000 --- a/src/main/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClient.java +++ /dev/null @@ -1,513 +0,0 @@ -package org.xbib.elasticsearch.extras.client.node; - -import com.google.common.collect.ImmutableSet; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.indices.create.CreateIndexAction; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; -import org.elasticsearch.action.bulk.BulkItemResponse; -import org.elasticsearch.action.bulk.BulkProcessor; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.env.Environment; -import org.elasticsearch.node.Node; -import org.elasticsearch.plugins.Plugin; -import org.xbib.elasticsearch.extras.client.AbstractClient; -import org.xbib.elasticsearch.extras.client.BulkControl; -import org.xbib.elasticsearch.extras.client.BulkMetric; -import org.xbib.elasticsearch.extras.client.ClientMethods; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Collection; -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; - -/** - * - */ -public class BulkNodeClient extends AbstractClient implements ClientMethods { - - private static final ESLogger logger = ESLoggerFactory.getLogger(BulkNodeClient.class.getName()); - - private int maxActionsPerRequest = DEFAULT_MAX_ACTIONS_PER_REQUEST; - - private int maxConcurrentRequests = DEFAULT_MAX_CONCURRENT_REQUESTS; - - private ByteSizeValue maxVolume; - - private TimeValue flushInterval; - - private Node node; - - private ElasticsearchClient client; - - private BulkProcessor bulkProcessor; - - private BulkMetric metric; - - private BulkControl control; - - private Throwable throwable; - - private boolean closed; - - @Override - public BulkNodeClient maxActionsPerRequest(int maxActionsPerRequest) { - this.maxActionsPerRequest = maxActionsPerRequest; - return this; - } - - @Override - public BulkNodeClient maxConcurrentRequests(int maxConcurrentRequests) { - this.maxConcurrentRequests = maxConcurrentRequests; - return this; - } - - @Override - public BulkNodeClient maxVolumePerRequest(String maxVolume) { - this.maxVolume = ByteSizeValue.parseBytesSizeValue(maxVolume, "maxVolumePerRequest"); - return this; - } - - @Override - public BulkNodeClient flushIngestInterval(String flushInterval) { - this.flushInterval = TimeValue.parseTimeValue(flushInterval, TimeValue.timeValueSeconds(5), "flushIngestInterval"); - return this; - } - - @Override - public BulkNodeClient init(ElasticsearchClient client, - final BulkMetric metric, final BulkControl control) { - this.client = client; - this.metric = metric; - this.control = control; - if (metric != null) { - metric.start(); - } - BulkProcessor.Listener listener = new BulkProcessor.Listener() { - @Override - public void beforeBulk(long executionId, BulkRequest request) { - long l = -1; - if (metric != null) { - metric.getCurrentIngest().inc(); - l = metric.getCurrentIngest().getCount(); - int n = request.numberOfActions(); - metric.getSubmitted().inc(n); - metric.getCurrentIngestNumDocs().inc(n); - metric.getTotalIngestSizeInBytes().inc(request.estimatedSizeInBytes()); - } - logger.debug("before bulk [{}] [actions={}] [bytes={}] [concurrent requests={}]", - executionId, - request.numberOfActions(), - request.estimatedSizeInBytes(), - l); - } - - @Override - public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { - long l = -1; - if (metric != null) { - metric.getCurrentIngest().dec(); - l = metric.getCurrentIngest().getCount(); - metric.getSucceeded().inc(response.getItems().length); - } - int n = 0; - for (BulkItemResponse itemResponse : response.getItems()) { - if (metric != null) { - metric.getCurrentIngest().dec(itemResponse.getIndex(), itemResponse.getType(), itemResponse.getId()); - } - if (itemResponse.isFailed()) { - n++; - if (metric != null) { - metric.getSucceeded().dec(1); - metric.getFailed().inc(1); - } - } - } - if (metric != null) { - logger.debug("after bulk [{}] [succeeded={}] [failed={}] [{}ms] {} concurrent requests", - executionId, - metric.getSucceeded().getCount(), - metric.getFailed().getCount(), - response.getTook().millis(), - l); - } - if (n > 0) { - logger.error("bulk [{}] failed with {} failed items, failure message = {}", - executionId, n, response.buildFailureMessage()); - } else { - if (metric != null) { - metric.getCurrentIngestNumDocs().dec(response.getItems().length); - } - } - } - - @Override - public void afterBulk(long executionId, BulkRequest request, Throwable failure) { - if (metric != null) { - metric.getCurrentIngest().dec(); - } - throwable = failure; - closed = true; - logger.error("after bulk [" + executionId + "] error", failure); - } - }; - BulkProcessor.Builder builder = BulkProcessor.builder((Client) client, listener) - .setBulkActions(maxActionsPerRequest) - .setConcurrentRequests(maxConcurrentRequests) - .setFlushInterval(flushInterval); - if (maxVolume != null) { - builder.setBulkSize(maxVolume); - } - this.bulkProcessor = builder.build(); - this.closed = false; - return this; - } - - @Override - public BulkNodeClient init(Settings settings, BulkMetric metric, BulkControl control) throws IOException { - createClient(settings); - this.metric = metric; - this.control = control; - return this; - } - - @Override - public ElasticsearchClient client() { - return client; - } - - @Override - protected synchronized void createClient(Settings settings) throws IOException { - if (client != null) { - logger.warn("client is open, closing..."); - client.threadPool().shutdown(); - client = null; - node.close(); - } - if (settings != null) { - String version = System.getProperty("os.name") - + " " + System.getProperty("java.vm.name") - + " " + System.getProperty("java.vm.vendor") - + " " + System.getProperty("java.runtime.version") - + " " + System.getProperty("java.vm.version"); - Settings effectiveSettings = Settings.builder().put(settings) - .put("node.client", true) - .put("node.master", false) - .put("node.data", false).build(); - logger.info("creating node client on {} with effective settings {}", - version, effectiveSettings.getAsMap()); - Collection> plugins = Collections.emptyList(); - this.node = new BulkNode(new Environment(effectiveSettings), plugins); - node.start(); - this.client = node.client(); - } - } - - @Override - public BulkMetric getMetric() { - return metric; - } - - @Override - public BulkNodeClient index(String index, String type, String id, String source) { - if (closed) { - throwClose(); - } - try { - if (metric != null) { - metric.getCurrentIngest().inc(index, type, id); - } - bulkProcessor.add(new IndexRequest(index).type(type).id(id).create(false).source(source)); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of index request failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkNodeClient bulkIndex(IndexRequest indexRequest) { - if (closed) { - throwClose(); - } - try { - if (metric != null) { - metric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id()); - } - bulkProcessor.add(indexRequest); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of index request failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkNodeClient delete(String index, String type, String id) { - if (closed) { - throwClose(); - } - try { - if (metric != null) { - metric.getCurrentIngest().inc(index, type, id); - } - bulkProcessor.add(new DeleteRequest(index).type(type).id(id)); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of delete failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkNodeClient bulkDelete(DeleteRequest deleteRequest) { - if (closed) { - throwClose(); - } - try { - if (metric != null) { - metric.getCurrentIngest().inc(deleteRequest.index(), deleteRequest.type(), deleteRequest.id()); - } - bulkProcessor.add(deleteRequest); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of delete failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkNodeClient update(String index, String type, String id, String source) { - if (closed) { - throwClose(); - } - try { - if (metric != null) { - metric.getCurrentIngest().inc(index, type, id); - } - bulkProcessor.add(new UpdateRequest().index(index).type(type).id(id).upsert(source)); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of update request failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkNodeClient bulkUpdate(UpdateRequest updateRequest) { - if (closed) { - throwClose(); - } - try { - if (metric != null) { - metric.getCurrentIngest().inc(updateRequest.index(), updateRequest.type(), updateRequest.id()); - } - bulkProcessor.add(updateRequest); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of update request failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkNodeClient flushIngest() { - if (closed) { - throwClose(); - } - logger.debug("flushing bulk processor"); - bulkProcessor.flush(); - return this; - } - - @Override - public BulkNodeClient waitForResponses(String maxWaitTime) throws InterruptedException, ExecutionException { - if (closed) { - throwClose(); - } - while (!bulkProcessor.awaitClose(TimeValue.parseTimeValue(maxWaitTime, TimeValue.timeValueSeconds(30), - "maxWaitTime").getMillis(), TimeUnit.MILLISECONDS)) { - logger.warn("still waiting for responses"); - } - return this; - } - - @Override - public BulkNodeClient startBulk(String index, long startRefreshIntervalMillis, long stopRefreshItervalMillis) - throws IOException { - if (control == null) { - return this; - } - if (!control.isBulk(index)) { - control.startBulk(index, startRefreshIntervalMillis, stopRefreshItervalMillis); - updateIndexSetting(index, "refresh_interval", startRefreshIntervalMillis + "ms"); - } - return this; - } - - @Override - public BulkNodeClient stopBulk(String index) throws IOException { - if (control == null) { - return this; - } - if (control.isBulk(index)) { - updateIndexSetting(index, "refresh_interval", control.getStopBulkRefreshIntervals().get(index) + "ms"); - control.finishBulk(index); - } - return this; - } - - @Override - public synchronized void shutdown() { - try { - if (bulkProcessor != null) { - logger.debug("closing bulk processor..."); - bulkProcessor.close(); - } - if (control != null && control.indices() != null && !control.indices().isEmpty()) { - logger.debug("stopping bulk mode for indices {}...", control.indices()); - for (String index : ImmutableSet.copyOf(control.indices())) { - stopBulk(index); - } - metric.stop(); - } - if (node != null) { - logger.debug("closing node..."); - node.close(); - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - @Override - public BulkNodeClient newIndex(String index) { - return newIndex(index, null, null); - } - - @Override - public BulkNodeClient newIndex(String index, String type, InputStream settings, InputStream mappings) throws IOException { - resetSettings(); - setting(settings); - mapping(type, mappings); - return newIndex(index, settings(), mappings()); - } - - @Override - public BulkNodeClient newIndex(String index, Settings settings, Map mappings) { - if (closed) { - throwClose(); - } - if (client == null) { - logger.warn("no client for create index"); - return this; - } - if (index == null) { - logger.warn("no index name given to create index"); - return this; - } - CreateIndexRequestBuilder createIndexRequestBuilder = - new CreateIndexRequestBuilder(client(), CreateIndexAction.INSTANCE).setIndex(index); - if (settings != null) { - logger.info("settings = {}", settings.getAsStructuredMap()); - createIndexRequestBuilder.setSettings(settings); - } - if (mappings != null) { - for (Map.Entry entry : mappings.entrySet()) { - String type = entry.getKey(); - String mapping = entry.getValue(); - logger.info("found mapping for {}", type); - createIndexRequestBuilder.addMapping(type, mapping); - } - } - createIndexRequestBuilder.execute().actionGet(); - logger.info("index {} created", index); - return this; - } - - @Override - public BulkNodeClient newMapping(String index, String type, Map mapping) { - PutMappingRequestBuilder putMappingRequestBuilder = - new PutMappingRequestBuilder(client(), PutMappingAction.INSTANCE) - .setIndices(index) - .setType(type) - .setSource(mapping); - putMappingRequestBuilder.execute().actionGet(); - logger.info("mapping created for index {} and type {}", index, type); - return this; - } - - @Override - public BulkNodeClient deleteIndex(String index) { - if (closed) { - throwClose(); - } - if (client == null) { - logger.warn("no client"); - return this; - } - if (index == null) { - logger.warn("no index name given to delete index"); - return this; - } - DeleteIndexRequestBuilder deleteIndexRequestBuilder = - new DeleteIndexRequestBuilder(client(), DeleteIndexAction.INSTANCE, index); - deleteIndexRequestBuilder.execute().actionGet(); - return this; - } - - @Override - public boolean hasThrowable() { - return throwable != null; - } - - @Override - public Throwable getThrowable() { - return throwable; - } - - public Settings getSettings() { - return settings(); - } - - @Override - public Settings.Builder getSettingsBuilder() { - return settingsBuilder(); - } - - private static void throwClose() { - throw new ElasticsearchException("client is closed"); - } - - private class BulkNode extends Node { - - BulkNode(Environment env, Collection> classpathPlugins) { - super(env, Version.CURRENT, classpathPlugins); - } - } - -} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/node/package-info.java b/src/main/java/org/xbib/elasticsearch/extras/client/node/package-info.java deleted file mode 100644 index c5c0895..0000000 --- a/src/main/java/org/xbib/elasticsearch/extras/client/node/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Classes for Elasticsearch node client extras. - */ -package org.xbib.elasticsearch.extras.client.node; diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/package-info.java b/src/main/java/org/xbib/elasticsearch/extras/client/package-info.java deleted file mode 100644 index c231c60..0000000 --- a/src/main/java/org/xbib/elasticsearch/extras/client/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Classes for Elasticsearch client extras. - */ -package org.xbib.elasticsearch.extras.client; diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClient.java b/src/main/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClient.java deleted file mode 100644 index ac37781..0000000 --- a/src/main/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClient.java +++ /dev/null @@ -1,564 +0,0 @@ -package org.xbib.elasticsearch.extras.client.transport; - -import com.google.common.collect.ImmutableSet; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.action.admin.indices.create.CreateIndexAction; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; -import org.elasticsearch.action.bulk.BulkItemResponse; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.logging.ESLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.InetSocketTransportAddress; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.xbib.elasticsearch.extras.client.AbstractClient; -import org.xbib.elasticsearch.extras.client.BulkControl; -import org.xbib.elasticsearch.extras.client.BulkMetric; -import org.xbib.elasticsearch.extras.client.BulkProcessor; -import org.xbib.elasticsearch.extras.client.ClientMethods; -import org.xbib.elasticsearch.extras.client.NetworkUtils; - -import java.io.IOException; -import java.io.InputStream; -import java.net.InetAddress; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; - -/** - * Transport client with addtitional methods using the BulkProcessor. - */ -public class BulkTransportClient extends AbstractClient implements ClientMethods { - - private static final ESLogger logger = ESLoggerFactory.getLogger(BulkTransportClient.class.getName()); - - private int maxActionsPerRequest = DEFAULT_MAX_ACTIONS_PER_REQUEST; - - private int maxConcurrentRequests = DEFAULT_MAX_CONCURRENT_REQUESTS; - - private ByteSizeValue maxVolumePerRequest; - - private TimeValue flushInterval; - - private BulkProcessor bulkProcessor; - - private Throwable throwable; - - private boolean closed; - - private TransportClient client; - - private BulkMetric metric; - - private BulkControl control; - - private boolean ignoreBulkErrors; - - private boolean isShutdown; - - @Override - public BulkTransportClient init(ElasticsearchClient client, BulkMetric metric, BulkControl control) throws IOException { - return init(findSettings(), metric, control); - } - - @Override - public BulkTransportClient init(Settings settings, final BulkMetric metric, final BulkControl control) { - createClient(settings); - this.metric = metric; - this.control = control; - if (metric != null) { - metric.start(); - } - resetSettings(); - BulkProcessor.Listener listener = new BulkProcessor.Listener() { - @Override - public void beforeBulk(long executionId, BulkRequest request) { - long l = -1L; - if (metric != null) { - metric.getCurrentIngest().inc(); - l = metric.getCurrentIngest().getCount(); - int n = request.numberOfActions(); - metric.getSubmitted().inc(n); - metric.getCurrentIngestNumDocs().inc(n); - metric.getTotalIngestSizeInBytes().inc(request.estimatedSizeInBytes()); - } - logger.debug("before bulk [{}] [actions={}] [bytes={}] [concurrent requests={}]", - executionId, - request.numberOfActions(), - request.estimatedSizeInBytes(), - l); - } - - @Override - public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { - long l = -1L; - if (metric != null) { - metric.getCurrentIngest().dec(); - l = metric.getCurrentIngest().getCount(); - metric.getSucceeded().inc(response.getItems().length); - } - int n = 0; - for (BulkItemResponse itemResponse : response.getItems()) { - if (metric != null) { - metric.getCurrentIngest().dec(itemResponse.getIndex(), itemResponse.getType(), itemResponse.getId()); - if (itemResponse.isFailed()) { - n++; - metric.getSucceeded().dec(1); - metric.getFailed().inc(1); - } - } - } - if (metric != null) { - logger.debug("after bulk [{}] [succeeded={}] [failed={}] [{}ms] [concurrent requests={}]", - executionId, - metric.getSucceeded().getCount(), - metric.getFailed().getCount(), - response.getTook().millis(), - l); - } - if (n > 0) { - logger.error("bulk [{}] failed with {} failed items, failure message = {}", - executionId, n, response.buildFailureMessage()); - } else { - if (metric != null) { - metric.getCurrentIngestNumDocs().dec(response.getItems().length); - } - } - } - - @Override - public void afterBulk(long executionId, BulkRequest requst, Throwable failure) { - if (metric != null) { - metric.getCurrentIngest().dec(); - } - throwable = failure; - if (!ignoreBulkErrors) { - closed = true; - } - logger.error("bulk [" + executionId + "] error", failure); - } - }; - BulkProcessor.Builder builder = BulkProcessor.builder(client, listener) - .setBulkActions(maxActionsPerRequest) - .setConcurrentRequests(maxConcurrentRequests) - .setFlushInterval(flushInterval); - if (maxVolumePerRequest != null) { - builder.setBulkSize(maxVolumePerRequest); - } - this.bulkProcessor = builder.build(); - try { - Collection addrs = findAddresses(settings); - if (!connect(addrs, settings.getAsBoolean("autodiscover", false))) { - throw new NoNodeAvailableException("no cluster nodes available, check settings " - + settings.getAsMap()); - } - } catch (IOException e) { - logger.error(e.getMessage(), e); - } - this.closed = false; - return this; - } - - @Override - public ClientMethods newMapping(String index, String type, Map mapping) { - new PutMappingRequestBuilder(client(), PutMappingAction.INSTANCE) - .setIndices(index) - .setType(type) - .setSource(mapping) - .execute().actionGet(); - logger.info("mapping created for index {} and type {}", index, type); - return this; - } - - @Override - protected void createClient(Settings settings) { - if (client != null) { - logger.warn("client is open, closing..."); - client.close(); - client.threadPool().shutdown(); - client = null; - } - if (settings != null) { - String version = System.getProperty("os.name") - + " " + System.getProperty("java.vm.name") - + " " + System.getProperty("java.vm.vendor") - + " " + System.getProperty("java.runtime.version") - + " " + System.getProperty("java.vm.version"); - logger.info("creating transport client on {} with effective settings {}", - version, settings.getAsMap()); - this.client = TransportClient.builder() - .settings(settings) - .build(); - this.ignoreBulkErrors = settings.getAsBoolean("ignoreBulkErrors", true); - } - } - - public boolean isShutdown() { - return isShutdown; - } - - @Override - public BulkTransportClient maxActionsPerRequest(int maxActionsPerRequest) { - this.maxActionsPerRequest = maxActionsPerRequest; - return this; - } - - @Override - public BulkTransportClient maxConcurrentRequests(int maxConcurrentRequests) { - this.maxConcurrentRequests = maxConcurrentRequests; - return this; - } - - @Override - public BulkTransportClient maxVolumePerRequest(String maxVolumePerRequest) { - this.maxVolumePerRequest = ByteSizeValue.parseBytesSizeValue(maxVolumePerRequest, "maxVolumePerRequest"); - return this; - } - - @Override - public BulkTransportClient flushIngestInterval(String flushInterval) { - this.flushInterval = TimeValue.parseTimeValue(flushInterval, TimeValue.timeValueSeconds(5), "flushIngestInterval"); - return this; - } - - @Override - public ElasticsearchClient client() { - return client; - } - - @Override - public BulkMetric getMetric() { - return metric; - } - - @Override - public ClientMethods newIndex(String index) { - if (closed) { - throwClose(); - } - return newIndex(index, null, null); - } - - @Override - public ClientMethods newIndex(String index, String type, InputStream settings, InputStream mappings) throws IOException { - resetSettings(); - setting(settings); - mapping(type, mappings); - return newIndex(index, settings(), mappings()); - } - - @Override - public ClientMethods newIndex(String index, Settings settings, Map mappings) { - if (closed) { - throwClose(); - } - if (index == null) { - logger.warn("no index name given to create index"); - return this; - } - CreateIndexRequestBuilder createIndexRequestBuilder = - new CreateIndexRequestBuilder(client(), CreateIndexAction.INSTANCE).setIndex(index); - if (settings != null) { - logger.info("settings = {}", settings.getAsStructuredMap()); - createIndexRequestBuilder.setSettings(settings); - } - if (mappings != null) { - for (Map.Entry entry : mappings.entrySet()) { - String type = entry.getKey(); - String mapping = entry.getValue(); - logger.info("found mapping for {}", type); - createIndexRequestBuilder.addMapping(type, mapping); - } - } - createIndexRequestBuilder.execute().actionGet(); - logger.info("index {} created", index); - return this; - } - - @Override - public ClientMethods deleteIndex(String index) { - if (closed) { - throwClose(); - } - if (index == null) { - logger.warn("no index name given to delete index"); - return this; - } - new DeleteIndexRequestBuilder(client(), DeleteIndexAction.INSTANCE, index).execute().actionGet(); - return this; - } - - @Override - public ClientMethods startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds) - throws IOException { - if (control == null) { - return this; - } - if (!control.isBulk(index)) { - control.startBulk(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds); - updateIndexSetting(index, "refresh_interval", startRefreshIntervalSeconds + "s"); - } - return this; - } - - @Override - public ClientMethods stopBulk(String index) throws IOException { - if (control == null) { - return this; - } - if (control.isBulk(index)) { - updateIndexSetting(index, "refresh_interval", control.getStopBulkRefreshIntervals().get(index) + "s"); - control.finishBulk(index); - } - return this; - } - - @Override - public BulkTransportClient index(String index, String type, String id, String source) { - if (closed) { - throwClose(); - } - try { - metric.getCurrentIngest().inc(index, type, id); - bulkProcessor.add(new IndexRequest().index(index).type(type).id(id).create(false).source(source)); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of index request failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkTransportClient bulkIndex(IndexRequest indexRequest) { - if (closed) { - throwClose(); - } - try { - metric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id()); - bulkProcessor.add(indexRequest); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of index request failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkTransportClient delete(String index, String type, String id) { - if (closed) { - throwClose(); - } - try { - metric.getCurrentIngest().inc(index, type, id); - bulkProcessor.add(new DeleteRequest().index(index).type(type).id(id)); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of delete request failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkTransportClient bulkDelete(DeleteRequest deleteRequest) { - if (closed) { - throwClose(); - } - try { - metric.getCurrentIngest().inc(deleteRequest.index(), deleteRequest.type(), deleteRequest.id()); - bulkProcessor.add(deleteRequest); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of delete request failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkTransportClient update(String index, String type, String id, String source) { - if (closed) { - throwClose(); - } - try { - metric.getCurrentIngest().inc(index, type, id); - bulkProcessor.add(new UpdateRequest().index(index).type(type).id(id).upsert(source)); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of update request failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public BulkTransportClient bulkUpdate(UpdateRequest updateRequest) { - if (closed) { - throwClose(); - } - try { - metric.getCurrentIngest().inc(updateRequest.index(), updateRequest.type(), updateRequest.id()); - bulkProcessor.add(updateRequest); - } catch (Exception e) { - throwable = e; - closed = true; - logger.error("bulk add of update request failed: " + e.getMessage(), e); - } - return this; - } - - @Override - public synchronized BulkTransportClient flushIngest() { - if (closed) { - throwClose(); - } - logger.debug("flushing bulk processor"); - bulkProcessor.flush(); - return this; - } - - @Override - public synchronized BulkTransportClient waitForResponses(String maxWaitTime) - throws InterruptedException, ExecutionException { - if (closed) { - throwClose(); - } - bulkProcessor.awaitClose(TimeValue.parseTimeValue(maxWaitTime, - TimeValue.timeValueSeconds(30), "maxWaitTime").getMillis(), TimeUnit.MILLISECONDS); - return this; - } - - @Override - public synchronized void shutdown() { - if (closed) { - shutdownClient(); - throwClose(); - } - try { - if (bulkProcessor != null) { - logger.debug("closing bulk processor..."); - bulkProcessor.close(); - } - if (control != null && control.indices() != null && !control.indices().isEmpty()) { - logger.debug("stopping bulk mode for indices {}...", control.indices()); - for (String index : ImmutableSet.copyOf(control.indices())) { - stopBulk(index); - } - metric.stop(); - } - logger.debug("shutting down..."); - shutdownClient(); - logger.debug("shutting down completed"); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - @Override - public boolean hasThrowable() { - return throwable != null; - } - - @Override - public Throwable getThrowable() { - return throwable; - } - - private Settings findSettings() { - Settings.Builder settingsBuilder = Settings.settingsBuilder(); - settingsBuilder.put("host", "localhost"); - try { - String hostname = NetworkUtils.getLocalAddress().getHostName(); - logger.debug("the hostname is {}", hostname); - settingsBuilder.put("host", hostname) - .put("port", 9300); - } catch (Exception e) { - logger.warn(e.getMessage(), e); - } - return settingsBuilder.build(); - } - - private Collection findAddresses(Settings settings) throws IOException { - String[] hostnames = settings.getAsArray("host", new String[]{"localhost"}); - int port = settings.getAsInt("port", 9300); - Collection addresses = new ArrayList<>(); - for (String hostname : hostnames) { - String[] splitHost = hostname.split(":", 2); - if (splitHost.length == 2) { - String host = splitHost[0]; - InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null); - try { - port = Integer.parseInt(splitHost[1]); - } catch (Exception e) { - logger.warn(e.getMessage(), e); - } - addresses.add(new InetSocketTransportAddress(inetAddress, port)); - } - if (splitHost.length == 1) { - String host = splitHost[0]; - InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null); - addresses.add(new InetSocketTransportAddress(inetAddress, port)); - } - } - return addresses; - } - - private static void throwClose() { - throw new ElasticsearchException("client is closed"); - } - - private void shutdownClient() { - if (client != null) { - logger.debug("shutdown started"); - client.close(); - client.threadPool().shutdown(); - client = null; - logger.debug("shutdown complete"); - } - isShutdown = true; - } - - private boolean connect(Collection addresses, boolean autodiscover) { - logger.info("trying to connect to {}", addresses); - client.addTransportAddresses(addresses); - if (client.connectedNodes() != null) { - List nodes = client.connectedNodes(); - if (!nodes.isEmpty()) { - logger.info("connected to {}", nodes); - if (autodiscover) { - logger.info("trying to auto-discover all cluster nodes..."); - ClusterStateRequestBuilder clusterStateRequestBuilder = - new ClusterStateRequestBuilder(client, ClusterStateAction.INSTANCE); - ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); - DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes(); - client.addDiscoveryNodes(discoveryNodes); - logger.info("after auto-discovery connected to {}", client.connectedNodes()); - } - return true; - } - return false; - } - return false; - } -} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/transport/MockTransportClient.java b/src/main/java/org/xbib/elasticsearch/extras/client/transport/MockTransportClient.java deleted file mode 100644 index ed0fcc7..0000000 --- a/src/main/java/org/xbib/elasticsearch/extras/client/transport/MockTransportClient.java +++ /dev/null @@ -1,155 +0,0 @@ -package org.xbib.elasticsearch.extras.client.transport; - -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.settings.Settings; -import org.xbib.elasticsearch.extras.client.BulkControl; -import org.xbib.elasticsearch.extras.client.BulkMetric; - -import java.io.IOException; -import java.util.Map; - -/** - * Mock client, it does not perform actions on a cluster. - * Useful for testing or dry runs. - */ -public class MockTransportClient extends BulkTransportClient { - - @Override - public ElasticsearchClient client() { - return null; - } - - @Override - public MockTransportClient init(ElasticsearchClient client, BulkMetric metric, BulkControl control) { - return this; - } - - @Override - public MockTransportClient init(Settings settings, BulkMetric metric, BulkControl control) { - return this; - } - - @Override - public MockTransportClient maxActionsPerRequest(int maxActions) { - return this; - } - - @Override - public MockTransportClient maxConcurrentRequests(int maxConcurrentRequests) { - return this; - } - - @Override - public MockTransportClient maxVolumePerRequest(String maxVolumePerRequest) { - return this; - } - - @Override - public MockTransportClient flushIngestInterval(String interval) { - return this; - } - - @Override - public MockTransportClient index(String index, String type, String id, String source) { - return this; - } - - @Override - public MockTransportClient delete(String index, String type, String id) { - return this; - } - - @Override - public MockTransportClient update(String index, String type, String id, String source) { - return this; - } - - @Override - public MockTransportClient bulkIndex(IndexRequest indexRequest) { - return this; - } - - @Override - public MockTransportClient bulkDelete(DeleteRequest deleteRequest) { - return this; - } - - @Override - public MockTransportClient bulkUpdate(UpdateRequest updateRequest) { - return this; - } - - @Override - public MockTransportClient flushIngest() { - return this; - } - - @Override - public MockTransportClient waitForResponses(String timeValue) throws InterruptedException { - return this; - } - - @Override - public MockTransportClient startBulk(String index, long startRefreshInterval, long stopRefreshIterval) { - return this; - } - - @Override - public MockTransportClient stopBulk(String index) { - return this; - } - - @Override - public MockTransportClient deleteIndex(String index) { - return this; - } - - @Override - public MockTransportClient newIndex(String index) { - return this; - } - - @Override - public MockTransportClient newMapping(String index, String type, Map mapping) { - return this; - } - - @Override - public void putMapping(String index) { - // mockup method - } - - @Override - public void refreshIndex(String index) { - // mockup method - } - - @Override - public void flushIndex(String index) { - // mockup method - } - - @Override - public void waitForCluster(String healthColor, String timeValue) throws IOException { - // mockup method - } - - @Override - public int waitForRecovery(String index) throws IOException { - return -1; - } - - @Override - public int updateReplicaLevel(String index, int level) throws IOException { - return -1; - } - - @Override - public void shutdown() { - // mockup method - } - -} diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/transport/package-info.java b/src/main/java/org/xbib/elasticsearch/extras/client/transport/package-info.java deleted file mode 100644 index ac6a50d..0000000 --- a/src/main/java/org/xbib/elasticsearch/extras/client/transport/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Classes for Elasticsearch transport client extras. - */ -package org.xbib.elasticsearch.extras.client.transport;