large refactoring, new subpackage structure, Gradle 5.2.1, Java 11
parent
17d114feb1
commit
c0dfb9a617
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,13 @@
|
||||
#
|
||||
# BSD-style license; for more info see http://pmd.sourceforge.net/license.html
|
||||
#
|
||||
|
||||
rulesets.filenames=\
|
||||
category/java/bestpractices.xml,\
|
||||
category/java/codestyle.xml,\
|
||||
category/java/design.xml,\
|
||||
category/java/documentation.xml,\
|
||||
category/java/errorprone.xml,\
|
||||
category/java/multithreading.xml,\
|
||||
category/java/performance.xml,\
|
||||
category/java/security.xml
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,144 @@
|
||||
<?xml version="1.0"?>
|
||||
|
||||
<ruleset name="Documentation"
|
||||
xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 https://pmd.sourceforge.io/ruleset_2_0_0.xsd">
|
||||
|
||||
<description>
|
||||
Rules that are related to code documentation.
|
||||
</description>
|
||||
|
||||
<rule name="CommentContent"
|
||||
since="5.0"
|
||||
message="Invalid words or phrases found"
|
||||
class="net.sourceforge.pmd.lang.java.rule.documentation.CommentContentRule"
|
||||
externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_documentation.html#commentcontent">
|
||||
<description>
|
||||
A rule for the politically correct... we don't want to offend anyone.
|
||||
</description>
|
||||
<priority>3</priority>
|
||||
<example>
|
||||
<![CDATA[
|
||||
//OMG, this is horrible, Bob is an idiot !!!
|
||||
]]>
|
||||
</example>
|
||||
</rule>
|
||||
|
||||
<rule name="CommentRequired"
|
||||
since="5.1"
|
||||
message="Comment is required"
|
||||
class="net.sourceforge.pmd.lang.java.rule.documentation.CommentRequiredRule"
|
||||
externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_documentation.html#commentrequired">
|
||||
<description>
|
||||
Denotes whether comments are required (or unwanted) for specific language elements.
|
||||
</description>
|
||||
<priority>3</priority>
|
||||
<example>
|
||||
<![CDATA[
|
||||
/**
|
||||
*
|
||||
*
|
||||
* @author Jon Doe
|
||||
*/
|
||||
]]>
|
||||
</example>
|
||||
</rule>
|
||||
|
||||
<rule name="CommentSize"
|
||||
since="5.0"
|
||||
message="Comment is too large"
|
||||
class="net.sourceforge.pmd.lang.java.rule.documentation.CommentSizeRule"
|
||||
externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_documentation.html#commentsize">
|
||||
<description>
|
||||
Determines whether the dimensions of non-header comments found are within the specified limits.
|
||||
</description>
|
||||
<priority>3</priority>
|
||||
<example>
|
||||
<![CDATA[
|
||||
/**
|
||||
*
|
||||
* too many lines!
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
*/
|
||||
]]>
|
||||
</example>
|
||||
</rule>
|
||||
|
||||
<rule name="UncommentedEmptyConstructor"
|
||||
language="java"
|
||||
since="3.4"
|
||||
message="Document empty constructor"
|
||||
class="net.sourceforge.pmd.lang.rule.XPathRule"
|
||||
typeResolution="true"
|
||||
externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_documentation.html#uncommentedemptyconstructor">
|
||||
<description>
|
||||
Uncommented Empty Constructor finds instances where a constructor does not
|
||||
contain statements, but there is no comment. By explicitly commenting empty
|
||||
constructors it is easier to distinguish between intentional (commented)
|
||||
and unintentional empty constructors.
|
||||
</description>
|
||||
<priority>3</priority>
|
||||
<properties>
|
||||
<property name="xpath">
|
||||
<value>
|
||||
<![CDATA[
|
||||
//ConstructorDeclaration[@Private='false']
|
||||
[count(BlockStatement) = 0 and ($ignoreExplicitConstructorInvocation = 'true' or not(ExplicitConstructorInvocation)) and @containsComment = 'false']
|
||||
[not(../Annotation/MarkerAnnotation/Name[pmd-java:typeIs('javax.inject.Inject')])]
|
||||
]]>
|
||||
</value>
|
||||
</property>
|
||||
<property name="ignoreExplicitConstructorInvocation" type="Boolean" description="Ignore explicit constructor invocation when deciding whether constructor is empty or not" value="false"/>
|
||||
</properties>
|
||||
<example>
|
||||
<![CDATA[
|
||||
public Foo() {
|
||||
// This constructor is intentionally empty. Nothing special is needed here.
|
||||
}
|
||||
]]>
|
||||
</example>
|
||||
</rule>
|
||||
|
||||
<rule name="UncommentedEmptyMethodBody"
|
||||
language="java"
|
||||
since="3.4"
|
||||
message="Document empty method body"
|
||||
class="net.sourceforge.pmd.lang.rule.XPathRule"
|
||||
externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_documentation.html#uncommentedemptymethodbody">
|
||||
<description>
|
||||
Uncommented Empty Method Body finds instances where a method body does not contain
|
||||
statements, but there is no comment. By explicitly commenting empty method bodies
|
||||
it is easier to distinguish between intentional (commented) and unintentional
|
||||
empty methods.
|
||||
</description>
|
||||
<priority>3</priority>
|
||||
<properties>
|
||||
<property name="xpath">
|
||||
<value>
|
||||
<![CDATA[
|
||||
//MethodDeclaration/Block[count(BlockStatement) = 0 and @containsComment = 'false']
|
||||
]]>
|
||||
</value>
|
||||
</property>
|
||||
</properties>
|
||||
<example>
|
||||
<![CDATA[
|
||||
public void doSomething() {
|
||||
}
|
||||
]]>
|
||||
</example>
|
||||
</rule>
|
||||
|
||||
</ruleset>
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,393 @@
|
||||
<?xml version="1.0"?>
|
||||
|
||||
<ruleset name="Multithreading"
|
||||
xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 https://pmd.sourceforge.io/ruleset_2_0_0.xsd">
|
||||
|
||||
<description>
|
||||
Rules that flag issues when dealing with multiple threads of execution.
|
||||
</description>
|
||||
|
||||
<rule name="AvoidSynchronizedAtMethodLevel"
|
||||
language="java"
|
||||
since="3.0"
|
||||
message="Use block level rather than method level synchronization"
|
||||
class="net.sourceforge.pmd.lang.rule.XPathRule"
|
||||
externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_multithreading.html#avoidsynchronizedatmethodlevel">
|
||||
<description>
|
||||
Method-level synchronization can cause problems when new code is added to the method.
|
||||
Block-level synchronization helps to ensure that only the code that needs synchronization
|
||||
gets it.
|
||||
</description>
|
||||
<priority>3</priority>
|
||||
<properties>
|
||||
<property name="xpath">
|
||||
<value>//MethodDeclaration[@Synchronized='true']</value>
|
||||
</property>
|
||||
</properties>
|
||||
<example>
|
||||
<![CDATA[
|
||||
public class Foo {
|
||||
// Try to avoid this:
|
||||
synchronized void foo() {
|
||||
}
|
||||
// Prefer this:
|
||||
void bar() {
|
||||
synchronized(this) {
|
||||
}
|
||||
}
|
||||
|
||||
// Try to avoid this for static methods:
|
||||
static synchronized void fooStatic() {
|
||||
}
|
||||
|
||||
// Prefer this:
|
||||
static void barStatic() {
|
||||
synchronized(Foo.class) {
|
||||
}
|
||||
}
|
||||
}
|
||||
]]>
|
||||
</example>
|
||||
</rule>
|
||||
|
||||
<rule name="AvoidThreadGroup"
|
||||
language="java"
|
||||
since="3.6"
|
||||
message="Avoid using java.lang.ThreadGroup; it is not thread safe"
|
||||
class="net.sourceforge.pmd.lang.rule.XPathRule"
|
||||
typeResolution="true"
|
||||
externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_multithreading.html#avoidthreadgroup">
|
||||
<description>
|
||||
Avoid using java.lang.ThreadGroup; although it is intended to be used in a threaded environment
|
||||
it contains methods that are not thread-safe.
|
||||
</description>
|
||||
<priority>3</priority>
|
||||
<properties>
|
||||
<property name="xpath">
|
||||
<value>
|
||||
<![CDATA[
|
||||
//AllocationExpression/ClassOrInterfaceType[pmd-java:typeIs('java.lang.ThreadGroup')]|
|
||||
//PrimarySuffix[contains(@Image, 'getThreadGroup')]
|
||||
]]>
|
||||
</value>
|
||||
</property>
|
||||
</properties>
|
||||
<example>
|
||||
<![CDATA[
|
||||
public class Bar {
|
||||
void buz() {
|
||||
ThreadGroup tg = new ThreadGroup("My threadgroup");
|
||||
tg = new ThreadGroup(tg, "my thread group");
|
||||
tg = Thread.currentThread().getThreadGroup();
|
||||
tg = System.getSecurityManager().getThreadGroup();
|
||||
}
|
||||
}
|
||||
]]>
|
||||
</example>
|
||||
</rule>
|
||||
|
||||
<rule name="AvoidUsingVolatile"
|
||||
language="java"
|
||||
since="4.1"
|
||||
class="net.sourceforge.pmd.lang.rule.XPathRule"
|
||||
message="Use of modifier volatile is not recommended."
|
||||
externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_multithreading.html#avoidusingvolatile">
|
||||
<description>
|
||||
Use of the keyword 'volatile' is generally used to fine tune a Java application, and therefore, requires
|
||||
a good expertise of the Java Memory Model. Moreover, its range of action is somewhat misknown. Therefore,
|
||||
the volatile keyword should not be used for maintenance purpose and portability.
|
||||
</description>
|
||||
<priority>2</priority>
|
||||
<properties>
|
||||
<property name="xpath">
|
||||
<value>//FieldDeclaration[contains(@Volatile,'true')]</value>
|
||||
</property>
|
||||
</properties>
|
||||
<example>
|
||||
<![CDATA[
|
||||
public class ThrDeux {
|
||||
private volatile String var1; // not suggested
|
||||
private String var2; // preferred
|
||||
}
|
||||
]]>
|
||||
</example>
|
||||
</rule>
|
||||
|
||||
<rule name="DoNotUseThreads"
|
||||
language="java"
|
||||
since="4.1"
|
||||
class="net.sourceforge.pmd.lang.rule.XPathRule"
|
||||
message="To be compliant to J2EE, a webapp should not use any thread."
|
||||
externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_multithreading.html#donotusethreads">
|
||||
<description>
|
||||
The J2EE specification explicitly forbids the use of threads.
|
||||
</description>
|
||||
<priority>3</priority>
|
||||
<properties>
|
||||
<property name="xpath">
|
||||
<value>//ClassOrInterfaceType[@Image = 'Thread' or @Image = 'Runnable']</value>
|
||||
</property>
|
||||
</properties>
|
||||
<example>
|
||||
<![CDATA[
|
||||
// This is not allowed
|
||||
public class UsingThread extends Thread {
|
||||
|
||||
}
|
||||
|
||||
// Neither this,
|
||||
public class OtherThread implements Runnable {
|
||||
// Nor this ...
|
||||
public void methode() {
|
||||
Runnable thread = new Thread(); thread.run();
|
||||
}
|
||||
}
|
||||
]]>
|
||||
</example>
|
||||
</rule>
|
||||
|
||||
<rule name="DontCallThreadRun"
|
||||
language="java"
|
||||
since="4.3"
|
||||
message="Don't call Thread.run() explicitly, use Thread.start()"
|
||||
class="net.sourceforge.pmd.lang.rule.XPathRule"
|
||||
typeResolution="true"
|
||||
externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_multithreading.html#dontcallthreadrun">
|
||||
<description>
|
||||
Explicitly calling Thread.run() method will execute in the caller's thread of control. Instead, call Thread.start() for the intended behavior.
|
||||
</description>
|
||||
<priority>4</priority>
|
||||
<properties>
|
||||
<property name="xpath">
|
||||
<value>
|
||||
<![CDATA[
|
||||
//StatementExpression/PrimaryExpression
|
||||
[
|
||||
PrimaryPrefix
|
||||
[
|
||||
./Name[ends-with(@Image, '.run') or @Image = 'run']
|
||||
and substring-before(Name/@Image, '.') =//VariableDeclarator/VariableDeclaratorId/@Image
|
||||
[../../../Type/ReferenceType/ClassOrInterfaceType[pmd-java:typeIs('java.lang.Thread')]]
|
||||
or (./AllocationExpression/ClassOrInterfaceType[pmd-java:typeIs('java.lang.Thread')]
|
||||
and ../PrimarySuffix[@Image = 'run'])
|
||||
]
|
||||
]
|
||||
]]>
|
||||
</value>
|
||||
</property>
|
||||
</properties>
|
||||
<example>
|
||||
<![CDATA[
|
||||
Thread t = new Thread();
|
||||
t.run(); // use t.start() instead
|
||||
new Thread().run(); // same violation
|
||||
]]>
|
||||
</example>
|
||||
</rule>
|
||||
|
||||
<rule name="DoubleCheckedLocking"
|
||||
language="java"
|
||||
since="1.04"
|
||||
message="Double checked locking is not thread safe in Java."
|
||||
class="net.sourceforge.pmd.lang.java.rule.multithreading.DoubleCheckedLockingRule"
|
||||
externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_multithreading.html#doublecheckedlocking">
|
||||
<description>
|
||||
Partially created objects can be returned by the Double Checked Locking pattern when used in Java.
|
||||
An optimizing JRE may assign a reference to the baz variable before it calls the constructor of the object the
|
||||
reference points to.
|
||||
|
||||
Note: With Java 5, you can make Double checked locking work, if you declare the variable to be `volatile`.
|
||||
|
||||
For more details refer to: <http://www.javaworld.com/javaworld/jw-02-2001/jw-0209-double.html>
|
||||
or <http://www.cs.umd.edu/~pugh/java/memoryModel/DoubleCheckedLocking.html>
|
||||
</description>
|
||||
<priority>1</priority>
|
||||
<example>
|
||||
<![CDATA[
|
||||
public class Foo {
|
||||
/*volatile */ Object baz = null; // fix for Java5 and later: volatile
|
||||
Object bar() {
|
||||
if (baz == null) { // baz may be non-null yet not fully created
|
||||
synchronized(this) {
|
||||
if (baz == null) {
|
||||
baz = new Object();
|
||||
}
|
||||
}
|
||||
}
|
||||
return baz;
|
||||
}
|
||||
}
|
||||
]]>
|
||||
</example>
|
||||
</rule>
|
||||
|
||||
<rule name="NonThreadSafeSingleton"
|
||||
since="3.4"
|
||||
message="Singleton is not thread safe"
|
||||
class="net.sourceforge.pmd.lang.java.rule.multithreading.NonThreadSafeSingletonRule"
|
||||
externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_multithreading.html#nonthreadsafesingleton">
|
||||
<description>
|
||||
Non-thread safe singletons can result in bad state changes. Eliminate
|
||||
static singletons if possible by instantiating the object directly. Static
|
||||
singletons are usually not needed as only a single instance exists anyway.
|
||||
Other possible fixes are to synchronize the entire method or to use an
|
||||
[initialize-on-demand holder class](https://en.wikipedia.org/wiki/Initialization-on-demand_holder_idiom).
|
||||
|
||||
Refrain from using the double-checked locking pattern. The Java Memory Model doesn't
|
||||
guarantee it to work unless the variable is declared as `volatile`, adding an uneeded
|
||||
performance penalty. [Reference](http://www.cs.umd.edu/~pugh/java/memoryModel/DoubleCheckedLocking.html)
|
||||
|
||||
See Effective Java, item 48.
|
||||
</description>
|
||||
<priority>3</priority>
|
||||
<example>
|
||||
<![CDATA[
|
||||
private static Foo foo = null;
|
||||
|
||||
//multiple simultaneous callers may see partially initialized objects
|
||||
public static Foo getFoo() {
|
||||
if (foo==null) {
|
||||
foo = new Foo();
|
||||
}
|
||||
return foo;
|
||||
}
|
||||
]]>
|
||||
</example>
|
||||
</rule>
|
||||
|
||||
<rule name="UnsynchronizedStaticDateFormatter"
|
||||
since="3.6"
|
||||
deprecated="true"
|
||||
message="Static DateFormatter objects should be accessed in a synchronized manner"
|
||||
class="net.sourceforge.pmd.lang.java.rule.multithreading.UnsynchronizedStaticDateFormatterRule"
|
||||
externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_multithreading.html#unsynchronizedstaticdateformatter">
|
||||
<description>
|
||||
SimpleDateFormat instances are not synchronized. Sun recommends using separate format instances
|
||||
for each thread. If multiple threads must access a static formatter, the formatter must be
|
||||
synchronized either on method or block level.
|
||||
|
||||
This rule has been deprecated in favor of the rule {% rule UnsynchronizedStaticFormatter %}.
|
||||
</description>
|
||||
<priority>3</priority>
|
||||
<example>
|
||||
<![CDATA[
|
||||
public class Foo {
|
||||
private static final SimpleDateFormat sdf = new SimpleDateFormat();
|
||||
void bar() {
|
||||
sdf.format(); // poor, no thread-safety
|
||||
}
|
||||
synchronized void foo() {
|
||||
sdf.format(); // preferred
|
||||
}
|
||||
}
|
||||
]]>
|
||||
</example>
|
||||
</rule>
|
||||
|
||||
<rule name="UnsynchronizedStaticFormatter"
|
||||
since="6.11.0"
|
||||
message="Static Formatter objects should be accessed in a synchronized manner"
|
||||
class="net.sourceforge.pmd.lang.java.rule.multithreading.UnsynchronizedStaticFormatterRule"
|
||||
externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_multithreading.html#unsynchronizedstaticformatter">
|
||||
<description>
|
||||
Instances of `java.text.Format` are generally not synchronized.
|
||||
Sun recommends using separate format instances for each thread.
|
||||
If multiple threads must access a static formatter, the formatter must be
|
||||
synchronized either on method or block level.
|
||||
</description>
|
||||
<priority>3</priority>
|
||||
<example>
|
||||
<![CDATA[
|
||||
public class Foo {
|
||||
private static final SimpleDateFormat sdf = new SimpleDateFormat();
|
||||
void bar() {
|
||||
sdf.format(); // poor, no thread-safety
|
||||
}
|
||||
synchronized void foo() {
|
||||
sdf.format(); // preferred
|
||||
}
|
||||
}
|
||||
]]>
|
||||
</example>
|
||||
</rule>
|
||||
|
||||
<rule name="UseConcurrentHashMap"
|
||||
language="java"
|
||||
minimumLanguageVersion="1.5"
|
||||
since="4.2.6"
|
||||
message="If you run in Java5 or newer and have concurrent access, you should use the ConcurrentHashMap implementation"
|
||||
class="net.sourceforge.pmd.lang.rule.XPathRule"
|
||||
externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_multithreading.html#useconcurrenthashmap">
|
||||
<description>
|
||||
Since Java5 brought a new implementation of the Map designed for multi-threaded access, you can
|
||||
perform efficient map reads without blocking other threads.
|
||||
</description>
|
||||
<priority>3</priority>
|
||||
<properties>
|
||||
<property name="xpath">
|
||||
<value>
|
||||
<![CDATA[
|
||||
//Type[../VariableDeclarator/VariableInitializer//AllocationExpression/ClassOrInterfaceType[@Image != 'ConcurrentHashMap']]
|
||||
/ReferenceType/ClassOrInterfaceType[@Image = 'Map']
|
||||
]]>
|
||||
</value>
|
||||
</property>
|
||||
</properties>
|
||||
<example>
|
||||
<![CDATA[
|
||||
public class ConcurrentApp {
|
||||
public void getMyInstance() {
|
||||
Map map1 = new HashMap(); // fine for single-threaded access
|
||||
Map map2 = new ConcurrentHashMap(); // preferred for use with multiple threads
|
||||
|
||||
// the following case will be ignored by this rule
|
||||
Map map3 = someModule.methodThatReturnMap(); // might be OK, if the returned map is already thread-safe
|
||||
}
|
||||
}
|
||||
]]>
|
||||
</example>
|
||||
</rule>
|
||||
|
||||
<rule name="UseNotifyAllInsteadOfNotify"
|
||||
language="java"
|
||||
since="3.0"
|
||||
message="Call Thread.notifyAll() rather than Thread.notify()"
|
||||
class="net.sourceforge.pmd.lang.rule.XPathRule"
|
||||
externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_multithreading.html#usenotifyallinsteadofnotify">
|
||||
<description>
|
||||
Thread.notify() awakens a thread monitoring the object. If more than one thread is monitoring, then only
|
||||
one is chosen. The thread chosen is arbitrary; thus its usually safer to call notifyAll() instead.
|
||||
</description>
|
||||
<priority>3</priority>
|
||||
<properties>
|
||||
<property name="xpath">
|
||||
<value>
|
||||
<![CDATA[
|
||||
//StatementExpression/PrimaryExpression
|
||||
[PrimarySuffix/Arguments[@ArgumentCount = '0']]
|
||||
[
|
||||
PrimaryPrefix[
|
||||
./Name[@Image='notify' or ends-with(@Image,'.notify')]
|
||||
or ../PrimarySuffix/@Image='notify'
|
||||
or (./AllocationExpression and ../PrimarySuffix[@Image='notify'])
|
||||
]
|
||||
]
|
||||
]]>
|
||||
</value>
|
||||
</property>
|
||||
</properties>
|
||||
<example>
|
||||
<![CDATA[
|
||||
void bar() {
|
||||
x.notify();
|
||||
// If many threads are monitoring x, only one (and you won't know which) will be notified.
|
||||
// use instead:
|
||||
x.notifyAll();
|
||||
}
|
||||
]]>
|
||||
</example>
|
||||
</rule>
|
||||
|
||||
</ruleset>
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,65 @@
|
||||
<?xml version="1.0"?>
|
||||
|
||||
<ruleset name="Security" xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 https://pmd.sourceforge.io/ruleset_2_0_0.xsd">
|
||||
|
||||
<description>
|
||||
Rules that flag potential security flaws.
|
||||
</description>
|
||||
|
||||
<rule name="HardCodedCryptoKey"
|
||||
since="6.4.0"
|
||||
message="Do not use hard coded encryption keys"
|
||||
class="net.sourceforge.pmd.lang.java.rule.security.HardCodedCryptoKeyRule"
|
||||
externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_security.html#hardcodedcryptokey">
|
||||
<description>
|
||||
Do not use hard coded values for cryptographic operations. Please store keys outside of source code.
|
||||
</description>
|
||||
<priority>3</priority>
|
||||
<example>
|
||||
<![CDATA[
|
||||
public class Foo {
|
||||
void good() {
|
||||
SecretKeySpec secretKeySpec = new SecretKeySpec(Properties.getKey(), "AES");
|
||||
}
|
||||
|
||||
void bad() {
|
||||
SecretKeySpec secretKeySpec = new SecretKeySpec("my secret here".getBytes(), "AES");
|
||||
}
|
||||
}
|
||||
]]>
|
||||
</example>
|
||||
</rule>
|
||||
|
||||
<rule name="InsecureCryptoIv"
|
||||
since="6.3.0"
|
||||
message="Do not use hard coded initialization vector in crypto operations"
|
||||
class="net.sourceforge.pmd.lang.java.rule.security.InsecureCryptoIvRule"
|
||||
externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_security.html#insecurecryptoiv">
|
||||
<description>
|
||||
Do not use hard coded initialization vector in cryptographic operations. Please use a randomly generated IV.
|
||||
</description>
|
||||
<priority>3</priority>
|
||||
<example>
|
||||
<![CDATA[
|
||||
public class Foo {
|
||||
void good() {
|
||||
SecureRandom random = new SecureRandom();
|
||||
byte iv[] = new byte[16];
|
||||
random.nextBytes(bytes);
|
||||
}
|
||||
|
||||
void bad() {
|
||||
byte[] iv = new byte[] { 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, };
|
||||
}
|
||||
|
||||
void alsoBad() {
|
||||
byte[] iv = "secret iv in here".getBytes();
|
||||
}
|
||||
}
|
||||
]]>
|
||||
</example>
|
||||
</rule>
|
||||
|
||||
</ruleset>
|
||||
@ -0,0 +1,19 @@
|
||||
dependencies {
|
||||
compile "org.xbib:metrics:${project.property('xbib-metrics.version')}"
|
||||
compile("org.elasticsearch:elasticsearch:${project.property('elasticsearch.version')}") {
|
||||
// exclude ES jackson yaml, cbor, smile versions
|
||||
exclude group: 'com.fasterxml.jackson.dataformat'
|
||||
// dependencies that are not meant for client
|
||||
exclude module: 'securesm'
|
||||
// we use log4j2, not log4j
|
||||
exclude group: 'log4j'
|
||||
}
|
||||
// override log4j2 of Elastic with ours
|
||||
compile "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}"
|
||||
// for Elasticsearch session, ES uses SMILE when encoding source for SearchRequest
|
||||
compile "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${project.property('jackson-dataformat.version')}"
|
||||
// CBOR ist default JSON content compression encoding in ES 2.2.1
|
||||
compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${project.property('jackson-dataformat.version')}"
|
||||
// not used, but maybe in other projects
|
||||
compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${project.property('jackson-dataformat.version')}"
|
||||
}
|
||||
@ -0,0 +1,18 @@
|
||||
|
||||
dependencies {
|
||||
compile("org.elasticsearch.client:transport:${rootProject.property('elasticsearch.version')}") {
|
||||
exclude group: 'org.elasticsearch', module: 'securesm'
|
||||
exclude group: 'org.elasticsearch.plugin', module: 'transport-netty3-client'
|
||||
exclude group: 'org.elasticsearch.plugin', module: 'reindex-client'
|
||||
exclude group: 'org.elasticsearch.plugin', module: 'percolator-client'
|
||||
exclude group: 'org.elasticsearch.plugin', module: 'lang-mustache-client'
|
||||
}
|
||||
// we try to override the Elasticsearch netty by our netty version which might be more recent
|
||||
compile "io.netty:netty-buffer:${rootProject.property('netty.version')}"
|
||||
compile "io.netty:netty-codec-http:${rootProject.property('netty.version')}"
|
||||
compile "io.netty:netty-handler:${rootProject.property('netty.version')}"
|
||||
}
|
||||
|
||||
jar {
|
||||
baseName "${rootProject.name}-api"
|
||||
}
|
||||
@ -0,0 +1,7 @@
|
||||
package org.xbib.elx.api;
|
||||
|
||||
@FunctionalInterface
|
||||
public interface ExtendedClientProvider<C extends ExtendedClient> {
|
||||
|
||||
C getExtendedClient();
|
||||
}
|
||||
@ -1,10 +1,7 @@
|
||||
package org.xbib.elasticsearch.extras.client;
|
||||
package org.xbib.elx.api;
|
||||
|
||||
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
@FunctionalInterface
|
||||
public interface IndexAliasAdder {
|
||||
|
||||
@ -0,0 +1,4 @@
|
||||
/**
|
||||
* The API of the Elasticsearch extensions.
|
||||
*/
|
||||
package org.xbib.elx.api;
|
||||
@ -0,0 +1,9 @@
|
||||
dependencies {
|
||||
compile project(':elx-api')
|
||||
compile "org.xbib:guice:${project.property('xbib-guice.version')}"
|
||||
// add all dependencies to runtime source set, even that which are excluded by Elasticsearch jar,
|
||||
// for metaprogramming. We are in Groovyland.
|
||||
runtime "com.vividsolutions:jts:${project.property('jts.version')}"
|
||||
runtime "com.github.spullara.mustache.java:compiler:${project.property('mustache.version')}"
|
||||
runtime "net.java.dev.jna:jna:${project.property('jna.version')}"
|
||||
}
|
||||
@ -0,0 +1,65 @@
|
||||
buildscript {
|
||||
repositories {
|
||||
jcenter()
|
||||
maven {
|
||||
url 'http://xbib.org/repository'
|
||||
}
|
||||
}
|
||||
dependencies {
|
||||
classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.2.2.0"
|
||||
}
|
||||
}
|
||||
|
||||
apply plugin: 'org.xbib.gradle.plugin.elasticsearch.build'
|
||||
|
||||
configurations {
|
||||
main
|
||||
tests
|
||||
}
|
||||
|
||||
dependencies {
|
||||
compile project(':api')
|
||||
compile "org.xbib:metrics:${project.property('xbib-metrics.version')}"
|
||||
compileOnly "org.apache.logging.log4j:log4j-api:${project.property('log4j.version')}"
|
||||
testCompile "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}"
|
||||
testRuntime "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}"
|
||||
}
|
||||
|
||||
jar {
|
||||
baseName "${rootProject.name}-common"
|
||||
}
|
||||
|
||||
/*
|
||||
task testJar(type: Jar, dependsOn: testClasses) {
|
||||
baseName = "${project.archivesBaseName}-tests"
|
||||
from sourceSets.test.output
|
||||
}
|
||||
*/
|
||||
|
||||
artifacts {
|
||||
main jar
|
||||
tests testJar
|
||||
archives sourcesJar, javadocJar
|
||||
}
|
||||
|
||||
test {
|
||||
enabled = false
|
||||
jvmArgs "-javaagent:" + configurations.alpnagent.asPath
|
||||
systemProperty 'path.home', project.buildDir.absolutePath
|
||||
testLogging {
|
||||
showStandardStreams = true
|
||||
exceptionFormat = 'full'
|
||||
}
|
||||
}
|
||||
|
||||
randomizedTest {
|
||||
enabled = false
|
||||
}
|
||||
|
||||
esTest {
|
||||
// test with the jars, not the classes, for security manager
|
||||
// classpath = files(configurations.testRuntime) + configurations.main.artifacts.files + configurations.tests.artifacts.files
|
||||
systemProperty 'tests.security.manager', 'true'
|
||||
}
|
||||
esTest.dependsOn jar, testJar
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,124 @@
|
||||
package org.xbib.elx.common;
|
||||
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.xbib.elx.api.BulkControl;
|
||||
import org.xbib.elx.api.BulkMetric;
|
||||
import org.xbib.elx.api.ExtendedClient;
|
||||
import org.xbib.elx.api.ExtendedClientProvider;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.ServiceLoader;
|
||||
|
||||
@SuppressWarnings("rawtypes")
|
||||
public class ClientBuilder {
|
||||
|
||||
private final ElasticsearchClient client;
|
||||
|
||||
private final Settings.Builder settingsBuilder;
|
||||
|
||||
private Map<Class<? extends ExtendedClientProvider>, ExtendedClientProvider> providerMap;
|
||||
|
||||
private Class<? extends ExtendedClientProvider> provider;
|
||||
|
||||
private BulkMetric metric;
|
||||
|
||||
private BulkControl control;
|
||||
|
||||
public ClientBuilder() {
|
||||
this(null);
|
||||
}
|
||||
|
||||
public ClientBuilder(ElasticsearchClient client) {
|
||||
this(client, Thread.currentThread().getContextClassLoader());
|
||||
}
|
||||
|
||||
public ClientBuilder(ElasticsearchClient client, ClassLoader classLoader) {
|
||||
this.client = client;
|
||||
this.settingsBuilder = Settings.builder();
|
||||
settingsBuilder.put("node.name", "elx-client-" + Version.CURRENT);
|
||||
this.providerMap = new HashMap<>();
|
||||
ServiceLoader<ExtendedClientProvider> serviceLoader = ServiceLoader.load(ExtendedClientProvider.class,
|
||||
classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader());
|
||||
for (ExtendedClientProvider provider : serviceLoader) {
|
||||
providerMap.put(provider.getClass(), provider);
|
||||
}
|
||||
this.metric = new SimpleBulkMetric();
|
||||
this.control = new SimpleBulkControl();
|
||||
}
|
||||
|
||||
public static ClientBuilder builder() {
|
||||
return new ClientBuilder();
|
||||
}
|
||||
|
||||
public static ClientBuilder builder(ElasticsearchClient client) {
|
||||
return new ClientBuilder(client);
|
||||
}
|
||||
|
||||
public ClientBuilder provider(Class<? extends ExtendedClientProvider> provider) {
|
||||
this.provider = provider;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ClientBuilder put(String key, String value) {
|
||||
settingsBuilder.put(key, value);
|
||||
return this;
|
||||
}
|
||||
|
||||
public ClientBuilder put(String key, Integer value) {
|
||||
settingsBuilder.put(key, value);
|
||||
return this;
|
||||
}
|
||||
|
||||
public ClientBuilder put(String key, Long value) {
|
||||
settingsBuilder.put(key, value);
|
||||
return this;
|
||||
}
|
||||
|
||||
public ClientBuilder put(String key, Double value) {
|
||||
settingsBuilder.put(key, value);
|
||||
return this;
|
||||
}
|
||||
|
||||
public ClientBuilder put(String key, ByteSizeValue value) {
|
||||
settingsBuilder.put(key, value);
|
||||
return this;
|
||||
}
|
||||
|
||||
public ClientBuilder put(String key, TimeValue value) {
|
||||
settingsBuilder.put(key, value);
|
||||
return this;
|
||||
}
|
||||
|
||||
public ClientBuilder put(Settings settings) {
|
||||
settingsBuilder.put(settings);
|
||||
return this;
|
||||
}
|
||||
|
||||
public ClientBuilder setMetric(BulkMetric metric) {
|
||||
this.metric = metric;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ClientBuilder setControl(BulkControl control) {
|
||||
this.control = control;
|
||||
return this;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public <C extends ExtendedClient> C build() throws IOException {
|
||||
if (provider == null) {
|
||||
throw new IllegalArgumentException("no provider");
|
||||
}
|
||||
return (C) providerMap.get(provider).getExtendedClient()
|
||||
.setClient(client)
|
||||
.setBulkMetric(metric)
|
||||
.setBulkControl(control)
|
||||
.init(settingsBuilder.build());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,146 @@
|
||||
package org.xbib.elx.common;
|
||||
|
||||
import org.elasticsearch.action.delete.DeleteRequest;
|
||||
import org.elasticsearch.action.index.IndexRequest;
|
||||
import org.elasticsearch.action.update.UpdateRequest;
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Mock client, it does not perform actions on a cluster. Useful for testing or dry runs.
|
||||
*/
|
||||
public class MockExtendedClient extends AbstractExtendedClient {
|
||||
|
||||
@Override
|
||||
public ElasticsearchClient getClient() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient init(Settings settings) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ElasticsearchClient createClient(Settings settings) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient maxActionsPerRequest(int maxActions) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient maxConcurrentRequests(int maxConcurrentRequests) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient maxVolumePerRequest(String maxVolumePerRequest) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient flushIngestInterval(String interval) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient index(String index, String type, String id, boolean create, String source) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient delete(String index, String type, String id) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient update(String index, String type, String id, String source) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient indexRequest(IndexRequest indexRequest) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient deleteRequest(DeleteRequest deleteRequest) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient updateRequest(UpdateRequest updateRequest) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient flushIngest() {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient waitForResponses(String timeValue) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient startBulk(String index, long startRefreshInterval, long stopRefreshIterval) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient stopBulk(String index) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient deleteIndex(String index) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient newIndex(String index) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MockExtendedClient newMapping(String index, String type, Map<String, Object> mapping) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putMapping(String index) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void refreshIndex(String index) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void flushIndex(String index) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void waitForCluster(String healthColor, String timeValue) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public int waitForRecovery(String index) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int updateReplicaLevel(String index, int level) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void shutdown() {
|
||||
// nothing to do
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
package org.xbib.elx.common;
|
||||
|
||||
import org.xbib.elx.api.ExtendedClientProvider;
|
||||
|
||||
public class MockExtendedClientProvider implements ExtendedClientProvider<MockExtendedClient> {
|
||||
@Override
|
||||
public MockExtendedClient getExtendedClient() {
|
||||
return new MockExtendedClient();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,40 @@
|
||||
package org.xbib.elx.common;
|
||||
|
||||
public enum Parameters {
|
||||
|
||||
DEFAULT_MAX_ACTIONS_PER_REQUEST(1000),
|
||||
|
||||
DEFAULT_MAX_CONCURRENT_REQUESTS(Runtime.getRuntime().availableProcessors()),
|
||||
|
||||
DEFAULT_MAX_VOLUME_PER_REQUEST("10mb"),
|
||||
|
||||
DEFAULT_FLUSH_INTERVAL("30s"),
|
||||
|
||||
MAX_ACTIONS_PER_REQUEST ("max_actions_per_request"),
|
||||
|
||||
MAX_CONCURRENT_REQUESTS("max_concurrent_requests"),
|
||||
|
||||
MAX_VOLUME_PER_REQUEST("max_volume_per_request"),
|
||||
|
||||
FLUSH_INTERVAL("flush_interval");
|
||||
|
||||
int num;
|
||||
|
||||
String string;
|
||||
|
||||
Parameters(int num) {
|
||||
this.num = num;
|
||||
}
|
||||
|
||||
Parameters(String string) {
|
||||
this.string = string;
|
||||
}
|
||||
|
||||
int getNum() {
|
||||
return num;
|
||||
}
|
||||
|
||||
String getString() {
|
||||
return string;
|
||||
}
|
||||
}
|
||||
@ -1,4 +1,6 @@
|
||||
package org.xbib.elasticsearch.extras.client;
|
||||
package org.xbib.elx.common;
|
||||
|
||||
import org.xbib.elx.api.BulkControl;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
@ -0,0 +1,25 @@
|
||||
package org.xbib.elx.common.io;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.net.URLConnection;
|
||||
import java.net.URLStreamHandler;
|
||||
|
||||
public class ClasspathURLStreamHandler extends URLStreamHandler {
|
||||
|
||||
private final ClassLoader classLoader;
|
||||
|
||||
public ClasspathURLStreamHandler() {
|
||||
this.classLoader = getClass().getClassLoader();
|
||||
}
|
||||
|
||||
public ClasspathURLStreamHandler(ClassLoader classLoader) {
|
||||
this.classLoader = classLoader;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected URLConnection openConnection(URL u) throws IOException {
|
||||
final URL resourceUrl = classLoader.getResource(u.getPath());
|
||||
return resourceUrl != null ? resourceUrl.openConnection() : null;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
package org.xbib.elx.common.io;
|
||||
|
||||
import java.net.URLStreamHandler;
|
||||
import java.net.URLStreamHandlerFactory;
|
||||
|
||||
public class ClasspathURLStreamHandlerFactory implements URLStreamHandlerFactory {
|
||||
|
||||
@Override
|
||||
public URLStreamHandler createURLStreamHandler(String protocol) {
|
||||
return "classpath".equals(protocol) ? new ClasspathURLStreamHandler() : null;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1 @@
|
||||
package org.xbib.elx.common.io;
|
||||
@ -0,0 +1,139 @@
|
||||
package org.xbib.elx.common.management;
|
||||
|
||||
import java.net.URL;
|
||||
|
||||
public class IndexDefinition {
|
||||
|
||||
private String index;
|
||||
|
||||
private String type;
|
||||
|
||||
private String fullIndexName;
|
||||
|
||||
private String dateTimePattern;
|
||||
|
||||
private URL settingsUrl;
|
||||
|
||||
private URL mappingsUrl;
|
||||
|
||||
private boolean enabled;
|
||||
|
||||
private boolean ignoreErrors;
|
||||
|
||||
private boolean switchAliases;
|
||||
|
||||
private boolean hasForceMerge;
|
||||
|
||||
private int replicaLevel;
|
||||
|
||||
private IndexRetention indexRetention;
|
||||
|
||||
public IndexDefinition setIndex(String index) {
|
||||
this.index = index;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getIndex() {
|
||||
return index;
|
||||
}
|
||||
|
||||
public IndexDefinition setFullIndexName(String fullIndexName) {
|
||||
this.fullIndexName = fullIndexName;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getFullIndexName() {
|
||||
return fullIndexName;
|
||||
}
|
||||
|
||||
public IndexDefinition setType(String type) {
|
||||
this.type = type;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public IndexDefinition setSettingsUrl(URL settingsUrl) {
|
||||
this.settingsUrl = settingsUrl;
|
||||
return this;
|
||||
}
|
||||
|
||||
public URL getSettingsUrl() {
|
||||
return settingsUrl;
|
||||
}
|
||||
|
||||
public IndexDefinition setMappingsUrl(URL mappingsUrl) {
|
||||
this.mappingsUrl = mappingsUrl;
|
||||
return this;
|
||||
}
|
||||
|
||||
public URL getMappingsUrl() {
|
||||
return mappingsUrl;
|
||||
}
|
||||
|
||||
public IndexDefinition setDateTimePattern(String timeWindow) {
|
||||
this.dateTimePattern = timeWindow;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getDateTimePattern() {
|
||||
return dateTimePattern;
|
||||
}
|
||||
|
||||
public IndexDefinition setEnabled(boolean enabled) {
|
||||
this.enabled = enabled;
|
||||
return this;
|
||||
}
|
||||
|
||||
public boolean isEnabled() {
|
||||
return enabled;
|
||||
}
|
||||
|
||||
public IndexDefinition setIgnoreErrors(boolean ignoreErrors) {
|
||||
this.ignoreErrors = ignoreErrors;
|
||||
return this;
|
||||
}
|
||||
|
||||
public boolean ignoreErrors() {
|
||||
return ignoreErrors;
|
||||
}
|
||||
|
||||
public IndexDefinition setSwitchAliases(boolean switchAliases) {
|
||||
this.switchAliases = switchAliases;
|
||||
return this;
|
||||
}
|
||||
|
||||
public boolean isSwitchAliases() {
|
||||
return switchAliases;
|
||||
}
|
||||
|
||||
public IndexDefinition setForceMerge(boolean hasForceMerge) {
|
||||
this.hasForceMerge = hasForceMerge;
|
||||
return this;
|
||||
}
|
||||
|
||||
public boolean hasForceMerge() {
|
||||
return hasForceMerge;
|
||||
}
|
||||
|
||||
public IndexDefinition setReplicaLevel(int replicaLevel) {
|
||||
this.replicaLevel = replicaLevel;
|
||||
return this;
|
||||
}
|
||||
|
||||
public int getReplicaLevel() {
|
||||
return replicaLevel;
|
||||
}
|
||||
|
||||
public IndexDefinition setRetention(IndexRetention indexRetention) {
|
||||
this.indexRetention = indexRetention;
|
||||
return this;
|
||||
}
|
||||
|
||||
public IndexRetention getRetention() {
|
||||
return indexRetention;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,27 @@
|
||||
package org.xbib.elx.common.management;
|
||||
|
||||
public class IndexRetention {
|
||||
|
||||
private int timestampDiff;
|
||||
|
||||
private int minToKeep;
|
||||
|
||||
public IndexRetention setTimestampDiff(int timestampDiff) {
|
||||
this.timestampDiff = timestampDiff;
|
||||
return this;
|
||||
}
|
||||
|
||||
public int getTimestampDiff() {
|
||||
return timestampDiff;
|
||||
}
|
||||
|
||||
public IndexRetention setMinToKeep(int minToKeep) {
|
||||
this.minToKeep = minToKeep;
|
||||
return this;
|
||||
}
|
||||
|
||||
public int getMinToKeep() {
|
||||
return minToKeep;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1 @@
|
||||
package org.xbib.elx.common.management;
|
||||
@ -0,0 +1,4 @@
|
||||
/**
|
||||
* Common classes for Elasticsearch client extensions.
|
||||
*/
|
||||
package org.xbib.elx.common;
|
||||
@ -0,0 +1 @@
|
||||
package org.xbib.elx.common.util;
|
||||
@ -0,0 +1 @@
|
||||
org.xbib.elx.common.io.ClasspathURLStreamHandlerFactory
|
||||
@ -0,0 +1 @@
|
||||
org.xbib.elx.common.MockExtendedClientProvider
|
||||
@ -0,0 +1 @@
|
||||
package org.elasticsearch.node;
|
||||
@ -0,0 +1,16 @@
|
||||
package org.xbib.elx.common;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
|
||||
public class MockExtendedClientProviderTest {
|
||||
|
||||
@Test
|
||||
public void testMockExtendedProvider() throws IOException {
|
||||
MockExtendedClient client = ClientBuilder.builder().provider(MockExtendedClientProvider.class).build();
|
||||
assertNotNull(client);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,213 @@
|
||||
package org.xbib.elx.common;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.ElasticsearchTimeoutException;
|
||||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
|
||||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
|
||||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
|
||||
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
|
||||
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
|
||||
import org.elasticsearch.client.support.AbstractClient;
|
||||
import org.elasticsearch.cluster.health.ClusterHealthStatus;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.transport.InetSocketTransportAddress;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.node.MockNode;
|
||||
import org.elasticsearch.node.Node;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.xbib.elx.common.util.NetworkUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.FileVisitResult;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.nio.file.SimpleFileVisitor;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Random;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
|
||||
|
||||
public class NodeTestUtils {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger("test");
|
||||
|
||||
private static Random random = new Random();
|
||||
|
||||
private static char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray();
|
||||
|
||||
private Map<String, Node> nodes = new HashMap<>();
|
||||
|
||||
private Map<String, AbstractClient> clients = new HashMap<>();
|
||||
|
||||
private AtomicInteger counter = new AtomicInteger();
|
||||
|
||||
private String cluster;
|
||||
|
||||
private String host;
|
||||
|
||||
private int port;
|
||||
|
||||
private static void deleteFiles() throws IOException {
|
||||
Path directory = Paths.get(getHome() + "/data");
|
||||
Files.walkFileTree(directory, new SimpleFileVisitor<Path>() {
|
||||
@Override
|
||||
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
|
||||
Files.delete(file);
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
|
||||
Files.delete(dir);
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
@Before
|
||||
public void startNodes() {
|
||||
try {
|
||||
logger.info("starting");
|
||||
setClusterName();
|
||||
startNode("1");
|
||||
findNodeAddress();
|
||||
try {
|
||||
ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE,
|
||||
new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)
|
||||
.timeout(TimeValue.timeValueSeconds(30))).actionGet();
|
||||
if (healthResponse != null && healthResponse.isTimedOut()) {
|
||||
throw new IOException("cluster state is " + healthResponse.getStatus().name()
|
||||
+ ", from here on, everything will fail!");
|
||||
}
|
||||
} catch (ElasticsearchTimeoutException e) {
|
||||
throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
logger.error("startNodes failed", t);
|
||||
}
|
||||
}
|
||||
|
||||
@After
|
||||
public void stopNodes() {
|
||||
try {
|
||||
closeNodes();
|
||||
} catch (Exception e) {
|
||||
logger.error("can not close nodes", e);
|
||||
} finally {
|
||||
try {
|
||||
deleteFiles();
|
||||
logger.info("data files wiped");
|
||||
Thread.sleep(2000L); // let OS commit changes
|
||||
} catch (IOException e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
} catch (InterruptedException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void setClusterName() {
|
||||
this.cluster = "test-helper-cluster-"
|
||||
+ NetworkUtils.getLocalAddress().getHostName()
|
||||
+ "-" + System.getProperty("user.name")
|
||||
+ "-" + counter.incrementAndGet();
|
||||
}
|
||||
|
||||
protected String getClusterName() {
|
||||
return cluster;
|
||||
}
|
||||
|
||||
protected Settings getSettings() {
|
||||
return settingsBuilder()
|
||||
.put("host", host)
|
||||
.put("port", port)
|
||||
.put("cluster.name", cluster)
|
||||
.put("path.home", getHome())
|
||||
.build();
|
||||
}
|
||||
|
||||
protected Settings getNodeSettings() {
|
||||
return settingsBuilder()
|
||||
.put("cluster.name", cluster)
|
||||
.put("cluster.routing.schedule", "50ms")
|
||||
.put("cluster.routing.allocation.disk.threshold_enabled", false)
|
||||
.put("discovery.zen.multicast.enabled", true)
|
||||
.put("discovery.zen.multicast.ping_timeout", "5s")
|
||||
.put("http.enabled", true)
|
||||
.put("threadpool.bulk.size", Runtime.getRuntime().availableProcessors())
|
||||
.put("threadpool.bulk.queue_size", 16 * Runtime.getRuntime().availableProcessors()) // default is 50, too low
|
||||
.put("index.number_of_replicas", 0)
|
||||
.put("path.home", getHome())
|
||||
.build();
|
||||
}
|
||||
|
||||
protected static String getHome() {
|
||||
return System.getProperty("path.home", System.getProperty("user.dir"));
|
||||
}
|
||||
|
||||
public void startNode(String id) {
|
||||
buildNode(id).start();
|
||||
}
|
||||
|
||||
public AbstractClient client(String id) {
|
||||
return clients.get(id);
|
||||
}
|
||||
|
||||
private void closeNodes() {
|
||||
logger.info("closing all clients");
|
||||
for (AbstractClient client : clients.values()) {
|
||||
client.close();
|
||||
}
|
||||
clients.clear();
|
||||
logger.info("closing all nodes");
|
||||
for (Node node : nodes.values()) {
|
||||
if (node != null) {
|
||||
node.close();
|
||||
}
|
||||
}
|
||||
nodes.clear();
|
||||
logger.info("all nodes closed");
|
||||
}
|
||||
|
||||
protected void findNodeAddress() {
|
||||
NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true);
|
||||
NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet();
|
||||
Object obj = response.iterator().next().getTransport().getAddress()
|
||||
.publishAddress();
|
||||
if (obj instanceof InetSocketTransportAddress) {
|
||||
InetSocketTransportAddress address = (InetSocketTransportAddress) obj;
|
||||
host = address.address().getHostName();
|
||||
port = address.address().getPort();
|
||||
}
|
||||
}
|
||||
|
||||
private Node buildNode(String id) {
|
||||
Settings nodeSettings = settingsBuilder()
|
||||
.put(getNodeSettings())
|
||||
.put("name", id)
|
||||
.build();
|
||||
Node node = new MockNode(nodeSettings);
|
||||
AbstractClient client = (AbstractClient) node.client();
|
||||
nodes.put(id, node);
|
||||
clients.put(id, client);
|
||||
logger.info("clients={}", clients);
|
||||
return node;
|
||||
}
|
||||
|
||||
protected String randomString(int len) {
|
||||
final char[] buf = new char[len];
|
||||
final int n = numbersAndLetters.length - 1;
|
||||
for (int i = 0; i < buf.length; i++) {
|
||||
buf[i] = numbersAndLetters[random.nextInt(n)];
|
||||
}
|
||||
return new String(buf);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,56 @@
|
||||
package org.xbib.elx.common;
|
||||
|
||||
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
|
||||
import org.elasticsearch.action.bulk.BulkAction;
|
||||
import org.elasticsearch.action.bulk.BulkRequestBuilder;
|
||||
import org.elasticsearch.action.index.IndexRequest;
|
||||
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.index.query.QueryBuilder;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.search.sort.SortOrder;
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
public class SearchTest extends NodeTestUtils {
|
||||
|
||||
@Test
|
||||
public void testSearch() throws Exception {
|
||||
Client client = client("1");
|
||||
BulkRequestBuilder builder = new BulkRequestBuilder(client, BulkAction.INSTANCE);
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
IndexRequest indexRequest = new IndexRequest("pages", "row")
|
||||
.source(XContentFactory.jsonBuilder()
|
||||
.startObject()
|
||||
.field("user1", "joerg")
|
||||
.field("user2", "joerg")
|
||||
.field("user3", "joerg")
|
||||
.field("user4", "joerg")
|
||||
.field("user5", "joerg")
|
||||
.field("user6", "joerg")
|
||||
.field("user7", "joerg")
|
||||
.field("user8", "joerg")
|
||||
.field("user9", "joerg")
|
||||
.field("rowcount", i)
|
||||
.field("rs", 1234));
|
||||
builder.add(indexRequest);
|
||||
}
|
||||
client.bulk(builder.request()).actionGet();
|
||||
client.admin().indices().refresh(new RefreshRequest()).actionGet();
|
||||
|
||||
for (int i = 0; i < 100; i++) {
|
||||
QueryBuilder queryStringBuilder = QueryBuilders.queryStringQuery("rs:" + 1234);
|
||||
SearchRequestBuilder requestBuilder = client.prepareSearch()
|
||||
.setIndices("pages")
|
||||
.setTypes("row")
|
||||
.setQuery(queryStringBuilder)
|
||||
.addSort("rowcount", SortOrder.DESC)
|
||||
.setFrom(i * 10).setSize(10);
|
||||
SearchResponse searchResponse = requestBuilder.execute().actionGet();
|
||||
assertTrue(searchResponse.getHits().getTotalHits() > 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,62 @@
|
||||
package org.xbib.elx.common;
|
||||
|
||||
import org.elasticsearch.action.index.IndexRequest;
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.index.query.QueryBuilder;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class WildcardTest extends NodeTestUtils {
|
||||
|
||||
protected Settings getNodeSettings() {
|
||||
return Settings.settingsBuilder()
|
||||
.put(super.getNodeSettings())
|
||||
.put("cluster.routing.allocation.disk.threshold_enabled", false)
|
||||
.put("discovery.zen.multicast.enabled", false)
|
||||
.put("http.enabled", false)
|
||||
.put("index.number_of_shards", 1)
|
||||
.put("index.number_of_replicas", 0)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWildcard() throws Exception {
|
||||
index(client("1"), "1", "010");
|
||||
index(client("1"), "2", "0*0");
|
||||
// exact
|
||||
validateCount(client("1"), QueryBuilders.queryStringQuery("010").defaultField("field"), 1);
|
||||
validateCount(client("1"), QueryBuilders.queryStringQuery("0\\*0").defaultField("field"), 1);
|
||||
// pattern
|
||||
validateCount(client("1"), QueryBuilders.queryStringQuery("0*0").defaultField("field"), 1); // 2?
|
||||
validateCount(client("1"), QueryBuilders.queryStringQuery("0?0").defaultField("field"), 1); // 2?
|
||||
validateCount(client("1"), QueryBuilders.queryStringQuery("0**0").defaultField("field"), 1); // 2?
|
||||
validateCount(client("1"), QueryBuilders.queryStringQuery("0??0").defaultField("field"), 0);
|
||||
validateCount(client("1"), QueryBuilders.queryStringQuery("*10").defaultField("field"), 1);
|
||||
validateCount(client("1"), QueryBuilders.queryStringQuery("*1*").defaultField("field"), 1);
|
||||
validateCount(client("1"), QueryBuilders.queryStringQuery("*\\*0").defaultField("field"), 0); // 1?
|
||||
validateCount(client("1"), QueryBuilders.queryStringQuery("*\\**").defaultField("field"), 0); // 1?
|
||||
}
|
||||
|
||||
private void index(Client client, String id, String fieldValue) throws IOException {
|
||||
client.index(new IndexRequest("index", "type", id)
|
||||
.source(XContentFactory.jsonBuilder().startObject().field("field", fieldValue).endObject())
|
||||
.refresh(true)).actionGet();
|
||||
}
|
||||
|
||||
private long count(Client client, QueryBuilder queryBuilder) {
|
||||
return client.prepareSearch("index").setTypes("type")
|
||||
.setQuery(queryBuilder)
|
||||
.execute().actionGet().getHits().getTotalHits();
|
||||
}
|
||||
|
||||
private void validateCount(Client client, QueryBuilder queryBuilder, long expectedHits) {
|
||||
final long actualHits = count(client, queryBuilder);
|
||||
if (actualHits != expectedHits) {
|
||||
throw new RuntimeException("actualHits=" + actualHits + ", expectedHits=" + expectedHits);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1 @@
|
||||
package org.xbib.elx.common;
|
||||
@ -0,0 +1,13 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<configuration status="OFF">
|
||||
<appenders>
|
||||
<Console name="Console" target="SYSTEM_OUT">
|
||||
<PatternLayout pattern="[%d{ISO8601}][%-5p][%-25c][%t] %m%n"/>
|
||||
</Console>
|
||||
</appenders>
|
||||
<Loggers>
|
||||
<Root level="info">
|
||||
<AppenderRef ref="Console" />
|
||||
</Root>
|
||||
</Loggers>
|
||||
</configuration>
|
||||
@ -0,0 +1,65 @@
|
||||
buildscript {
|
||||
repositories {
|
||||
jcenter()
|
||||
maven {
|
||||
url 'http://xbib.org/repository'
|
||||
}
|
||||
}
|
||||
dependencies {
|
||||
classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.2.2.0"
|
||||
}
|
||||
}
|
||||
|
||||
apply plugin: 'org.xbib.gradle.plugin.elasticsearch.build'
|
||||
|
||||
configurations {
|
||||
main
|
||||
tests
|
||||
}
|
||||
|
||||
dependencies {
|
||||
compile project(':common')
|
||||
compile "org.xbib:netty-http-client:${project.property('xbib-netty-http-client.version')}"
|
||||
testCompile "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}"
|
||||
testRuntime "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}"
|
||||
}
|
||||
|
||||
jar {
|
||||
baseName "${rootProject.name}-common"
|
||||
}
|
||||
|
||||
/*
|
||||
task testJar(type: Jar, dependsOn: testClasses) {
|
||||
baseName = "${project.archivesBaseName}-tests"
|
||||
from sourceSets.test.output
|
||||
}
|
||||
*/
|
||||
|
||||
artifacts {
|
||||
main jar
|
||||
tests testJar
|
||||
archives sourcesJar, javadocJar
|
||||
}
|
||||
|
||||
test {
|
||||
enabled = true
|
||||
include '**/SimpleTest.*'
|
||||
testLogging {
|
||||
showStandardStreams = true
|
||||
exceptionFormat = 'full'
|
||||
}
|
||||
}
|
||||
|
||||
randomizedTest {
|
||||
enabled = false
|
||||
}
|
||||
|
||||
esTest {
|
||||
enabled = true
|
||||
// test with the jars, not the classes, for security manager
|
||||
// classpath = files(configurations.testRuntime) + configurations.main.artifacts.files + configurations.tests.artifacts.files
|
||||
systemProperty 'tests.security.manager', 'true'
|
||||
// maybe we like some extra security policy for our code
|
||||
systemProperty 'tests.security.policy', '/extra-security.policy'
|
||||
}
|
||||
esTest.dependsOn jar, testJar
|
||||
@ -0,0 +1,3 @@
|
||||
dependencies {
|
||||
compile project(':elx-common')
|
||||
}
|
||||
@ -0,0 +1,65 @@
|
||||
buildscript {
|
||||
repositories {
|
||||
jcenter()
|
||||
maven {
|
||||
url 'http://xbib.org/repository'
|
||||
}
|
||||
}
|
||||
dependencies {
|
||||
classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.2.3.4"
|
||||
}
|
||||
}
|
||||
|
||||
apply plugin: 'org.xbib.gradle.plugin.elasticsearch.build'
|
||||
|
||||
configurations {
|
||||
main
|
||||
tests
|
||||
}
|
||||
|
||||
dependencies {
|
||||
compile project(':common')
|
||||
testCompile "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}"
|
||||
testRuntime "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}"
|
||||
}
|
||||
|
||||
jar {
|
||||
baseName "${rootProject.name}-node"
|
||||
}
|
||||
|
||||
/*
|
||||
task testJar(type: Jar, dependsOn: testClasses) {
|
||||
baseName = "${project.archivesBaseName}-tests"
|
||||
from sourceSets.test.output
|
||||
}
|
||||
*/
|
||||
|
||||
artifacts {
|
||||
main jar
|
||||
tests testJar
|
||||
archives sourcesJar, javadocJar
|
||||
}
|
||||
|
||||
test {
|
||||
enabled = false
|
||||
jvmArgs "-javaagent:" + configurations.alpnagent.asPath
|
||||
systemProperty 'path.home', projectDir.absolutePath
|
||||
testLogging {
|
||||
showStandardStreams = true
|
||||
exceptionFormat = 'full'
|
||||
}
|
||||
}
|
||||
|
||||
randomizedTest {
|
||||
enabled = false
|
||||
}
|
||||
|
||||
|
||||
esTest {
|
||||
// test with the jars, not the classes, for security manager
|
||||
// classpath = files(configurations.testRuntime) + configurations.main.artifacts.files + configurations.tests.artifacts.files
|
||||
systemProperty 'tests.security.manager', 'true'
|
||||
// maybe we like some extra security policy for our code
|
||||
systemProperty 'tests.security.policy', '/extra-security.policy'
|
||||
}
|
||||
esTest.dependsOn jar, testJar
|
||||
@ -0,0 +1,70 @@
|
||||
package org.xbib.elx.node;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.env.Environment;
|
||||
import org.elasticsearch.node.Node;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.xbib.elx.common.AbstractExtendedClient;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
|
||||
public class ExtendedNodeClient extends AbstractExtendedClient {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(ExtendedNodeClient.class.getName());
|
||||
|
||||
private Node node;
|
||||
|
||||
@Override
|
||||
protected ElasticsearchClient createClient(Settings settings) throws IOException {
|
||||
if (settings != null) {
|
||||
String version = System.getProperty("os.name")
|
||||
+ " " + System.getProperty("java.vm.name")
|
||||
+ " " + System.getProperty("java.vm.vendor")
|
||||
+ " " + System.getProperty("java.runtime.version")
|
||||
+ " " + System.getProperty("java.vm.version");
|
||||
Settings effectiveSettings = Settings.builder().put(settings)
|
||||
.put("node.client", true)
|
||||
.put("node.master", false)
|
||||
.put("node.data", false)
|
||||
.build();
|
||||
logger.info("creating node client on {} with effective settings {}",
|
||||
version, effectiveSettings.toString());
|
||||
Collection<Class<? extends Plugin>> plugins = Collections.emptyList();
|
||||
this.node = new BulkNode(new Environment(effectiveSettings), plugins);
|
||||
try {
|
||||
node.start();
|
||||
} catch (Exception e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
return node.client();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void shutdown() throws IOException {
|
||||
super.shutdown();
|
||||
try {
|
||||
if (node != null) {
|
||||
logger.debug("closing node...");
|
||||
node.close();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
private static class BulkNode extends Node {
|
||||
|
||||
BulkNode(Environment env, Collection<Class<? extends Plugin>> classpathPlugins) {
|
||||
super(env, Version.CURRENT, classpathPlugins);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
package org.xbib.elx.node;
|
||||
|
||||
import org.xbib.elx.api.ExtendedClientProvider;
|
||||
|
||||
public class ExtendedNodeClientProvider implements ExtendedClientProvider<ExtendedNodeClient> {
|
||||
@Override
|
||||
public ExtendedNodeClient getExtendedClient() {
|
||||
return new ExtendedNodeClient();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1 @@
|
||||
org.xbib.elx.node.ExtendedNodeClientProvider
|
||||
@ -0,0 +1,34 @@
|
||||
package org.elasticsearch.node;
|
||||
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.node.internal.InternalSettingsPreparer;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
|
||||
public class MockNode extends Node {
|
||||
|
||||
public MockNode() {
|
||||
super(Settings.EMPTY);
|
||||
}
|
||||
|
||||
public MockNode(Settings settings) {
|
||||
super(settings);
|
||||
}
|
||||
|
||||
public MockNode(Settings settings, Collection<Class<? extends Plugin>> classpathPlugins) {
|
||||
super(InternalSettingsPreparer.prepareEnvironment(settings, null), Version.CURRENT, classpathPlugins);
|
||||
}
|
||||
|
||||
public MockNode(Settings settings, Class<? extends Plugin> classpathPlugin) {
|
||||
this(settings, list(classpathPlugin));
|
||||
}
|
||||
|
||||
private static Collection<Class<? extends Plugin>> list(Class<? extends Plugin> classpathPlugin) {
|
||||
Collection<Class<? extends Plugin>> list = new ArrayList<>();
|
||||
list.add(classpathPlugin);
|
||||
return list;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,58 @@
|
||||
package org.xbib.elx.node;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.action.search.SearchAction;
|
||||
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
import org.xbib.elx.common.Parameters;
|
||||
|
||||
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
@Ignore
|
||||
public class ExtendeNodeDuplicateIDTest extends NodeTestUtils {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(ExtendeNodeDuplicateIDTest.class.getSimpleName());
|
||||
|
||||
private static final Long MAX_ACTIONS_PER_REQUEST = 1000L;
|
||||
|
||||
private static final Long ACTIONS = 12345L;
|
||||
|
||||
@Test
|
||||
public void testDuplicateDocIDs() throws Exception {
|
||||
long numactions = ACTIONS;
|
||||
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
|
||||
.provider(ExtendedNodeClientProvider.class)
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||
.build();
|
||||
try {
|
||||
client.newIndex("test");
|
||||
for (int i = 0; i < ACTIONS; i++) {
|
||||
client.index("test", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
|
||||
}
|
||||
client.flushIngest();
|
||||
client.waitForResponses("30s");
|
||||
client.refreshIndex("test");
|
||||
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
|
||||
.setIndices("test")
|
||||
.setTypes("test")
|
||||
.setQuery(matchAllQuery());
|
||||
long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
|
||||
logger.info("hits = {}", hits);
|
||||
assertTrue(hits < ACTIONS);
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} finally {
|
||||
client.shutdown();
|
||||
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
|
||||
if (client.hasThrowable()) {
|
||||
logger.error("error", client.getThrowable());
|
||||
}
|
||||
assertFalse(client.hasThrowable());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,39 @@
|
||||
package org.xbib.elx.node;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.junit.Test;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
|
||||
public class ExtendedNodeClientSingleNodeTest extends NodeTestUtils {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(ExtendedNodeClientSingleNodeTest.class.getSimpleName());
|
||||
|
||||
@Test
|
||||
public void testSingleDocNodeClient() throws Exception {
|
||||
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
|
||||
.provider(ExtendedNodeClientProvider.class)
|
||||
.build();
|
||||
try {
|
||||
client.newIndex("test");
|
||||
client.index("test", "test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
||||
client.flushIngest();
|
||||
client.waitForResponses("30s");
|
||||
} catch (InterruptedException e) {
|
||||
// ignore
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} finally {
|
||||
assertEquals(1, client.getBulkMetric().getSucceeded().getCount());
|
||||
if (client.hasThrowable()) {
|
||||
logger.error("error", client.getThrowable());
|
||||
}
|
||||
assertFalse(client.hasThrowable());
|
||||
client.shutdown();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,54 +1,47 @@
|
||||
package org.xbib.elasticsearch.extras.client.transport;
|
||||
package org.xbib.elx.node;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.elasticsearch.common.logging.ESLogger;
|
||||
import org.elasticsearch.common.logging.ESLoggerFactory;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import org.xbib.elasticsearch.NodeTestUtils;
|
||||
import org.xbib.elasticsearch.extras.client.Clients;
|
||||
import org.xbib.elasticsearch.extras.client.SimpleBulkControl;
|
||||
import org.xbib.elasticsearch.extras.client.SimpleBulkMetric;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
public class BulkTransportUpdateReplicaLevelTest extends NodeTestUtils {
|
||||
@Ignore
|
||||
public class ExtendedNodeUpdateReplicaLevelTest extends NodeTestUtils {
|
||||
|
||||
private static final ESLogger logger =
|
||||
ESLoggerFactory.getLogger(BulkTransportUpdateReplicaLevelTest.class.getSimpleName());
|
||||
private static final Logger logger = LogManager.getLogger(ExtendedNodeUpdateReplicaLevelTest.class.getSimpleName());
|
||||
|
||||
@Test
|
||||
public void testUpdateReplicaLevel() throws Exception {
|
||||
|
||||
int numberOfShards = 2;
|
||||
long numberOfShards = 2;
|
||||
int replicaLevel = 3;
|
||||
|
||||
// we need 3 nodes for replica level 3
|
||||
startNode("2");
|
||||
startNode("3");
|
||||
|
||||
int shardsAfterReplica;
|
||||
long shardsAfterReplica;
|
||||
|
||||
Settings settings = Settings.settingsBuilder()
|
||||
.put("index.number_of_shards", numberOfShards)
|
||||
.put("index.number_of_replicas", 0)
|
||||
.build();
|
||||
|
||||
final BulkTransportClient client = Clients.builder()
|
||||
.put(getSettings())
|
||||
.setMetric(new SimpleBulkMetric())
|
||||
.setControl(new SimpleBulkControl())
|
||||
.toBulkTransportClient();
|
||||
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
|
||||
.provider(ExtendedNodeClientProvider.class)
|
||||
.build();
|
||||
|
||||
try {
|
||||
client.newIndex("replicatest", settings, null);
|
||||
client.waitForCluster("GREEN", "30s");
|
||||
for (int i = 0; i < 12345; i++) {
|
||||
client.index("replicatest", "replicatest", null, "{ \"name\" : \"" + randomString(32) + "\"}");
|
||||
client.index("replicatest", "replicatest", null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
|
||||
}
|
||||
client.flushIngest();
|
||||
client.waitForResponses("30s");
|
||||
@ -0,0 +1,201 @@
|
||||
package org.xbib.elx.node;
|
||||
|
||||
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.ElasticsearchTimeoutException;
|
||||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
|
||||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
|
||||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
|
||||
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
|
||||
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
|
||||
import org.elasticsearch.client.support.AbstractClient;
|
||||
import org.elasticsearch.cluster.health.ClusterHealthStatus;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.transport.InetSocketTransportAddress;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.node.MockNode;
|
||||
import org.elasticsearch.node.Node;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.xbib.elx.common.util.NetworkUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.FileVisitResult;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.nio.file.SimpleFileVisitor;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Random;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
public class NodeTestUtils {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger("test");
|
||||
|
||||
private static Random random = new Random();
|
||||
|
||||
private static char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray();
|
||||
|
||||
private Map<String, Node> nodes = new HashMap<>();
|
||||
|
||||
private Map<String, AbstractClient> clients = new HashMap<>();
|
||||
|
||||
private AtomicInteger counter = new AtomicInteger();
|
||||
|
||||
private String cluster;
|
||||
|
||||
private String host;
|
||||
|
||||
private int port;
|
||||
|
||||
private static void deleteFiles() throws IOException {
|
||||
Path directory = Paths.get(getHome() + "/data");
|
||||
Files.walkFileTree(directory, new SimpleFileVisitor<Path>() {
|
||||
@Override
|
||||
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
|
||||
Files.delete(file);
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
|
||||
Files.delete(dir);
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
@Before
|
||||
public void startNodes() {
|
||||
try {
|
||||
logger.info("starting");
|
||||
setClusterName();
|
||||
startNode("1");
|
||||
findNodeAddress();
|
||||
try {
|
||||
ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE,
|
||||
new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)
|
||||
.timeout(TimeValue.timeValueSeconds(30))).actionGet();
|
||||
if (healthResponse != null && healthResponse.isTimedOut()) {
|
||||
throw new IOException("cluster state is " + healthResponse.getStatus().name()
|
||||
+ ", from here on, everything will fail!");
|
||||
}
|
||||
} catch (ElasticsearchTimeoutException e) {
|
||||
throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
logger.error("startNodes failed", t);
|
||||
}
|
||||
}
|
||||
|
||||
@After
|
||||
public void stopNodes() {
|
||||
try {
|
||||
closeNodes();
|
||||
} catch (Exception e) {
|
||||
logger.error("can not close nodes", e);
|
||||
} finally {
|
||||
try {
|
||||
deleteFiles();
|
||||
logger.info("data files wiped");
|
||||
Thread.sleep(2000L); // let OS commit changes
|
||||
} catch (IOException e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
} catch (InterruptedException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void setClusterName() {
|
||||
this.cluster = "test-helper-cluster-"
|
||||
+ NetworkUtils.getLocalAddress().getHostName()
|
||||
+ "-" + System.getProperty("user.name")
|
||||
+ "-" + counter.incrementAndGet();
|
||||
}
|
||||
|
||||
protected Settings getNodeSettings() {
|
||||
return settingsBuilder()
|
||||
.put("cluster.name", cluster)
|
||||
.put("cluster.routing.schedule", "50ms")
|
||||
.put("cluster.routing.allocation.disk.threshold_enabled", false)
|
||||
.put("discovery.zen.multicast.enabled", true)
|
||||
.put("discovery.zen.multicast.ping_timeout", "5s")
|
||||
.put("http.enabled", true)
|
||||
.put("threadpool.bulk.size", Runtime.getRuntime().availableProcessors())
|
||||
.put("threadpool.bulk.queue_size", 16 * Runtime.getRuntime().availableProcessors()) // default is 50, too low
|
||||
.put("index.number_of_replicas", 0)
|
||||
.put("path.home", getHome())
|
||||
.build();
|
||||
}
|
||||
|
||||
protected static String getHome() {
|
||||
return System.getProperty("path.home", System.getProperty("user.dir"));
|
||||
}
|
||||
|
||||
public void startNode(String id) {
|
||||
buildNode(id).start();
|
||||
}
|
||||
|
||||
public AbstractClient client(String id) {
|
||||
return clients.get(id);
|
||||
}
|
||||
|
||||
private void closeNodes() {
|
||||
logger.info("closing all clients");
|
||||
for (AbstractClient client : clients.values()) {
|
||||
client.close();
|
||||
}
|
||||
clients.clear();
|
||||
logger.info("closing all nodes");
|
||||
for (Node node : nodes.values()) {
|
||||
if (node != null) {
|
||||
node.close();
|
||||
}
|
||||
}
|
||||
nodes.clear();
|
||||
logger.info("all nodes closed");
|
||||
}
|
||||
|
||||
protected void findNodeAddress() {
|
||||
NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true);
|
||||
NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet();
|
||||
Object obj = response.iterator().next().getTransport().getAddress()
|
||||
.publishAddress();
|
||||
if (obj instanceof InetSocketTransportAddress) {
|
||||
InetSocketTransportAddress address = (InetSocketTransportAddress) obj;
|
||||
host = address.address().getHostName();
|
||||
port = address.address().getPort();
|
||||
}
|
||||
}
|
||||
|
||||
private Node buildNode(String id) {
|
||||
Settings nodeSettings = settingsBuilder()
|
||||
.put(getNodeSettings())
|
||||
.put("name", id)
|
||||
.build();
|
||||
logger.info("settings={}", nodeSettings.getAsMap());
|
||||
Node node = new MockNode(nodeSettings);
|
||||
AbstractClient client = (AbstractClient) node.client();
|
||||
nodes.put(id, node);
|
||||
clients.put(id, client);
|
||||
logger.info("clients={}", clients);
|
||||
return node;
|
||||
}
|
||||
|
||||
protected String randomString(int len) {
|
||||
final char[] buf = new char[len];
|
||||
final int n = numbersAndLetters.length - 1;
|
||||
for (int i = 0; i < buf.length; i++) {
|
||||
buf[i] = numbersAndLetters[random.nextInt(n)];
|
||||
}
|
||||
return new String(buf);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,3 @@
|
||||
dependencies {
|
||||
compile project(':elx-common')
|
||||
}
|
||||
@ -0,0 +1,63 @@
|
||||
buildscript {
|
||||
repositories {
|
||||
jcenter()
|
||||
maven {
|
||||
url 'http://xbib.org/repository'
|
||||
}
|
||||
}
|
||||
dependencies {
|
||||
classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.2.2.0"
|
||||
}
|
||||
}
|
||||
|
||||
apply plugin: 'org.xbib.gradle.plugin.elasticsearch.build'
|
||||
|
||||
configurations {
|
||||
main
|
||||
tests
|
||||
}
|
||||
|
||||
dependencies {
|
||||
compile project(':common')
|
||||
testCompile "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}"
|
||||
testRuntime "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}"
|
||||
}
|
||||
|
||||
jar {
|
||||
baseName "${rootProject.name}-transport"
|
||||
}
|
||||
|
||||
task testJar(type: Jar, dependsOn: testClasses) {
|
||||
baseName = "${project.archivesBaseName}-tests"
|
||||
from sourceSets.test.output
|
||||
}
|
||||
|
||||
artifacts {
|
||||
main jar
|
||||
tests testJar
|
||||
archives sourcesJar, javadocJar
|
||||
}
|
||||
|
||||
esTest {
|
||||
enabled = true
|
||||
// test with the jars, not the classes, for security manager
|
||||
classpath = files(configurations.testRuntime) + configurations.main.artifacts.files + configurations.tests.artifacts.files
|
||||
systemProperty 'tests.security.manager', 'true'
|
||||
// maybe we like some extra security policy for our code
|
||||
systemProperty 'tests.security.policy', '/extra-security.policy'
|
||||
}
|
||||
esTest.dependsOn jar, testJar
|
||||
|
||||
randomizedTest {
|
||||
enabled = false
|
||||
}
|
||||
|
||||
test {
|
||||
enabled = false
|
||||
jvmArgs "-javaagent:" + configurations.alpnagent.asPath
|
||||
systemProperty 'path.home', projectDir.absolutePath
|
||||
testLogging {
|
||||
showStandardStreams = true
|
||||
exceptionFormat = 'full'
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,129 @@
|
||||
package org.xbib.elx.transport;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
|
||||
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder;
|
||||
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.elasticsearch.cluster.node.DiscoveryNode;
|
||||
import org.elasticsearch.cluster.node.DiscoveryNodes;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.transport.InetSocketTransportAddress;
|
||||
import org.xbib.elx.common.AbstractExtendedClient;
|
||||
import org.xbib.elx.common.util.NetworkUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.InetAddress;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Transport client with additional methods using the BulkProcessor.
|
||||
*/
|
||||
public class ExtendedTransportClient extends AbstractExtendedClient {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(ExtendedTransportClient.class.getName());
|
||||
|
||||
@Override
|
||||
protected ElasticsearchClient createClient(Settings settings) {
|
||||
if (settings != null) {
|
||||
String systemIdentifier = System.getProperty("os.name")
|
||||
+ " " + System.getProperty("java.vm.name")
|
||||
+ " " + System.getProperty("java.vm.vendor")
|
||||
+ " " + System.getProperty("java.vm.version")
|
||||
+ " Elasticsearch " + Version.CURRENT.toString();
|
||||
logger.info("creating transport client on {} with effective settings {}",
|
||||
systemIdentifier, settings.getAsMap());
|
||||
TransportClient.Builder builder = TransportClient.builder()
|
||||
.settings(Settings.builder()
|
||||
.put("cluster.name", settings.get("cluster.name"))
|
||||
.put("processors", settings.getAsInt("processors", Runtime.getRuntime().availableProcessors()))
|
||||
.put("client.transport.ignore_cluster_name", true)
|
||||
.build());
|
||||
return builder.build();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExtendedTransportClient init(Settings settings) throws IOException {
|
||||
super.init(settings);
|
||||
// additional auto-connect
|
||||
try {
|
||||
Collection<InetSocketTransportAddress> addrs = findAddresses(settings);
|
||||
if (!connect(addrs, settings.getAsBoolean("autodiscover", false))) {
|
||||
throw new NoNodeAvailableException("no cluster nodes available, check settings "
|
||||
+ settings.toString());
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void shutdown() throws IOException {
|
||||
super.shutdown();
|
||||
logger.info("shutting down...");
|
||||
if (getClient() != null) {
|
||||
TransportClient client = (TransportClient) getClient();
|
||||
client.close();
|
||||
client.threadPool().shutdown();
|
||||
}
|
||||
logger.info("shutting down completed");
|
||||
}
|
||||
|
||||
private Collection<InetSocketTransportAddress> findAddresses(Settings settings) throws IOException {
|
||||
final int defaultPort = settings.getAsInt("port", 9300);
|
||||
Collection<InetSocketTransportAddress> addresses = new ArrayList<>();
|
||||
for (String hostname : settings.getAsArray("host")) {
|
||||
String[] splitHost = hostname.split(":", 2);
|
||||
if (splitHost.length == 2) {
|
||||
try {
|
||||
String host = splitHost[0];
|
||||
InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null);
|
||||
int port = Integer.parseInt(splitHost[1]);
|
||||
InetSocketTransportAddress address = new InetSocketTransportAddress(inetAddress, port);
|
||||
addresses.add(address);
|
||||
} catch (NumberFormatException e) {
|
||||
logger.warn(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
if (splitHost.length == 1) {
|
||||
String host = splitHost[0];
|
||||
InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null);
|
||||
InetSocketTransportAddress address = new InetSocketTransportAddress(inetAddress, defaultPort);
|
||||
addresses.add(address);
|
||||
}
|
||||
}
|
||||
return addresses;
|
||||
}
|
||||
|
||||
private boolean connect(Collection<InetSocketTransportAddress> addresses, boolean autodiscover) {
|
||||
if (getClient() == null) {
|
||||
throw new IllegalStateException("no client present");
|
||||
}
|
||||
logger.debug("trying to connect to {}", addresses);
|
||||
TransportClient transportClient = (TransportClient) getClient();
|
||||
transportClient.addTransportAddresses(addresses);
|
||||
List<DiscoveryNode> nodes = transportClient.connectedNodes();
|
||||
logger.info("connected to nodes = {}", nodes);
|
||||
if (nodes != null && !nodes.isEmpty()) {
|
||||
if (autodiscover) {
|
||||
logger.debug("trying to auto-discover all nodes...");
|
||||
ClusterStateRequestBuilder clusterStateRequestBuilder =
|
||||
new ClusterStateRequestBuilder(getClient(), ClusterStateAction.INSTANCE);
|
||||
ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet();
|
||||
DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes();
|
||||
transportClient.addDiscoveryNodes(discoveryNodes);
|
||||
logger.info("after auto-discovery: connected to {}", transportClient.connectedNodes());
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
package org.xbib.elx.transport;
|
||||
|
||||
import org.xbib.elx.api.ExtendedClientProvider;
|
||||
|
||||
public class ExtendedTransportClientProvider implements ExtendedClientProvider<ExtendedTransportClient> {
|
||||
|
||||
@Override
|
||||
public ExtendedTransportClient getExtendedClient() {
|
||||
return new ExtendedTransportClient();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,4 @@
|
||||
/**
|
||||
* Classes for Elasticsearch transport client extensions.
|
||||
*/
|
||||
package org.xbib.elx.transport;
|
||||
@ -0,0 +1 @@
|
||||
org.xbib.elx.transport.ExtendedTransportClientProvider
|
||||
@ -0,0 +1,34 @@
|
||||
package org.elasticsearch.node;
|
||||
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.node.internal.InternalSettingsPreparer;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
|
||||
public class MockNode extends Node {
|
||||
|
||||
public MockNode() {
|
||||
super(Settings.EMPTY);
|
||||
}
|
||||
|
||||
public MockNode(Settings settings) {
|
||||
super(settings);
|
||||
}
|
||||
|
||||
public MockNode(Settings settings, Collection<Class<? extends Plugin>> classpathPlugins) {
|
||||
super(InternalSettingsPreparer.prepareEnvironment(settings, null), Version.CURRENT, classpathPlugins);
|
||||
}
|
||||
|
||||
public MockNode(Settings settings, Class<? extends Plugin> classpathPlugin) {
|
||||
this(settings, list(classpathPlugin));
|
||||
}
|
||||
|
||||
private static Collection<Class<? extends Plugin>> list(Class<? extends Plugin> classpathPlugin) {
|
||||
Collection<Class<? extends Plugin>> list = new ArrayList<>();
|
||||
list.add(classpathPlugin);
|
||||
return list;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1 @@
|
||||
package org.elasticsearch.node;
|
||||
@ -0,0 +1,40 @@
|
||||
package org.xbib.elx.transport;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.junit.Test;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
|
||||
public class ExtendedTransportClientSingleNodeTest extends NodeTestUtils {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(ExtendedTransportClientSingleNodeTest.class.getSimpleName());
|
||||
|
||||
@Test
|
||||
public void testSingleDocNodeClient() throws Exception {
|
||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
||||
.provider(ExtendedTransportClientProvider.class)
|
||||
.put(getSettings())
|
||||
.build();
|
||||
try {
|
||||
client.newIndex("test");
|
||||
client.index("test", "test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
|
||||
client.flushIngest();
|
||||
client.waitForResponses("30s");
|
||||
} catch (InterruptedException e) {
|
||||
// ignore
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} finally {
|
||||
assertEquals(1, client.getBulkMetric().getSucceeded().getCount());
|
||||
if (client.hasThrowable()) {
|
||||
logger.error("error", client.getThrowable());
|
||||
}
|
||||
assertFalse(client.hasThrowable());
|
||||
client.shutdown();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,57 @@
|
||||
package org.xbib.elx.transport;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.action.search.SearchAction;
|
||||
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.junit.Test;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
import org.xbib.elx.common.Parameters;
|
||||
|
||||
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
public class ExtendedTransportDuplicateIDTest extends NodeTestUtils {
|
||||
|
||||
private final static Logger logger = LogManager.getLogger(ExtendedTransportDuplicateIDTest.class.getSimpleName());
|
||||
|
||||
private final static Long MAX_ACTIONS_PER_REQUEST = 1000L;
|
||||
|
||||
private final static Long ACTIONS = 12345L;
|
||||
|
||||
@Test
|
||||
public void testDuplicateDocIDs() throws Exception {
|
||||
long numactions = ACTIONS;
|
||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
||||
.provider(ExtendedTransportClientProvider.class)
|
||||
.put(getSettings())
|
||||
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
|
||||
.build();
|
||||
try {
|
||||
client.newIndex("test");
|
||||
for (int i = 0; i < ACTIONS; i++) {
|
||||
client.index("test", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
|
||||
}
|
||||
client.flushIngest();
|
||||
client.waitForResponses("30s");
|
||||
client.refreshIndex("test");
|
||||
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
|
||||
.setIndices("test")
|
||||
.setTypes("test")
|
||||
.setQuery(matchAllQuery());
|
||||
long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
|
||||
logger.info("hits = {}", hits);
|
||||
assertTrue(hits < ACTIONS);
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} finally {
|
||||
client.shutdown();
|
||||
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
|
||||
if (client.hasThrowable()) {
|
||||
logger.error("error", client.getThrowable());
|
||||
}
|
||||
assertFalse(client.hasThrowable());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,65 @@
|
||||
package org.xbib.elx.transport;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import org.xbib.elx.common.ClientBuilder;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.Assert.assertFalse;
|
||||
|
||||
@Ignore
|
||||
public class ExtendedTransportIndexAliasTest extends NodeTestUtils {
|
||||
|
||||
private static final Logger logger = LogManager.getLogger(ExtendedTransportIndexAliasTest.class.getSimpleName());
|
||||
|
||||
@Test
|
||||
public void testIndexAlias() throws Exception {
|
||||
final ExtendedTransportClient client = ClientBuilder.builder()
|
||||
.provider(ExtendedTransportClientProvider.class)
|
||||
.build();
|
||||
try {
|
||||
client.newIndex("test1234");
|
||||
for (int i = 0; i < 1; i++) {
|
||||
client.index("test1234", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
|
||||
}
|
||||
client.flushIngest();
|
||||
client.refreshIndex("test1234");
|
||||
|
||||
List<String> simpleAliases = Arrays.asList("a", "b", "c");
|
||||
client.switchAliases("test", "test1234", simpleAliases);
|
||||
|
||||
client.newIndex("test5678");
|
||||
for (int i = 0; i < 1; i++) {
|
||||
client.index("test5678", "test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
|
||||
}
|
||||
client.flushIngest();
|
||||
client.refreshIndex("test5678");
|
||||
|
||||
simpleAliases = Arrays.asList("d", "e", "f");
|
||||
client.switchAliases("test", "test5678", simpleAliases, (builder, index, alias) ->
|
||||
builder.addAlias(index, alias, QueryBuilders.termQuery("my_key", alias)));
|
||||
Map<String, String> aliases = client.getIndexFilters("test5678");
|
||||
logger.info("aliases of index test5678 = {}", aliases);
|
||||
|
||||
aliases = client.getAliasFilters("test");
|
||||
logger.info("aliases of alias test = {}", aliases);
|
||||
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} finally {
|
||||
client.waitForResponses("30s");
|
||||
client.shutdown();
|
||||
if (client.hasThrowable()) {
|
||||
logger.error("error", client.getThrowable());
|
||||
}
|
||||
assertFalse(client.hasThrowable());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1 @@
|
||||
package org.xbib.elx.transport;
|
||||
@ -0,0 +1,13 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<configuration status="OFF">
|
||||
<appenders>
|
||||
<Console name="Console" target="SYSTEM_OUT">
|
||||
<PatternLayout pattern="[%d{ISO8601}][%-5p][%-25c][%t] %m%n"/>
|
||||
</Console>
|
||||
</appenders>
|
||||
<Loggers>
|
||||
<Root level="info">
|
||||
<AppenderRef ref="Console" />
|
||||
</Root>
|
||||
</Loggers>
|
||||
</configuration>
|
||||
@ -1,3 +1,19 @@
|
||||
group = org.xbib
|
||||
name = elasticsearch-extras-client
|
||||
version = 2.2.1.2
|
||||
name = elx
|
||||
version = 2.2.1.3
|
||||
|
||||
xbib-metrics.version = 1.1.0
|
||||
xbib-guice.version = 4.0.4
|
||||
|
||||
elasticsearch.version = 2.2.1
|
||||
jna.version = 4.5.2
|
||||
log4j.version = 2.11.1
|
||||
mustache.version = 0.9.5
|
||||
jts.version = 1.13
|
||||
jackson-dataformat.version = 2.8.11
|
||||
|
||||
junit.version = 4.12
|
||||
wagon.version = 3.0.0
|
||||
asciidoclet.version = 1.5.4
|
||||
|
||||
org.gradle.warning.mode = all
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
ext {
|
||||
user = 'xbib'
|
||||
name = 'elasticsearch-extras-client'
|
||||
description = 'Some extras implemented for using Elasticsearch clients (node and transport)'
|
||||
scmUrl = 'https://github.com/' + user + '/' + name
|
||||
scmConnection = 'scm:git:git://github.com/' + user + '/' + name + '.git'
|
||||
scmDeveloperConnection = 'scm:git:git://github.com/' + user + '/' + name + '.git'
|
||||
}
|
||||
Binary file not shown.
@ -1,6 +1,6 @@
|
||||
#Tue Jan 03 14:13:22 CET 2017
|
||||
#Fri Feb 15 11:59:10 CET 2019
|
||||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-all.zip
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-5.2.1-all.zip
|
||||
|
||||
@ -1 +1,5 @@
|
||||
rootProject.name = 'elasticsearch-extras-client'
|
||||
include 'elx-api'
|
||||
include 'elx-common'
|
||||
include 'elx-node'
|
||||
include 'elx-transport'
|
||||
include 'elx-http'
|
||||
|
||||
@ -1,4 +0,0 @@
|
||||
/**
|
||||
* Classes to support Elasticsearch node creation.
|
||||
*/
|
||||
package org.elasticsearch.node;
|
||||
@ -1,70 +0,0 @@
|
||||
package org.xbib.elasticsearch;
|
||||
|
||||
import static org.elasticsearch.client.Requests.indexRequest;
|
||||
import static org.elasticsearch.client.Requests.refreshRequest;
|
||||
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
|
||||
|
||||
import org.elasticsearch.action.bulk.BulkAction;
|
||||
import org.elasticsearch.action.bulk.BulkRequestBuilder;
|
||||
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.elasticsearch.common.logging.ESLogger;
|
||||
import org.elasticsearch.common.logging.ESLoggerFactory;
|
||||
import org.elasticsearch.index.query.QueryBuilder;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.search.sort.SortOrder;
|
||||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
public class SearchTest extends NodeTestUtils {
|
||||
|
||||
private static final ESLogger logger = ESLoggerFactory.getLogger("test");
|
||||
|
||||
@Test
|
||||
public void testSearch() throws Exception {
|
||||
Client client = client("1");
|
||||
long t0 = System.currentTimeMillis();
|
||||
BulkRequestBuilder builder = new BulkRequestBuilder(client, BulkAction.INSTANCE);
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
builder.add(indexRequest()
|
||||
.index("pages").type("row")
|
||||
.source(jsonBuilder()
|
||||
.startObject()
|
||||
.field("user1", "kimchy")
|
||||
.field("user2", "kimchy")
|
||||
.field("user3", "kimchy")
|
||||
.field("user4", "kimchy")
|
||||
.field("user5", "kimchy")
|
||||
.field("user6", "kimchy")
|
||||
.field("user7", "kimchy")
|
||||
.field("user8", "kimchy")
|
||||
.field("user9", "kimchy")
|
||||
.field("rowcount", i)
|
||||
.field("rs", 1234)));
|
||||
}
|
||||
client.bulk(builder.request()).actionGet();
|
||||
|
||||
client.admin().indices().refresh(refreshRequest()).actionGet();
|
||||
|
||||
long t1 = System.currentTimeMillis();
|
||||
logger.info("t1-t0 = {}", t1 - t0);
|
||||
|
||||
for (int i = 0; i < 100; i++) {
|
||||
t1 = System.currentTimeMillis();
|
||||
QueryBuilder queryStringBuilder =
|
||||
QueryBuilders.queryStringQuery("rs:" + 1234);
|
||||
SearchRequestBuilder requestBuilder = client.prepareSearch()
|
||||
.setIndices("pages")
|
||||
.setTypes("row")
|
||||
.setQuery(queryStringBuilder)
|
||||
.addSort("rowcount", SortOrder.DESC)
|
||||
.setFrom(i * 10).setSize(10);
|
||||
SearchResponse response = requestBuilder.execute().actionGet();
|
||||
long t2 = System.currentTimeMillis();
|
||||
logger.info("t2-t1 = {}", t2 - t1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,69 +0,0 @@
|
||||
package org.xbib.elasticsearch;
|
||||
|
||||
import static org.elasticsearch.client.Requests.indexRequest;
|
||||
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
|
||||
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
|
||||
import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
|
||||
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.query.QueryBuilder;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
public class WildcardTest extends NodeTestUtils {
|
||||
|
||||
protected Settings getNodeSettings() {
|
||||
return settingsBuilder()
|
||||
.put("cluster.name", getClusterName())
|
||||
.put("cluster.routing.allocation.disk.threshold_enabled", false)
|
||||
.put("discovery.zen.multicast.enabled", false)
|
||||
.put("http.enabled", false)
|
||||
.put("path.home", System.getProperty("path.home"))
|
||||
.put("index.number_of_shards", 1)
|
||||
.put("index.number_of_replicas", 0)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWildcard() throws Exception {
|
||||
index(client("1"), "1", "010");
|
||||
index(client("1"), "2", "0*0");
|
||||
// exact
|
||||
validateCount(client("1"), queryStringQuery("010").defaultField("field"), 1);
|
||||
validateCount(client("1"), queryStringQuery("0\\*0").defaultField("field"), 1);
|
||||
// pattern
|
||||
validateCount(client("1"), queryStringQuery("0*0").defaultField("field"), 1); // 2?
|
||||
validateCount(client("1"), queryStringQuery("0?0").defaultField("field"), 1); // 2?
|
||||
validateCount(client("1"), queryStringQuery("0**0").defaultField("field"), 1); // 2?
|
||||
validateCount(client("1"), queryStringQuery("0??0").defaultField("field"), 0);
|
||||
validateCount(client("1"), queryStringQuery("*10").defaultField("field"), 1);
|
||||
validateCount(client("1"), queryStringQuery("*1*").defaultField("field"), 1);
|
||||
validateCount(client("1"), queryStringQuery("*\\*0").defaultField("field"), 0); // 1?
|
||||
validateCount(client("1"), queryStringQuery("*\\**").defaultField("field"), 0); // 1?
|
||||
}
|
||||
|
||||
private void index(Client client, String id, String fieldValue) throws IOException {
|
||||
client.index(indexRequest()
|
||||
.index("index").type("type").id(id)
|
||||
.source(jsonBuilder().startObject().field("field", fieldValue).endObject())
|
||||
.refresh(true)).actionGet();
|
||||
}
|
||||
|
||||
private long count(Client client, QueryBuilder queryBuilder) {
|
||||
return client.prepareSearch("index").setTypes("type")
|
||||
.setQuery(queryBuilder)
|
||||
.execute().actionGet().getHits().getTotalHits();
|
||||
}
|
||||
|
||||
private void validateCount(Client client, QueryBuilder queryBuilder, long expectedHits) {
|
||||
final long actualHits = count(client, queryBuilder);
|
||||
if (actualHits != expectedHits) {
|
||||
throw new RuntimeException("actualHits=" + actualHits + ", expectedHits=" + expectedHits);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,62 +0,0 @@
|
||||
package org.xbib.elasticsearch.extras.client.node;
|
||||
|
||||
import org.elasticsearch.action.search.SearchAction;
|
||||
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.elasticsearch.common.logging.ESLogger;
|
||||
import org.elasticsearch.common.logging.ESLoggerFactory;
|
||||
import org.junit.Test;
|
||||
import org.xbib.elasticsearch.NodeTestUtils;
|
||||
import org.xbib.elasticsearch.extras.client.Clients;
|
||||
import org.xbib.elasticsearch.extras.client.SimpleBulkControl;
|
||||
import org.xbib.elasticsearch.extras.client.SimpleBulkMetric;
|
||||
|
||||
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
public class BulkNodeDuplicateIDTest extends NodeTestUtils {
|
||||
|
||||
private static final ESLogger logger = ESLoggerFactory.getLogger(BulkNodeDuplicateIDTest.class.getSimpleName());
|
||||
|
||||
private static final Long MAX_ACTIONS = 1000L;
|
||||
|
||||
private static final Long NUM_ACTIONS = 12345L;
|
||||
|
||||
@Test
|
||||
public void testDuplicateDocIDs() throws Exception {
|
||||
long numactions = NUM_ACTIONS;
|
||||
final BulkNodeClient client = Clients.builder()
|
||||
.put(Clients.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS)
|
||||
.setMetric(new SimpleBulkMetric())
|
||||
.setControl(new SimpleBulkControl())
|
||||
.toBulkNodeClient(client("1"));
|
||||
try {
|
||||
client.newIndex("test");
|
||||
for (int i = 0; i < NUM_ACTIONS; i++) {
|
||||
client.index("test", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}");
|
||||
}
|
||||
client.flushIngest();
|
||||
client.waitForResponses("30s");
|
||||
client.refreshIndex("test");
|
||||
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE)
|
||||
.setIndices("test")
|
||||
.setTypes("test")
|
||||
.setQuery(matchAllQuery());
|
||||
long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
|
||||
logger.info("hits = {}", hits);
|
||||
assertTrue(hits < NUM_ACTIONS);
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} finally {
|
||||
client.shutdown();
|
||||
assertEquals(numactions, client.getMetric().getSucceeded().getCount());
|
||||
if (client.hasThrowable()) {
|
||||
logger.error("error", client.getThrowable());
|
||||
}
|
||||
assertFalse(client.hasThrowable());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,4 +0,0 @@
|
||||
/**
|
||||
* Classes for testing Elasticsearch node client extras.
|
||||
*/
|
||||
package org.xbib.elasticsearch.extras.client.node;
|
||||
@ -1,4 +0,0 @@
|
||||
/**
|
||||
* Classes to test Elasticsearch clients.
|
||||
*/
|
||||
package org.xbib.elasticsearch.extras.client;
|
||||
@ -1,60 +0,0 @@
|
||||
package org.xbib.elasticsearch.extras.client.transport;
|
||||
|
||||
import org.elasticsearch.action.search.SearchAction;
|
||||
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||
import org.elasticsearch.client.transport.NoNodeAvailableException;
|
||||
import org.elasticsearch.common.logging.ESLogger;
|
||||
import org.elasticsearch.common.logging.ESLoggerFactory;
|
||||
import org.junit.Test;
|
||||
import org.xbib.elasticsearch.NodeTestUtils;
|
||||
import org.xbib.elasticsearch.extras.client.Clients;
|
||||
import org.xbib.elasticsearch.extras.client.SimpleBulkControl;
|
||||
import org.xbib.elasticsearch.extras.client.SimpleBulkMetric;
|
||||
|
||||
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
public class BulkTransportDuplicateIDTest extends NodeTestUtils {
|
||||
|
||||
private final static ESLogger logger = ESLoggerFactory.getLogger(BulkTransportDuplicateIDTest.class.getSimpleName());
|
||||
|
||||
private final static Long MAX_ACTIONS = 1000L;
|
||||
|
||||
private final static Long NUM_ACTIONS = 12345L;
|
||||
|
||||
@Test
|
||||
public void testDuplicateDocIDs() throws Exception {
|
||||
long numactions = NUM_ACTIONS;
|
||||
final BulkTransportClient client = Clients.builder()
|
||||
.put(getSettings())
|
||||
.put(Clients.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS)
|
||||
.setMetric(new SimpleBulkMetric())
|
||||
.setControl(new SimpleBulkControl())
|
||||
.toBulkTransportClient();
|
||||
try {
|
||||
client.newIndex("test");
|
||||
for (int i = 0; i < NUM_ACTIONS; i++) {
|
||||
client.index("test", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}");
|
||||
}
|
||||
client.flushIngest();
|
||||
client.waitForResponses("30s");
|
||||
client.refreshIndex("test");
|
||||
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE)
|
||||
.setIndices("test")
|
||||
.setTypes("test")
|
||||
.setQuery(matchAllQuery());
|
||||
long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
|
||||
logger.info("hits = {}", hits);
|
||||
assertTrue(hits < NUM_ACTIONS);
|
||||
} catch (NoNodeAvailableException e) {
|
||||
logger.warn("skipping, no node available");
|
||||
} finally {
|
||||
client.shutdown();
|
||||
assertEquals(numactions, client.getMetric().getSucceeded().getCount());
|
||||
if (client.hasThrowable()) {
|
||||
logger.error("error", client.getThrowable());
|
||||
}
|
||||
assertFalse(client.hasThrowable());
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue