diff --git a/.gitignore b/.gitignore
index bf3e9b4..b92da43 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,9 +5,10 @@
/target
.DS_Store
*.iml
+*~
/.settings
/.classpath
/.project
/.gradle
-/build
-/plugins
\ No newline at end of file
+build
+plugins
diff --git a/.travis.yml b/.travis.yml
index a830350..ee1dfd1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,8 +5,3 @@ jdk:
cache:
directories:
- $HOME/.m2
-after_success:
- - ./gradlew sonarqube -Dsonar.host.url=https://sonarqube.com -Dsonar.login=$SONAR_TOKEN
-env:
- global:
- secure: n1Ai4q/yMLn/Pg5pA4lTavoJoe7mQYB1PSKnZAqwbgyla94ySzK6iyBCBiNs/foMPisB/x+DHvmUXTsjvquw9Ay48ZITCV3xhcWzD0eZM2TMoG19CpRAEe8L8LNuYiti9k89ijDdUGZ5ifsvQNTGNHksouayAuApC3PrTUejJfR6SYrp1ZsQTbsMlr+4XU3p7QknK5rGgOwATIMP28F+bVnB05WJtlJA3b0SeucCurn3wJ4FGBQXRYmdlT7bQhNE4QgZM1VzcUFD/K0TBxzzq/otb/lNRSifyoekktDmJwQnaT9uQ4R8R6KdQ2Kb38Rvgjur+TKm5i1G8qS2+6LnIxQJG1aw3JvKK6W0wWCgnAVVRrXaCLday9NuY59tuh1mfjQ10UcsMNKcTdcKEMrLow506wSETcXc7L/LEnneWQyJJeV4vhPqR7KJfsBbeqgz3yIfsCn1GZVWFlfegzYCN52YTl0Y0uRD2Z+TnzQu+Bf4DzaWXLge1rz31xkhyeNNspub4h024+XqBjcMm6M9mlMzmmK8t2DIwPy/BlQbFBUyhrxziuR/5/2NEDPyHltvWkRb4AUIa25WJqkV0gTBegbMadZ9DyOo6Ea7aoVFBae2WGR08F1kzABsWrd1S7UJmWxW35iyMEtoAIayXphIK98qO5aCutwZ+3iOQazxbAs=
diff --git a/README.adoc b/README.adoc
index 852224a..c5cdd92 100644
--- a/README.adoc
+++ b/README.adoc
@@ -1,4 +1,4 @@
-# Elasticsearch Extras - Client
+# Elasticsearch Clients
image:https://api.travis-ci.org/xbib/content.svg[title="Build status", link="https://travis-ci.org/jprante/elasticsearch-extras-client/"]
image:https://img.shields.io/sonar/http/nemo.sonarqube.com/org.xbib%3Aelasticsearch-extras-client/coverage.svg?style=flat-square[title="Coverage", link="https://sonarqube.com/dashboard/index?id=org.xbib%3Aelasticsearch-extras-client"]
@@ -99,7 +99,7 @@ You will need Java 8, although Elasticsearch 2.x requires Java 7. Java 7 is not
## Dependencies
This project depends only on https://github.com/xbib/metrics which is a slim version of Coda Hale's metrics library,
-and Elasticsearch.
+Elasticsearch, and Log4j2 API.
## How to decode the Elasticsearch version
diff --git a/api/build.gradle b/api/build.gradle
new file mode 100644
index 0000000..61be444
--- /dev/null
+++ b/api/build.gradle
@@ -0,0 +1,18 @@
+
+dependencies {
+ compile("org.elasticsearch.client:transport:${rootProject.property('elasticsearch.version')}") {
+ exclude group: 'org.elasticsearch', module: 'securesm'
+ exclude group: 'org.elasticsearch.plugin', module: 'transport-netty3-client'
+ exclude group: 'org.elasticsearch.plugin', module: 'reindex-client'
+ exclude group: 'org.elasticsearch.plugin', module: 'percolator-client'
+ exclude group: 'org.elasticsearch.plugin', module: 'lang-mustache-client'
+ }
+ // we try to override the Elasticsearch netty by our netty version which is more recent
+ compile "io.netty:netty-buffer:${rootProject.property('netty.version')}"
+ compile "io.netty:netty-codec-http:${rootProject.property('netty.version')}"
+ compile "io.netty:netty-handler:${rootProject.property('netty.version')}"
+}
+
+jar {
+ baseName "${rootProject.name}-api"
+}
diff --git a/api/config/checkstyle/checkstyle.xml b/api/config/checkstyle/checkstyle.xml
new file mode 100644
index 0000000..8cb4438
--- /dev/null
+++ b/api/config/checkstyle/checkstyle.xml
@@ -0,0 +1,321 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/api/src/docs/asciidoc/css/foundation.css b/api/src/docs/asciidoc/css/foundation.css
new file mode 100644
index 0000000..27be611
--- /dev/null
+++ b/api/src/docs/asciidoc/css/foundation.css
@@ -0,0 +1,684 @@
+/*! normalize.css v2.1.2 | MIT License | git.io/normalize */
+/* ========================================================================== HTML5 display definitions ========================================================================== */
+/** Correct `block` display not defined in IE 8/9. */
+article, aside, details, figcaption, figure, footer, header, hgroup, main, nav, section, summary { display: block; }
+
+/** Correct `inline-block` display not defined in IE 8/9. */
+audio, canvas, video { display: inline-block; }
+
+/** Prevent modern browsers from displaying `audio` without controls. Remove excess height in iOS 5 devices. */
+audio:not([controls]) { display: none; height: 0; }
+
+/** Address `[hidden]` styling not present in IE 8/9. Hide the `template` element in IE, Safari, and Firefox < 22. */
+[hidden], template { display: none; }
+
+script { display: none !important; }
+
+/* ========================================================================== Base ========================================================================== */
+/** 1. Set default font family to sans-serif. 2. Prevent iOS text size adjust after orientation change, without disabling user zoom. */
+html { font-family: sans-serif; /* 1 */ -ms-text-size-adjust: 100%; /* 2 */ -webkit-text-size-adjust: 100%; /* 2 */ }
+
+/** Remove default margin. */
+body { margin: 0; }
+
+/* ========================================================================== Links ========================================================================== */
+/** Remove the gray background color from active links in IE 10. */
+a { background: transparent; }
+
+/** Address `outline` inconsistency between Chrome and other browsers. */
+a:focus { outline: thin dotted; }
+
+/** Improve readability when focused and also mouse hovered in all browsers. */
+a:active, a:hover { outline: 0; }
+
+/* ========================================================================== Typography ========================================================================== */
+/** Address variable `h1` font-size and margin within `section` and `article` contexts in Firefox 4+, Safari 5, and Chrome. */
+h1 { font-size: 2em; margin: 0.67em 0; }
+
+/** Address styling not present in IE 8/9, Safari 5, and Chrome. */
+abbr[title] { border-bottom: 1px dotted; }
+
+/** Address style set to `bolder` in Firefox 4+, Safari 5, and Chrome. */
+b, strong { font-weight: bold; }
+
+/** Address styling not present in Safari 5 and Chrome. */
+dfn { font-style: italic; }
+
+/** Address differences between Firefox and other browsers. */
+hr { -moz-box-sizing: content-box; box-sizing: content-box; height: 0; }
+
+/** Address styling not present in IE 8/9. */
+mark { background: #ff0; color: #000; }
+
+/** Correct font family set oddly in Safari 5 and Chrome. */
+code, kbd, pre, samp { font-family: monospace, serif; font-size: 1em; }
+
+/** Improve readability of pre-formatted text in all browsers. */
+pre { white-space: pre-wrap; }
+
+/** Set consistent quote types. */
+q { quotes: "\201C" "\201D" "\2018" "\2019"; }
+
+/** Address inconsistent and variable font size in all browsers. */
+small { font-size: 80%; }
+
+/** Prevent `sub` and `sup` affecting `line-height` in all browsers. */
+sub, sup { font-size: 75%; line-height: 0; position: relative; vertical-align: baseline; }
+
+sup { top: -0.5em; }
+
+sub { bottom: -0.25em; }
+
+/* ========================================================================== Embedded content ========================================================================== */
+/** Remove border when inside `a` element in IE 8/9. */
+img { border: 0; }
+
+/** Correct overflow displayed oddly in IE 9. */
+svg:not(:root) { overflow: hidden; }
+
+/* ========================================================================== Figures ========================================================================== */
+/** Address margin not present in IE 8/9 and Safari 5. */
+figure { margin: 0; }
+
+/* ========================================================================== Forms ========================================================================== */
+/** Define consistent border, margin, and padding. */
+fieldset { border: 1px solid #c0c0c0; margin: 0 2px; padding: 0.35em 0.625em 0.75em; }
+
+/** 1. Correct `color` not being inherited in IE 8/9. 2. Remove padding so people aren't caught out if they zero out fieldsets. */
+legend { border: 0; /* 1 */ padding: 0; /* 2 */ }
+
+/** 1. Correct font family not being inherited in all browsers. 2. Correct font size not being inherited in all browsers. 3. Address margins set differently in Firefox 4+, Safari 5, and Chrome. */
+button, input, select, textarea { font-family: inherit; /* 1 */ font-size: 100%; /* 2 */ margin: 0; /* 3 */ }
+
+/** Address Firefox 4+ setting `line-height` on `input` using `!important` in the UA stylesheet. */
+button, input { line-height: normal; }
+
+/** Address inconsistent `text-transform` inheritance for `button` and `select`. All other form control elements do not inherit `text-transform` values. Correct `button` style inheritance in Chrome, Safari 5+, and IE 8+. Correct `select` style inheritance in Firefox 4+ and Opera. */
+button, select { text-transform: none; }
+
+/** 1. Avoid the WebKit bug in Android 4.0.* where (2) destroys native `audio` and `video` controls. 2. Correct inability to style clickable `input` types in iOS. 3. Improve usability and consistency of cursor style between image-type `input` and others. */
+button, html input[type="button"], input[type="reset"], input[type="submit"] { -webkit-appearance: button; /* 2 */ cursor: pointer; /* 3 */ }
+
+/** Re-set default cursor for disabled elements. */
+button[disabled], html input[disabled] { cursor: default; }
+
+/** 1. Address box sizing set to `content-box` in IE 8/9. 2. Remove excess padding in IE 8/9. */
+input[type="checkbox"], input[type="radio"] { box-sizing: border-box; /* 1 */ padding: 0; /* 2 */ }
+
+/** 1. Address `appearance` set to `searchfield` in Safari 5 and Chrome. 2. Address `box-sizing` set to `border-box` in Safari 5 and Chrome (include `-moz` to future-proof). */
+input[type="search"] { -webkit-appearance: textfield; /* 1 */ -moz-box-sizing: content-box; -webkit-box-sizing: content-box; /* 2 */ box-sizing: content-box; }
+
+/** Remove inner padding and search cancel button in Safari 5 and Chrome on OS X. */
+input[type="search"]::-webkit-search-cancel-button, input[type="search"]::-webkit-search-decoration { -webkit-appearance: none; }
+
+/** Remove inner padding and border in Firefox 4+. */
+button::-moz-focus-inner, input::-moz-focus-inner { border: 0; padding: 0; }
+
+/** 1. Remove default vertical scrollbar in IE 8/9. 2. Improve readability and alignment in all browsers. */
+textarea { overflow: auto; /* 1 */ vertical-align: top; /* 2 */ }
+
+/* ========================================================================== Tables ========================================================================== */
+/** Remove most spacing between table cells. */
+table { border-collapse: collapse; border-spacing: 0; }
+
+meta.foundation-mq-small { font-family: "only screen and (min-width: 768px)"; width: 768px; }
+
+meta.foundation-mq-medium { font-family: "only screen and (min-width:1280px)"; width: 1280px; }
+
+meta.foundation-mq-large { font-family: "only screen and (min-width:1440px)"; width: 1440px; }
+
+*, *:before, *:after { -moz-box-sizing: border-box; -webkit-box-sizing: border-box; box-sizing: border-box; }
+
+html, body { font-size: 100%; }
+
+body { background: white; color: #222222; padding: 0; margin: 0; font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; font-weight: normal; font-style: normal; line-height: 1; position: relative; cursor: auto; }
+
+a:hover { cursor: pointer; }
+
+img, object, embed { max-width: 100%; height: auto; }
+
+object, embed { height: 100%; }
+
+img { -ms-interpolation-mode: bicubic; }
+
+#map_canvas img, #map_canvas embed, #map_canvas object, .map_canvas img, .map_canvas embed, .map_canvas object { max-width: none !important; }
+
+.left { float: left !important; }
+
+.right { float: right !important; }
+
+.text-left { text-align: left !important; }
+
+.text-right { text-align: right !important; }
+
+.text-center { text-align: center !important; }
+
+.text-justify { text-align: justify !important; }
+
+.hide { display: none; }
+
+.antialiased { -webkit-font-smoothing: antialiased; }
+
+img { display: inline-block; vertical-align: middle; }
+
+textarea { height: auto; min-height: 50px; }
+
+select { width: 100%; }
+
+object, svg { display: inline-block; vertical-align: middle; }
+
+.center { margin-left: auto; margin-right: auto; }
+
+.spread { width: 100%; }
+
+p.lead, .paragraph.lead > p, #preamble > .sectionbody > .paragraph:first-of-type p { font-size: 1.21875em; line-height: 1.6; }
+
+.subheader, .admonitionblock td.content > .title, .audioblock > .title, .exampleblock > .title, .imageblock > .title, .listingblock > .title, .literalblock > .title, .stemblock > .title, .openblock > .title, .paragraph > .title, .quoteblock > .title, table.tableblock > .title, .verseblock > .title, .videoblock > .title, .dlist > .title, .olist > .title, .ulist > .title, .qlist > .title, .hdlist > .title { line-height: 1.4; color: #6f6f6f; font-weight: 300; margin-top: 0.2em; margin-bottom: 0.5em; }
+
+/* Typography resets */
+div, dl, dt, dd, ul, ol, li, h1, h2, h3, #toctitle, .sidebarblock > .content > .title, h4, h5, h6, pre, form, p, blockquote, th, td { margin: 0; padding: 0; direction: ltr; }
+
+/* Default Link Styles */
+a { color: #2ba6cb; text-decoration: none; line-height: inherit; }
+a:hover, a:focus { color: #2795b6; }
+a img { border: none; }
+
+/* Default paragraph styles */
+p { font-family: inherit; font-weight: normal; font-size: 1em; line-height: 1.6; margin-bottom: 1.25em; text-rendering: optimizeLegibility; }
+p aside { font-size: 0.875em; line-height: 1.35; font-style: italic; }
+
+/* Default header styles */
+h1, h2, h3, #toctitle, .sidebarblock > .content > .title, h4, h5, h6 { font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; font-weight: bold; font-style: normal; color: #222222; text-rendering: optimizeLegibility; margin-top: 1em; margin-bottom: 0.5em; line-height: 1.2125em; }
+h1 small, h2 small, h3 small, #toctitle small, .sidebarblock > .content > .title small, h4 small, h5 small, h6 small { font-size: 60%; color: #6f6f6f; line-height: 0; }
+
+h1 { font-size: 2.125em; }
+
+h2 { font-size: 1.6875em; }
+
+h3, #toctitle, .sidebarblock > .content > .title { font-size: 1.375em; }
+
+h4 { font-size: 1.125em; }
+
+h5 { font-size: 1.125em; }
+
+h6 { font-size: 1em; }
+
+hr { border: solid #dddddd; border-width: 1px 0 0; clear: both; margin: 1.25em 0 1.1875em; height: 0; }
+
+/* Helpful Typography Defaults */
+em, i { font-style: italic; line-height: inherit; }
+
+strong, b { font-weight: bold; line-height: inherit; }
+
+small { font-size: 60%; line-height: inherit; }
+
+code { font-family: Consolas, "Liberation Mono", Courier, monospace; font-weight: bold; color: #7f0a0c; }
+
+/* Lists */
+ul, ol, dl { font-size: 1em; line-height: 1.6; margin-bottom: 1.25em; list-style-position: outside; font-family: inherit; }
+
+ul, ol { margin-left: 1.5em; }
+ul.no-bullet, ol.no-bullet { margin-left: 1.5em; }
+
+/* Unordered Lists */
+ul li ul, ul li ol { margin-left: 1.25em; margin-bottom: 0; font-size: 1em; /* Override nested font-size change */ }
+ul.square li ul, ul.circle li ul, ul.disc li ul { list-style: inherit; }
+ul.square { list-style-type: square; }
+ul.circle { list-style-type: circle; }
+ul.disc { list-style-type: disc; }
+ul.no-bullet { list-style: none; }
+
+/* Ordered Lists */
+ol li ul, ol li ol { margin-left: 1.25em; margin-bottom: 0; }
+
+/* Definition Lists */
+dl dt { margin-bottom: 0.3125em; font-weight: bold; }
+dl dd { margin-bottom: 1.25em; }
+
+/* Abbreviations */
+abbr, acronym { text-transform: uppercase; font-size: 90%; color: #222222; border-bottom: 1px dotted #dddddd; cursor: help; }
+
+abbr { text-transform: none; }
+
+/* Blockquotes */
+blockquote { margin: 0 0 1.25em; padding: 0.5625em 1.25em 0 1.1875em; border-left: 1px solid #dddddd; }
+blockquote cite { display: block; font-size: 0.8125em; color: #555555; }
+blockquote cite:before { content: "\2014 \0020"; }
+blockquote cite a, blockquote cite a:visited { color: #555555; }
+
+blockquote, blockquote p { line-height: 1.6; color: #6f6f6f; }
+
+/* Microformats */
+.vcard { display: inline-block; margin: 0 0 1.25em 0; border: 1px solid #dddddd; padding: 0.625em 0.75em; }
+.vcard li { margin: 0; display: block; }
+.vcard .fn { font-weight: bold; font-size: 0.9375em; }
+
+.vevent .summary { font-weight: bold; }
+.vevent abbr { cursor: auto; text-decoration: none; font-weight: bold; border: none; padding: 0 0.0625em; }
+
+@media only screen and (min-width: 768px) { h1, h2, h3, #toctitle, .sidebarblock > .content > .title, h4, h5, h6 { line-height: 1.4; }
+ h1 { font-size: 2.75em; }
+ h2 { font-size: 2.3125em; }
+ h3, #toctitle, .sidebarblock > .content > .title { font-size: 1.6875em; }
+ h4 { font-size: 1.4375em; } }
+/* Tables */
+table { background: white; margin-bottom: 1.25em; border: solid 1px #dddddd; }
+table thead, table tfoot { background: whitesmoke; font-weight: bold; }
+table thead tr th, table thead tr td, table tfoot tr th, table tfoot tr td { padding: 0.5em 0.625em 0.625em; font-size: inherit; color: #222222; text-align: left; }
+table tr th, table tr td { padding: 0.5625em 0.625em; font-size: inherit; color: #222222; }
+table tr.even, table tr.alt, table tr:nth-of-type(even) { background: #f9f9f9; }
+table thead tr th, table tfoot tr th, table tbody tr td, table tr td, table tfoot tr td { display: table-cell; line-height: 1.4; }
+
+body { -moz-osx-font-smoothing: grayscale; -webkit-font-smoothing: antialiased; tab-size: 4; }
+
+h1, h2, h3, #toctitle, .sidebarblock > .content > .title, h4, h5, h6 { line-height: 1.4; }
+
+.clearfix:before, .clearfix:after, .float-group:before, .float-group:after { content: " "; display: table; }
+.clearfix:after, .float-group:after { clear: both; }
+
+*:not(pre) > code { font-size: inherit; font-style: normal !important; letter-spacing: 0; padding: 0; line-height: inherit; word-wrap: break-word; }
+*:not(pre) > code.nobreak { word-wrap: normal; }
+*:not(pre) > code.nowrap { white-space: nowrap; }
+
+pre, pre > code { line-height: 1.4; color: black; font-family: monospace, serif; font-weight: normal; }
+
+em em { font-style: normal; }
+
+strong strong { font-weight: normal; }
+
+.keyseq { color: #555555; }
+
+kbd { font-family: Consolas, "Liberation Mono", Courier, monospace; display: inline-block; color: #222222; font-size: 0.65em; line-height: 1.45; background-color: #f7f7f7; border: 1px solid #ccc; -webkit-border-radius: 3px; border-radius: 3px; -webkit-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.2), 0 0 0 0.1em white inset; box-shadow: 0 1px 0 rgba(0, 0, 0, 0.2), 0 0 0 0.1em white inset; margin: 0 0.15em; padding: 0.2em 0.5em; vertical-align: middle; position: relative; top: -0.1em; white-space: nowrap; }
+
+.keyseq kbd:first-child { margin-left: 0; }
+
+.keyseq kbd:last-child { margin-right: 0; }
+
+.menuseq, .menu { color: #090909; }
+
+b.button:before, b.button:after { position: relative; top: -1px; font-weight: normal; }
+
+b.button:before { content: "["; padding: 0 3px 0 2px; }
+
+b.button:after { content: "]"; padding: 0 2px 0 3px; }
+
+#header, #content, #footnotes, #footer { width: 100%; margin-left: auto; margin-right: auto; margin-top: 0; margin-bottom: 0; max-width: 62.5em; *zoom: 1; position: relative; padding-left: 0.9375em; padding-right: 0.9375em; }
+#header:before, #header:after, #content:before, #content:after, #footnotes:before, #footnotes:after, #footer:before, #footer:after { content: " "; display: table; }
+#header:after, #content:after, #footnotes:after, #footer:after { clear: both; }
+
+#content { margin-top: 1.25em; }
+
+#content:before { content: none; }
+
+#header > h1:first-child { color: black; margin-top: 2.25rem; margin-bottom: 0; }
+#header > h1:first-child + #toc { margin-top: 8px; border-top: 1px solid #dddddd; }
+#header > h1:only-child, body.toc2 #header > h1:nth-last-child(2) { border-bottom: 1px solid #dddddd; padding-bottom: 8px; }
+#header .details { border-bottom: 1px solid #dddddd; line-height: 1.45; padding-top: 0.25em; padding-bottom: 0.25em; padding-left: 0.25em; color: #555555; display: -ms-flexbox; display: -webkit-flex; display: flex; -ms-flex-flow: row wrap; -webkit-flex-flow: row wrap; flex-flow: row wrap; }
+#header .details span:first-child { margin-left: -0.125em; }
+#header .details span.email a { color: #6f6f6f; }
+#header .details br { display: none; }
+#header .details br + span:before { content: "\00a0\2013\00a0"; }
+#header .details br + span.author:before { content: "\00a0\22c5\00a0"; color: #6f6f6f; }
+#header .details br + span#revremark:before { content: "\00a0|\00a0"; }
+#header #revnumber { text-transform: capitalize; }
+#header #revnumber:after { content: "\00a0"; }
+
+#content > h1:first-child:not([class]) { color: black; border-bottom: 1px solid #dddddd; padding-bottom: 8px; margin-top: 0; padding-top: 1rem; margin-bottom: 1.25rem; }
+
+#toc { border-bottom: 1px solid #dddddd; padding-bottom: 0.5em; }
+#toc > ul { margin-left: 0.125em; }
+#toc ul.sectlevel0 > li > a { font-style: italic; }
+#toc ul.sectlevel0 ul.sectlevel1 { margin: 0.5em 0; }
+#toc ul { font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; list-style-type: none; }
+#toc li { line-height: 1.3334; margin-top: 0.3334em; }
+#toc a { text-decoration: none; }
+#toc a:active { text-decoration: underline; }
+
+#toctitle { color: #6f6f6f; font-size: 1.2em; }
+
+@media only screen and (min-width: 768px) { #toctitle { font-size: 1.375em; }
+ body.toc2 { padding-left: 15em; padding-right: 0; }
+ #toc.toc2 { margin-top: 0 !important; background-color: #f2f2f2; position: fixed; width: 15em; left: 0; top: 0; border-right: 1px solid #dddddd; border-top-width: 0 !important; border-bottom-width: 0 !important; z-index: 1000; padding: 1.25em 1em; height: 100%; overflow: auto; }
+ #toc.toc2 #toctitle { margin-top: 0; margin-bottom: 0.8rem; font-size: 1.2em; }
+ #toc.toc2 > ul { font-size: 0.9em; margin-bottom: 0; }
+ #toc.toc2 ul ul { margin-left: 0; padding-left: 1em; }
+ #toc.toc2 ul.sectlevel0 ul.sectlevel1 { padding-left: 0; margin-top: 0.5em; margin-bottom: 0.5em; }
+ body.toc2.toc-right { padding-left: 0; padding-right: 15em; }
+ body.toc2.toc-right #toc.toc2 { border-right-width: 0; border-left: 1px solid #dddddd; left: auto; right: 0; } }
+@media only screen and (min-width: 1280px) { body.toc2 { padding-left: 20em; padding-right: 0; }
+ #toc.toc2 { width: 20em; }
+ #toc.toc2 #toctitle { font-size: 1.375em; }
+ #toc.toc2 > ul { font-size: 0.95em; }
+ #toc.toc2 ul ul { padding-left: 1.25em; }
+ body.toc2.toc-right { padding-left: 0; padding-right: 20em; } }
+#content #toc { border-style: solid; border-width: 1px; border-color: #d9d9d9; margin-bottom: 1.25em; padding: 1.25em; background: #f2f2f2; -webkit-border-radius: 0; border-radius: 0; }
+#content #toc > :first-child { margin-top: 0; }
+#content #toc > :last-child { margin-bottom: 0; }
+
+#footer { max-width: 100%; background-color: #222222; padding: 1.25em; }
+
+#footer-text { color: #dddddd; line-height: 1.44; }
+
+.sect1 { padding-bottom: 0.625em; }
+
+@media only screen and (min-width: 768px) { .sect1 { padding-bottom: 1.25em; } }
+.sect1 + .sect1 { border-top: 1px solid #dddddd; }
+
+#content h1 > a.anchor, h2 > a.anchor, h3 > a.anchor, #toctitle > a.anchor, .sidebarblock > .content > .title > a.anchor, h4 > a.anchor, h5 > a.anchor, h6 > a.anchor { position: absolute; z-index: 1001; width: 1.5ex; margin-left: -1.5ex; display: block; text-decoration: none !important; visibility: hidden; text-align: center; font-weight: normal; }
+#content h1 > a.anchor:before, h2 > a.anchor:before, h3 > a.anchor:before, #toctitle > a.anchor:before, .sidebarblock > .content > .title > a.anchor:before, h4 > a.anchor:before, h5 > a.anchor:before, h6 > a.anchor:before { content: "\00A7"; font-size: 0.85em; display: block; padding-top: 0.1em; }
+#content h1:hover > a.anchor, #content h1 > a.anchor:hover, h2:hover > a.anchor, h2 > a.anchor:hover, h3:hover > a.anchor, #toctitle:hover > a.anchor, .sidebarblock > .content > .title:hover > a.anchor, h3 > a.anchor:hover, #toctitle > a.anchor:hover, .sidebarblock > .content > .title > a.anchor:hover, h4:hover > a.anchor, h4 > a.anchor:hover, h5:hover > a.anchor, h5 > a.anchor:hover, h6:hover > a.anchor, h6 > a.anchor:hover { visibility: visible; }
+#content h1 > a.link, h2 > a.link, h3 > a.link, #toctitle > a.link, .sidebarblock > .content > .title > a.link, h4 > a.link, h5 > a.link, h6 > a.link { color: #222222; text-decoration: none; }
+#content h1 > a.link:hover, h2 > a.link:hover, h3 > a.link:hover, #toctitle > a.link:hover, .sidebarblock > .content > .title > a.link:hover, h4 > a.link:hover, h5 > a.link:hover, h6 > a.link:hover { color: #151515; }
+
+.audioblock, .imageblock, .literalblock, .listingblock, .stemblock, .videoblock { margin-bottom: 1.25em; }
+
+.admonitionblock td.content > .title, .audioblock > .title, .exampleblock > .title, .imageblock > .title, .listingblock > .title, .literalblock > .title, .stemblock > .title, .openblock > .title, .paragraph > .title, .quoteblock > .title, table.tableblock > .title, .verseblock > .title, .videoblock > .title, .dlist > .title, .olist > .title, .ulist > .title, .qlist > .title, .hdlist > .title { text-rendering: optimizeLegibility; text-align: left; }
+
+table.tableblock > caption.title { white-space: nowrap; overflow: visible; max-width: 0; }
+
+.paragraph.lead > p, #preamble > .sectionbody > .paragraph:first-of-type p { color: black; }
+
+table.tableblock #preamble > .sectionbody > .paragraph:first-of-type p { font-size: inherit; }
+
+.admonitionblock > table { border-collapse: separate; border: 0; background: none; width: 100%; }
+.admonitionblock > table td.icon { text-align: center; width: 80px; }
+.admonitionblock > table td.icon img { max-width: initial; }
+.admonitionblock > table td.icon .title { font-weight: bold; font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; text-transform: uppercase; }
+.admonitionblock > table td.content { padding-left: 1.125em; padding-right: 1.25em; border-left: 1px solid #dddddd; color: #555555; }
+.admonitionblock > table td.content > :last-child > :last-child { margin-bottom: 0; }
+
+.exampleblock > .content { border-style: solid; border-width: 1px; border-color: #e6e6e6; margin-bottom: 1.25em; padding: 1.25em; background: white; -webkit-border-radius: 0; border-radius: 0; }
+.exampleblock > .content > :first-child { margin-top: 0; }
+.exampleblock > .content > :last-child { margin-bottom: 0; }
+
+.sidebarblock { border-style: solid; border-width: 1px; border-color: #d9d9d9; margin-bottom: 1.25em; padding: 1.25em; background: #f2f2f2; -webkit-border-radius: 0; border-radius: 0; }
+.sidebarblock > :first-child { margin-top: 0; }
+.sidebarblock > :last-child { margin-bottom: 0; }
+.sidebarblock > .content > .title { color: #6f6f6f; margin-top: 0; }
+
+.exampleblock > .content > :last-child > :last-child, .exampleblock > .content .olist > ol > li:last-child > :last-child, .exampleblock > .content .ulist > ul > li:last-child > :last-child, .exampleblock > .content .qlist > ol > li:last-child > :last-child, .sidebarblock > .content > :last-child > :last-child, .sidebarblock > .content .olist > ol > li:last-child > :last-child, .sidebarblock > .content .ulist > ul > li:last-child > :last-child, .sidebarblock > .content .qlist > ol > li:last-child > :last-child { margin-bottom: 0; }
+
+.literalblock pre, .listingblock pre:not(.highlight), .listingblock pre[class="highlight"], .listingblock pre[class^="highlight "], .listingblock pre.CodeRay, .listingblock pre.prettyprint { background: #eeeeee; }
+.sidebarblock .literalblock pre, .sidebarblock .listingblock pre:not(.highlight), .sidebarblock .listingblock pre[class="highlight"], .sidebarblock .listingblock pre[class^="highlight "], .sidebarblock .listingblock pre.CodeRay, .sidebarblock .listingblock pre.prettyprint { background: #f2f1f1; }
+
+.literalblock pre, .literalblock pre[class], .listingblock pre, .listingblock pre[class] { border: 1px solid #cccccc; -webkit-border-radius: 0; border-radius: 0; word-wrap: break-word; padding: 0.8em 0.8em 0.65em 0.8em; font-size: 0.8125em; }
+.literalblock pre.nowrap, .literalblock pre[class].nowrap, .listingblock pre.nowrap, .listingblock pre[class].nowrap { overflow-x: auto; white-space: pre; word-wrap: normal; }
+@media only screen and (min-width: 768px) { .literalblock pre, .literalblock pre[class], .listingblock pre, .listingblock pre[class] { font-size: 0.90625em; } }
+@media only screen and (min-width: 1280px) { .literalblock pre, .literalblock pre[class], .listingblock pre, .listingblock pre[class] { font-size: 1em; } }
+
+.literalblock.output pre { color: #eeeeee; background-color: black; }
+
+.listingblock pre.highlightjs { padding: 0; }
+.listingblock pre.highlightjs > code { padding: 0.8em 0.8em 0.65em 0.8em; -webkit-border-radius: 0; border-radius: 0; }
+
+.listingblock > .content { position: relative; }
+
+.listingblock code[data-lang]:before { display: none; content: attr(data-lang); position: absolute; font-size: 0.75em; top: 0.425rem; right: 0.5rem; line-height: 1; text-transform: uppercase; color: #999; }
+
+.listingblock:hover code[data-lang]:before { display: block; }
+
+.listingblock.terminal pre .command:before { content: attr(data-prompt); padding-right: 0.5em; color: #999; }
+
+.listingblock.terminal pre .command:not([data-prompt]):before { content: "$"; }
+
+table.pyhltable { border-collapse: separate; border: 0; margin-bottom: 0; background: none; }
+
+table.pyhltable td { vertical-align: top; padding-top: 0; padding-bottom: 0; line-height: 1.4; }
+
+table.pyhltable td.code { padding-left: .75em; padding-right: 0; }
+
+pre.pygments .lineno, table.pyhltable td:not(.code) { color: #999; padding-left: 0; padding-right: .5em; border-right: 1px solid #dddddd; }
+
+pre.pygments .lineno { display: inline-block; margin-right: .25em; }
+
+table.pyhltable .linenodiv { background: none !important; padding-right: 0 !important; }
+
+.quoteblock { margin: 0 1em 1.25em 1.5em; display: table; }
+.quoteblock > .title { margin-left: -1.5em; margin-bottom: 0.75em; }
+.quoteblock blockquote, .quoteblock blockquote p { color: #6f6f6f; font-size: 1.15rem; line-height: 1.75; word-spacing: 0.1em; letter-spacing: 0; font-style: italic; text-align: justify; }
+.quoteblock blockquote { margin: 0; padding: 0; border: 0; }
+.quoteblock blockquote:before { content: "\201c"; float: left; font-size: 2.75em; font-weight: bold; line-height: 0.6em; margin-left: -0.6em; color: #6f6f6f; text-shadow: 0 1px 2px rgba(0, 0, 0, 0.1); }
+.quoteblock blockquote > .paragraph:last-child p { margin-bottom: 0; }
+.quoteblock .attribution { margin-top: 0.5em; margin-right: 0.5ex; text-align: right; }
+.quoteblock .quoteblock { margin-left: 0; margin-right: 0; padding: 0.5em 0; border-left: 3px solid #555555; }
+.quoteblock .quoteblock blockquote { padding: 0 0 0 0.75em; }
+.quoteblock .quoteblock blockquote:before { display: none; }
+
+.verseblock { margin: 0 1em 1.25em 1em; }
+.verseblock pre { font-family: "Open Sans", "DejaVu Sans", sans; font-size: 1.15rem; color: #6f6f6f; font-weight: 300; text-rendering: optimizeLegibility; }
+.verseblock pre strong { font-weight: 400; }
+.verseblock .attribution { margin-top: 1.25rem; margin-left: 0.5ex; }
+
+.quoteblock .attribution, .verseblock .attribution { font-size: 0.8125em; line-height: 1.45; font-style: italic; }
+.quoteblock .attribution br, .verseblock .attribution br { display: none; }
+.quoteblock .attribution cite, .verseblock .attribution cite { display: block; letter-spacing: -0.025em; color: #555555; }
+
+.quoteblock.abstract { margin: 0 0 1.25em 0; display: block; }
+.quoteblock.abstract blockquote, .quoteblock.abstract blockquote p { text-align: left; word-spacing: 0; }
+.quoteblock.abstract blockquote:before, .quoteblock.abstract blockquote p:first-of-type:before { display: none; }
+
+table.tableblock { max-width: 100%; border-collapse: separate; }
+table.tableblock td > .paragraph:last-child p > p:last-child, table.tableblock th > p:last-child, table.tableblock td > p:last-child { margin-bottom: 0; }
+
+table.tableblock, th.tableblock, td.tableblock { border: 0 solid #dddddd; }
+
+table.grid-all th.tableblock, table.grid-all td.tableblock { border-width: 0 1px 1px 0; }
+
+table.grid-all tfoot > tr > th.tableblock, table.grid-all tfoot > tr > td.tableblock { border-width: 1px 1px 0 0; }
+
+table.grid-cols th.tableblock, table.grid-cols td.tableblock { border-width: 0 1px 0 0; }
+
+table.grid-all * > tr > .tableblock:last-child, table.grid-cols * > tr > .tableblock:last-child { border-right-width: 0; }
+
+table.grid-rows th.tableblock, table.grid-rows td.tableblock { border-width: 0 0 1px 0; }
+
+table.grid-all tbody > tr:last-child > th.tableblock, table.grid-all tbody > tr:last-child > td.tableblock, table.grid-all thead:last-child > tr > th.tableblock, table.grid-rows tbody > tr:last-child > th.tableblock, table.grid-rows tbody > tr:last-child > td.tableblock, table.grid-rows thead:last-child > tr > th.tableblock { border-bottom-width: 0; }
+
+table.grid-rows tfoot > tr > th.tableblock, table.grid-rows tfoot > tr > td.tableblock { border-width: 1px 0 0 0; }
+
+table.frame-all { border-width: 1px; }
+
+table.frame-sides { border-width: 0 1px; }
+
+table.frame-topbot { border-width: 1px 0; }
+
+th.halign-left, td.halign-left { text-align: left; }
+
+th.halign-right, td.halign-right { text-align: right; }
+
+th.halign-center, td.halign-center { text-align: center; }
+
+th.valign-top, td.valign-top { vertical-align: top; }
+
+th.valign-bottom, td.valign-bottom { vertical-align: bottom; }
+
+th.valign-middle, td.valign-middle { vertical-align: middle; }
+
+table thead th, table tfoot th { font-weight: bold; }
+
+tbody tr th { display: table-cell; line-height: 1.4; background: whitesmoke; }
+
+tbody tr th, tbody tr th p, tfoot tr th, tfoot tr th p { color: #222222; font-weight: bold; }
+
+p.tableblock > code:only-child { background: none; padding: 0; }
+
+p.tableblock { font-size: 1em; }
+
+td > div.verse { white-space: pre; }
+
+ol { margin-left: 1.75em; }
+
+ul li ol { margin-left: 1.5em; }
+
+dl dd { margin-left: 1.125em; }
+
+dl dd:last-child, dl dd:last-child > :last-child { margin-bottom: 0; }
+
+ol > li p, ul > li p, ul dd, ol dd, .olist .olist, .ulist .ulist, .ulist .olist, .olist .ulist { margin-bottom: 0.625em; }
+
+ul.unstyled, ol.unnumbered, ul.checklist, ul.none { list-style-type: none; }
+
+ul.unstyled, ol.unnumbered, ul.checklist { margin-left: 0.625em; }
+
+ul.checklist li > p:first-child > .fa-square-o:first-child, ul.checklist li > p:first-child > .fa-check-square-o:first-child { width: 1em; font-size: 0.85em; }
+
+ul.checklist li > p:first-child > input[type="checkbox"]:first-child { width: 1em; position: relative; top: 1px; }
+
+ul.inline { margin: 0 auto 0.625em auto; margin-left: -1.375em; margin-right: 0; padding: 0; list-style: none; overflow: hidden; }
+ul.inline > li { list-style: none; float: left; margin-left: 1.375em; display: block; }
+ul.inline > li > * { display: block; }
+
+.unstyled dl dt { font-weight: normal; font-style: normal; }
+
+ol.arabic { list-style-type: decimal; }
+
+ol.decimal { list-style-type: decimal-leading-zero; }
+
+ol.loweralpha { list-style-type: lower-alpha; }
+
+ol.upperalpha { list-style-type: upper-alpha; }
+
+ol.lowerroman { list-style-type: lower-roman; }
+
+ol.upperroman { list-style-type: upper-roman; }
+
+ol.lowergreek { list-style-type: lower-greek; }
+
+.hdlist > table, .colist > table { border: 0; background: none; }
+.hdlist > table > tbody > tr, .colist > table > tbody > tr { background: none; }
+
+td.hdlist1, td.hdlist2 { vertical-align: top; padding: 0 0.625em; }
+
+td.hdlist1 { font-weight: bold; padding-bottom: 1.25em; }
+
+.literalblock + .colist, .listingblock + .colist { margin-top: -0.5em; }
+
+.colist > table tr > td:first-of-type { padding: 0 0.75em; line-height: 1; }
+.colist > table tr > td:first-of-type img { max-width: initial; }
+.colist > table tr > td:last-of-type { padding: 0.25em 0; }
+
+.thumb, .th { line-height: 0; display: inline-block; border: solid 4px white; -webkit-box-shadow: 0 0 0 1px #dddddd; box-shadow: 0 0 0 1px #dddddd; }
+
+.imageblock.left, .imageblock[style*="float: left"] { margin: 0.25em 0.625em 1.25em 0; }
+.imageblock.right, .imageblock[style*="float: right"] { margin: 0.25em 0 1.25em 0.625em; }
+.imageblock > .title { margin-bottom: 0; }
+.imageblock.thumb, .imageblock.th { border-width: 6px; }
+.imageblock.thumb > .title, .imageblock.th > .title { padding: 0 0.125em; }
+
+.image.left, .image.right { margin-top: 0.25em; margin-bottom: 0.25em; display: inline-block; line-height: 0; }
+.image.left { margin-right: 0.625em; }
+.image.right { margin-left: 0.625em; }
+
+a.image { text-decoration: none; display: inline-block; }
+a.image object { pointer-events: none; }
+
+sup.footnote, sup.footnoteref { font-size: 0.875em; position: static; vertical-align: super; }
+sup.footnote a, sup.footnoteref a { text-decoration: none; }
+sup.footnote a:active, sup.footnoteref a:active { text-decoration: underline; }
+
+#footnotes { padding-top: 0.75em; padding-bottom: 0.75em; margin-bottom: 0.625em; }
+#footnotes hr { width: 20%; min-width: 6.25em; margin: -0.25em 0 0.75em 0; border-width: 1px 0 0 0; }
+#footnotes .footnote { padding: 0 0.375em 0 0.225em; line-height: 1.3334; font-size: 0.875em; margin-left: 1.2em; text-indent: -1.05em; margin-bottom: 0.2em; }
+#footnotes .footnote a:first-of-type { font-weight: bold; text-decoration: none; }
+#footnotes .footnote:last-of-type { margin-bottom: 0; }
+#content #footnotes { margin-top: -0.625em; margin-bottom: 0; padding: 0.75em 0; }
+
+.gist .file-data > table { border: 0; background: #fff; width: 100%; margin-bottom: 0; }
+.gist .file-data > table td.line-data { width: 99%; }
+
+div.unbreakable { page-break-inside: avoid; }
+
+.big { font-size: larger; }
+
+.small { font-size: smaller; }
+
+.underline { text-decoration: underline; }
+
+.overline { text-decoration: overline; }
+
+.line-through { text-decoration: line-through; }
+
+.aqua { color: #00bfbf; }
+
+.aqua-background { background-color: #00fafa; }
+
+.black { color: black; }
+
+.black-background { background-color: black; }
+
+.blue { color: #0000bf; }
+
+.blue-background { background-color: #0000fa; }
+
+.fuchsia { color: #bf00bf; }
+
+.fuchsia-background { background-color: #fa00fa; }
+
+.gray { color: #606060; }
+
+.gray-background { background-color: #7d7d7d; }
+
+.green { color: #006000; }
+
+.green-background { background-color: #007d00; }
+
+.lime { color: #00bf00; }
+
+.lime-background { background-color: #00fa00; }
+
+.maroon { color: #600000; }
+
+.maroon-background { background-color: #7d0000; }
+
+.navy { color: #000060; }
+
+.navy-background { background-color: #00007d; }
+
+.olive { color: #606000; }
+
+.olive-background { background-color: #7d7d00; }
+
+.purple { color: #600060; }
+
+.purple-background { background-color: #7d007d; }
+
+.red { color: #bf0000; }
+
+.red-background { background-color: #fa0000; }
+
+.silver { color: #909090; }
+
+.silver-background { background-color: #bcbcbc; }
+
+.teal { color: #006060; }
+
+.teal-background { background-color: #007d7d; }
+
+.white { color: #bfbfbf; }
+
+.white-background { background-color: #fafafa; }
+
+.yellow { color: #bfbf00; }
+
+.yellow-background { background-color: #fafa00; }
+
+span.icon > .fa { cursor: default; }
+
+.admonitionblock td.icon [class^="fa icon-"] { font-size: 2.5em; text-shadow: 1px 1px 2px rgba(0, 0, 0, 0.5); cursor: default; }
+.admonitionblock td.icon .icon-note:before { content: "\f05a"; color: #207c98; }
+.admonitionblock td.icon .icon-tip:before { content: "\f0eb"; text-shadow: 1px 1px 2px rgba(155, 155, 0, 0.8); color: #111; }
+.admonitionblock td.icon .icon-warning:before { content: "\f071"; color: #bf6900; }
+.admonitionblock td.icon .icon-caution:before { content: "\f06d"; color: #bf3400; }
+.admonitionblock td.icon .icon-important:before { content: "\f06a"; color: #bf0000; }
+
+.conum[data-value] { display: inline-block; color: #fff !important; background-color: #222222; -webkit-border-radius: 100px; border-radius: 100px; text-align: center; font-size: 0.75em; width: 1.67em; height: 1.67em; line-height: 1.67em; font-family: "Open Sans", "DejaVu Sans", sans-serif; font-style: normal; font-weight: bold; }
+.conum[data-value] * { color: #fff !important; }
+.conum[data-value] + b { display: none; }
+.conum[data-value]:after { content: attr(data-value); }
+pre .conum[data-value] { position: relative; top: -0.125em; }
+
+b.conum * { color: inherit !important; }
+
+.conum:not([data-value]):empty { display: none; }
+
+.literalblock pre, .listingblock pre { background: #eeeeee; }
diff --git a/api/src/docs/asciidoclet/overview.adoc b/api/src/docs/asciidoclet/overview.adoc
new file mode 100644
index 0000000..7947331
--- /dev/null
+++ b/api/src/docs/asciidoclet/overview.adoc
@@ -0,0 +1,4 @@
+= Elasticsearch Java client
+Jörg Prante
+Version 5.4.0.0
+
diff --git a/backup/XbibTransportService.java b/backup/XbibTransportService.java
new file mode 100644
index 0000000..c2dc502
--- /dev/null
+++ b/backup/XbibTransportService.java
@@ -0,0 +1,1047 @@
+package org.xbib.elasticsearch.client.transport;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.metrics.MeanMetric;
+import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.BoundTransportAddress;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.util.concurrent.ConcurrentMapLong;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.common.util.concurrent.FutureUtils;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.tasks.TaskManager;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.ActionNotFoundTransportException;
+import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.FutureTransportResponseHandler;
+import org.elasticsearch.transport.NodeDisconnectedException;
+import org.elasticsearch.transport.NodeNotConnectedException;
+import org.elasticsearch.transport.PlainTransportFuture;
+import org.elasticsearch.transport.ReceiveTimeoutTransportException;
+import org.elasticsearch.transport.RemoteTransportException;
+import org.elasticsearch.transport.RequestHandlerRegistry;
+import org.elasticsearch.transport.ResponseHandlerFailureTransportException;
+import org.elasticsearch.transport.SendRequestTransportException;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportChannel;
+import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TransportFuture;
+import org.elasticsearch.transport.TransportInterceptor;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportRequestHandler;
+import org.elasticsearch.transport.TransportRequestOptions;
+import org.elasticsearch.transport.TransportResponse;
+import org.elasticsearch.transport.TransportResponseHandler;
+import org.elasticsearch.transport.TransportResponseOptions;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ScheduledFuture;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+
+/**
+ *
+ */
+public class XbibTransportService extends AbstractLifecycleComponent {
+
+ private static final String HANDSHAKE_ACTION_NAME = "internal:transport/handshake";
+
+ private static final Setting> TRACE_LOG_INCLUDE_SETTING =
+ Setting.listSetting("transport.tracer.include", Collections.emptyList(), Function.identity(),
+ Property.Dynamic, Property.NodeScope);
+
+ private static final Setting> TRACE_LOG_EXCLUDE_SETTING =
+ Setting.listSetting("transport.tracer.exclude", Arrays.asList("internal:discovery/zen/fd*",
+ TransportLivenessAction.NAME), Function.identity(), Property.Dynamic, Property.NodeScope);
+
+ private final CountDownLatch blockIncomingRequestsLatch = new CountDownLatch(1);
+
+ private final Transport transport;
+
+ private final ThreadPool threadPool;
+
+ private final ClusterName clusterName;
+
+ private final TaskManager taskManager;
+
+ private final TransportInterceptor.AsyncSender asyncSender;
+
+ private final Function localNodeFactory;
+
+ private volatile Map> requestHandlers = Collections.emptyMap();
+
+ private final Object requestHandlerMutex = new Object();
+
+ private final ConcurrentMapLong> clientHandlers =
+ ConcurrentCollections.newConcurrentMapLongWithAggressiveConcurrency();
+
+ private final TransportInterceptor interceptor;
+
+ // An LRU (don't really care about concurrency here) that holds the latest timed out requests so if they
+ // do show up, we can print more descriptive information about them
+ private final Map timeoutInfoHandlers =
+ Collections.synchronizedMap(new LinkedHashMap(100, .75F, true) {
+ private static final long serialVersionUID = 9174428975922394994L;
+
+ @Override
+ protected boolean removeEldestEntry(Map.Entry eldest) {
+ return size() > 100;
+ }
+ });
+
+ private final Logger tracerLog;
+
+ private volatile String[] tracerLogInclude;
+
+ private volatile String[] tracerLogExclude;
+
+ private volatile DiscoveryNode localNode = null;
+
+ private final Transport.Connection localNodeConnection = new Transport.Connection() {
+ @Override
+ public DiscoveryNode getNode() {
+ return localNode;
+ }
+
+ @Override
+ public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
+ throws IOException, TransportException {
+ sendLocalRequest(requestId, action, request, options);
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+ };
+
+ /**
+ * Build the service.
+ *
+ * @param clusterSettings if non null the the {@linkplain XbibTransportService} will register
+ * with the {@link ClusterSettings} for settings updates for
+ * {@link #TRACE_LOG_EXCLUDE_SETTING} and {@link #TRACE_LOG_INCLUDE_SETTING}.
+ */
+ XbibTransportService(Settings settings, Transport transport, ThreadPool threadPool,
+ TransportInterceptor transportInterceptor,
+ Function localNodeFactory,
+ @Nullable ClusterSettings clusterSettings) {
+ super(settings);
+ this.transport = transport;
+ this.threadPool = threadPool;
+ this.localNodeFactory = localNodeFactory;
+ this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
+ setTracerLogInclude(TRACE_LOG_INCLUDE_SETTING.get(settings));
+ setTracerLogExclude(TRACE_LOG_EXCLUDE_SETTING.get(settings));
+ tracerLog = Loggers.getLogger(logger, ".tracer");
+ taskManager = createTaskManager();
+ this.interceptor = transportInterceptor;
+ this.asyncSender = interceptor.interceptSender(this::sendRequestInternal);
+ if (clusterSettings != null) {
+ clusterSettings.addSettingsUpdateConsumer(TRACE_LOG_INCLUDE_SETTING, this::setTracerLogInclude);
+ clusterSettings.addSettingsUpdateConsumer(TRACE_LOG_EXCLUDE_SETTING, this::setTracerLogExclude);
+ }
+ }
+
+ private TaskManager createTaskManager() {
+ return new TaskManager(settings);
+ }
+
+ private void setTracerLogInclude(List tracerLogInclude) {
+ this.tracerLogInclude = tracerLogInclude.toArray(Strings.EMPTY_ARRAY);
+ }
+
+ private void setTracerLogExclude(List tracerLogExclude) {
+ this.tracerLogExclude = tracerLogExclude.toArray(Strings.EMPTY_ARRAY);
+ }
+
+ @Override
+ protected void doStart() {
+ rxMetric.clear();
+ txMetric.clear();
+ transport.setTransportService(this);
+ transport.start();
+ if (transport.boundAddress() != null && logger.isInfoEnabled()) {
+ logger.info("{}", transport.boundAddress());
+ for (Map.Entry entry : transport.profileBoundAddresses().entrySet()) {
+ logger.info("profile [{}]: {}", entry.getKey(), entry.getValue());
+ }
+ }
+ localNode = localNodeFactory.apply(transport.boundAddress());
+ registerRequestHandler(HANDSHAKE_ACTION_NAME,
+ () -> HandshakeRequest.INSTANCE,
+ ThreadPool.Names.SAME,
+ (request, channel) -> channel.sendResponse(new HandshakeResponse(localNode, clusterName,
+ localNode.getVersion())));
+ }
+
+ @Override
+ protected void doStop() {
+ try {
+ transport.stop();
+ } finally {
+ // in case the transport is not connected to our local node (thus cleaned on node disconnect)
+ // make sure to clean any leftover on going handles
+ for (Map.Entry> entry : clientHandlers.entrySet()) {
+ final RequestHolder holderToNotify = clientHandlers.remove(entry.getKey());
+ if (holderToNotify != null) {
+ // callback that an exception happened, but on a different thread since we don't
+ // want handlers to worry about stack overflows
+ threadPool.generic().execute(new AbstractRunnable() {
+ @Override
+ public void onRejection(Exception e) {
+ // if we get rejected during node shutdown we don't wanna bubble it up
+ logger.debug((Supplier) () -> new ParameterizedMessage(
+ "failed to notify response handler on rejection, action: {}",
+ holderToNotify.action()),
+ e);
+ }
+ @Override
+ public void onFailure(Exception e) {
+ logger.warn((Supplier) () -> new ParameterizedMessage(
+ "failed to notify response handler on exception, action: {}",
+ holderToNotify.action()),
+ e);
+ }
+ @Override
+ public void doRun() {
+ TransportException ex = new TransportException("transport stopped, action: " +
+ holderToNotify.action());
+ holderToNotify.handler().handleException(ex);
+ }
+ });
+ }
+ }
+ }
+ }
+
+ @Override
+ protected void doClose() {
+ transport.close();
+ }
+
+ /**
+ * Start accepting incoming requests.
+ * when the transport layer starts up it will block any incoming requests until
+ * this method is called
+ */
+ final void acceptIncomingRequests() {
+ blockIncomingRequestsLatch.countDown();
+ }
+
+ /**
+ * Returns true iff the given node is already connected.
+ */
+ boolean nodeConnected(DiscoveryNode node) {
+ return isLocalNode(node) || transport.nodeConnected(node);
+ }
+
+ /**
+ * Connect to the specified node.
+ *
+ * @param node the node to connect to
+ */
+ void connectToNode(final DiscoveryNode node) {
+ if (isLocalNode(node)) {
+ return;
+ }
+ transport.connectToNode(node, null, (newConnection, actualProfile) ->
+ handshake(newConnection, actualProfile.getHandshakeTimeout().millis()));
+ }
+
+ /**
+ * Executes a high-level handshake using the given connection
+ * and returns the discovery node of the node the connection
+ * was established with. The handshake will fail if the cluster
+ * name on the target node mismatches the local cluster name.
+ *
+ * @param connection the connection to a specific node
+ * @param handshakeTimeout handshake timeout
+ * @return the connected node
+ * @throws ConnectTransportException if the connection failed
+ * @throws IllegalStateException if the handshake failed
+ */
+ private DiscoveryNode handshake(final Transport.Connection connection,
+ final long handshakeTimeout) throws ConnectTransportException {
+ return handshake(connection, handshakeTimeout, clusterName::equals);
+ }
+
+ /**
+ * Executes a high-level handshake using the given connection
+ * and returns the discovery node of the node the connection
+ * was established with. The handshake will fail if the cluster
+ * name on the target node doesn't match the local cluster name.
+ *
+ * @param connection the connection to a specific node
+ * @param handshakeTimeout handshake timeout
+ * @param clusterNamePredicate cluster name validation predicate
+ * @return the connected node
+ * @throws ConnectTransportException if the connection failed
+ * @throws IllegalStateException if the handshake failed
+ */
+ private DiscoveryNode handshake(final Transport.Connection connection,
+ final long handshakeTimeout, Predicate clusterNamePredicate)
+ throws ConnectTransportException {
+ final HandshakeResponse response;
+ final DiscoveryNode node = connection.getNode();
+ try {
+ PlainTransportFuture futureHandler = new PlainTransportFuture<>(
+ new FutureTransportResponseHandler() {
+ @Override
+ public HandshakeResponse newInstance() {
+ return new HandshakeResponse();
+ }
+ });
+ sendRequest(connection, HANDSHAKE_ACTION_NAME, HandshakeRequest.INSTANCE,
+ TransportRequestOptions.builder().withTimeout(handshakeTimeout).build(), futureHandler);
+ response = futureHandler.txGet();
+ } catch (Exception e) {
+ throw new IllegalStateException("handshake failed with " + node, e);
+ }
+ if (!clusterNamePredicate.test(response.clusterName)) {
+ throw new IllegalStateException("handshake failed, mismatched cluster name [" +
+ response.clusterName + "] - " + node);
+ } else if (!response.version.isCompatible(localNode.getVersion())) {
+ throw new IllegalStateException("handshake failed, incompatible version [" +
+ response.version + "] - " + node);
+ }
+ return response.discoveryNode;
+ }
+
+ void disconnectFromNode(DiscoveryNode node) {
+ if (isLocalNode(node)) {
+ return;
+ }
+ transport.disconnectFromNode(node);
+ }
+
+ TransportFuture submitRequest(DiscoveryNode node, String action,
+ TransportRequest request,
+ TransportRequestOptions options,
+ TransportResponseHandler handler)
+ throws TransportException {
+ PlainTransportFuture futureHandler = new PlainTransportFuture<>(handler);
+ try {
+ Transport.Connection connection = getConnection(node);
+ sendRequest(connection, action, request, options, futureHandler);
+ } catch (NodeNotConnectedException ex) {
+ futureHandler.handleException(ex);
+ }
+ return futureHandler;
+ }
+
+ final void sendRequest(final DiscoveryNode node, final String action,
+ final TransportRequest request,
+ final TransportRequestOptions options,
+ TransportResponseHandler handler) {
+ try {
+ Transport.Connection connection = getConnection(node);
+ sendRequest(connection, action, request, options, handler);
+ } catch (NodeNotConnectedException ex) {
+ handler.handleException(ex);
+ }
+ }
+
+ private void sendRequest(final Transport.Connection connection, final String action,
+ final TransportRequest request,
+ final TransportRequestOptions options,
+ TransportResponseHandler handler) {
+
+ asyncSender.sendRequest(connection, action, request, options, handler);
+ }
+
+ /**
+ * Returns either a real transport connection or a local node connection
+ * if we are using the local node optimization.
+ * @throws NodeNotConnectedException if the given node is not connected
+ */
+ private Transport.Connection getConnection(DiscoveryNode node) {
+ if (isLocalNode(node)) {
+ return localNodeConnection;
+ } else {
+ return transport.getConnection(node);
+ }
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private void sendRequestInternal(final Transport.Connection connection,
+ final String action,
+ final TransportRequest request,
+ final TransportRequestOptions options,
+ TransportResponseHandler handler) {
+ if (connection == null) {
+ throw new IllegalStateException("can't send request to a null connection");
+ }
+ DiscoveryNode node = connection.getNode();
+ final long requestId = transport.newRequestId();
+ final TimeoutHandler timeoutHandler;
+ try {
+ if (options.timeout() == null) {
+ timeoutHandler = null;
+ } else {
+ timeoutHandler = new TimeoutHandler(requestId);
+ }
+ Supplier storedContextSupplier =
+ threadPool.getThreadContext().newRestorableContext(true);
+ TransportResponseHandler responseHandler =
+ new ContextRestoreResponseHandler<>(storedContextSupplier, handler);
+ clientHandlers.put(requestId,
+ new RequestHolder(responseHandler, connection.getNode(), action, timeoutHandler));
+ if (lifecycle.stoppedOrClosed()) {
+ // if we are not started the exception handling will remove the RequestHolder again
+ // and calls the handler to notify the caller. It will only notify if the toStop code
+ // hasn't done the work yet.
+ throw new TransportException("TransportService is closed stopped can't send request");
+ }
+ if (timeoutHandler != null) {
+ assert options.timeout() != null;
+ timeoutHandler.future = threadPool.schedule(options.timeout(), ThreadPool.Names.GENERIC, timeoutHandler);
+ }
+ connection.sendRequest(requestId, action, request, options);
+ } catch (final Exception e) {
+ // usually happen either because we failed to connect to the node
+ // or because we failed serializing the message
+ final RequestHolder holderToNotify = clientHandlers.remove(requestId);
+ // If holderToNotify == null then handler has already been taken care of.
+ if (holderToNotify != null) {
+ holderToNotify.cancelTimeout();
+ // callback that an exception happened, but on a different thread since we don't
+ // want handlers to worry about stack overflows
+ final SendRequestTransportException sendRequestException =
+ new SendRequestTransportException(node, action, e);
+ threadPool.executor(ThreadPool.Names.GENERIC).execute(new AbstractRunnable() {
+ @Override
+ public void onRejection(Exception e) {
+ // if we get rejected during node shutdown we don't wanna bubble it up
+ logger.debug((Supplier) () -> new ParameterizedMessage(
+ "failed to notify response handler on rejection, action: {}",
+ holderToNotify.action()), e);
+ }
+ @Override
+ public void onFailure(Exception e) {
+ logger.warn((Supplier) () -> new ParameterizedMessage(
+ "failed to notify response handler on exception, action: {}",
+ holderToNotify.action()), e);
+ }
+ @Override
+ protected void doRun() throws Exception {
+ holderToNotify.handler().handleException(sendRequestException);
+ }
+ });
+ } else {
+ logger.debug("Exception while sending request, handler likely already notified due to timeout", e);
+ }
+ }
+ }
+
+ private void sendLocalRequest(long requestId, final String action, final TransportRequest request,
+ TransportRequestOptions options) {
+ final DirectResponseChannel channel = new DirectResponseChannel(logger, localNode, action, requestId, adapter,
+ threadPool);
+ try {
+ adapter.onRequestSent(localNode, requestId, action, request, options);
+ adapter.onRequestReceived(requestId, action);
+ final RequestHandlerRegistry reg = adapter.getRequestHandler(action);
+ if (reg == null) {
+ throw new ActionNotFoundTransportException("Action [" + action + "] not found");
+ }
+ final String executor = reg.getExecutor();
+ if (ThreadPool.Names.SAME.equals(executor)) {
+ reg.processMessageReceived(request, channel);
+ } else {
+ threadPool.executor(executor).execute(new AbstractRunnable() {
+ @Override
+ protected void doRun() throws Exception {
+ reg.processMessageReceived(request, channel);
+ }
+
+ @Override
+ public boolean isForceExecution() {
+ return reg.isForceExecution();
+ }
+
+ @Override
+ public void onFailure(Exception e) {
+ try {
+ channel.sendResponse(e);
+ } catch (Exception inner) {
+ inner.addSuppressed(e);
+ logger.warn((Supplier) () ->
+ new ParameterizedMessage("failed to notify channel of error message for action [{}]",
+ action), inner);
+ }
+ }
+ });
+ }
+
+ } catch (Exception e) {
+ try {
+ channel.sendResponse(e);
+ } catch (Exception inner) {
+ inner.addSuppressed(e);
+ logger.warn(
+ (Supplier) () -> new ParameterizedMessage(
+ "failed to notify channel of error message for action [{}]", action), inner);
+ }
+ }
+ }
+
+ private boolean shouldTraceAction(String action) {
+ if (tracerLogInclude.length > 0) {
+ if (!Regex.simpleMatch(tracerLogInclude, action)) {
+ return false;
+ }
+ }
+ return tracerLogExclude.length <= 0 || !Regex.simpleMatch(tracerLogExclude, action);
+ }
+
+ /**
+ * Registers a new request handler.
+ *
+ * @param action the action the request handler is associated with
+ * @param request the request class that will be used to construct new instances for streaming
+ * @param executor the executor the request handling will be executed on
+ * @param handler the handler itself that implements the request handling
+ */
+ private void registerRequestHandler(String action, Supplier request,
+ String executor,
+ TransportRequestHandler handler) {
+ handler = interceptor.interceptHandler(action, executor, false, handler);
+ RequestHandlerRegistry reg = new RequestHandlerRegistry<>(
+ action, request, taskManager, handler, executor, false, false);
+ registerRequestHandler(reg);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void registerRequestHandler(RequestHandlerRegistry reg) {
+ synchronized (requestHandlerMutex) {
+ if (requestHandlers.containsKey(reg.getAction())) {
+ throw new IllegalArgumentException("transport handlers for action " +
+ reg.getAction() + " is already registered");
+ }
+ requestHandlers = MapBuilder.newMapBuilder(requestHandlers).put(reg.getAction(),
+ (RequestHandlerRegistry) reg).immutableMap();
+ }
+ }
+
+ private boolean isLocalNode(DiscoveryNode discoveryNode) {
+ return Objects.requireNonNull(discoveryNode, "discovery node must not be null").equals(localNode);
+ }
+
+ static class HandshakeRequest extends TransportRequest {
+
+ static final HandshakeRequest INSTANCE = new HandshakeRequest();
+
+ private HandshakeRequest() {
+ }
+
+ }
+
+ /**
+ *
+ */
+ public static class HandshakeResponse extends TransportResponse {
+
+ private DiscoveryNode discoveryNode;
+
+ private ClusterName clusterName;
+
+ private Version version;
+
+ /**
+ * For extern construction.
+ */
+ public HandshakeResponse() {
+ }
+
+ HandshakeResponse(DiscoveryNode discoveryNode, ClusterName clusterName, Version version) {
+ this.discoveryNode = discoveryNode;
+ this.version = version;
+ this.clusterName = clusterName;
+ }
+
+ @Override
+ public void readFrom(StreamInput in) throws IOException {
+ super.readFrom(in);
+ discoveryNode = in.readOptionalWriteable(DiscoveryNode::new);
+ clusterName = new ClusterName(in);
+ version = Version.readVersion(in);
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ super.writeTo(out);
+ out.writeOptionalWriteable(discoveryNode);
+ clusterName.writeTo(out);
+ Version.writeVersion(version, out);
+ }
+ }
+
+ private final class Adapter implements TransportServiceAdapter {
+
+ final MeanMetric rxMetric = new MeanMetric();
+
+ final MeanMetric txMetric = new MeanMetric();
+
+ @Override
+ public void addBytesReceived(long size) {
+ rxMetric.inc(size);
+ }
+
+ @Override
+ public void addBytesSent(long size) {
+ txMetric.inc(size);
+ }
+
+ @Override
+ public void onRequestSent(DiscoveryNode node, long requestId, String action, TransportRequest request,
+ TransportRequestOptions options) {
+ if (traceEnabled() && shouldTraceAction(action)) {
+ traceRequestSent(node, requestId, action, options);
+ }
+ }
+
+ boolean traceEnabled() {
+ return tracerLog.isTraceEnabled();
+ }
+
+ @Override
+ public void onResponseSent(long requestId, String action, TransportResponse response,
+ TransportResponseOptions options) {
+ if (traceEnabled() && shouldTraceAction(action)) {
+ traceResponseSent(requestId, action);
+ }
+ }
+
+ @Override
+ public void onResponseSent(long requestId, String action, Exception e) {
+ if (traceEnabled() && shouldTraceAction(action)) {
+ traceResponseSent(requestId, action, e);
+ }
+ }
+
+ void traceResponseSent(long requestId, String action, Exception e) {
+ tracerLog.trace(
+ (org.apache.logging.log4j.util.Supplier)
+ () -> new ParameterizedMessage("[{}][{}] sent error response", requestId, action), e);
+ }
+
+ @Override
+ public void onRequestReceived(long requestId, String action) {
+ try {
+ blockIncomingRequestsLatch.await();
+ } catch (InterruptedException e) {
+ logger.trace("interrupted while waiting for incoming requests block to be removed");
+ }
+ if (traceEnabled() && shouldTraceAction(action)) {
+ traceReceivedRequest(requestId, action);
+ }
+ }
+
+ @Override
+ public RequestHandlerRegistry getRequestHandler(String action) {
+ return requestHandlers.get(action);
+ }
+
+ @Override
+ public TransportResponseHandler onResponseReceived(final long requestId) {
+ RequestHolder holder = clientHandlers.remove(requestId);
+ if (holder == null) {
+ checkForTimeout(requestId);
+ return null;
+ }
+ holder.cancelTimeout();
+ if (traceEnabled() && shouldTraceAction(holder.action())) {
+ traceReceivedResponse(requestId, holder.node(), holder.action());
+ }
+ return holder.handler();
+ }
+
+ void checkForTimeout(long requestId) {
+ // lets see if its in the timeout holder, but sync on mutex to make sure any ongoing timeout
+ // handling has finished
+ final DiscoveryNode sourceNode;
+ final String action;
+ if (clientHandlers.get(requestId) != null) {
+ throw new IllegalStateException();
+ }
+ TimeoutInfoHolder timeoutInfoHolder = timeoutInfoHandlers.remove(requestId);
+ if (timeoutInfoHolder != null) {
+ long time = System.currentTimeMillis();
+ logger.warn("Received response for a request that has timed out, sent [{}ms] ago, timed out [{}ms] ago, " +
+ "action [{}], node [{}], id [{}]", time - timeoutInfoHolder.sentTime(),
+ time - timeoutInfoHolder.timeoutTime(),
+ timeoutInfoHolder.action(), timeoutInfoHolder.node(), requestId);
+ action = timeoutInfoHolder.action();
+ sourceNode = timeoutInfoHolder.node();
+ } else {
+ logger.warn("Transport response handler not found of id [{}]", requestId);
+ action = null;
+ sourceNode = null;
+ }
+ // call tracer out of lock
+ if (!traceEnabled()) {
+ return;
+ }
+ if (action == null) {
+ assert sourceNode == null;
+ traceUnresolvedResponse(requestId);
+ } else if (shouldTraceAction(action)) {
+ traceReceivedResponse(requestId, sourceNode, action);
+ }
+ }
+
+ @Override
+ public void onNodeConnected(final DiscoveryNode node) {
+ }
+
+ @Override
+ public void onConnectionOpened(DiscoveryNode node) {
+ }
+
+ @Override
+ public void onNodeDisconnected(final DiscoveryNode node) {
+ try {
+ for (Map.Entry> entry : clientHandlers.entrySet()) {
+ RequestHolder holder = entry.getValue();
+ if (holder.node().equals(node)) {
+ final RequestHolder holderToNotify = clientHandlers.remove(entry.getKey());
+ if (holderToNotify != null) {
+ // callback that an exception happened, but on a different thread since we don't
+ // want handlers to worry about stack overflows
+ threadPool.generic().execute(() -> holderToNotify.handler()
+ .handleException(new NodeDisconnectedException(node,
+ holderToNotify.action())));
+ }
+ }
+ }
+ } catch (EsRejectedExecutionException ex) {
+ logger.debug("Rejected execution on NodeDisconnected", ex);
+ }
+ }
+
+ void traceReceivedRequest(long requestId, String action) {
+ tracerLog.trace("[{}][{}] received request", requestId, action);
+ }
+
+ void traceResponseSent(long requestId, String action) {
+ tracerLog.trace("[{}][{}] sent response", requestId, action);
+ }
+
+ void traceReceivedResponse(long requestId, DiscoveryNode sourceNode, String action) {
+ tracerLog.trace("[{}][{}] received response from [{}]", requestId, action, sourceNode);
+ }
+
+ void traceUnresolvedResponse(long requestId) {
+ tracerLog.trace("[{}] received response but can't resolve it to a request", requestId);
+ }
+
+ void traceRequestSent(DiscoveryNode node, long requestId, String action, TransportRequestOptions options) {
+ tracerLog.trace("[{}][{}] sent to [{}] (timeout: [{}])", requestId, action, node, options.timeout());
+ }
+ }
+
+ private final class TimeoutHandler implements Runnable {
+
+ private final long requestId;
+
+ private final long sentTime = System.currentTimeMillis();
+
+ volatile ScheduledFuture future;
+
+ TimeoutHandler(long requestId) {
+ this.requestId = requestId;
+ }
+
+ @Override
+ public void run() {
+ // we get first to make sure we only add the TimeoutInfoHandler if needed.
+ final RequestHolder holder = clientHandlers.get(requestId);
+ if (holder != null) {
+ // add it to the timeout information holder, in case we are going to get a response later
+ long timeoutTime = System.currentTimeMillis();
+ timeoutInfoHandlers.put(requestId, new TimeoutInfoHolder(holder.node(), holder.action(), sentTime,
+ timeoutTime));
+ // now that we have the information visible via timeoutInfoHandlers, we try to remove the request id
+ final RequestHolder removedHolder = clientHandlers.remove(requestId);
+ if (removedHolder != null) {
+ assert removedHolder == holder : "two different holder instances for request [" + requestId + "]";
+ removedHolder.handler().handleException(
+ new ReceiveTimeoutTransportException(holder.node(), holder.action(),
+ "request_id [" + requestId + "] timed out after [" + (timeoutTime - sentTime) + "ms]"));
+ } else {
+ // response was processed, remove timeout info.
+ timeoutInfoHandlers.remove(requestId);
+ }
+ }
+ }
+
+ /**
+ * Cancels timeout handling. This is a best effort only to avoid running it.
+ * Remove the requestId from {@link #clientHandlers} to make sure this doesn't run.
+ */
+ void cancel() {
+ if (clientHandlers.get(requestId) != null) {
+ throw new IllegalStateException("cancel must be called after the requestId [" +
+ requestId + "] has been removed from clientHandlers");
+ }
+ FutureUtils.cancel(future);
+ }
+ }
+
+ private static class TimeoutInfoHolder {
+
+ private final DiscoveryNode node;
+ private final String action;
+ private final long sentTime;
+ private final long timeoutTime;
+
+ TimeoutInfoHolder(DiscoveryNode node, String action, long sentTime, long timeoutTime) {
+ this.node = node;
+ this.action = action;
+ this.sentTime = sentTime;
+ this.timeoutTime = timeoutTime;
+ }
+
+ public DiscoveryNode node() {
+ return node;
+ }
+
+ String action() {
+ return action;
+ }
+
+ long sentTime() {
+ return sentTime;
+ }
+
+ long timeoutTime() {
+ return timeoutTime;
+ }
+ }
+
+ private static class RequestHolder {
+
+ private final TransportResponseHandler handler;
+
+ private final DiscoveryNode node;
+
+ private final String action;
+
+ private final TimeoutHandler timeoutHandler;
+
+ RequestHolder(TransportResponseHandler handler, DiscoveryNode node, String action,
+ TimeoutHandler timeoutHandler) {
+ this.handler = handler;
+ this.node = node;
+ this.action = action;
+ this.timeoutHandler = timeoutHandler;
+ }
+
+ TransportResponseHandler handler() {
+ return handler;
+ }
+
+ public DiscoveryNode node() {
+ return this.node;
+ }
+
+ String action() {
+ return this.action;
+ }
+
+ void cancelTimeout() {
+ if (timeoutHandler != null) {
+ timeoutHandler.cancel();
+ }
+ }
+ }
+
+ /**
+ * This handler wrapper ensures that the response thread executes with the correct thread context.
+ * Before any of the handle methods are invoked we restore the context.
+ * @param thr transport response type
+ */
+ public static final class ContextRestoreResponseHandler
+ implements TransportResponseHandler {
+
+ private final TransportResponseHandler delegate;
+
+ private final Supplier contextSupplier;
+
+ ContextRestoreResponseHandler(Supplier contextSupplier,
+ TransportResponseHandler delegate) {
+ this.delegate = delegate;
+ this.contextSupplier = contextSupplier;
+ }
+
+ @Override
+ public T newInstance() {
+ return delegate.newInstance();
+ }
+
+ @SuppressWarnings("try")
+ @Override
+ public void handleResponse(T response) {
+ try (ThreadContext.StoredContext ignore = contextSupplier.get()) {
+ delegate.handleResponse(response);
+ }
+ }
+
+ @SuppressWarnings("try")
+ @Override
+ public void handleException(TransportException exp) {
+ try (ThreadContext.StoredContext ignore = contextSupplier.get()) {
+ delegate.handleException(exp);
+ }
+ }
+
+ @Override
+ public String executor() {
+ return delegate.executor();
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getName() + "/" + delegate.toString();
+ }
+
+ }
+
+ static class DirectResponseChannel implements TransportChannel {
+
+ private static final String DIRECT_RESPONSE_PROFILE = ".direct";
+
+ private final Logger logger;
+
+ private final DiscoveryNode localNode;
+
+ private final String action;
+
+ private final long requestId;
+
+ private final TransportServiceAdapter adapter;
+
+ private final ThreadPool threadPool;
+
+ DirectResponseChannel(Logger logger, DiscoveryNode localNode, String action, long requestId,
+ TransportServiceAdapter adapter, ThreadPool threadPool) {
+ this.logger = logger;
+ this.localNode = localNode;
+ this.action = action;
+ this.requestId = requestId;
+ this.adapter = adapter;
+ this.threadPool = threadPool;
+ }
+
+ @Override
+ public String action() {
+ return action;
+ }
+
+ @Override
+ public String getProfileName() {
+ return DIRECT_RESPONSE_PROFILE;
+ }
+
+ @Override
+ public void sendResponse(TransportResponse response) throws IOException {
+ sendResponse(response, TransportResponseOptions.EMPTY);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void sendResponse(final TransportResponse response, TransportResponseOptions options)
+ throws IOException {
+ adapter.onResponseSent(requestId, action, response, options);
+ final TransportResponseHandler handler = adapter.onResponseReceived(requestId);
+ if (handler != null) {
+ final String executor = handler.executor();
+ if (ThreadPool.Names.SAME.equals(executor)) {
+ processResponse(handler, response);
+ } else {
+ threadPool.executor(executor).execute(() -> processResponse(handler, response));
+ }
+ }
+ }
+
+ void processResponse(TransportResponseHandler handler, TransportResponse response) {
+ try {
+ handler.handleResponse(response);
+ } catch (Exception e) {
+ processException(handler, wrapInRemote(new ResponseHandlerFailureTransportException(e)));
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void sendResponse(Exception exception) throws IOException {
+ adapter.onResponseSent(requestId, action, exception);
+ final TransportResponseHandler handler = adapter.onResponseReceived(requestId);
+ if (handler != null) {
+ final RemoteTransportException rtx = wrapInRemote(exception);
+ final String executor = handler.executor();
+ if (ThreadPool.Names.SAME.equals(executor)) {
+ processException(handler, rtx);
+ } else {
+ threadPool.executor(handler.executor()).execute(() -> processException(handler, rtx));
+ }
+ }
+ }
+
+ RemoteTransportException wrapInRemote(Exception e) {
+ if (e instanceof RemoteTransportException) {
+ return (RemoteTransportException) e;
+ }
+ return new RemoteTransportException(localNode.getName(), localNode.getAddress(), action, e);
+ }
+
+ void processException(final TransportResponseHandler handler, final RemoteTransportException rtx) {
+ try {
+ handler.handleException(rtx);
+ } catch (Exception e) {
+ logger.error((Supplier) () -> new ParameterizedMessage(
+ "failed to handle exception for action [{}], handler [{}]", action, handler), e);
+ }
+ }
+
+ @Override
+ public long getRequestId() {
+ return requestId;
+ }
+
+ @Override
+ public String getChannelType() {
+ return "direct";
+ }
+
+ @Override
+ public Version getVersion() {
+ return localNode.getVersion();
+ }
+ }
+}
diff --git a/build.gradle b/build.gradle
index 2f8478a..49dfe7d 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1,10 +1,27 @@
+import java.time.ZonedDateTime
+import java.time.format.DateTimeFormatter
+
+buildscript {
+ repositories {
+ jcenter()
+ maven {
+ url 'http://xbib.org/repository'
+ }
+ }
+ dependencies {
+ classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.3.2.4"
+ }
+}
plugins {
- id "org.sonarqube" version "2.2"
+ id "org.sonarqube" version "2.6.1"
+ id "io.codearte.nexus-staging" version "0.11.0"
+ id "org.xbib.gradle.plugin.asciidoctor" version "1.6.0.1"
}
-printf "Host: %s\nOS: %s %s %s\nJVM: %s %s %s %s\nGroovy: %s\nGradle: %s\n" +
+printf "Date: %s\nHost: %s\nOS: %s %s %s\nJava: %s %s %s %s\nGradle: %s Groovy: %s Java: %s\n" +
"Build: group: ${project.group} name: ${project.name} version: ${project.version}\n",
+ ZonedDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME),
InetAddress.getLocalHost(),
System.getProperty("os.name"),
System.getProperty("os.arch"),
@@ -13,102 +30,121 @@ printf "Host: %s\nOS: %s %s %s\nJVM: %s %s %s %s\nGroovy: %s\nGradle: %s\n" +
System.getProperty("java.vm.version"),
System.getProperty("java.vm.vendor"),
System.getProperty("java.vm.name"),
- GroovySystem.getVersion(),
- gradle.gradleVersion
-
-apply plugin: 'java'
-apply plugin: 'maven'
-apply plugin: 'signing'
-apply plugin: 'findbugs'
-apply plugin: 'pmd'
-apply plugin: 'checkstyle'
-apply plugin: "jacoco"
-
-apply from: 'gradle/ext.gradle'
-
-sourceSets {
- integrationTest {
- java {
- srcDir file('src/integration-test/java')
- compileClasspath += main.output
- compileClasspath += test.output
- }
- resources {
- srcDir file('src/integration-test/resources')
- }
- }
-}
+ gradle.gradleVersion, GroovySystem.getVersion(), JavaVersion.current()
-configurations {
- wagon
- integrationTestCompile.extendsFrom testCompile
- integrationTestRuntime.extendsFrom testRuntime
+
+apply plugin: "io.codearte.nexus-staging"
+apply plugin: 'org.xbib.gradle.plugin.asciidoctor'
+
+ext {
+ user = 'jprante'
+ name = 'elx'
+ description = 'Elasticsearch extensions'
+ scmUrl = 'https://github.com/' + user + '/' + name
+ scmConnection = 'scm:git:git://github.com/' + user + '/' + name + '.git'
+ scmDeveloperConnection = 'scm:git:git://github.com/' + user + '/' + name + '.git'
}
-dependencies {
- compile "org.xbib:metrics:1.0.0"
- compile("org.elasticsearch:elasticsearch:2.2.1") {
- exclude module: "securesm"
+subprojects {
+ apply plugin: 'java'
+ apply plugin: 'maven'
+ apply plugin: 'signing'
+
+ configurations {
+ wagon
+ alpnagent
+ asciidoclet
}
- testCompile "net.java.dev.jna:jna:4.1.0"
- testCompile "junit:junit:4.12"
- testCompile "org.apache.logging.log4j:log4j-core:2.7"
- testCompile "org.apache.logging.log4j:log4j-slf4j-impl:2.7"
- wagon 'org.apache.maven.wagon:wagon-ssh-external:2.10'
-}
-sourceCompatibility = JavaVersion.VERSION_1_8
-targetCompatibility = JavaVersion.VERSION_1_8
+ dependencies {
+ alpnagent "org.mortbay.jetty.alpn:jetty-alpn-agent:${project.property('alpnagent.version')}"
+ asciidoclet "org.xbib:asciidoclet:${project.property('asciidoclet.version')}"
+ wagon "org.apache.maven.wagon:wagon-ssh:${project.property('wagon.version')}"
+ }
-[compileJava, compileTestJava]*.options*.encoding = 'UTF-8'
-tasks.withType(JavaCompile) {
- options.compilerArgs << "-Xlint:all"
-}
+ compileJava {
+ sourceCompatibility = JavaVersion.VERSION_11
+ targetCompatibility = JavaVersion.VERSION_11
+ }
+ compileTestJava {
+ sourceCompatibility = JavaVersion.VERSION_11
+ targetCompatibility = JavaVersion.VERSION_11
+ }
-task integrationTest(type: Test) {
- include '**/MiscTestSuite.class'
- include '**/BulkNodeTestSuite.class'
- include '**/BulkTransportTestSuite.class'
- testClassesDir = sourceSets.integrationTest.output.classesDir
- classpath = configurations.integrationTestCompile
- classpath += configurations.integrationTestRuntime
- classpath += sourceSets.main.output
- classpath += sourceSets.test.output
- classpath += sourceSets.integrationTest.output
- outputs.upToDateWhen { false }
- systemProperty 'path.home', projectDir.absolutePath
- testLogging.showStandardStreams = true
-}
+ jar {
+ baseName "${rootProject.name}-${project.name}"
+ }
-integrationTest.mustRunAfter test
-check.dependsOn integrationTest
+ javadoc {
+ options.docletpath = configurations.asciidoclet.files.asType(List)
+ options.doclet = 'org.xbib.asciidoclet.Asciidoclet'
+ options.overview = "src/docs/asciidoclet/overview.adoc"
+ options.addStringOption "-base-dir", "${projectDir}"
+ options.addStringOption "-attribute",
+ "name=${project.name},version=${project.version},title-link=https://github.com/jprante/${project.name}"
+ configure(options) {
+ noTimestamp = true
+ }
+ }
-clean {
- delete "plugins"
- delete "logs"
-}
+ /*task javadocJar(type: Jar, dependsOn: classes) {
+ baseName "${rootProject.name}-${project.name}"
+ from javadoc
+ into "build/tmp"
+ classifier 'javadoc'
+ }
-task javadocJar(type: Jar, dependsOn: classes) {
- from javadoc
- into "build/tmp"
- classifier 'javadoc'
-}
+ task sourcesJar(type: Jar, dependsOn: classes) {
+ baseName "${rootProject.name}-${project.name}"
+ from sourceSets.main.allSource
+ into "build/tmp"
+ classifier 'sources'
+ }
-task sourcesJar(type: Jar, dependsOn: classes) {
- from sourceSets.main.allSource
- into "build/tmp"
- classifier 'sources'
-}
+ artifacts {
+ archives javadocJar, sourcesJar
+ }*/
+
+ if (project.hasProperty('signing.keyId')) {
+ signing {
+ sign configurations.archives
+ }
+ }
+
+ apply from: "${rootProject.projectDir}/gradle/ext.gradle"
+ apply from: "${rootProject.projectDir}/gradle/publish.gradle"
+ //apply from: "${rootProject.projectDir}/gradle/sonarqube.gradle"
-artifacts {
- archives javadocJar, sourcesJar
}
-if (project.hasProperty('signing.keyId')) {
- signing {
- sign configurations.archives
+/*asciidoctor {
+ attributes toc: 'left',
+ doctype: 'book',
+ icons: 'font',
+ encoding: 'utf-8',
+ sectlink: true,
+ sectanchors: true,
+ linkattrs: true,
+ imagesdir: 'img',
+ 'source-highlighter': 'coderay'
+}*/
+
+/*
+task aggregatedJavadoc(type: Javadoc) {
+ group = 'aggregation'
+ description = 'Generates aggregated Javadoc API documentation.'
+ title = "$description $version API"
+ destinationDir = file("$buildDir/docs/javadoc")
+ def sourceProjects = subprojects.findAll {
+ it.plugins.hasPlugin('java') || it.plugins.hasPlugin('groovy')
+ }
+ source sourceProjects.collect {
+ it.sourceSets.main.allJava
}
+ classpath = files(sourceProjects.collect {
+ it.sourceSets.main.runtimeClasspath
+ })
+ //options.overview = 'gradle/api/overview.html'
+ options.showFromProtected()
}
-
-apply from: 'gradle/publish.gradle'
-apply from: 'gradle/sonarqube.gradle'
+*/
diff --git a/common/build.gradle b/common/build.gradle
new file mode 100644
index 0000000..7e0f3cb
--- /dev/null
+++ b/common/build.gradle
@@ -0,0 +1,65 @@
+buildscript {
+ repositories {
+ jcenter()
+ maven {
+ url 'http://xbib.org/repository'
+ }
+ }
+ dependencies {
+ classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.3.2.4"
+ }
+}
+
+apply plugin: 'org.xbib.gradle.plugin.elasticsearch.build'
+
+configurations {
+ main
+ tests
+}
+
+dependencies {
+ compile project(':api')
+ compile "org.xbib:metrics:${project.property('xbib-metrics.version')}"
+ compileOnly "org.apache.logging.log4j:log4j-api:${project.property('log4j.version')}"
+ testCompile "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}"
+ testRuntime "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}"
+}
+
+jar {
+ baseName "${rootProject.name}-common"
+}
+
+/*
+task testJar(type: Jar, dependsOn: testClasses) {
+ baseName = "${project.archivesBaseName}-tests"
+ from sourceSets.test.output
+}
+*/
+
+artifacts {
+ main jar
+ tests testJar
+ archives sourcesJar, javadocJar
+}
+
+test {
+ enabled = false
+ jvmArgs "-javaagent:" + configurations.alpnagent.asPath
+ systemProperty 'path.home', project.buildDir.absolutePath
+ testLogging {
+ showStandardStreams = true
+ exceptionFormat = 'full'
+ }
+}
+
+randomizedTest {
+ enabled = false
+}
+
+esTest {
+ // test with the jars, not the classes, for security manager
+ // classpath = files(configurations.testRuntime) + configurations.main.artifacts.files + configurations.tests.artifacts.files
+ systemProperty 'tests.security.manager', 'true'
+}
+esTest.dependsOn jar, testJar
+
diff --git a/common/config/checkstyle/checkstyle.xml b/common/config/checkstyle/checkstyle.xml
new file mode 100644
index 0000000..8cb4438
--- /dev/null
+++ b/common/config/checkstyle/checkstyle.xml
@@ -0,0 +1,321 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/common/src/docs/asciidoc/css/foundation.css b/common/src/docs/asciidoc/css/foundation.css
new file mode 100644
index 0000000..27be611
--- /dev/null
+++ b/common/src/docs/asciidoc/css/foundation.css
@@ -0,0 +1,684 @@
+/*! normalize.css v2.1.2 | MIT License | git.io/normalize */
+/* ========================================================================== HTML5 display definitions ========================================================================== */
+/** Correct `block` display not defined in IE 8/9. */
+article, aside, details, figcaption, figure, footer, header, hgroup, main, nav, section, summary { display: block; }
+
+/** Correct `inline-block` display not defined in IE 8/9. */
+audio, canvas, video { display: inline-block; }
+
+/** Prevent modern browsers from displaying `audio` without controls. Remove excess height in iOS 5 devices. */
+audio:not([controls]) { display: none; height: 0; }
+
+/** Address `[hidden]` styling not present in IE 8/9. Hide the `template` element in IE, Safari, and Firefox < 22. */
+[hidden], template { display: none; }
+
+script { display: none !important; }
+
+/* ========================================================================== Base ========================================================================== */
+/** 1. Set default font family to sans-serif. 2. Prevent iOS text size adjust after orientation change, without disabling user zoom. */
+html { font-family: sans-serif; /* 1 */ -ms-text-size-adjust: 100%; /* 2 */ -webkit-text-size-adjust: 100%; /* 2 */ }
+
+/** Remove default margin. */
+body { margin: 0; }
+
+/* ========================================================================== Links ========================================================================== */
+/** Remove the gray background color from active links in IE 10. */
+a { background: transparent; }
+
+/** Address `outline` inconsistency between Chrome and other browsers. */
+a:focus { outline: thin dotted; }
+
+/** Improve readability when focused and also mouse hovered in all browsers. */
+a:active, a:hover { outline: 0; }
+
+/* ========================================================================== Typography ========================================================================== */
+/** Address variable `h1` font-size and margin within `section` and `article` contexts in Firefox 4+, Safari 5, and Chrome. */
+h1 { font-size: 2em; margin: 0.67em 0; }
+
+/** Address styling not present in IE 8/9, Safari 5, and Chrome. */
+abbr[title] { border-bottom: 1px dotted; }
+
+/** Address style set to `bolder` in Firefox 4+, Safari 5, and Chrome. */
+b, strong { font-weight: bold; }
+
+/** Address styling not present in Safari 5 and Chrome. */
+dfn { font-style: italic; }
+
+/** Address differences between Firefox and other browsers. */
+hr { -moz-box-sizing: content-box; box-sizing: content-box; height: 0; }
+
+/** Address styling not present in IE 8/9. */
+mark { background: #ff0; color: #000; }
+
+/** Correct font family set oddly in Safari 5 and Chrome. */
+code, kbd, pre, samp { font-family: monospace, serif; font-size: 1em; }
+
+/** Improve readability of pre-formatted text in all browsers. */
+pre { white-space: pre-wrap; }
+
+/** Set consistent quote types. */
+q { quotes: "\201C" "\201D" "\2018" "\2019"; }
+
+/** Address inconsistent and variable font size in all browsers. */
+small { font-size: 80%; }
+
+/** Prevent `sub` and `sup` affecting `line-height` in all browsers. */
+sub, sup { font-size: 75%; line-height: 0; position: relative; vertical-align: baseline; }
+
+sup { top: -0.5em; }
+
+sub { bottom: -0.25em; }
+
+/* ========================================================================== Embedded content ========================================================================== */
+/** Remove border when inside `a` element in IE 8/9. */
+img { border: 0; }
+
+/** Correct overflow displayed oddly in IE 9. */
+svg:not(:root) { overflow: hidden; }
+
+/* ========================================================================== Figures ========================================================================== */
+/** Address margin not present in IE 8/9 and Safari 5. */
+figure { margin: 0; }
+
+/* ========================================================================== Forms ========================================================================== */
+/** Define consistent border, margin, and padding. */
+fieldset { border: 1px solid #c0c0c0; margin: 0 2px; padding: 0.35em 0.625em 0.75em; }
+
+/** 1. Correct `color` not being inherited in IE 8/9. 2. Remove padding so people aren't caught out if they zero out fieldsets. */
+legend { border: 0; /* 1 */ padding: 0; /* 2 */ }
+
+/** 1. Correct font family not being inherited in all browsers. 2. Correct font size not being inherited in all browsers. 3. Address margins set differently in Firefox 4+, Safari 5, and Chrome. */
+button, input, select, textarea { font-family: inherit; /* 1 */ font-size: 100%; /* 2 */ margin: 0; /* 3 */ }
+
+/** Address Firefox 4+ setting `line-height` on `input` using `!important` in the UA stylesheet. */
+button, input { line-height: normal; }
+
+/** Address inconsistent `text-transform` inheritance for `button` and `select`. All other form control elements do not inherit `text-transform` values. Correct `button` style inheritance in Chrome, Safari 5+, and IE 8+. Correct `select` style inheritance in Firefox 4+ and Opera. */
+button, select { text-transform: none; }
+
+/** 1. Avoid the WebKit bug in Android 4.0.* where (2) destroys native `audio` and `video` controls. 2. Correct inability to style clickable `input` types in iOS. 3. Improve usability and consistency of cursor style between image-type `input` and others. */
+button, html input[type="button"], input[type="reset"], input[type="submit"] { -webkit-appearance: button; /* 2 */ cursor: pointer; /* 3 */ }
+
+/** Re-set default cursor for disabled elements. */
+button[disabled], html input[disabled] { cursor: default; }
+
+/** 1. Address box sizing set to `content-box` in IE 8/9. 2. Remove excess padding in IE 8/9. */
+input[type="checkbox"], input[type="radio"] { box-sizing: border-box; /* 1 */ padding: 0; /* 2 */ }
+
+/** 1. Address `appearance` set to `searchfield` in Safari 5 and Chrome. 2. Address `box-sizing` set to `border-box` in Safari 5 and Chrome (include `-moz` to future-proof). */
+input[type="search"] { -webkit-appearance: textfield; /* 1 */ -moz-box-sizing: content-box; -webkit-box-sizing: content-box; /* 2 */ box-sizing: content-box; }
+
+/** Remove inner padding and search cancel button in Safari 5 and Chrome on OS X. */
+input[type="search"]::-webkit-search-cancel-button, input[type="search"]::-webkit-search-decoration { -webkit-appearance: none; }
+
+/** Remove inner padding and border in Firefox 4+. */
+button::-moz-focus-inner, input::-moz-focus-inner { border: 0; padding: 0; }
+
+/** 1. Remove default vertical scrollbar in IE 8/9. 2. Improve readability and alignment in all browsers. */
+textarea { overflow: auto; /* 1 */ vertical-align: top; /* 2 */ }
+
+/* ========================================================================== Tables ========================================================================== */
+/** Remove most spacing between table cells. */
+table { border-collapse: collapse; border-spacing: 0; }
+
+meta.foundation-mq-small { font-family: "only screen and (min-width: 768px)"; width: 768px; }
+
+meta.foundation-mq-medium { font-family: "only screen and (min-width:1280px)"; width: 1280px; }
+
+meta.foundation-mq-large { font-family: "only screen and (min-width:1440px)"; width: 1440px; }
+
+*, *:before, *:after { -moz-box-sizing: border-box; -webkit-box-sizing: border-box; box-sizing: border-box; }
+
+html, body { font-size: 100%; }
+
+body { background: white; color: #222222; padding: 0; margin: 0; font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; font-weight: normal; font-style: normal; line-height: 1; position: relative; cursor: auto; }
+
+a:hover { cursor: pointer; }
+
+img, object, embed { max-width: 100%; height: auto; }
+
+object, embed { height: 100%; }
+
+img { -ms-interpolation-mode: bicubic; }
+
+#map_canvas img, #map_canvas embed, #map_canvas object, .map_canvas img, .map_canvas embed, .map_canvas object { max-width: none !important; }
+
+.left { float: left !important; }
+
+.right { float: right !important; }
+
+.text-left { text-align: left !important; }
+
+.text-right { text-align: right !important; }
+
+.text-center { text-align: center !important; }
+
+.text-justify { text-align: justify !important; }
+
+.hide { display: none; }
+
+.antialiased { -webkit-font-smoothing: antialiased; }
+
+img { display: inline-block; vertical-align: middle; }
+
+textarea { height: auto; min-height: 50px; }
+
+select { width: 100%; }
+
+object, svg { display: inline-block; vertical-align: middle; }
+
+.center { margin-left: auto; margin-right: auto; }
+
+.spread { width: 100%; }
+
+p.lead, .paragraph.lead > p, #preamble > .sectionbody > .paragraph:first-of-type p { font-size: 1.21875em; line-height: 1.6; }
+
+.subheader, .admonitionblock td.content > .title, .audioblock > .title, .exampleblock > .title, .imageblock > .title, .listingblock > .title, .literalblock > .title, .stemblock > .title, .openblock > .title, .paragraph > .title, .quoteblock > .title, table.tableblock > .title, .verseblock > .title, .videoblock > .title, .dlist > .title, .olist > .title, .ulist > .title, .qlist > .title, .hdlist > .title { line-height: 1.4; color: #6f6f6f; font-weight: 300; margin-top: 0.2em; margin-bottom: 0.5em; }
+
+/* Typography resets */
+div, dl, dt, dd, ul, ol, li, h1, h2, h3, #toctitle, .sidebarblock > .content > .title, h4, h5, h6, pre, form, p, blockquote, th, td { margin: 0; padding: 0; direction: ltr; }
+
+/* Default Link Styles */
+a { color: #2ba6cb; text-decoration: none; line-height: inherit; }
+a:hover, a:focus { color: #2795b6; }
+a img { border: none; }
+
+/* Default paragraph styles */
+p { font-family: inherit; font-weight: normal; font-size: 1em; line-height: 1.6; margin-bottom: 1.25em; text-rendering: optimizeLegibility; }
+p aside { font-size: 0.875em; line-height: 1.35; font-style: italic; }
+
+/* Default header styles */
+h1, h2, h3, #toctitle, .sidebarblock > .content > .title, h4, h5, h6 { font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; font-weight: bold; font-style: normal; color: #222222; text-rendering: optimizeLegibility; margin-top: 1em; margin-bottom: 0.5em; line-height: 1.2125em; }
+h1 small, h2 small, h3 small, #toctitle small, .sidebarblock > .content > .title small, h4 small, h5 small, h6 small { font-size: 60%; color: #6f6f6f; line-height: 0; }
+
+h1 { font-size: 2.125em; }
+
+h2 { font-size: 1.6875em; }
+
+h3, #toctitle, .sidebarblock > .content > .title { font-size: 1.375em; }
+
+h4 { font-size: 1.125em; }
+
+h5 { font-size: 1.125em; }
+
+h6 { font-size: 1em; }
+
+hr { border: solid #dddddd; border-width: 1px 0 0; clear: both; margin: 1.25em 0 1.1875em; height: 0; }
+
+/* Helpful Typography Defaults */
+em, i { font-style: italic; line-height: inherit; }
+
+strong, b { font-weight: bold; line-height: inherit; }
+
+small { font-size: 60%; line-height: inherit; }
+
+code { font-family: Consolas, "Liberation Mono", Courier, monospace; font-weight: bold; color: #7f0a0c; }
+
+/* Lists */
+ul, ol, dl { font-size: 1em; line-height: 1.6; margin-bottom: 1.25em; list-style-position: outside; font-family: inherit; }
+
+ul, ol { margin-left: 1.5em; }
+ul.no-bullet, ol.no-bullet { margin-left: 1.5em; }
+
+/* Unordered Lists */
+ul li ul, ul li ol { margin-left: 1.25em; margin-bottom: 0; font-size: 1em; /* Override nested font-size change */ }
+ul.square li ul, ul.circle li ul, ul.disc li ul { list-style: inherit; }
+ul.square { list-style-type: square; }
+ul.circle { list-style-type: circle; }
+ul.disc { list-style-type: disc; }
+ul.no-bullet { list-style: none; }
+
+/* Ordered Lists */
+ol li ul, ol li ol { margin-left: 1.25em; margin-bottom: 0; }
+
+/* Definition Lists */
+dl dt { margin-bottom: 0.3125em; font-weight: bold; }
+dl dd { margin-bottom: 1.25em; }
+
+/* Abbreviations */
+abbr, acronym { text-transform: uppercase; font-size: 90%; color: #222222; border-bottom: 1px dotted #dddddd; cursor: help; }
+
+abbr { text-transform: none; }
+
+/* Blockquotes */
+blockquote { margin: 0 0 1.25em; padding: 0.5625em 1.25em 0 1.1875em; border-left: 1px solid #dddddd; }
+blockquote cite { display: block; font-size: 0.8125em; color: #555555; }
+blockquote cite:before { content: "\2014 \0020"; }
+blockquote cite a, blockquote cite a:visited { color: #555555; }
+
+blockquote, blockquote p { line-height: 1.6; color: #6f6f6f; }
+
+/* Microformats */
+.vcard { display: inline-block; margin: 0 0 1.25em 0; border: 1px solid #dddddd; padding: 0.625em 0.75em; }
+.vcard li { margin: 0; display: block; }
+.vcard .fn { font-weight: bold; font-size: 0.9375em; }
+
+.vevent .summary { font-weight: bold; }
+.vevent abbr { cursor: auto; text-decoration: none; font-weight: bold; border: none; padding: 0 0.0625em; }
+
+@media only screen and (min-width: 768px) { h1, h2, h3, #toctitle, .sidebarblock > .content > .title, h4, h5, h6 { line-height: 1.4; }
+ h1 { font-size: 2.75em; }
+ h2 { font-size: 2.3125em; }
+ h3, #toctitle, .sidebarblock > .content > .title { font-size: 1.6875em; }
+ h4 { font-size: 1.4375em; } }
+/* Tables */
+table { background: white; margin-bottom: 1.25em; border: solid 1px #dddddd; }
+table thead, table tfoot { background: whitesmoke; font-weight: bold; }
+table thead tr th, table thead tr td, table tfoot tr th, table tfoot tr td { padding: 0.5em 0.625em 0.625em; font-size: inherit; color: #222222; text-align: left; }
+table tr th, table tr td { padding: 0.5625em 0.625em; font-size: inherit; color: #222222; }
+table tr.even, table tr.alt, table tr:nth-of-type(even) { background: #f9f9f9; }
+table thead tr th, table tfoot tr th, table tbody tr td, table tr td, table tfoot tr td { display: table-cell; line-height: 1.4; }
+
+body { -moz-osx-font-smoothing: grayscale; -webkit-font-smoothing: antialiased; tab-size: 4; }
+
+h1, h2, h3, #toctitle, .sidebarblock > .content > .title, h4, h5, h6 { line-height: 1.4; }
+
+.clearfix:before, .clearfix:after, .float-group:before, .float-group:after { content: " "; display: table; }
+.clearfix:after, .float-group:after { clear: both; }
+
+*:not(pre) > code { font-size: inherit; font-style: normal !important; letter-spacing: 0; padding: 0; line-height: inherit; word-wrap: break-word; }
+*:not(pre) > code.nobreak { word-wrap: normal; }
+*:not(pre) > code.nowrap { white-space: nowrap; }
+
+pre, pre > code { line-height: 1.4; color: black; font-family: monospace, serif; font-weight: normal; }
+
+em em { font-style: normal; }
+
+strong strong { font-weight: normal; }
+
+.keyseq { color: #555555; }
+
+kbd { font-family: Consolas, "Liberation Mono", Courier, monospace; display: inline-block; color: #222222; font-size: 0.65em; line-height: 1.45; background-color: #f7f7f7; border: 1px solid #ccc; -webkit-border-radius: 3px; border-radius: 3px; -webkit-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.2), 0 0 0 0.1em white inset; box-shadow: 0 1px 0 rgba(0, 0, 0, 0.2), 0 0 0 0.1em white inset; margin: 0 0.15em; padding: 0.2em 0.5em; vertical-align: middle; position: relative; top: -0.1em; white-space: nowrap; }
+
+.keyseq kbd:first-child { margin-left: 0; }
+
+.keyseq kbd:last-child { margin-right: 0; }
+
+.menuseq, .menu { color: #090909; }
+
+b.button:before, b.button:after { position: relative; top: -1px; font-weight: normal; }
+
+b.button:before { content: "["; padding: 0 3px 0 2px; }
+
+b.button:after { content: "]"; padding: 0 2px 0 3px; }
+
+#header, #content, #footnotes, #footer { width: 100%; margin-left: auto; margin-right: auto; margin-top: 0; margin-bottom: 0; max-width: 62.5em; *zoom: 1; position: relative; padding-left: 0.9375em; padding-right: 0.9375em; }
+#header:before, #header:after, #content:before, #content:after, #footnotes:before, #footnotes:after, #footer:before, #footer:after { content: " "; display: table; }
+#header:after, #content:after, #footnotes:after, #footer:after { clear: both; }
+
+#content { margin-top: 1.25em; }
+
+#content:before { content: none; }
+
+#header > h1:first-child { color: black; margin-top: 2.25rem; margin-bottom: 0; }
+#header > h1:first-child + #toc { margin-top: 8px; border-top: 1px solid #dddddd; }
+#header > h1:only-child, body.toc2 #header > h1:nth-last-child(2) { border-bottom: 1px solid #dddddd; padding-bottom: 8px; }
+#header .details { border-bottom: 1px solid #dddddd; line-height: 1.45; padding-top: 0.25em; padding-bottom: 0.25em; padding-left: 0.25em; color: #555555; display: -ms-flexbox; display: -webkit-flex; display: flex; -ms-flex-flow: row wrap; -webkit-flex-flow: row wrap; flex-flow: row wrap; }
+#header .details span:first-child { margin-left: -0.125em; }
+#header .details span.email a { color: #6f6f6f; }
+#header .details br { display: none; }
+#header .details br + span:before { content: "\00a0\2013\00a0"; }
+#header .details br + span.author:before { content: "\00a0\22c5\00a0"; color: #6f6f6f; }
+#header .details br + span#revremark:before { content: "\00a0|\00a0"; }
+#header #revnumber { text-transform: capitalize; }
+#header #revnumber:after { content: "\00a0"; }
+
+#content > h1:first-child:not([class]) { color: black; border-bottom: 1px solid #dddddd; padding-bottom: 8px; margin-top: 0; padding-top: 1rem; margin-bottom: 1.25rem; }
+
+#toc { border-bottom: 1px solid #dddddd; padding-bottom: 0.5em; }
+#toc > ul { margin-left: 0.125em; }
+#toc ul.sectlevel0 > li > a { font-style: italic; }
+#toc ul.sectlevel0 ul.sectlevel1 { margin: 0.5em 0; }
+#toc ul { font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; list-style-type: none; }
+#toc li { line-height: 1.3334; margin-top: 0.3334em; }
+#toc a { text-decoration: none; }
+#toc a:active { text-decoration: underline; }
+
+#toctitle { color: #6f6f6f; font-size: 1.2em; }
+
+@media only screen and (min-width: 768px) { #toctitle { font-size: 1.375em; }
+ body.toc2 { padding-left: 15em; padding-right: 0; }
+ #toc.toc2 { margin-top: 0 !important; background-color: #f2f2f2; position: fixed; width: 15em; left: 0; top: 0; border-right: 1px solid #dddddd; border-top-width: 0 !important; border-bottom-width: 0 !important; z-index: 1000; padding: 1.25em 1em; height: 100%; overflow: auto; }
+ #toc.toc2 #toctitle { margin-top: 0; margin-bottom: 0.8rem; font-size: 1.2em; }
+ #toc.toc2 > ul { font-size: 0.9em; margin-bottom: 0; }
+ #toc.toc2 ul ul { margin-left: 0; padding-left: 1em; }
+ #toc.toc2 ul.sectlevel0 ul.sectlevel1 { padding-left: 0; margin-top: 0.5em; margin-bottom: 0.5em; }
+ body.toc2.toc-right { padding-left: 0; padding-right: 15em; }
+ body.toc2.toc-right #toc.toc2 { border-right-width: 0; border-left: 1px solid #dddddd; left: auto; right: 0; } }
+@media only screen and (min-width: 1280px) { body.toc2 { padding-left: 20em; padding-right: 0; }
+ #toc.toc2 { width: 20em; }
+ #toc.toc2 #toctitle { font-size: 1.375em; }
+ #toc.toc2 > ul { font-size: 0.95em; }
+ #toc.toc2 ul ul { padding-left: 1.25em; }
+ body.toc2.toc-right { padding-left: 0; padding-right: 20em; } }
+#content #toc { border-style: solid; border-width: 1px; border-color: #d9d9d9; margin-bottom: 1.25em; padding: 1.25em; background: #f2f2f2; -webkit-border-radius: 0; border-radius: 0; }
+#content #toc > :first-child { margin-top: 0; }
+#content #toc > :last-child { margin-bottom: 0; }
+
+#footer { max-width: 100%; background-color: #222222; padding: 1.25em; }
+
+#footer-text { color: #dddddd; line-height: 1.44; }
+
+.sect1 { padding-bottom: 0.625em; }
+
+@media only screen and (min-width: 768px) { .sect1 { padding-bottom: 1.25em; } }
+.sect1 + .sect1 { border-top: 1px solid #dddddd; }
+
+#content h1 > a.anchor, h2 > a.anchor, h3 > a.anchor, #toctitle > a.anchor, .sidebarblock > .content > .title > a.anchor, h4 > a.anchor, h5 > a.anchor, h6 > a.anchor { position: absolute; z-index: 1001; width: 1.5ex; margin-left: -1.5ex; display: block; text-decoration: none !important; visibility: hidden; text-align: center; font-weight: normal; }
+#content h1 > a.anchor:before, h2 > a.anchor:before, h3 > a.anchor:before, #toctitle > a.anchor:before, .sidebarblock > .content > .title > a.anchor:before, h4 > a.anchor:before, h5 > a.anchor:before, h6 > a.anchor:before { content: "\00A7"; font-size: 0.85em; display: block; padding-top: 0.1em; }
+#content h1:hover > a.anchor, #content h1 > a.anchor:hover, h2:hover > a.anchor, h2 > a.anchor:hover, h3:hover > a.anchor, #toctitle:hover > a.anchor, .sidebarblock > .content > .title:hover > a.anchor, h3 > a.anchor:hover, #toctitle > a.anchor:hover, .sidebarblock > .content > .title > a.anchor:hover, h4:hover > a.anchor, h4 > a.anchor:hover, h5:hover > a.anchor, h5 > a.anchor:hover, h6:hover > a.anchor, h6 > a.anchor:hover { visibility: visible; }
+#content h1 > a.link, h2 > a.link, h3 > a.link, #toctitle > a.link, .sidebarblock > .content > .title > a.link, h4 > a.link, h5 > a.link, h6 > a.link { color: #222222; text-decoration: none; }
+#content h1 > a.link:hover, h2 > a.link:hover, h3 > a.link:hover, #toctitle > a.link:hover, .sidebarblock > .content > .title > a.link:hover, h4 > a.link:hover, h5 > a.link:hover, h6 > a.link:hover { color: #151515; }
+
+.audioblock, .imageblock, .literalblock, .listingblock, .stemblock, .videoblock { margin-bottom: 1.25em; }
+
+.admonitionblock td.content > .title, .audioblock > .title, .exampleblock > .title, .imageblock > .title, .listingblock > .title, .literalblock > .title, .stemblock > .title, .openblock > .title, .paragraph > .title, .quoteblock > .title, table.tableblock > .title, .verseblock > .title, .videoblock > .title, .dlist > .title, .olist > .title, .ulist > .title, .qlist > .title, .hdlist > .title { text-rendering: optimizeLegibility; text-align: left; }
+
+table.tableblock > caption.title { white-space: nowrap; overflow: visible; max-width: 0; }
+
+.paragraph.lead > p, #preamble > .sectionbody > .paragraph:first-of-type p { color: black; }
+
+table.tableblock #preamble > .sectionbody > .paragraph:first-of-type p { font-size: inherit; }
+
+.admonitionblock > table { border-collapse: separate; border: 0; background: none; width: 100%; }
+.admonitionblock > table td.icon { text-align: center; width: 80px; }
+.admonitionblock > table td.icon img { max-width: initial; }
+.admonitionblock > table td.icon .title { font-weight: bold; font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; text-transform: uppercase; }
+.admonitionblock > table td.content { padding-left: 1.125em; padding-right: 1.25em; border-left: 1px solid #dddddd; color: #555555; }
+.admonitionblock > table td.content > :last-child > :last-child { margin-bottom: 0; }
+
+.exampleblock > .content { border-style: solid; border-width: 1px; border-color: #e6e6e6; margin-bottom: 1.25em; padding: 1.25em; background: white; -webkit-border-radius: 0; border-radius: 0; }
+.exampleblock > .content > :first-child { margin-top: 0; }
+.exampleblock > .content > :last-child { margin-bottom: 0; }
+
+.sidebarblock { border-style: solid; border-width: 1px; border-color: #d9d9d9; margin-bottom: 1.25em; padding: 1.25em; background: #f2f2f2; -webkit-border-radius: 0; border-radius: 0; }
+.sidebarblock > :first-child { margin-top: 0; }
+.sidebarblock > :last-child { margin-bottom: 0; }
+.sidebarblock > .content > .title { color: #6f6f6f; margin-top: 0; }
+
+.exampleblock > .content > :last-child > :last-child, .exampleblock > .content .olist > ol > li:last-child > :last-child, .exampleblock > .content .ulist > ul > li:last-child > :last-child, .exampleblock > .content .qlist > ol > li:last-child > :last-child, .sidebarblock > .content > :last-child > :last-child, .sidebarblock > .content .olist > ol > li:last-child > :last-child, .sidebarblock > .content .ulist > ul > li:last-child > :last-child, .sidebarblock > .content .qlist > ol > li:last-child > :last-child { margin-bottom: 0; }
+
+.literalblock pre, .listingblock pre:not(.highlight), .listingblock pre[class="highlight"], .listingblock pre[class^="highlight "], .listingblock pre.CodeRay, .listingblock pre.prettyprint { background: #eeeeee; }
+.sidebarblock .literalblock pre, .sidebarblock .listingblock pre:not(.highlight), .sidebarblock .listingblock pre[class="highlight"], .sidebarblock .listingblock pre[class^="highlight "], .sidebarblock .listingblock pre.CodeRay, .sidebarblock .listingblock pre.prettyprint { background: #f2f1f1; }
+
+.literalblock pre, .literalblock pre[class], .listingblock pre, .listingblock pre[class] { border: 1px solid #cccccc; -webkit-border-radius: 0; border-radius: 0; word-wrap: break-word; padding: 0.8em 0.8em 0.65em 0.8em; font-size: 0.8125em; }
+.literalblock pre.nowrap, .literalblock pre[class].nowrap, .listingblock pre.nowrap, .listingblock pre[class].nowrap { overflow-x: auto; white-space: pre; word-wrap: normal; }
+@media only screen and (min-width: 768px) { .literalblock pre, .literalblock pre[class], .listingblock pre, .listingblock pre[class] { font-size: 0.90625em; } }
+@media only screen and (min-width: 1280px) { .literalblock pre, .literalblock pre[class], .listingblock pre, .listingblock pre[class] { font-size: 1em; } }
+
+.literalblock.output pre { color: #eeeeee; background-color: black; }
+
+.listingblock pre.highlightjs { padding: 0; }
+.listingblock pre.highlightjs > code { padding: 0.8em 0.8em 0.65em 0.8em; -webkit-border-radius: 0; border-radius: 0; }
+
+.listingblock > .content { position: relative; }
+
+.listingblock code[data-lang]:before { display: none; content: attr(data-lang); position: absolute; font-size: 0.75em; top: 0.425rem; right: 0.5rem; line-height: 1; text-transform: uppercase; color: #999; }
+
+.listingblock:hover code[data-lang]:before { display: block; }
+
+.listingblock.terminal pre .command:before { content: attr(data-prompt); padding-right: 0.5em; color: #999; }
+
+.listingblock.terminal pre .command:not([data-prompt]):before { content: "$"; }
+
+table.pyhltable { border-collapse: separate; border: 0; margin-bottom: 0; background: none; }
+
+table.pyhltable td { vertical-align: top; padding-top: 0; padding-bottom: 0; line-height: 1.4; }
+
+table.pyhltable td.code { padding-left: .75em; padding-right: 0; }
+
+pre.pygments .lineno, table.pyhltable td:not(.code) { color: #999; padding-left: 0; padding-right: .5em; border-right: 1px solid #dddddd; }
+
+pre.pygments .lineno { display: inline-block; margin-right: .25em; }
+
+table.pyhltable .linenodiv { background: none !important; padding-right: 0 !important; }
+
+.quoteblock { margin: 0 1em 1.25em 1.5em; display: table; }
+.quoteblock > .title { margin-left: -1.5em; margin-bottom: 0.75em; }
+.quoteblock blockquote, .quoteblock blockquote p { color: #6f6f6f; font-size: 1.15rem; line-height: 1.75; word-spacing: 0.1em; letter-spacing: 0; font-style: italic; text-align: justify; }
+.quoteblock blockquote { margin: 0; padding: 0; border: 0; }
+.quoteblock blockquote:before { content: "\201c"; float: left; font-size: 2.75em; font-weight: bold; line-height: 0.6em; margin-left: -0.6em; color: #6f6f6f; text-shadow: 0 1px 2px rgba(0, 0, 0, 0.1); }
+.quoteblock blockquote > .paragraph:last-child p { margin-bottom: 0; }
+.quoteblock .attribution { margin-top: 0.5em; margin-right: 0.5ex; text-align: right; }
+.quoteblock .quoteblock { margin-left: 0; margin-right: 0; padding: 0.5em 0; border-left: 3px solid #555555; }
+.quoteblock .quoteblock blockquote { padding: 0 0 0 0.75em; }
+.quoteblock .quoteblock blockquote:before { display: none; }
+
+.verseblock { margin: 0 1em 1.25em 1em; }
+.verseblock pre { font-family: "Open Sans", "DejaVu Sans", sans; font-size: 1.15rem; color: #6f6f6f; font-weight: 300; text-rendering: optimizeLegibility; }
+.verseblock pre strong { font-weight: 400; }
+.verseblock .attribution { margin-top: 1.25rem; margin-left: 0.5ex; }
+
+.quoteblock .attribution, .verseblock .attribution { font-size: 0.8125em; line-height: 1.45; font-style: italic; }
+.quoteblock .attribution br, .verseblock .attribution br { display: none; }
+.quoteblock .attribution cite, .verseblock .attribution cite { display: block; letter-spacing: -0.025em; color: #555555; }
+
+.quoteblock.abstract { margin: 0 0 1.25em 0; display: block; }
+.quoteblock.abstract blockquote, .quoteblock.abstract blockquote p { text-align: left; word-spacing: 0; }
+.quoteblock.abstract blockquote:before, .quoteblock.abstract blockquote p:first-of-type:before { display: none; }
+
+table.tableblock { max-width: 100%; border-collapse: separate; }
+table.tableblock td > .paragraph:last-child p > p:last-child, table.tableblock th > p:last-child, table.tableblock td > p:last-child { margin-bottom: 0; }
+
+table.tableblock, th.tableblock, td.tableblock { border: 0 solid #dddddd; }
+
+table.grid-all th.tableblock, table.grid-all td.tableblock { border-width: 0 1px 1px 0; }
+
+table.grid-all tfoot > tr > th.tableblock, table.grid-all tfoot > tr > td.tableblock { border-width: 1px 1px 0 0; }
+
+table.grid-cols th.tableblock, table.grid-cols td.tableblock { border-width: 0 1px 0 0; }
+
+table.grid-all * > tr > .tableblock:last-child, table.grid-cols * > tr > .tableblock:last-child { border-right-width: 0; }
+
+table.grid-rows th.tableblock, table.grid-rows td.tableblock { border-width: 0 0 1px 0; }
+
+table.grid-all tbody > tr:last-child > th.tableblock, table.grid-all tbody > tr:last-child > td.tableblock, table.grid-all thead:last-child > tr > th.tableblock, table.grid-rows tbody > tr:last-child > th.tableblock, table.grid-rows tbody > tr:last-child > td.tableblock, table.grid-rows thead:last-child > tr > th.tableblock { border-bottom-width: 0; }
+
+table.grid-rows tfoot > tr > th.tableblock, table.grid-rows tfoot > tr > td.tableblock { border-width: 1px 0 0 0; }
+
+table.frame-all { border-width: 1px; }
+
+table.frame-sides { border-width: 0 1px; }
+
+table.frame-topbot { border-width: 1px 0; }
+
+th.halign-left, td.halign-left { text-align: left; }
+
+th.halign-right, td.halign-right { text-align: right; }
+
+th.halign-center, td.halign-center { text-align: center; }
+
+th.valign-top, td.valign-top { vertical-align: top; }
+
+th.valign-bottom, td.valign-bottom { vertical-align: bottom; }
+
+th.valign-middle, td.valign-middle { vertical-align: middle; }
+
+table thead th, table tfoot th { font-weight: bold; }
+
+tbody tr th { display: table-cell; line-height: 1.4; background: whitesmoke; }
+
+tbody tr th, tbody tr th p, tfoot tr th, tfoot tr th p { color: #222222; font-weight: bold; }
+
+p.tableblock > code:only-child { background: none; padding: 0; }
+
+p.tableblock { font-size: 1em; }
+
+td > div.verse { white-space: pre; }
+
+ol { margin-left: 1.75em; }
+
+ul li ol { margin-left: 1.5em; }
+
+dl dd { margin-left: 1.125em; }
+
+dl dd:last-child, dl dd:last-child > :last-child { margin-bottom: 0; }
+
+ol > li p, ul > li p, ul dd, ol dd, .olist .olist, .ulist .ulist, .ulist .olist, .olist .ulist { margin-bottom: 0.625em; }
+
+ul.unstyled, ol.unnumbered, ul.checklist, ul.none { list-style-type: none; }
+
+ul.unstyled, ol.unnumbered, ul.checklist { margin-left: 0.625em; }
+
+ul.checklist li > p:first-child > .fa-square-o:first-child, ul.checklist li > p:first-child > .fa-check-square-o:first-child { width: 1em; font-size: 0.85em; }
+
+ul.checklist li > p:first-child > input[type="checkbox"]:first-child { width: 1em; position: relative; top: 1px; }
+
+ul.inline { margin: 0 auto 0.625em auto; margin-left: -1.375em; margin-right: 0; padding: 0; list-style: none; overflow: hidden; }
+ul.inline > li { list-style: none; float: left; margin-left: 1.375em; display: block; }
+ul.inline > li > * { display: block; }
+
+.unstyled dl dt { font-weight: normal; font-style: normal; }
+
+ol.arabic { list-style-type: decimal; }
+
+ol.decimal { list-style-type: decimal-leading-zero; }
+
+ol.loweralpha { list-style-type: lower-alpha; }
+
+ol.upperalpha { list-style-type: upper-alpha; }
+
+ol.lowerroman { list-style-type: lower-roman; }
+
+ol.upperroman { list-style-type: upper-roman; }
+
+ol.lowergreek { list-style-type: lower-greek; }
+
+.hdlist > table, .colist > table { border: 0; background: none; }
+.hdlist > table > tbody > tr, .colist > table > tbody > tr { background: none; }
+
+td.hdlist1, td.hdlist2 { vertical-align: top; padding: 0 0.625em; }
+
+td.hdlist1 { font-weight: bold; padding-bottom: 1.25em; }
+
+.literalblock + .colist, .listingblock + .colist { margin-top: -0.5em; }
+
+.colist > table tr > td:first-of-type { padding: 0 0.75em; line-height: 1; }
+.colist > table tr > td:first-of-type img { max-width: initial; }
+.colist > table tr > td:last-of-type { padding: 0.25em 0; }
+
+.thumb, .th { line-height: 0; display: inline-block; border: solid 4px white; -webkit-box-shadow: 0 0 0 1px #dddddd; box-shadow: 0 0 0 1px #dddddd; }
+
+.imageblock.left, .imageblock[style*="float: left"] { margin: 0.25em 0.625em 1.25em 0; }
+.imageblock.right, .imageblock[style*="float: right"] { margin: 0.25em 0 1.25em 0.625em; }
+.imageblock > .title { margin-bottom: 0; }
+.imageblock.thumb, .imageblock.th { border-width: 6px; }
+.imageblock.thumb > .title, .imageblock.th > .title { padding: 0 0.125em; }
+
+.image.left, .image.right { margin-top: 0.25em; margin-bottom: 0.25em; display: inline-block; line-height: 0; }
+.image.left { margin-right: 0.625em; }
+.image.right { margin-left: 0.625em; }
+
+a.image { text-decoration: none; display: inline-block; }
+a.image object { pointer-events: none; }
+
+sup.footnote, sup.footnoteref { font-size: 0.875em; position: static; vertical-align: super; }
+sup.footnote a, sup.footnoteref a { text-decoration: none; }
+sup.footnote a:active, sup.footnoteref a:active { text-decoration: underline; }
+
+#footnotes { padding-top: 0.75em; padding-bottom: 0.75em; margin-bottom: 0.625em; }
+#footnotes hr { width: 20%; min-width: 6.25em; margin: -0.25em 0 0.75em 0; border-width: 1px 0 0 0; }
+#footnotes .footnote { padding: 0 0.375em 0 0.225em; line-height: 1.3334; font-size: 0.875em; margin-left: 1.2em; text-indent: -1.05em; margin-bottom: 0.2em; }
+#footnotes .footnote a:first-of-type { font-weight: bold; text-decoration: none; }
+#footnotes .footnote:last-of-type { margin-bottom: 0; }
+#content #footnotes { margin-top: -0.625em; margin-bottom: 0; padding: 0.75em 0; }
+
+.gist .file-data > table { border: 0; background: #fff; width: 100%; margin-bottom: 0; }
+.gist .file-data > table td.line-data { width: 99%; }
+
+div.unbreakable { page-break-inside: avoid; }
+
+.big { font-size: larger; }
+
+.small { font-size: smaller; }
+
+.underline { text-decoration: underline; }
+
+.overline { text-decoration: overline; }
+
+.line-through { text-decoration: line-through; }
+
+.aqua { color: #00bfbf; }
+
+.aqua-background { background-color: #00fafa; }
+
+.black { color: black; }
+
+.black-background { background-color: black; }
+
+.blue { color: #0000bf; }
+
+.blue-background { background-color: #0000fa; }
+
+.fuchsia { color: #bf00bf; }
+
+.fuchsia-background { background-color: #fa00fa; }
+
+.gray { color: #606060; }
+
+.gray-background { background-color: #7d7d7d; }
+
+.green { color: #006000; }
+
+.green-background { background-color: #007d00; }
+
+.lime { color: #00bf00; }
+
+.lime-background { background-color: #00fa00; }
+
+.maroon { color: #600000; }
+
+.maroon-background { background-color: #7d0000; }
+
+.navy { color: #000060; }
+
+.navy-background { background-color: #00007d; }
+
+.olive { color: #606000; }
+
+.olive-background { background-color: #7d7d00; }
+
+.purple { color: #600060; }
+
+.purple-background { background-color: #7d007d; }
+
+.red { color: #bf0000; }
+
+.red-background { background-color: #fa0000; }
+
+.silver { color: #909090; }
+
+.silver-background { background-color: #bcbcbc; }
+
+.teal { color: #006060; }
+
+.teal-background { background-color: #007d7d; }
+
+.white { color: #bfbfbf; }
+
+.white-background { background-color: #fafafa; }
+
+.yellow { color: #bfbf00; }
+
+.yellow-background { background-color: #fafa00; }
+
+span.icon > .fa { cursor: default; }
+
+.admonitionblock td.icon [class^="fa icon-"] { font-size: 2.5em; text-shadow: 1px 1px 2px rgba(0, 0, 0, 0.5); cursor: default; }
+.admonitionblock td.icon .icon-note:before { content: "\f05a"; color: #207c98; }
+.admonitionblock td.icon .icon-tip:before { content: "\f0eb"; text-shadow: 1px 1px 2px rgba(155, 155, 0, 0.8); color: #111; }
+.admonitionblock td.icon .icon-warning:before { content: "\f071"; color: #bf6900; }
+.admonitionblock td.icon .icon-caution:before { content: "\f06d"; color: #bf3400; }
+.admonitionblock td.icon .icon-important:before { content: "\f06a"; color: #bf0000; }
+
+.conum[data-value] { display: inline-block; color: #fff !important; background-color: #222222; -webkit-border-radius: 100px; border-radius: 100px; text-align: center; font-size: 0.75em; width: 1.67em; height: 1.67em; line-height: 1.67em; font-family: "Open Sans", "DejaVu Sans", sans-serif; font-style: normal; font-weight: bold; }
+.conum[data-value] * { color: #fff !important; }
+.conum[data-value] + b { display: none; }
+.conum[data-value]:after { content: attr(data-value); }
+pre .conum[data-value] { position: relative; top: -0.125em; }
+
+b.conum * { color: inherit !important; }
+
+.conum:not([data-value]):empty { display: none; }
+
+.literalblock pre, .listingblock pre { background: #eeeeee; }
diff --git a/common/src/docs/asciidoclet/overview.adoc b/common/src/docs/asciidoclet/overview.adoc
new file mode 100644
index 0000000..7947331
--- /dev/null
+++ b/common/src/docs/asciidoclet/overview.adoc
@@ -0,0 +1,4 @@
+= Elasticsearch Java client
+Jörg Prante
+Version 5.4.0.0
+
diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/AbstractClient.java b/common/src/main/java/org/xbib/elasticsearch/client/AbstractClient.java
similarity index 52%
rename from src/main/java/org/xbib/elasticsearch/extras/client/AbstractClient.java
rename to common/src/main/java/org/xbib/elasticsearch/client/AbstractClient.java
index aed7be0..79a9336 100644
--- a/src/main/java/org/xbib/elasticsearch/extras/client/AbstractClient.java
+++ b/common/src/main/java/org/xbib/elasticsearch/client/AbstractClient.java
@@ -1,7 +1,10 @@
-package org.xbib.elasticsearch.extras.client;
+package org.xbib.elasticsearch.client;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
@@ -14,6 +17,9 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
+import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
@@ -24,6 +30,7 @@ import org.elasticsearch.action.admin.indices.get.GetIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder;
import org.elasticsearch.action.admin.indices.recovery.RecoveryAction;
import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest;
import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
@@ -31,18 +38,25 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.Streams;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.ESLoggerFactory;
import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
@@ -52,6 +66,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -60,34 +75,188 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
*
*/
-public abstract class AbstractClient {
+public abstract class AbstractClient implements ClientMethods {
- private static final ESLogger logger = ESLoggerFactory.getLogger(AbstractClient.class.getName());
+ private static final Logger logger = LogManager.getLogger(AbstractClient.class.getName());
private Settings.Builder settingsBuilder;
private Settings settings;
- private Map mappings = new HashMap<>();
+ private Map mappings;
- public abstract ElasticsearchClient client();
+ private ElasticsearchClient client;
- protected abstract void createClient(Settings settings) throws IOException;
+ protected BulkProcessor bulkProcessor;
- public abstract void shutdown();
+ protected BulkMetric metric;
- public Settings.Builder getSettingsBuilder() {
- return settingsBuilder();
+ protected BulkControl control;
+
+ protected Throwable throwable;
+
+ protected boolean closed;
+
+ protected int maxActionsPerRequest = DEFAULT_MAX_ACTIONS_PER_REQUEST;
+
+ protected int maxConcurrentRequests = DEFAULT_MAX_CONCURRENT_REQUESTS;
+
+ protected ByteSizeValue maxVolume = DEFAULT_MAX_VOLUME_PER_REQUEST;
+
+ protected TimeValue flushInterval = DEFAULT_FLUSH_INTERVAL;
+
+ @Override
+ public AbstractClient init(ElasticsearchClient client, Settings settings,
+ final BulkMetric metric, final BulkControl control) {
+ this.client = client;
+ this.mappings = new HashMap<>();
+ if (settings == null) {
+ settings = findSettings();
+ }
+ if (client == null && settings != null) {
+ try {
+ this.client = createClient(settings);
+ } catch (IOException e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+ this.metric = metric;
+ this.control = control;
+ if (metric != null) {
+ metric.start();
+ }
+ resetSettings();
+ BulkProcessor.Listener listener = new BulkProcessor.Listener() {
+
+ private final Logger logger = LogManager.getLogger(getClass().getName() + ".Listener");
+
+ @Override
+ public void beforeBulk(long executionId, BulkRequest request) {
+ long l = -1;
+ if (metric != null) {
+ metric.getCurrentIngest().inc();
+ l = metric.getCurrentIngest().getCount();
+ int n = request.numberOfActions();
+ metric.getSubmitted().inc(n);
+ metric.getCurrentIngestNumDocs().inc(n);
+ metric.getTotalIngestSizeInBytes().inc(request.estimatedSizeInBytes());
+ }
+ logger.debug("before bulk [{}] [actions={}] [bytes={}] [concurrent requests={}]",
+ executionId,
+ request.numberOfActions(),
+ request.estimatedSizeInBytes(),
+ l);
+ }
+
+ @Override
+ public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
+ long l = -1;
+ if (metric != null) {
+ metric.getCurrentIngest().dec();
+ l = metric.getCurrentIngest().getCount();
+ metric.getSucceeded().inc(response.getItems().length);
+ }
+ int n = 0;
+ for (BulkItemResponse itemResponse : response.getItems()) {
+ if (metric != null) {
+ metric.getCurrentIngest().dec(itemResponse.getIndex(), itemResponse.getType(), itemResponse.getId());
+ }
+ if (itemResponse.isFailed()) {
+ n++;
+ if (metric != null) {
+ metric.getSucceeded().dec(1);
+ metric.getFailed().inc(1);
+ }
+ }
+ }
+ if (metric != null) {
+ logger.debug("after bulk [{}] [succeeded={}] [failed={}] [{}ms] {} concurrent requests",
+ executionId,
+ metric.getSucceeded().getCount(),
+ metric.getFailed().getCount(),
+ response.getTook().millis(),
+ l);
+ }
+ if (n > 0) {
+ logger.error("bulk [{}] failed with {} failed items, failure message = {}",
+ executionId, n, response.buildFailureMessage());
+ } else {
+ if (metric != null) {
+ metric.getCurrentIngestNumDocs().dec(response.getItems().length);
+ }
+ }
+ }
+
+ @Override
+ public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
+ if (metric != null) {
+ metric.getCurrentIngest().dec();
+ }
+ throwable = failure;
+ closed = true;
+ logger.error("after bulk [" + executionId + "] error", failure);
+ }
+ };
+ if (this.client != null) {
+ BulkProcessor.Builder builder = BulkProcessor.builder(this.client, listener)
+ .setBulkActions(maxActionsPerRequest)
+ .setConcurrentRequests(maxConcurrentRequests)
+ .setFlushInterval(flushInterval);
+ if (maxVolume != null) {
+ builder.setBulkSize(maxVolume);
+ }
+ this.bulkProcessor = builder.build();
+ }
+ this.closed = false;
+ return this;
+ }
+
+ protected abstract ElasticsearchClient createClient(Settings settings) throws IOException;
+
+ @Override
+ public ElasticsearchClient client() {
+ return client;
+ }
+
+ @Override
+ public ClientMethods maxActionsPerRequest(int maxActionsPerRequest) {
+ this.maxActionsPerRequest = maxActionsPerRequest;
+ return this;
+ }
+
+ @Override
+ public ClientMethods maxConcurrentRequests(int maxConcurrentRequests) {
+ this.maxConcurrentRequests = maxConcurrentRequests;
+ return this;
+ }
+
+ @Override
+ public ClientMethods maxVolumePerRequest(ByteSizeValue maxVolume) {
+ this.maxVolume = maxVolume;
+ return this;
+ }
+
+ @Override
+ public ClientMethods flushIngestInterval(TimeValue flushInterval) {
+ this.flushInterval = flushInterval;
+ return this;
+ }
+
+ @Override
+ public BulkMetric getMetric() {
+ return metric;
}
public void resetSettings() {
- this.settingsBuilder = Settings.settingsBuilder();
+ this.settingsBuilder = Settings.builder();
settings = null;
mappings = new HashMap<>();
}
@@ -98,31 +267,31 @@ public abstract class AbstractClient {
public void setting(String key, String value) {
if (settingsBuilder == null) {
- settingsBuilder = Settings.settingsBuilder();
+ settingsBuilder = Settings.builder();
}
settingsBuilder.put(key, value);
}
public void setting(String key, Boolean value) {
if (settingsBuilder == null) {
- settingsBuilder = Settings.settingsBuilder();
+ settingsBuilder = Settings.builder();
}
settingsBuilder.put(key, value);
}
public void setting(String key, Integer value) {
if (settingsBuilder == null) {
- settingsBuilder = Settings.settingsBuilder();
+ settingsBuilder = Settings.builder();
}
settingsBuilder.put(key, value);
}
public void setting(InputStream in) throws IOException {
- settingsBuilder = Settings.settingsBuilder().loadFromStream(".json", in);
+ settingsBuilder = Settings.builder().loadFromStream(".json", in, true);
}
public Settings.Builder settingsBuilder() {
- return settingsBuilder != null ? settingsBuilder : Settings.settingsBuilder();
+ return settingsBuilder != null ? settingsBuilder : Settings.builder();
}
public Settings settings() {
@@ -130,47 +299,253 @@ public abstract class AbstractClient {
return settings;
}
if (settingsBuilder == null) {
- settingsBuilder = Settings.settingsBuilder();
+ settingsBuilder = Settings.builder();
}
return settingsBuilder.build();
}
+ @Override
public void mapping(String type, String mapping) throws IOException {
mappings.put(type, mapping);
}
+ @Override
public void mapping(String type, InputStream in) throws IOException {
if (type == null) {
return;
}
StringWriter sw = new StringWriter();
- Streams.copy(new InputStreamReader(in), sw);
+ Streams.copy(new InputStreamReader(in, StandardCharsets.UTF_8), sw);
mappings.put(type, sw.toString());
}
- public Map mappings() {
- return mappings.isEmpty() ? null : mappings;
+ @Override
+ public ClientMethods index(String index, String type, String id, boolean create, BytesReference source) {
+ return indexRequest(new IndexRequest(index).type(type).id(id).create(create).source(source, XContentType.JSON));
+ }
+
+ @Override
+ public ClientMethods index(String index, String type, String id, boolean create, String source) {
+ return indexRequest(new IndexRequest(index).type(type).id(id).create(create).source(source, XContentType.JSON));
+ }
+
+ @Override
+ public ClientMethods indexRequest(IndexRequest indexRequest) {
+ if (closed) {
+ throwClose();
+ }
+ try {
+ if (metric != null) {
+ metric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id());
+ }
+ bulkProcessor.add(indexRequest);
+ } catch (Exception e) {
+ throwable = e;
+ closed = true;
+ logger.error("bulk add of index request failed: " + e.getMessage(), e);
+ }
+ return this;
+ }
+
+ @Override
+ public ClientMethods delete(String index, String type, String id) {
+ return deleteRequest(new DeleteRequest(index).type(type).id(id));
+ }
+
+ @Override
+ public ClientMethods deleteRequest(DeleteRequest deleteRequest) {
+ if (closed) {
+ throwClose();
+ }
+ try {
+ if (metric != null) {
+ metric.getCurrentIngest().inc(deleteRequest.index(), deleteRequest.type(), deleteRequest.id());
+ }
+ bulkProcessor.add(deleteRequest);
+ } catch (Exception e) {
+ throwable = e;
+ closed = true;
+ logger.error("bulk add of delete failed: " + e.getMessage(), e);
+ }
+ return this;
+ }
+
+ @Override
+ public ClientMethods update(String index, String type, String id, BytesReference source) {
+ return updateRequest(new UpdateRequest().index(index).type(type).id(id).upsert(source, XContentType.JSON));
+ }
+
+ @Override
+ public ClientMethods update(String index, String type, String id, String source) {
+ return updateRequest(new UpdateRequest().index(index).type(type).id(id).upsert(source, XContentType.JSON));
+ }
+
+ @Override
+ public ClientMethods updateRequest(UpdateRequest updateRequest) {
+ if (closed) {
+ throwClose();
+ }
+ try {
+ if (metric != null) {
+ metric.getCurrentIngest().inc(updateRequest.index(), updateRequest.type(), updateRequest.id());
+ }
+ bulkProcessor.add(updateRequest);
+ } catch (Exception e) {
+ throwable = e;
+ closed = true;
+ logger.error("bulk add of update request failed: " + e.getMessage(), e);
+ }
+ return this;
+ }
+
+ @Override
+ public ClientMethods startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds)
+ throws IOException {
+ if (control == null) {
+ return this;
+ }
+ if (!control.isBulk(index) && startRefreshIntervalSeconds > 0L && stopRefreshIntervalSeconds > 0L) {
+ control.startBulk(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds);
+ updateIndexSetting(index, "refresh_interval", startRefreshIntervalSeconds + "s");
+ }
+ return this;
+ }
+
+ @Override
+ public ClientMethods stopBulk(String index) throws IOException {
+ if (control == null) {
+ return this;
+ }
+ if (control.isBulk(index)) {
+ long secs = control.getStopBulkRefreshIntervals().get(index);
+ if (secs > 0L) {
+ updateIndexSetting(index, "refresh_interval", secs + "s");
+ }
+ control.finishBulk(index);
+ }
+ return this;
}
+ @Override
+ public ClientMethods flushIngest() {
+ if (closed) {
+ throwClose();
+ }
+ logger.debug("flushing bulk processor");
+ bulkProcessor.flush();
+ return this;
+ }
- public void updateIndexSetting(String index, String key, Object value) throws IOException {
+ @Override
+ public synchronized void shutdown() throws IOException {
+ if (closed) {
+ throwClose();
+ }
+ if (bulkProcessor != null) {
+ logger.info("closing bulk processor...");
+ bulkProcessor.close();
+ }
+ if (metric != null) {
+ logger.info("stopping metric");
+ metric.stop();
+ }
+ if (control != null && control.indices() != null && !control.indices().isEmpty()) {
+ logger.info("stopping bulk mode for indices {}...", control.indices());
+ for (String index : control.indices()) {
+ stopBulk(index);
+ }
+ }
+ }
+
+ @Override
+ public ClientMethods newIndex(String index) {
+ if (closed) {
+ throwClose();
+ }
+ return newIndex(index, null, null);
+ }
+
+ @Override
+ public ClientMethods newIndex(String index, String type, InputStream settings, InputStream mappings) throws IOException {
+ resetSettings();
+ setting(settings);
+ mapping(type, mappings);
+ return newIndex(index, settings(), this.mappings);
+ }
+
+ @Override
+ public ClientMethods newIndex(String index, Settings settings, Map mappings) {
+ if (closed) {
+ throwClose();
+ }
if (client() == null) {
- return;
+ logger.warn("no client for create index");
+ return this;
}
if (index == null) {
- throw new IOException("no index name given");
+ logger.warn("no index name given to create index");
+ return this;
}
- if (key == null) {
- throw new IOException("no key given");
+ CreateIndexRequestBuilder createIndexRequestBuilder =
+ new CreateIndexRequestBuilder(client(), CreateIndexAction.INSTANCE).setIndex(index);
+ if (settings != null) {
+ logger.info("found settings {}", settings.toString());
+ createIndexRequestBuilder.setSettings(settings);
+ }
+ if (mappings != null) {
+ for (Map.Entry entry : mappings.entrySet()) {
+ String type = entry.getKey();
+ String mapping = entry.getValue();
+ logger.info("found mapping for {}", type);
+ createIndexRequestBuilder.addMapping(type, mapping, XContentType.JSON);
+ }
}
- if (value == null) {
- throw new IOException("no value given");
+ CreateIndexResponse createIndexResponse = createIndexRequestBuilder.execute().actionGet();
+ logger.info("index {} created: {}", index, createIndexResponse);
+ return this;
+ }
+
+
+ @Override
+ public ClientMethods newMapping(String index, String type, Map mapping) {
+ PutMappingRequestBuilder putMappingRequestBuilder =
+ new PutMappingRequestBuilder(client(), PutMappingAction.INSTANCE)
+ .setIndices(index)
+ .setType(type)
+ .setSource(mapping);
+ putMappingRequestBuilder.execute().actionGet();
+ logger.info("mapping created for index {} and type {}", index, type);
+ return this;
+ }
+
+ @Override
+ public ClientMethods deleteIndex(String index) {
+ if (closed) {
+ throwClose();
}
- Settings.Builder updateSettingsBuilder = Settings.settingsBuilder();
- updateSettingsBuilder.put(key, value.toString());
- UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index)
- .settings(updateSettingsBuilder);
- client().execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet();
+ if (client == null) {
+ logger.warn("no client");
+ return this;
+ }
+ if (index == null) {
+ logger.warn("no index name given to delete index");
+ return this;
+ }
+ DeleteIndexRequestBuilder deleteIndexRequestBuilder =
+ new DeleteIndexRequestBuilder(client(), DeleteIndexAction.INSTANCE, index);
+ deleteIndexRequestBuilder.execute().actionGet();
+ return this;
+ }
+
+ @Override
+ public ClientMethods waitForResponses(TimeValue maxWaitTime) throws InterruptedException, ExecutionException {
+ if (closed) {
+ throwClose();
+ }
+ while (!bulkProcessor.awaitClose(maxWaitTime.getMillis(), TimeUnit.MILLISECONDS)) {
+ logger.warn("still waiting for responses");
+ }
+ return this;
}
public void waitForRecovery() throws IOException {
@@ -180,6 +555,7 @@ public abstract class AbstractClient {
client().execute(RecoveryAction.INSTANCE, new RecoveryRequest()).actionGet();
}
+ @Override
public int waitForRecovery(String index) throws IOException {
if (client() == null) {
return -1;
@@ -194,7 +570,8 @@ public abstract class AbstractClient {
return shards;
}
- public void waitForCluster(String statusString, String timeout) throws IOException {
+ @Override
+ public void waitForCluster(String statusString, TimeValue timeout) throws IOException {
if (client() == null) {
return;
}
@@ -218,7 +595,7 @@ public abstract class AbstractClient {
new ClusterStateRequestBuilder(client(), ClusterStateAction.INSTANCE).all();
ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet();
String name = clusterStateResponse.getClusterName().value();
- int nodeCount = clusterStateResponse.getState().getNodes().size();
+ int nodeCount = clusterStateResponse.getState().getNodes().getSize();
return name + " (" + nodeCount + " nodes connected)";
} catch (ElasticsearchTimeoutException e) {
logger.warn(e.getMessage(), e);
@@ -255,7 +632,7 @@ public abstract class AbstractClient {
}
public int updateReplicaLevel(String index, int level) throws IOException {
- waitForCluster("YELLOW", "30s");
+ waitForCluster("YELLOW", TimeValue.timeValueSeconds(30));
updateIndexSetting(index, "number_of_replicas", level);
return waitForRecovery(index);
}
@@ -282,10 +659,10 @@ public abstract class AbstractClient {
if (client() == null) {
return;
}
- if (!mappings().isEmpty()) {
- for (Map.Entry me : mappings().entrySet()) {
+ if (!mappings.isEmpty()) {
+ for (Map.Entry me : mappings.entrySet()) {
client().execute(PutMappingAction.INSTANCE,
- new PutMappingRequest(index).type(me.getKey()).source(me.getValue())).actionGet();
+ new PutMappingRequest(index).type(me.getKey()).source(me.getValue(), XContentType.JSON)).actionGet();
}
}
}
@@ -332,25 +709,13 @@ public abstract class AbstractClient {
return getFilters(getAliasesRequestBuilder.setIndices(index).execute().actionGet());
}
- private Map getFilters(GetAliasesResponse getAliasesResponse) {
- Map result = new HashMap<>();
- for (ObjectObjectCursor> object : getAliasesResponse.getAliases()) {
- List aliasMetaDataList = object.value;
- for (AliasMetaData aliasMetaData : aliasMetaDataList) {
- if (aliasMetaData.filteringRequired()) {
- result.put(aliasMetaData.alias(), new String(aliasMetaData.getFilter().uncompressed()));
- } else {
- result.put(aliasMetaData.alias(), null);
- }
- }
- }
- return result;
- }
+ @Override
public void switchAliases(String index, String concreteIndex, List extraAliases) {
switchAliases(index, concreteIndex, extraAliases, null);
}
+ @Override
public void switchAliases(String index, String concreteIndex,
List extraAliases, IndexAliasAdder adder) {
if (client() == null) {
@@ -413,6 +778,7 @@ public abstract class AbstractClient {
}
}
+ @Override
public void performRetentionPolicy(String index, String concreteIndex, int timestampdiff, int mintokeep) {
if (client() == null) {
return;
@@ -471,14 +837,15 @@ public abstract class AbstractClient {
}
}
+ @Override
public Long mostRecentDocument(String index, String timestampfieldname) {
if (client() == null) {
return null;
}
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client(), SearchAction.INSTANCE);
- SortBuilder sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC);
+ SortBuilder sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC);
SearchResponse searchResponse = searchRequestBuilder.setIndices(index)
- .addField(timestampfieldname)
+ .addStoredField(timestampfieldname)
.setSize(1)
.addSort(sort)
.execute().actionGet();
@@ -493,4 +860,68 @@ public abstract class AbstractClient {
return null;
}
+ @Override
+ public boolean hasThrowable() {
+ return throwable != null;
+ }
+
+ @Override
+ public Throwable getThrowable() {
+ return throwable;
+ }
+
+ protected static void throwClose() {
+ throw new ElasticsearchException("client is closed");
+ }
+
+
+ protected void updateIndexSetting(String index, String key, Object value) throws IOException {
+ if (client() == null) {
+ return;
+ }
+ if (index == null) {
+ throw new IOException("no index name given");
+ }
+ if (key == null) {
+ throw new IOException("no key given");
+ }
+ if (value == null) {
+ throw new IOException("no value given");
+ }
+ Settings.Builder updateSettingsBuilder = Settings.builder();
+ updateSettingsBuilder.put(key, value.toString());
+ UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index)
+ .settings(updateSettingsBuilder);
+ client().execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet();
+ }
+
+ private Map getFilters(GetAliasesResponse getAliasesResponse) {
+ Map result = new HashMap<>();
+ for (ObjectObjectCursor> object : getAliasesResponse.getAliases()) {
+ List aliasMetaDataList = object.value;
+ for (AliasMetaData aliasMetaData : aliasMetaDataList) {
+ if (aliasMetaData.filteringRequired()) {
+ String metaData = new String(aliasMetaData.getFilter().uncompressed(), StandardCharsets.UTF_8);
+ result.put(aliasMetaData.alias(), metaData);
+ } else {
+ result.put(aliasMetaData.alias(), null);
+ }
+ }
+ }
+ return result;
+ }
+
+ private Settings findSettings() {
+ Settings.Builder settingsBuilder = Settings.builder();
+ settingsBuilder.put("host", "localhost");
+ try {
+ String hostname = NetworkUtils.getLocalAddress().getHostName();
+ logger.debug("the hostname is {}", hostname);
+ settingsBuilder.put("host", hostname)
+ .put("port", 9300);
+ } catch (Exception e) {
+ logger.warn(e.getMessage(), e);
+ }
+ return settingsBuilder.build();
+ }
}
diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/BulkControl.java b/common/src/main/java/org/xbib/elasticsearch/client/BulkControl.java
similarity index 89%
rename from src/main/java/org/xbib/elasticsearch/extras/client/BulkControl.java
rename to common/src/main/java/org/xbib/elasticsearch/client/BulkControl.java
index 910f2f2..5fe6311 100644
--- a/src/main/java/org/xbib/elasticsearch/extras/client/BulkControl.java
+++ b/common/src/main/java/org/xbib/elasticsearch/client/BulkControl.java
@@ -1,4 +1,4 @@
-package org.xbib.elasticsearch.extras.client;
+package org.xbib.elasticsearch.client;
import java.util.Map;
import java.util.Set;
@@ -18,5 +18,4 @@ public interface BulkControl {
Map getStartBulkRefreshIntervals();
Map getStopBulkRefreshIntervals();
-
}
diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/BulkMetric.java b/common/src/main/java/org/xbib/elasticsearch/client/BulkMetric.java
similarity index 89%
rename from src/main/java/org/xbib/elasticsearch/extras/client/BulkMetric.java
rename to common/src/main/java/org/xbib/elasticsearch/client/BulkMetric.java
index a45e9c2..e7a60d2 100644
--- a/src/main/java/org/xbib/elasticsearch/extras/client/BulkMetric.java
+++ b/common/src/main/java/org/xbib/elasticsearch/client/BulkMetric.java
@@ -1,4 +1,4 @@
-package org.xbib.elasticsearch.extras.client;
+package org.xbib.elasticsearch.client;
import org.xbib.metrics.Count;
import org.xbib.metrics.Metered;
@@ -27,5 +27,4 @@ public interface BulkMetric {
void stop();
long elapsed();
-
}
diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/BulkProcessor.java b/common/src/main/java/org/xbib/elasticsearch/client/BulkProcessor.java
similarity index 77%
rename from src/main/java/org/xbib/elasticsearch/extras/client/BulkProcessor.java
rename to common/src/main/java/org/xbib/elasticsearch/client/BulkProcessor.java
index b32637e..7de2a3b 100644
--- a/src/main/java/org/xbib/elasticsearch/extras/client/BulkProcessor.java
+++ b/common/src/main/java/org/xbib/elasticsearch/client/BulkProcessor.java
@@ -1,18 +1,17 @@
-package org.xbib.elasticsearch.extras.client;
+package org.xbib.elasticsearch.client;
import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.bulk.BulkAction;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.common.util.concurrent.FutureUtils;
import java.io.Closeable;
import java.util.concurrent.Executors;
@@ -30,9 +29,9 @@ import java.util.concurrent.atomic.AtomicLong;
*/
public class BulkProcessor implements Closeable {
- private final int bulkActions;
+ private final int maximumBulkActionsPerRequest;
- private final long bulkSize;
+ private final long maximumBulkRequestByteSize;
private final ScheduledThreadPoolExecutor scheduler;
@@ -40,26 +39,24 @@ public class BulkProcessor implements Closeable {
private final AtomicLong executionIdGen = new AtomicLong();
- private final BulkRequestHandler bulkRequestHandler;
+ private final BulkExecutor bulkExecutor;
private BulkRequest bulkRequest;
private volatile boolean closed = false;
- private BulkProcessor(Client client, Listener listener, String name, int concurrentRequests,
- int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval) {
- this.bulkActions = bulkActions;
- this.bulkSize = bulkSize.bytes();
-
+ private BulkProcessor(ElasticsearchClient client, Listener listener, int maximumConcurrentBulkRequests,
+ int maximumBulkActionsPerRequest, ByteSizeValue maximumBulkRequestByteSize,
+ @Nullable TimeValue flushInterval) {
+ this.maximumBulkActionsPerRequest = maximumBulkActionsPerRequest;
+ this.maximumBulkRequestByteSize = maximumBulkRequestByteSize.getBytes();
this.bulkRequest = new BulkRequest();
- this.bulkRequestHandler = concurrentRequests == 0 ?
- new SyncBulkRequestHandler(client, listener) :
- new AsyncBulkRequestHandler(client, listener, concurrentRequests);
+ this.bulkExecutor = maximumConcurrentBulkRequests == 0 ?
+ new SyncBulkExecutor(client, listener) :
+ new AsyncBulkExecutor(client, listener, maximumConcurrentBulkRequests);
if (flushInterval != null) {
- this.scheduler = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(1,
- EsExecutors.daemonThreadFactory(client.settings(),
- name != null ? "[" + name + "]" : "" + "bulk_processor"));
+ this.scheduler = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(1);
this.scheduler.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
this.scheduler.setContinueExistingPeriodicTasksAfterShutdownPolicy(false);
this.scheduledFuture = this.scheduler.scheduleWithFixedDelay(new Flush(), flushInterval.millis(),
@@ -70,7 +67,7 @@ public class BulkProcessor implements Closeable {
}
}
- public static Builder builder(Client client, Listener listener) {
+ public static Builder builder(ElasticsearchClient client, Listener listener) {
if (client == null) {
throw new NullPointerException("The client you specified while building a BulkProcessor is null");
}
@@ -110,13 +107,13 @@ public class BulkProcessor implements Closeable {
}
closed = true;
if (this.scheduledFuture != null) {
- FutureUtils.cancel(this.scheduledFuture);
+ this.scheduledFuture.cancel(false);
this.scheduler.shutdown();
}
if (bulkRequest.numberOfActions() > 0) {
execute();
}
- return this.bulkRequestHandler.awaitClose(timeout, unit);
+ return bulkExecutor.awaitClose(timeout, unit);
}
/**
@@ -126,8 +123,16 @@ public class BulkProcessor implements Closeable {
* @param request request
* @return his bulk processor
*/
- public BulkProcessor add(IndexRequest request) {
- return add((ActionRequest) request);
+ public synchronized BulkProcessor add(IndexRequest request) {
+ if (request == null) {
+ return this;
+ }
+ ensureOpen();
+ bulkRequest.add(request);
+ if (isOverTheLimit()) {
+ execute();
+ }
+ return this;
}
/**
@@ -136,29 +141,33 @@ public class BulkProcessor implements Closeable {
* @param request request
* @return his bulk processor
*/
- public BulkProcessor add(DeleteRequest request) {
- return add((ActionRequest) request);
- }
-
- /**
- * Adds either a delete or an index request.
- *
- * @param request request
- * @return his bulk processor
- */
- public BulkProcessor add(ActionRequest request) {
- return add(request, null);
+ public synchronized BulkProcessor add(DeleteRequest request) {
+ if (request == null) {
+ return this;
+ }
+ ensureOpen();
+ bulkRequest.add(request);
+ if (isOverTheLimit()) {
+ execute();
+ }
+ return this;
}
/**
- * Adds either a delete or an index request with a payload.
+ * Adds an {@link UpdateRequest} to the list of actions to execute.
*
* @param request request
- * @param payload payload
* @return his bulk processor
*/
- public BulkProcessor add(ActionRequest request, Object payload) {
- internalAdd(request, payload);
+ public synchronized BulkProcessor add(UpdateRequest request) {
+ if (request == null) {
+ return this;
+ }
+ ensureOpen();
+ bulkRequest.add(request);
+ if (isOverTheLimit()) {
+ execute();
+ }
return this;
}
@@ -168,32 +177,17 @@ public class BulkProcessor implements Closeable {
}
}
- private synchronized void internalAdd(ActionRequest request, Object payload) {
- ensureOpen();
- bulkRequest.add(request, payload);
- executeIfNeeded();
- }
-
- private void executeIfNeeded() {
- ensureOpen();
- if (!isOverTheLimit()) {
- return;
- }
- execute();
+ private boolean isOverTheLimit() {
+ final int count = bulkRequest.numberOfActions();
+ return count > 0 &&
+ (maximumBulkActionsPerRequest != -1 && count >= maximumBulkActionsPerRequest) ||
+ (maximumBulkRequestByteSize != -1 && bulkRequest.estimatedSizeInBytes() >= maximumBulkRequestByteSize);
}
private void execute() {
final BulkRequest myBulkRequest = this.bulkRequest;
- final long executionId = executionIdGen.incrementAndGet();
+ bulkExecutor.execute(myBulkRequest, executionIdGen.incrementAndGet());
this.bulkRequest = new BulkRequest();
- this.bulkRequestHandler.execute(myBulkRequest, executionId);
- }
-
- private boolean isOverTheLimit() {
- return bulkActions != -1 &&
- bulkRequest.numberOfActions() >= bulkActions ||
- bulkSize != -1 &&
- bulkRequest.estimatedSizeInBytes() >= bulkSize;
}
/**
@@ -247,9 +241,8 @@ public class BulkProcessor implements Closeable {
*/
public static class Builder {
- private final Client client;
+ private final ElasticsearchClient client;
private final Listener listener;
- private String name;
private int concurrentRequests = 1;
private int bulkActions = 1000;
private ByteSizeValue bulkSize = new ByteSizeValue(5, ByteSizeUnit.MB);
@@ -262,22 +255,11 @@ public class BulkProcessor implements Closeable {
* @param client the client
* @param listener the listener
*/
- Builder(Client client, Listener listener) {
+ Builder(ElasticsearchClient client, Listener listener) {
this.client = client;
this.listener = listener;
}
- /**
- * Sets an optional name to identify this bulk processor.
- *
- * @param name name
- * @return this builder
- */
- public Builder setName(String name) {
- this.name = name;
- return this;
- }
-
/**
* Sets the number of concurrent requests allowed to be executed. A value of 0 means that only a single
* request will be allowed to be executed. A value of 1 means 1 concurrent request is allowed to be executed
@@ -334,7 +316,7 @@ public class BulkProcessor implements Closeable {
* @return a bulk processor
*/
public BulkProcessor build() {
- return new BulkProcessor(client, listener, name, concurrentRequests, bulkActions, bulkSize, flushInterval);
+ return new BulkProcessor(client, listener, concurrentRequests, bulkActions, bulkSize, flushInterval);
}
}
@@ -346,18 +328,14 @@ public class BulkProcessor implements Closeable {
if (closed) {
return;
}
- if (bulkRequest.numberOfActions() == 0) {
- return;
+ if (bulkRequest.numberOfActions() > 0) {
+ execute();
}
- execute();
}
}
}
- /**
- * Abstracts the low-level details of bulk request handling.
- */
- interface BulkRequestHandler {
+ interface BulkExecutor {
void execute(BulkRequest bulkRequest, long executionId);
@@ -365,11 +343,13 @@ public class BulkProcessor implements Closeable {
}
- private class SyncBulkRequestHandler implements BulkRequestHandler {
- private final Client client;
+ private static class SyncBulkExecutor implements BulkExecutor {
+
+ private final ElasticsearchClient client;
+
private final BulkProcessor.Listener listener;
- SyncBulkRequestHandler(Client client, BulkProcessor.Listener listener) {
+ SyncBulkExecutor(ElasticsearchClient client, BulkProcessor.Listener listener) {
this.client = client;
this.listener = listener;
}
@@ -395,13 +375,17 @@ public class BulkProcessor implements Closeable {
}
}
- private class AsyncBulkRequestHandler implements BulkRequestHandler {
- private final Client client;
+ private static class AsyncBulkExecutor implements BulkExecutor {
+
+ private final ElasticsearchClient client;
+
private final BulkProcessor.Listener listener;
+
private final Semaphore semaphore;
+
private final int concurrentRequests;
- private AsyncBulkRequestHandler(Client client, BulkProcessor.Listener listener, int concurrentRequests) {
+ private AsyncBulkExecutor(ElasticsearchClient client, BulkProcessor.Listener listener, int concurrentRequests) {
this.client = client;
this.listener = listener;
this.concurrentRequests = concurrentRequests;
@@ -427,7 +411,7 @@ public class BulkProcessor implements Closeable {
}
@Override
- public void onFailure(Throwable e) {
+ public void onFailure(Exception e) {
try {
listener.afterBulk(executionId, bulkRequest, e);
} finally {
diff --git a/common/src/main/java/org/xbib/elasticsearch/client/ClientBuilder.java b/common/src/main/java/org/xbib/elasticsearch/client/ClientBuilder.java
new file mode 100644
index 0000000..441c8e2
--- /dev/null
+++ b/common/src/main/java/org/xbib/elasticsearch/client/ClientBuilder.java
@@ -0,0 +1,104 @@
+package org.xbib.elasticsearch.client;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.ServiceLoader;
+
+/**
+ *
+ */
+public final class ClientBuilder implements Parameters {
+
+ private final Settings.Builder settingsBuilder;
+
+ private Map, ClientMethods> clientMethodsMap;
+
+ private BulkMetric metric;
+
+ private BulkControl control;
+
+ public ClientBuilder() {
+ this(Thread.currentThread().getContextClassLoader());
+ }
+
+ public ClientBuilder(ClassLoader classLoader) {
+ this.settingsBuilder = Settings.builder();
+ //settingsBuilder.put("node.name", "clientnode");
+ this.clientMethodsMap = new HashMap<>();
+ ServiceLoader serviceLoader = ServiceLoader.load(ClientMethods.class,
+ classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader());
+ for (ClientMethods clientMethods : serviceLoader) {
+ clientMethodsMap.put(clientMethods.getClass(), clientMethods);
+ }
+ }
+
+ public static ClientBuilder builder() {
+ return new ClientBuilder();
+ }
+
+ public ClientBuilder put(String key, String value) {
+ settingsBuilder.put(key, value);
+ return this;
+ }
+
+ public ClientBuilder put(String key, Integer value) {
+ settingsBuilder.put(key, value);
+ return this;
+ }
+
+ public ClientBuilder put(String key, Long value) {
+ settingsBuilder.put(key, value);
+ return this;
+ }
+
+ public ClientBuilder put(String key, Double value) {
+ settingsBuilder.put(key, value);
+ return this;
+ }
+
+ public ClientBuilder put(String key, ByteSizeValue value) {
+ settingsBuilder.put(key, value);
+ return this;
+ }
+
+ public ClientBuilder put(String key, TimeValue value) {
+ settingsBuilder.put(key, value);
+ return this;
+ }
+
+ public ClientBuilder put(Settings settings) {
+ settingsBuilder.put(settings);
+ return this;
+ }
+
+ public ClientBuilder setMetric(BulkMetric metric) {
+ this.metric = metric;
+ return this;
+ }
+
+ public ClientBuilder setControl(BulkControl control) {
+ this.control = control;
+ return this;
+ }
+
+ public C getClient(Class clientClass) throws IOException {
+ return getClient(null, clientClass);
+ }
+
+ @SuppressWarnings("unchecked")
+ public C getClient(Client client, Class clientClass) throws IOException {
+ Settings settings = settingsBuilder.build();
+ return (C) clientMethodsMap.get(clientClass)
+ .maxActionsPerRequest(settings.getAsInt(MAX_ACTIONS_PER_REQUEST, DEFAULT_MAX_ACTIONS_PER_REQUEST))
+ .maxConcurrentRequests(settings.getAsInt(MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_REQUESTS))
+ .maxVolumePerRequest(settings.getAsBytesSize(MAX_VOLUME_PER_REQUEST, DEFAULT_MAX_VOLUME_PER_REQUEST))
+ .flushIngestInterval(settings.getAsTime(FLUSH_INTERVAL, DEFAULT_FLUSH_INTERVAL))
+ .init(client, settings, metric, control);
+ }
+}
diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/ClientMethods.java b/common/src/main/java/org/xbib/elasticsearch/client/ClientMethods.java
similarity index 83%
rename from src/main/java/org/xbib/elasticsearch/extras/client/ClientMethods.java
rename to common/src/main/java/org/xbib/elasticsearch/client/ClientMethods.java
index a683b63..df199ce 100644
--- a/src/main/java/org/xbib/elasticsearch/extras/client/ClientMethods.java
+++ b/common/src/main/java/org/xbib/elasticsearch/client/ClientMethods.java
@@ -1,9 +1,10 @@
-package org.xbib.elasticsearch.extras.client;
+package org.xbib.elasticsearch.client;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.Settings;
import java.io.IOException;
@@ -17,45 +18,49 @@ import java.util.concurrent.ExecutionException;
*/
public interface ClientMethods extends Parameters {
- /**
- * Initialize new ingest client, wrap an existing Elasticsearch client, and set up metrics.
- *
- * @param client the Elasticsearch client
- * @param metric metric
- * @param control control
- * @return this ingest
- * @throws IOException if client could not get created
- */
- ClientMethods init(ElasticsearchClient client, BulkMetric metric, BulkControl control) throws IOException;
+ ClientMethods init(ElasticsearchClient client, Settings settings, BulkMetric metric, BulkControl control);
/**
- * Initialize, create new ingest client, and set up metrics.
+ * Return Elasticsearch client.
*
- * @param settings settings
- * @param metric metric
- * @param control control
- * @return this ingest
- * @throws IOException if client could not get created
+ * @return Elasticsearch client
*/
- ClientMethods init(Settings settings, BulkMetric metric, BulkControl control) throws IOException;
+ ElasticsearchClient client();
/**
- * Return Elasticsearch client.
+ * Bulked index request. Each request will be added to a queue for bulking requests.
+ * Submitting request will be done when bulk limits are exceeded.
*
- * @return Elasticsearch client
+ * @param index the index
+ * @param type the type
+ * @param id the id
+ * @param create true if document must be created
+ * @param source the source
+ * @return this
*/
- ElasticsearchClient client();
+ ClientMethods index(String index, String type, String id, boolean create, BytesReference source);
/**
- * Index document.
+ * Bulked index request. Each request will be added to a queue for bulking requests.
+ * Submitting request will be done when bulk limits are exceeded.
*
* @param index the index
* @param type the type
* @param id the id
+ * @param create true if document must be created
* @param source the source
* @return this
*/
- ClientMethods index(String index, String type, String id, String source);
+ ClientMethods index(String index, String type, String id, boolean create, String source);
+
+ /**
+ * Bulked index request. Each request will be added to a queue for bulking requests.
+ * Submitting request will be done when bulk limits are exceeded.
+ *
+ * @param indexRequest the index request to add
+ * @return this ingest
+ */
+ ClientMethods indexRequest(IndexRequest indexRequest);
/**
* Delete document.
@@ -68,7 +73,31 @@ public interface ClientMethods extends Parameters {
ClientMethods delete(String index, String type, String id);
/**
- * Update document. Use with precaution! Does not work in all cases.
+ * Bulked delete request. Each request will be added to a queue for bulking requests.
+ * Submitting request will be done when bulk limits are exceeded.
+ *
+ * @param deleteRequest the delete request to add
+ * @return this ingest
+ */
+ ClientMethods deleteRequest(DeleteRequest deleteRequest);
+
+ /**
+ * Bulked update request. Each request will be added to a queue for bulking requests.
+ * Submitting request will be done when bulk limits are exceeded.
+ * Note that updates only work correctly when all operations between nodes are synchronized.
+ *
+ * @param index the index
+ * @param type the type
+ * @param id the id
+ * @param source the source
+ * @return this
+ */
+ ClientMethods update(String index, String type, String id, BytesReference source);
+
+ /**
+ * Bulked update request. Each request will be added to a queue for bulking requests.
+ * Submitting request will be done when bulk limits are exceeded.
+ * Note that updates only work correctly when all operations between nodes are synchronized.
*
* @param index the index
* @param type the type
@@ -78,6 +107,16 @@ public interface ClientMethods extends Parameters {
*/
ClientMethods update(String index, String type, String id, String source);
+ /**
+ * Bulked update request. Each request will be added to a queue for bulking requests.
+ * Submitting request will be done when bulk limits are exceeded.
+ * Note that updates only work correctly when all operations between nodes are synchronized.
+ *
+ * @param updateRequest the update request to add
+ * @return this ingest
+ */
+ ClientMethods updateRequest(UpdateRequest updateRequest);
+
/**
* Set the maximum number of actions per request.
*
@@ -203,34 +242,6 @@ public interface ClientMethods extends Parameters {
*/
ClientMethods stopBulk(String index) throws IOException;
- /**
- * Bulked index request. Each request will be added to a queue for bulking requests.
- * Submitting request will be done when bulk limits are exceeded.
- *
- * @param indexRequest the index request to add
- * @return this ingest
- */
- ClientMethods bulkIndex(IndexRequest indexRequest);
-
- /**
- * Bulked delete request. Each request will be added to a queue for bulking requests.
- * Submitting request will be done when bulk limits are exceeded.
- *
- * @param deleteRequest the delete request to add
- * @return this ingest
- */
- ClientMethods bulkDelete(DeleteRequest deleteRequest);
-
- /**
- * Bulked update request. Each request will be added to a queue for bulking requests.
- * Submitting request will be done when bulk limits are exceeded.
- * Note that updates only work correctly when all operations between nodes are synchronized!
- *
- * @param updateRequest the update request to add
- * @return this ingest
- */
- ClientMethods bulkUpdate(UpdateRequest updateRequest);
-
/**
* Flush ingest, move all pending documents to the cluster.
*
@@ -386,5 +397,5 @@ public interface ClientMethods extends Parameters {
/**
* Shutdown the ingesting.
*/
- void shutdown();
+ void shutdown() throws IOException;
}
diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/IndexAliasAdder.java b/common/src/main/java/org/xbib/elasticsearch/client/IndexAliasAdder.java
similarity index 84%
rename from src/main/java/org/xbib/elasticsearch/extras/client/IndexAliasAdder.java
rename to common/src/main/java/org/xbib/elasticsearch/client/IndexAliasAdder.java
index a659ab4..7c93d22 100644
--- a/src/main/java/org/xbib/elasticsearch/extras/client/IndexAliasAdder.java
+++ b/common/src/main/java/org/xbib/elasticsearch/client/IndexAliasAdder.java
@@ -1,4 +1,4 @@
-package org.xbib.elasticsearch.extras.client;
+package org.xbib.elasticsearch.client;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/NetworkUtils.java b/common/src/main/java/org/xbib/elasticsearch/client/NetworkUtils.java
similarity index 96%
rename from src/main/java/org/xbib/elasticsearch/extras/client/NetworkUtils.java
rename to common/src/main/java/org/xbib/elasticsearch/client/NetworkUtils.java
index 9c5ffc2..847e414 100644
--- a/src/main/java/org/xbib/elasticsearch/extras/client/NetworkUtils.java
+++ b/common/src/main/java/org/xbib/elasticsearch/client/NetworkUtils.java
@@ -1,7 +1,7 @@
-package org.xbib.elasticsearch.extras.client;
+package org.xbib.elasticsearch.client;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.net.Inet4Address;
@@ -11,6 +11,7 @@ import java.net.NetworkInterface;
import java.net.SocketException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Comparator;
import java.util.Enumeration;
import java.util.List;
import java.util.Locale;
@@ -20,7 +21,7 @@ import java.util.Locale;
*/
public class NetworkUtils {
- private static final ESLogger logger = ESLoggerFactory.getLogger(NetworkUtils.class.getName());
+ private static final Logger logger = LogManager.getLogger(NetworkUtils.class.getName());
private static final String IPV4_SETTING = "java.net.preferIPv4Stack";
@@ -234,7 +235,7 @@ public class NetworkUtils {
}
private static void sortInterfaces(List interfaces) {
- Collections.sort(interfaces, (o1, o2) -> Integer.compare(o1.getIndex(), o2.getIndex()));
+ Collections.sort(interfaces, Comparator.comparingInt(NetworkInterface::getIndex));
}
private static void sortAddresses(List addressList) {
diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/Parameters.java b/common/src/main/java/org/xbib/elasticsearch/client/Parameters.java
similarity index 91%
rename from src/main/java/org/xbib/elasticsearch/extras/client/Parameters.java
rename to common/src/main/java/org/xbib/elasticsearch/client/Parameters.java
index d77ce24..a9dd7c8 100644
--- a/src/main/java/org/xbib/elasticsearch/extras/client/Parameters.java
+++ b/common/src/main/java/org/xbib/elasticsearch/client/Parameters.java
@@ -1,4 +1,4 @@
-package org.xbib.elasticsearch.extras.client;
+package org.xbib.elasticsearch.client;
/**
*
@@ -20,5 +20,4 @@ public interface Parameters {
String MAX_VOLUME_PER_REQUEST = "max_volume_per_request";
String FLUSH_INTERVAL = "flush_interval";
-
}
diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkControl.java b/common/src/main/java/org/xbib/elasticsearch/client/SimpleBulkControl.java
similarity index 96%
rename from src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkControl.java
rename to common/src/main/java/org/xbib/elasticsearch/client/SimpleBulkControl.java
index b9a92d6..b8257f7 100644
--- a/src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkControl.java
+++ b/common/src/main/java/org/xbib/elasticsearch/client/SimpleBulkControl.java
@@ -1,4 +1,4 @@
-package org.xbib.elasticsearch.extras.client;
+package org.xbib.elasticsearch.client;
import java.util.HashMap;
import java.util.HashSet;
diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkMetric.java b/common/src/main/java/org/xbib/elasticsearch/client/SimpleBulkMetric.java
similarity index 53%
rename from src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkMetric.java
rename to common/src/main/java/org/xbib/elasticsearch/client/SimpleBulkMetric.java
index e836816..9b82444 100644
--- a/src/main/java/org/xbib/elasticsearch/extras/client/SimpleBulkMetric.java
+++ b/common/src/main/java/org/xbib/elasticsearch/client/SimpleBulkMetric.java
@@ -1,32 +1,53 @@
-package org.xbib.elasticsearch.extras.client;
+package org.xbib.elasticsearch.client;
import org.xbib.metrics.Count;
import org.xbib.metrics.CountMetric;
import org.xbib.metrics.Meter;
import org.xbib.metrics.Metered;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+
/**
*
*/
public class SimpleBulkMetric implements BulkMetric {
- private final Meter totalIngest = new Meter();
+ private final ScheduledExecutorService executorService;
- private final Count totalIngestSizeInBytes = new CountMetric();
+ private final Meter totalIngest;
- private final Count currentIngest = new CountMetric();
+ private final Count totalIngestSizeInBytes;
- private final Count currentIngestNumDocs = new CountMetric();
+ private final Count currentIngest;
- private final Count submitted = new CountMetric();
+ private final Count currentIngestNumDocs;
- private final Count succeeded = new CountMetric();
+ private final Count submitted;
- private final Count failed = new CountMetric();
+ private final Count succeeded;
+
+ private final Count failed;
private Long started;
private Long stopped;
+ public SimpleBulkMetric() {
+ this(Executors.newSingleThreadScheduledExecutor());
+ }
+
+ public SimpleBulkMetric(ScheduledExecutorService executorService) {
+ this.executorService = executorService;
+ totalIngest = new Meter(executorService);
+ totalIngestSizeInBytes = new CountMetric();
+ currentIngest = new CountMetric();
+ currentIngestNumDocs = new CountMetric();
+ submitted = new CountMetric();
+ succeeded = new CountMetric();
+ failed = new CountMetric();
+ }
+
@Override
public Metered getTotalIngest() {
return totalIngest;
@@ -65,13 +86,14 @@ public class SimpleBulkMetric implements BulkMetric {
@Override
public void start() {
this.started = System.nanoTime();
- totalIngest.spawn(5L);
+ totalIngest.start(5L);
}
@Override
public void stop() {
this.stopped = System.nanoTime();
totalIngest.stop();
+ executorService.shutdownNow();
}
@Override
diff --git a/common/src/main/java/org/xbib/elasticsearch/client/package-info.java b/common/src/main/java/org/xbib/elasticsearch/client/package-info.java
new file mode 100644
index 0000000..941a500
--- /dev/null
+++ b/common/src/main/java/org/xbib/elasticsearch/client/package-info.java
@@ -0,0 +1,4 @@
+/**
+ * Classes for Elasticsearch client.
+ */
+package org.xbib.elasticsearch.client;
diff --git a/src/integration-test/java/org/xbib/elasticsearch/AliasTest.java b/common/src/test/java/org/xbib/elasticsearch/client/common/AliasTests.java
similarity index 53%
rename from src/integration-test/java/org/xbib/elasticsearch/AliasTest.java
rename to common/src/test/java/org/xbib/elasticsearch/client/common/AliasTests.java
index 970268e..37db0de 100644
--- a/src/integration-test/java/org/xbib/elasticsearch/AliasTest.java
+++ b/common/src/test/java/org/xbib/elasticsearch/client/common/AliasTests.java
@@ -1,22 +1,17 @@
-package org.xbib.elasticsearch;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+package org.xbib.elasticsearch.client.common;
import com.carrotsearch.hppc.cursors.ObjectCursor;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.cluster.metadata.AliasAction;
import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.junit.Test;
+import org.elasticsearch.test.ESSingleNodeTestCase;
-import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.Set;
@@ -24,62 +19,52 @@ import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-/**
- *
- */
-public class AliasTest extends NodeTestUtils {
+public class AliasTests extends ESSingleNodeTestCase {
- private static final ESLogger logger = ESLoggerFactory.getLogger(AliasTest.class.getName());
+ private static final Logger logger = LogManager.getLogger(AliasTests.class.getName());
- @Test
- public void testAlias() throws IOException {
+ public void testAlias() {
CreateIndexRequest indexRequest = new CreateIndexRequest("test");
- client("1").admin().indices().create(indexRequest).actionGet();
+ client().admin().indices().create(indexRequest).actionGet();
// put alias
IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest();
- String[] indices = new String[]{"test"};
- String[] aliases = new String[]{"test_alias"};
- IndicesAliasesRequest.AliasActions aliasAction =
- new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, indices, aliases);
- indicesAliasesRequest.addAliasAction(aliasAction);
- client("1").admin().indices().aliases(indicesAliasesRequest).actionGet();
+ indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add()
+ .index("test").alias("test_alias")
+ );
+ client().admin().indices().aliases(indicesAliasesRequest).actionGet();
// get alias
GetAliasesRequest getAliasesRequest = new GetAliasesRequest(Strings.EMPTY_ARRAY);
long t0 = System.nanoTime();
- GetAliasesResponse getAliasesResponse = client("1").admin().indices().getAliases(getAliasesRequest).actionGet();
+ GetAliasesResponse getAliasesResponse = client().admin().indices().getAliases(getAliasesRequest).actionGet();
long t1 = (System.nanoTime() - t0) / 1000000;
logger.info("{} time(ms) = {}", getAliasesResponse.getAliases(), t1);
assertTrue(t1 >= 0);
}
- @Test
- public void testMostRecentIndex() throws IOException {
+ public void testMostRecentIndex() {
String alias = "test";
CreateIndexRequest indexRequest = new CreateIndexRequest("test20160101");
- client("1").admin().indices().create(indexRequest).actionGet();
+ client().admin().indices().create(indexRequest).actionGet();
indexRequest = new CreateIndexRequest("test20160102");
- client("1").admin().indices().create(indexRequest).actionGet();
+ client().admin().indices().create(indexRequest).actionGet();
indexRequest = new CreateIndexRequest("test20160103");
- client("1").admin().indices().create(indexRequest).actionGet();
+ client().admin().indices().create(indexRequest).actionGet();
IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest();
- String[] indices = new String[]{"test20160101", "test20160102", "test20160103"};
- String[] aliases = new String[]{alias};
- IndicesAliasesRequest.AliasActions aliasAction =
- new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, indices, aliases);
- indicesAliasesRequest.addAliasAction(aliasAction);
- client("1").admin().indices().aliases(indicesAliasesRequest).actionGet();
+ indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add()
+ .indices("test20160101", "test20160102", "test20160103")
+ .alias(alias)
+ );
+ client().admin().indices().aliases(indicesAliasesRequest).actionGet();
- GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client("1"),
+ GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client(),
GetAliasesAction.INSTANCE);
GetAliasesResponse getAliasesResponse = getAliasesRequestBuilder.setAliases(alias).execute().actionGet();
Pattern pattern = Pattern.compile("^(.*?)(\\d+)$");
Set result = new TreeSet<>(Collections.reverseOrder());
for (ObjectCursor indexName : getAliasesResponse.getAliases().keys()) {
Matcher m = pattern.matcher(indexName.value);
- if (m.matches()) {
- if (alias.equals(m.group(1))) {
- result.add(indexName.value);
- }
+ if (m.matches() && alias.equals(m.group(1))) {
+ result.add(indexName.value);
}
}
Iterator it = result.iterator();
@@ -88,5 +73,4 @@ public class AliasTest extends NodeTestUtils {
assertEquals("test20160101", it.next());
logger.info("result={}", result);
}
-
}
diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/NetworkTest.java b/common/src/test/java/org/xbib/elasticsearch/client/common/NetworkTest.java
similarity index 81%
rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/NetworkTest.java
rename to common/src/test/java/org/xbib/elasticsearch/client/common/NetworkTest.java
index b9e7a87..0ed4fc8 100644
--- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/NetworkTest.java
+++ b/common/src/test/java/org/xbib/elasticsearch/client/common/NetworkTest.java
@@ -1,4 +1,4 @@
-package org.xbib.elasticsearch.extras.client;
+package org.xbib.elasticsearch.client.common;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -9,18 +9,21 @@ import java.net.NetworkInterface;
import java.util.Collections;
import java.util.Enumeration;
-/**
- *
- */
public class NetworkTest {
private static final Logger logger = LogManager.getLogger(NetworkTest.class);
+ /**
+ * Demonstrates the slowness oj Java network interface lookup on certain environments.
+ * May be a killer for ES node startup - so avoid automatic traversal of NICs at all costs.
+ *
+ * @throws Exception if test fails
+ */
@Test
public void testNetwork() throws Exception {
Enumeration nets = NetworkInterface.getNetworkInterfaces();
for (NetworkInterface netint : Collections.list(nets)) {
- System.out.println("checking network interface = " + netint.getName());
+ logger.info("checking network interface = " + netint.getName());
Enumeration inetAddresses = netint.getInetAddresses();
for (InetAddress addr : Collections.list(inetAddresses)) {
logger.info("found address = " + addr.getHostAddress()
diff --git a/src/integration-test/java/org/xbib/elasticsearch/SearchTest.java b/common/src/test/java/org/xbib/elasticsearch/client/common/SearchTests.java
similarity index 67%
rename from src/integration-test/java/org/xbib/elasticsearch/SearchTest.java
rename to common/src/test/java/org/xbib/elasticsearch/client/common/SearchTests.java
index 8d1276a..09771c5 100644
--- a/src/integration-test/java/org/xbib/elasticsearch/SearchTest.java
+++ b/common/src/test/java/org/xbib/elasticsearch/client/common/SearchTests.java
@@ -1,37 +1,29 @@
-package org.xbib.elasticsearch;
-
-import static org.elasticsearch.client.Requests.indexRequest;
-import static org.elasticsearch.client.Requests.refreshRequest;
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+package org.xbib.elasticsearch.client.common;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.bulk.BulkAction;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.elasticsearch.client.Requests;
+import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.sort.SortOrder;
-import org.junit.Test;
+import org.elasticsearch.test.ESSingleNodeTestCase;
-/**
- *
- */
-public class SearchTest extends NodeTestUtils {
+public class SearchTests extends ESSingleNodeTestCase {
- private static final ESLogger logger = ESLoggerFactory.getLogger("test");
+ private static final Logger logger = LogManager.getLogger(SearchTests.class.getName());
- @Test
public void testSearch() throws Exception {
- Client client = client("1");
long t0 = System.currentTimeMillis();
- BulkRequestBuilder builder = new BulkRequestBuilder(client, BulkAction.INSTANCE);
+ BulkRequestBuilder builder = new BulkRequestBuilder(client(), BulkAction.INSTANCE);
for (int i = 0; i < 1000; i++) {
- builder.add(indexRequest()
+ builder.add(Requests.indexRequest()
.index("pages").type("row")
- .source(jsonBuilder()
+ .source(XContentFactory.jsonBuilder()
.startObject()
.field("user1", "kimchy")
.field("user2", "kimchy")
@@ -43,20 +35,18 @@ public class SearchTest extends NodeTestUtils {
.field("user8", "kimchy")
.field("user9", "kimchy")
.field("rowcount", i)
- .field("rs", 1234)));
+ .field("rs", 1234)
+ .endObject()));
}
- client.bulk(builder.request()).actionGet();
-
- client.admin().indices().refresh(refreshRequest()).actionGet();
-
+ client().bulk(builder.request()).actionGet();
+ client().admin().indices().refresh(Requests.refreshRequest()).actionGet();
long t1 = System.currentTimeMillis();
logger.info("t1-t0 = {}", t1 - t0);
-
for (int i = 0; i < 100; i++) {
t1 = System.currentTimeMillis();
QueryBuilder queryStringBuilder =
QueryBuilders.queryStringQuery("rs:" + 1234);
- SearchRequestBuilder requestBuilder = client.prepareSearch()
+ SearchRequestBuilder requestBuilder = client().prepareSearch()
.setIndices("pages")
.setTypes("row")
.setQuery(queryStringBuilder)
diff --git a/common/src/test/java/org/xbib/elasticsearch/client/common/SimpleTests.java b/common/src/test/java/org/xbib/elasticsearch/client/common/SimpleTests.java
new file mode 100644
index 0000000..8b89df6
--- /dev/null
+++ b/common/src/test/java/org/xbib/elasticsearch/client/common/SimpleTests.java
@@ -0,0 +1,61 @@
+package org.xbib.elasticsearch.client.common;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder;
+import org.elasticsearch.action.index.IndexAction;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+public class SimpleTests extends ESSingleNodeTestCase {
+
+ private static final Logger logger = LogManager.getLogger(SimpleTests.class.getName());
+
+ public void test() throws Exception {
+ try {
+ DeleteIndexRequestBuilder deleteIndexRequestBuilder =
+ new DeleteIndexRequestBuilder(client(), DeleteIndexAction.INSTANCE, "test");
+ deleteIndexRequestBuilder.execute().actionGet();
+ } catch (Exception e) {
+ logger.warn(e.getMessage(), e);
+ }
+ CreateIndexRequestBuilder createIndexRequestBuilder = new CreateIndexRequestBuilder(client(),
+ CreateIndexAction.INSTANCE)
+ .setIndex("test")
+ .setSettings(Settings.builder()
+ .put("index.analysis.analyzer.default.filter.0", "lowercase")
+ // where is the trim token filter???
+ //.put("index.analysis.analyzer.default.filter.1", "trim")
+ .put("index.analysis.analyzer.default.tokenizer", "keyword")
+ .build());
+ createIndexRequestBuilder.execute().actionGet();
+
+ IndexRequestBuilder indexRequestBuilder = new IndexRequestBuilder(client(), IndexAction.INSTANCE);
+ indexRequestBuilder
+ .setIndex("test")
+ .setType("test")
+ .setId("1")
+ .setSource(XContentFactory.jsonBuilder().startObject().field("field",
+ "1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8").endObject())
+ .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+ .execute()
+ .actionGet();
+ String doc = client().prepareSearch("test")
+ .setTypes("test")
+ .setQuery(QueryBuilders.matchQuery("field",
+ "1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8"))
+ .execute()
+ .actionGet()
+ .getHits().getAt(0).getSourceAsString();
+
+ assertEquals(doc,
+ "{\"field\":\"1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8\"}");
+ }
+}
diff --git a/common/src/test/java/org/xbib/elasticsearch/client/common/WildcardTests.java b/common/src/test/java/org/xbib/elasticsearch/client/common/WildcardTests.java
new file mode 100644
index 0000000..bb84bb9
--- /dev/null
+++ b/common/src/test/java/org/xbib/elasticsearch/client/common/WildcardTests.java
@@ -0,0 +1,51 @@
+package org.xbib.elasticsearch.client.common;
+
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.client.Requests;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import java.io.IOException;
+
+public class WildcardTests extends ESSingleNodeTestCase {
+
+ public void testWildcard() throws Exception {
+ index("1", "010");
+ index("2", "0*0");
+ // exact
+ validateCount(QueryBuilders.queryStringQuery("010").defaultField("field"), 1);
+ validateCount(QueryBuilders.queryStringQuery("0\\*0").defaultField("field"), 1);
+ // pattern
+ validateCount(QueryBuilders.queryStringQuery("0*0").defaultField("field"), 1); // 2?
+ validateCount(QueryBuilders.queryStringQuery("0?0").defaultField("field"), 1); // 2?
+ validateCount(QueryBuilders.queryStringQuery("0**0").defaultField("field"), 1); // 2?
+ validateCount(QueryBuilders.queryStringQuery("0??0").defaultField("field"), 0);
+ validateCount(QueryBuilders.queryStringQuery("*10").defaultField("field"), 1);
+ validateCount(QueryBuilders.queryStringQuery("*1*").defaultField("field"), 1);
+ validateCount(QueryBuilders.queryStringQuery("*\\*0").defaultField("field"), 0); // 1?
+ validateCount(QueryBuilders.queryStringQuery("*\\**").defaultField("field"), 0); // 1?
+ }
+
+ private void index(String id, String fieldValue) throws IOException {
+ client().index(Requests.indexRequest()
+ .index("index").type("type").id(id)
+ .source(XContentFactory.jsonBuilder().startObject().field("field", fieldValue).endObject())
+ .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE))
+ .actionGet();
+ }
+
+ private void validateCount(QueryBuilder queryBuilder, long expectedHits) {
+ final long actualHits = count(queryBuilder);
+ if (actualHits != expectedHits) {
+ throw new RuntimeException("actualHits=" + actualHits + ", expectedHits=" + expectedHits);
+ }
+ }
+
+ private long count(QueryBuilder queryBuilder) {
+ return client().prepareSearch("index").setTypes("type")
+ .setQuery(queryBuilder)
+ .execute().actionGet().getHits().getTotalHits();
+ }
+}
diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/package-info.java b/common/src/test/java/org/xbib/elasticsearch/client/common/package-info.java
similarity index 52%
rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/package-info.java
rename to common/src/test/java/org/xbib/elasticsearch/client/common/package-info.java
index 2bfc45c..af3209f 100644
--- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/package-info.java
+++ b/common/src/test/java/org/xbib/elasticsearch/client/common/package-info.java
@@ -1,4 +1,4 @@
/**
* Classes to test Elasticsearch clients.
*/
-package org.xbib.elasticsearch.extras.client;
+package org.xbib.elasticsearch.client.common;
diff --git a/src/integration-test/resources/log4j2.xml b/common/src/test/resources/log4j2.xml
similarity index 100%
rename from src/integration-test/resources/log4j2.xml
rename to common/src/test/resources/log4j2.xml
diff --git a/gradle.properties b/gradle.properties
index 3f32f32..24bd424 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1,3 +1,30 @@
-group = org.xbib
-name = elasticsearch-extras-client
-version = 2.2.1.2
+group = org.xbib.elasticsearch
+name = elasticsearch-client
+version = 6.3.2.0
+profile = default
+release = 0
+
+elasticsearch.version = 6.3.2
+lucene.version = 7.3.1
+
+netty.version = 4.1.29.Final
+tcnative.version = 2.0.15.Final
+alpnagent.version = 2.0.7
+xbib-netty-http-client.version = 4.1.29.0
+xbib-metrics.version = 1.1.0
+
+# elasticsearch build plugin
+elasticsearch-libs.version = 6.3.2.1
+elasticsearch-devkit.version = 6.3.2.4
+spatial4j.version = 0.7
+jts.version = 1.15.1
+jna.version = 4.5.2
+log4j.version = 2.11.1
+checkstyle.version = 8.13
+
+# test
+junit.version = 4.12
+wagon.version = 3.0.0
+asciidoclet.version = 1.6.0.0
+
+org.gradle.warning.mode=all
diff --git a/gradle/ext.gradle b/gradle/ext.gradle
index 7bb7c73..e69de29 100644
--- a/gradle/ext.gradle
+++ b/gradle/ext.gradle
@@ -1,8 +0,0 @@
-ext {
- user = 'xbib'
- name = 'elasticsearch-extras-client'
- description = 'Some extras implemented for using Elasticsearch clients (node and transport)'
- scmUrl = 'https://github.com/' + user + '/' + name
- scmConnection = 'scm:git:git://github.com/' + user + '/' + name + '.git'
- scmDeveloperConnection = 'scm:git:git://github.com/' + user + '/' + name + '.git'
-}
diff --git a/gradle/publish.gradle b/gradle/publish.gradle
index 0337849..c05a223 100644
--- a/gradle/publish.gradle
+++ b/gradle/publish.gradle
@@ -6,7 +6,7 @@ task xbibUpload(type: Upload) {
if (project.hasProperty("xbibUsername")) {
mavenDeployer {
configuration = configurations.wagon
- repository(url: 'scpexe://xbib.org/repository') {
+ repository(url: 'sftp://xbib.org/repository') {
authentication(userName: xbibUsername, privateKey: xbibPrivateKey)
}
}
diff --git a/gradle/sonarqube.gradle b/gradle/sonarqube.gradle
index 5de408d..d759e4c 100644
--- a/gradle/sonarqube.gradle
+++ b/gradle/sonarqube.gradle
@@ -1,8 +1,8 @@
tasks.withType(FindBugs) {
ignoreFailures = true
reports {
- xml.enabled = true
- html.enabled = false
+ xml.enabled = false
+ html.enabled = true
}
}
tasks.withType(Pmd) {
@@ -20,22 +20,11 @@ tasks.withType(Checkstyle) {
}
}
-jacocoTestReport {
- reports {
- xml.enabled true
- csv.enabled false
- xml.destination "${buildDir}/reports/jacoco-xml"
- html.destination "${buildDir}/reports/jacoco-html"
- }
-}
-
sonarqube {
properties {
property "sonar.projectName", "${project.group} ${project.name}"
property "sonar.sourceEncoding", "UTF-8"
- property "sonar.tests", "src/integration-test/java"
property "sonar.scm.provider", "git"
- property "sonar.java.coveragePlugin", "jacoco"
property "sonar.junit.reportsPath", "build/test-results/test/"
}
}
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index 27b5466..6b3851a 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -1,6 +1,5 @@
-#Tue Jan 03 14:13:22 CET 2017
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.1-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-all.zip
diff --git a/gradlew b/gradlew
index 4453cce..2c74879 100755
--- a/gradlew
+++ b/gradlew
@@ -28,16 +28,16 @@ APP_NAME="Gradle"
APP_BASE_NAME=`basename "$0"`
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
+DEFAULT_JVM_OPTS='"-Xmx64m"'
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"
-warn ( ) {
+warn () {
echo "$*"
}
-die ( ) {
+die () {
echo
echo "$*"
echo
@@ -155,7 +155,11 @@ if $cygwin ; then
fi
# Escape application args
+<<<<<<< HEAD
save ( ) {
+=======
+save () {
+>>>>>>> 6.3
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
echo " "
}
diff --git a/gradlew.bat b/gradlew.bat
index e95643d..0f8d593 100644
--- a/gradlew.bat
+++ b/gradlew.bat
@@ -14,7 +14,7 @@ set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
+set DEFAULT_JVM_OPTS="-Xmx64m"
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
diff --git a/http/build.gradle b/http/build.gradle
new file mode 100644
index 0000000..f5d73a1
--- /dev/null
+++ b/http/build.gradle
@@ -0,0 +1,65 @@
+buildscript {
+ repositories {
+ jcenter()
+ maven {
+ url 'http://xbib.org/repository'
+ }
+ }
+ dependencies {
+ classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.3.2.4"
+ }
+}
+
+apply plugin: 'org.xbib.gradle.plugin.elasticsearch.build'
+
+configurations {
+ main
+ tests
+}
+
+dependencies {
+ compile project(':common')
+ compile "org.xbib:netty-http-client:${project.property('xbib-netty-http-client.version')}"
+ testCompile "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}"
+ testRuntime "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}"
+}
+
+jar {
+ baseName "${rootProject.name}-common"
+}
+
+/*
+task testJar(type: Jar, dependsOn: testClasses) {
+ baseName = "${project.archivesBaseName}-tests"
+ from sourceSets.test.output
+}
+*/
+
+artifacts {
+ main jar
+ tests testJar
+ archives sourcesJar, javadocJar
+}
+
+test {
+ enabled = true
+ include '**/SimpleTest.*'
+ testLogging {
+ showStandardStreams = true
+ exceptionFormat = 'full'
+ }
+}
+
+randomizedTest {
+ enabled = false
+}
+
+esTest {
+ enabled = true
+ // test with the jars, not the classes, for security manager
+ // classpath = files(configurations.testRuntime) + configurations.main.artifacts.files + configurations.tests.artifacts.files
+ systemProperty 'tests.security.manager', 'true'
+ // maybe we like some extra security policy for our code
+ systemProperty 'tests.security.policy', '/extra-security.policy'
+}
+esTest.dependsOn jar, testJar
diff --git a/http/config/checkstyle/checkstyle.xml b/http/config/checkstyle/checkstyle.xml
new file mode 100644
index 0000000..7af8d6d
--- /dev/null
+++ b/http/config/checkstyle/checkstyle.xml
@@ -0,0 +1,323 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/http/src/docs/asciidoc/css/foundation.css b/http/src/docs/asciidoc/css/foundation.css
new file mode 100644
index 0000000..27be611
--- /dev/null
+++ b/http/src/docs/asciidoc/css/foundation.css
@@ -0,0 +1,684 @@
+/*! normalize.css v2.1.2 | MIT License | git.io/normalize */
+/* ========================================================================== HTML5 display definitions ========================================================================== */
+/** Correct `block` display not defined in IE 8/9. */
+article, aside, details, figcaption, figure, footer, header, hgroup, main, nav, section, summary { display: block; }
+
+/** Correct `inline-block` display not defined in IE 8/9. */
+audio, canvas, video { display: inline-block; }
+
+/** Prevent modern browsers from displaying `audio` without controls. Remove excess height in iOS 5 devices. */
+audio:not([controls]) { display: none; height: 0; }
+
+/** Address `[hidden]` styling not present in IE 8/9. Hide the `template` element in IE, Safari, and Firefox < 22. */
+[hidden], template { display: none; }
+
+script { display: none !important; }
+
+/* ========================================================================== Base ========================================================================== */
+/** 1. Set default font family to sans-serif. 2. Prevent iOS text size adjust after orientation change, without disabling user zoom. */
+html { font-family: sans-serif; /* 1 */ -ms-text-size-adjust: 100%; /* 2 */ -webkit-text-size-adjust: 100%; /* 2 */ }
+
+/** Remove default margin. */
+body { margin: 0; }
+
+/* ========================================================================== Links ========================================================================== */
+/** Remove the gray background color from active links in IE 10. */
+a { background: transparent; }
+
+/** Address `outline` inconsistency between Chrome and other browsers. */
+a:focus { outline: thin dotted; }
+
+/** Improve readability when focused and also mouse hovered in all browsers. */
+a:active, a:hover { outline: 0; }
+
+/* ========================================================================== Typography ========================================================================== */
+/** Address variable `h1` font-size and margin within `section` and `article` contexts in Firefox 4+, Safari 5, and Chrome. */
+h1 { font-size: 2em; margin: 0.67em 0; }
+
+/** Address styling not present in IE 8/9, Safari 5, and Chrome. */
+abbr[title] { border-bottom: 1px dotted; }
+
+/** Address style set to `bolder` in Firefox 4+, Safari 5, and Chrome. */
+b, strong { font-weight: bold; }
+
+/** Address styling not present in Safari 5 and Chrome. */
+dfn { font-style: italic; }
+
+/** Address differences between Firefox and other browsers. */
+hr { -moz-box-sizing: content-box; box-sizing: content-box; height: 0; }
+
+/** Address styling not present in IE 8/9. */
+mark { background: #ff0; color: #000; }
+
+/** Correct font family set oddly in Safari 5 and Chrome. */
+code, kbd, pre, samp { font-family: monospace, serif; font-size: 1em; }
+
+/** Improve readability of pre-formatted text in all browsers. */
+pre { white-space: pre-wrap; }
+
+/** Set consistent quote types. */
+q { quotes: "\201C" "\201D" "\2018" "\2019"; }
+
+/** Address inconsistent and variable font size in all browsers. */
+small { font-size: 80%; }
+
+/** Prevent `sub` and `sup` affecting `line-height` in all browsers. */
+sub, sup { font-size: 75%; line-height: 0; position: relative; vertical-align: baseline; }
+
+sup { top: -0.5em; }
+
+sub { bottom: -0.25em; }
+
+/* ========================================================================== Embedded content ========================================================================== */
+/** Remove border when inside `a` element in IE 8/9. */
+img { border: 0; }
+
+/** Correct overflow displayed oddly in IE 9. */
+svg:not(:root) { overflow: hidden; }
+
+/* ========================================================================== Figures ========================================================================== */
+/** Address margin not present in IE 8/9 and Safari 5. */
+figure { margin: 0; }
+
+/* ========================================================================== Forms ========================================================================== */
+/** Define consistent border, margin, and padding. */
+fieldset { border: 1px solid #c0c0c0; margin: 0 2px; padding: 0.35em 0.625em 0.75em; }
+
+/** 1. Correct `color` not being inherited in IE 8/9. 2. Remove padding so people aren't caught out if they zero out fieldsets. */
+legend { border: 0; /* 1 */ padding: 0; /* 2 */ }
+
+/** 1. Correct font family not being inherited in all browsers. 2. Correct font size not being inherited in all browsers. 3. Address margins set differently in Firefox 4+, Safari 5, and Chrome. */
+button, input, select, textarea { font-family: inherit; /* 1 */ font-size: 100%; /* 2 */ margin: 0; /* 3 */ }
+
+/** Address Firefox 4+ setting `line-height` on `input` using `!important` in the UA stylesheet. */
+button, input { line-height: normal; }
+
+/** Address inconsistent `text-transform` inheritance for `button` and `select`. All other form control elements do not inherit `text-transform` values. Correct `button` style inheritance in Chrome, Safari 5+, and IE 8+. Correct `select` style inheritance in Firefox 4+ and Opera. */
+button, select { text-transform: none; }
+
+/** 1. Avoid the WebKit bug in Android 4.0.* where (2) destroys native `audio` and `video` controls. 2. Correct inability to style clickable `input` types in iOS. 3. Improve usability and consistency of cursor style between image-type `input` and others. */
+button, html input[type="button"], input[type="reset"], input[type="submit"] { -webkit-appearance: button; /* 2 */ cursor: pointer; /* 3 */ }
+
+/** Re-set default cursor for disabled elements. */
+button[disabled], html input[disabled] { cursor: default; }
+
+/** 1. Address box sizing set to `content-box` in IE 8/9. 2. Remove excess padding in IE 8/9. */
+input[type="checkbox"], input[type="radio"] { box-sizing: border-box; /* 1 */ padding: 0; /* 2 */ }
+
+/** 1. Address `appearance` set to `searchfield` in Safari 5 and Chrome. 2. Address `box-sizing` set to `border-box` in Safari 5 and Chrome (include `-moz` to future-proof). */
+input[type="search"] { -webkit-appearance: textfield; /* 1 */ -moz-box-sizing: content-box; -webkit-box-sizing: content-box; /* 2 */ box-sizing: content-box; }
+
+/** Remove inner padding and search cancel button in Safari 5 and Chrome on OS X. */
+input[type="search"]::-webkit-search-cancel-button, input[type="search"]::-webkit-search-decoration { -webkit-appearance: none; }
+
+/** Remove inner padding and border in Firefox 4+. */
+button::-moz-focus-inner, input::-moz-focus-inner { border: 0; padding: 0; }
+
+/** 1. Remove default vertical scrollbar in IE 8/9. 2. Improve readability and alignment in all browsers. */
+textarea { overflow: auto; /* 1 */ vertical-align: top; /* 2 */ }
+
+/* ========================================================================== Tables ========================================================================== */
+/** Remove most spacing between table cells. */
+table { border-collapse: collapse; border-spacing: 0; }
+
+meta.foundation-mq-small { font-family: "only screen and (min-width: 768px)"; width: 768px; }
+
+meta.foundation-mq-medium { font-family: "only screen and (min-width:1280px)"; width: 1280px; }
+
+meta.foundation-mq-large { font-family: "only screen and (min-width:1440px)"; width: 1440px; }
+
+*, *:before, *:after { -moz-box-sizing: border-box; -webkit-box-sizing: border-box; box-sizing: border-box; }
+
+html, body { font-size: 100%; }
+
+body { background: white; color: #222222; padding: 0; margin: 0; font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; font-weight: normal; font-style: normal; line-height: 1; position: relative; cursor: auto; }
+
+a:hover { cursor: pointer; }
+
+img, object, embed { max-width: 100%; height: auto; }
+
+object, embed { height: 100%; }
+
+img { -ms-interpolation-mode: bicubic; }
+
+#map_canvas img, #map_canvas embed, #map_canvas object, .map_canvas img, .map_canvas embed, .map_canvas object { max-width: none !important; }
+
+.left { float: left !important; }
+
+.right { float: right !important; }
+
+.text-left { text-align: left !important; }
+
+.text-right { text-align: right !important; }
+
+.text-center { text-align: center !important; }
+
+.text-justify { text-align: justify !important; }
+
+.hide { display: none; }
+
+.antialiased { -webkit-font-smoothing: antialiased; }
+
+img { display: inline-block; vertical-align: middle; }
+
+textarea { height: auto; min-height: 50px; }
+
+select { width: 100%; }
+
+object, svg { display: inline-block; vertical-align: middle; }
+
+.center { margin-left: auto; margin-right: auto; }
+
+.spread { width: 100%; }
+
+p.lead, .paragraph.lead > p, #preamble > .sectionbody > .paragraph:first-of-type p { font-size: 1.21875em; line-height: 1.6; }
+
+.subheader, .admonitionblock td.content > .title, .audioblock > .title, .exampleblock > .title, .imageblock > .title, .listingblock > .title, .literalblock > .title, .stemblock > .title, .openblock > .title, .paragraph > .title, .quoteblock > .title, table.tableblock > .title, .verseblock > .title, .videoblock > .title, .dlist > .title, .olist > .title, .ulist > .title, .qlist > .title, .hdlist > .title { line-height: 1.4; color: #6f6f6f; font-weight: 300; margin-top: 0.2em; margin-bottom: 0.5em; }
+
+/* Typography resets */
+div, dl, dt, dd, ul, ol, li, h1, h2, h3, #toctitle, .sidebarblock > .content > .title, h4, h5, h6, pre, form, p, blockquote, th, td { margin: 0; padding: 0; direction: ltr; }
+
+/* Default Link Styles */
+a { color: #2ba6cb; text-decoration: none; line-height: inherit; }
+a:hover, a:focus { color: #2795b6; }
+a img { border: none; }
+
+/* Default paragraph styles */
+p { font-family: inherit; font-weight: normal; font-size: 1em; line-height: 1.6; margin-bottom: 1.25em; text-rendering: optimizeLegibility; }
+p aside { font-size: 0.875em; line-height: 1.35; font-style: italic; }
+
+/* Default header styles */
+h1, h2, h3, #toctitle, .sidebarblock > .content > .title, h4, h5, h6 { font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; font-weight: bold; font-style: normal; color: #222222; text-rendering: optimizeLegibility; margin-top: 1em; margin-bottom: 0.5em; line-height: 1.2125em; }
+h1 small, h2 small, h3 small, #toctitle small, .sidebarblock > .content > .title small, h4 small, h5 small, h6 small { font-size: 60%; color: #6f6f6f; line-height: 0; }
+
+h1 { font-size: 2.125em; }
+
+h2 { font-size: 1.6875em; }
+
+h3, #toctitle, .sidebarblock > .content > .title { font-size: 1.375em; }
+
+h4 { font-size: 1.125em; }
+
+h5 { font-size: 1.125em; }
+
+h6 { font-size: 1em; }
+
+hr { border: solid #dddddd; border-width: 1px 0 0; clear: both; margin: 1.25em 0 1.1875em; height: 0; }
+
+/* Helpful Typography Defaults */
+em, i { font-style: italic; line-height: inherit; }
+
+strong, b { font-weight: bold; line-height: inherit; }
+
+small { font-size: 60%; line-height: inherit; }
+
+code { font-family: Consolas, "Liberation Mono", Courier, monospace; font-weight: bold; color: #7f0a0c; }
+
+/* Lists */
+ul, ol, dl { font-size: 1em; line-height: 1.6; margin-bottom: 1.25em; list-style-position: outside; font-family: inherit; }
+
+ul, ol { margin-left: 1.5em; }
+ul.no-bullet, ol.no-bullet { margin-left: 1.5em; }
+
+/* Unordered Lists */
+ul li ul, ul li ol { margin-left: 1.25em; margin-bottom: 0; font-size: 1em; /* Override nested font-size change */ }
+ul.square li ul, ul.circle li ul, ul.disc li ul { list-style: inherit; }
+ul.square { list-style-type: square; }
+ul.circle { list-style-type: circle; }
+ul.disc { list-style-type: disc; }
+ul.no-bullet { list-style: none; }
+
+/* Ordered Lists */
+ol li ul, ol li ol { margin-left: 1.25em; margin-bottom: 0; }
+
+/* Definition Lists */
+dl dt { margin-bottom: 0.3125em; font-weight: bold; }
+dl dd { margin-bottom: 1.25em; }
+
+/* Abbreviations */
+abbr, acronym { text-transform: uppercase; font-size: 90%; color: #222222; border-bottom: 1px dotted #dddddd; cursor: help; }
+
+abbr { text-transform: none; }
+
+/* Blockquotes */
+blockquote { margin: 0 0 1.25em; padding: 0.5625em 1.25em 0 1.1875em; border-left: 1px solid #dddddd; }
+blockquote cite { display: block; font-size: 0.8125em; color: #555555; }
+blockquote cite:before { content: "\2014 \0020"; }
+blockquote cite a, blockquote cite a:visited { color: #555555; }
+
+blockquote, blockquote p { line-height: 1.6; color: #6f6f6f; }
+
+/* Microformats */
+.vcard { display: inline-block; margin: 0 0 1.25em 0; border: 1px solid #dddddd; padding: 0.625em 0.75em; }
+.vcard li { margin: 0; display: block; }
+.vcard .fn { font-weight: bold; font-size: 0.9375em; }
+
+.vevent .summary { font-weight: bold; }
+.vevent abbr { cursor: auto; text-decoration: none; font-weight: bold; border: none; padding: 0 0.0625em; }
+
+@media only screen and (min-width: 768px) { h1, h2, h3, #toctitle, .sidebarblock > .content > .title, h4, h5, h6 { line-height: 1.4; }
+ h1 { font-size: 2.75em; }
+ h2 { font-size: 2.3125em; }
+ h3, #toctitle, .sidebarblock > .content > .title { font-size: 1.6875em; }
+ h4 { font-size: 1.4375em; } }
+/* Tables */
+table { background: white; margin-bottom: 1.25em; border: solid 1px #dddddd; }
+table thead, table tfoot { background: whitesmoke; font-weight: bold; }
+table thead tr th, table thead tr td, table tfoot tr th, table tfoot tr td { padding: 0.5em 0.625em 0.625em; font-size: inherit; color: #222222; text-align: left; }
+table tr th, table tr td { padding: 0.5625em 0.625em; font-size: inherit; color: #222222; }
+table tr.even, table tr.alt, table tr:nth-of-type(even) { background: #f9f9f9; }
+table thead tr th, table tfoot tr th, table tbody tr td, table tr td, table tfoot tr td { display: table-cell; line-height: 1.4; }
+
+body { -moz-osx-font-smoothing: grayscale; -webkit-font-smoothing: antialiased; tab-size: 4; }
+
+h1, h2, h3, #toctitle, .sidebarblock > .content > .title, h4, h5, h6 { line-height: 1.4; }
+
+.clearfix:before, .clearfix:after, .float-group:before, .float-group:after { content: " "; display: table; }
+.clearfix:after, .float-group:after { clear: both; }
+
+*:not(pre) > code { font-size: inherit; font-style: normal !important; letter-spacing: 0; padding: 0; line-height: inherit; word-wrap: break-word; }
+*:not(pre) > code.nobreak { word-wrap: normal; }
+*:not(pre) > code.nowrap { white-space: nowrap; }
+
+pre, pre > code { line-height: 1.4; color: black; font-family: monospace, serif; font-weight: normal; }
+
+em em { font-style: normal; }
+
+strong strong { font-weight: normal; }
+
+.keyseq { color: #555555; }
+
+kbd { font-family: Consolas, "Liberation Mono", Courier, monospace; display: inline-block; color: #222222; font-size: 0.65em; line-height: 1.45; background-color: #f7f7f7; border: 1px solid #ccc; -webkit-border-radius: 3px; border-radius: 3px; -webkit-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.2), 0 0 0 0.1em white inset; box-shadow: 0 1px 0 rgba(0, 0, 0, 0.2), 0 0 0 0.1em white inset; margin: 0 0.15em; padding: 0.2em 0.5em; vertical-align: middle; position: relative; top: -0.1em; white-space: nowrap; }
+
+.keyseq kbd:first-child { margin-left: 0; }
+
+.keyseq kbd:last-child { margin-right: 0; }
+
+.menuseq, .menu { color: #090909; }
+
+b.button:before, b.button:after { position: relative; top: -1px; font-weight: normal; }
+
+b.button:before { content: "["; padding: 0 3px 0 2px; }
+
+b.button:after { content: "]"; padding: 0 2px 0 3px; }
+
+#header, #content, #footnotes, #footer { width: 100%; margin-left: auto; margin-right: auto; margin-top: 0; margin-bottom: 0; max-width: 62.5em; *zoom: 1; position: relative; padding-left: 0.9375em; padding-right: 0.9375em; }
+#header:before, #header:after, #content:before, #content:after, #footnotes:before, #footnotes:after, #footer:before, #footer:after { content: " "; display: table; }
+#header:after, #content:after, #footnotes:after, #footer:after { clear: both; }
+
+#content { margin-top: 1.25em; }
+
+#content:before { content: none; }
+
+#header > h1:first-child { color: black; margin-top: 2.25rem; margin-bottom: 0; }
+#header > h1:first-child + #toc { margin-top: 8px; border-top: 1px solid #dddddd; }
+#header > h1:only-child, body.toc2 #header > h1:nth-last-child(2) { border-bottom: 1px solid #dddddd; padding-bottom: 8px; }
+#header .details { border-bottom: 1px solid #dddddd; line-height: 1.45; padding-top: 0.25em; padding-bottom: 0.25em; padding-left: 0.25em; color: #555555; display: -ms-flexbox; display: -webkit-flex; display: flex; -ms-flex-flow: row wrap; -webkit-flex-flow: row wrap; flex-flow: row wrap; }
+#header .details span:first-child { margin-left: -0.125em; }
+#header .details span.email a { color: #6f6f6f; }
+#header .details br { display: none; }
+#header .details br + span:before { content: "\00a0\2013\00a0"; }
+#header .details br + span.author:before { content: "\00a0\22c5\00a0"; color: #6f6f6f; }
+#header .details br + span#revremark:before { content: "\00a0|\00a0"; }
+#header #revnumber { text-transform: capitalize; }
+#header #revnumber:after { content: "\00a0"; }
+
+#content > h1:first-child:not([class]) { color: black; border-bottom: 1px solid #dddddd; padding-bottom: 8px; margin-top: 0; padding-top: 1rem; margin-bottom: 1.25rem; }
+
+#toc { border-bottom: 1px solid #dddddd; padding-bottom: 0.5em; }
+#toc > ul { margin-left: 0.125em; }
+#toc ul.sectlevel0 > li > a { font-style: italic; }
+#toc ul.sectlevel0 ul.sectlevel1 { margin: 0.5em 0; }
+#toc ul { font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; list-style-type: none; }
+#toc li { line-height: 1.3334; margin-top: 0.3334em; }
+#toc a { text-decoration: none; }
+#toc a:active { text-decoration: underline; }
+
+#toctitle { color: #6f6f6f; font-size: 1.2em; }
+
+@media only screen and (min-width: 768px) { #toctitle { font-size: 1.375em; }
+ body.toc2 { padding-left: 15em; padding-right: 0; }
+ #toc.toc2 { margin-top: 0 !important; background-color: #f2f2f2; position: fixed; width: 15em; left: 0; top: 0; border-right: 1px solid #dddddd; border-top-width: 0 !important; border-bottom-width: 0 !important; z-index: 1000; padding: 1.25em 1em; height: 100%; overflow: auto; }
+ #toc.toc2 #toctitle { margin-top: 0; margin-bottom: 0.8rem; font-size: 1.2em; }
+ #toc.toc2 > ul { font-size: 0.9em; margin-bottom: 0; }
+ #toc.toc2 ul ul { margin-left: 0; padding-left: 1em; }
+ #toc.toc2 ul.sectlevel0 ul.sectlevel1 { padding-left: 0; margin-top: 0.5em; margin-bottom: 0.5em; }
+ body.toc2.toc-right { padding-left: 0; padding-right: 15em; }
+ body.toc2.toc-right #toc.toc2 { border-right-width: 0; border-left: 1px solid #dddddd; left: auto; right: 0; } }
+@media only screen and (min-width: 1280px) { body.toc2 { padding-left: 20em; padding-right: 0; }
+ #toc.toc2 { width: 20em; }
+ #toc.toc2 #toctitle { font-size: 1.375em; }
+ #toc.toc2 > ul { font-size: 0.95em; }
+ #toc.toc2 ul ul { padding-left: 1.25em; }
+ body.toc2.toc-right { padding-left: 0; padding-right: 20em; } }
+#content #toc { border-style: solid; border-width: 1px; border-color: #d9d9d9; margin-bottom: 1.25em; padding: 1.25em; background: #f2f2f2; -webkit-border-radius: 0; border-radius: 0; }
+#content #toc > :first-child { margin-top: 0; }
+#content #toc > :last-child { margin-bottom: 0; }
+
+#footer { max-width: 100%; background-color: #222222; padding: 1.25em; }
+
+#footer-text { color: #dddddd; line-height: 1.44; }
+
+.sect1 { padding-bottom: 0.625em; }
+
+@media only screen and (min-width: 768px) { .sect1 { padding-bottom: 1.25em; } }
+.sect1 + .sect1 { border-top: 1px solid #dddddd; }
+
+#content h1 > a.anchor, h2 > a.anchor, h3 > a.anchor, #toctitle > a.anchor, .sidebarblock > .content > .title > a.anchor, h4 > a.anchor, h5 > a.anchor, h6 > a.anchor { position: absolute; z-index: 1001; width: 1.5ex; margin-left: -1.5ex; display: block; text-decoration: none !important; visibility: hidden; text-align: center; font-weight: normal; }
+#content h1 > a.anchor:before, h2 > a.anchor:before, h3 > a.anchor:before, #toctitle > a.anchor:before, .sidebarblock > .content > .title > a.anchor:before, h4 > a.anchor:before, h5 > a.anchor:before, h6 > a.anchor:before { content: "\00A7"; font-size: 0.85em; display: block; padding-top: 0.1em; }
+#content h1:hover > a.anchor, #content h1 > a.anchor:hover, h2:hover > a.anchor, h2 > a.anchor:hover, h3:hover > a.anchor, #toctitle:hover > a.anchor, .sidebarblock > .content > .title:hover > a.anchor, h3 > a.anchor:hover, #toctitle > a.anchor:hover, .sidebarblock > .content > .title > a.anchor:hover, h4:hover > a.anchor, h4 > a.anchor:hover, h5:hover > a.anchor, h5 > a.anchor:hover, h6:hover > a.anchor, h6 > a.anchor:hover { visibility: visible; }
+#content h1 > a.link, h2 > a.link, h3 > a.link, #toctitle > a.link, .sidebarblock > .content > .title > a.link, h4 > a.link, h5 > a.link, h6 > a.link { color: #222222; text-decoration: none; }
+#content h1 > a.link:hover, h2 > a.link:hover, h3 > a.link:hover, #toctitle > a.link:hover, .sidebarblock > .content > .title > a.link:hover, h4 > a.link:hover, h5 > a.link:hover, h6 > a.link:hover { color: #151515; }
+
+.audioblock, .imageblock, .literalblock, .listingblock, .stemblock, .videoblock { margin-bottom: 1.25em; }
+
+.admonitionblock td.content > .title, .audioblock > .title, .exampleblock > .title, .imageblock > .title, .listingblock > .title, .literalblock > .title, .stemblock > .title, .openblock > .title, .paragraph > .title, .quoteblock > .title, table.tableblock > .title, .verseblock > .title, .videoblock > .title, .dlist > .title, .olist > .title, .ulist > .title, .qlist > .title, .hdlist > .title { text-rendering: optimizeLegibility; text-align: left; }
+
+table.tableblock > caption.title { white-space: nowrap; overflow: visible; max-width: 0; }
+
+.paragraph.lead > p, #preamble > .sectionbody > .paragraph:first-of-type p { color: black; }
+
+table.tableblock #preamble > .sectionbody > .paragraph:first-of-type p { font-size: inherit; }
+
+.admonitionblock > table { border-collapse: separate; border: 0; background: none; width: 100%; }
+.admonitionblock > table td.icon { text-align: center; width: 80px; }
+.admonitionblock > table td.icon img { max-width: initial; }
+.admonitionblock > table td.icon .title { font-weight: bold; font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; text-transform: uppercase; }
+.admonitionblock > table td.content { padding-left: 1.125em; padding-right: 1.25em; border-left: 1px solid #dddddd; color: #555555; }
+.admonitionblock > table td.content > :last-child > :last-child { margin-bottom: 0; }
+
+.exampleblock > .content { border-style: solid; border-width: 1px; border-color: #e6e6e6; margin-bottom: 1.25em; padding: 1.25em; background: white; -webkit-border-radius: 0; border-radius: 0; }
+.exampleblock > .content > :first-child { margin-top: 0; }
+.exampleblock > .content > :last-child { margin-bottom: 0; }
+
+.sidebarblock { border-style: solid; border-width: 1px; border-color: #d9d9d9; margin-bottom: 1.25em; padding: 1.25em; background: #f2f2f2; -webkit-border-radius: 0; border-radius: 0; }
+.sidebarblock > :first-child { margin-top: 0; }
+.sidebarblock > :last-child { margin-bottom: 0; }
+.sidebarblock > .content > .title { color: #6f6f6f; margin-top: 0; }
+
+.exampleblock > .content > :last-child > :last-child, .exampleblock > .content .olist > ol > li:last-child > :last-child, .exampleblock > .content .ulist > ul > li:last-child > :last-child, .exampleblock > .content .qlist > ol > li:last-child > :last-child, .sidebarblock > .content > :last-child > :last-child, .sidebarblock > .content .olist > ol > li:last-child > :last-child, .sidebarblock > .content .ulist > ul > li:last-child > :last-child, .sidebarblock > .content .qlist > ol > li:last-child > :last-child { margin-bottom: 0; }
+
+.literalblock pre, .listingblock pre:not(.highlight), .listingblock pre[class="highlight"], .listingblock pre[class^="highlight "], .listingblock pre.CodeRay, .listingblock pre.prettyprint { background: #eeeeee; }
+.sidebarblock .literalblock pre, .sidebarblock .listingblock pre:not(.highlight), .sidebarblock .listingblock pre[class="highlight"], .sidebarblock .listingblock pre[class^="highlight "], .sidebarblock .listingblock pre.CodeRay, .sidebarblock .listingblock pre.prettyprint { background: #f2f1f1; }
+
+.literalblock pre, .literalblock pre[class], .listingblock pre, .listingblock pre[class] { border: 1px solid #cccccc; -webkit-border-radius: 0; border-radius: 0; word-wrap: break-word; padding: 0.8em 0.8em 0.65em 0.8em; font-size: 0.8125em; }
+.literalblock pre.nowrap, .literalblock pre[class].nowrap, .listingblock pre.nowrap, .listingblock pre[class].nowrap { overflow-x: auto; white-space: pre; word-wrap: normal; }
+@media only screen and (min-width: 768px) { .literalblock pre, .literalblock pre[class], .listingblock pre, .listingblock pre[class] { font-size: 0.90625em; } }
+@media only screen and (min-width: 1280px) { .literalblock pre, .literalblock pre[class], .listingblock pre, .listingblock pre[class] { font-size: 1em; } }
+
+.literalblock.output pre { color: #eeeeee; background-color: black; }
+
+.listingblock pre.highlightjs { padding: 0; }
+.listingblock pre.highlightjs > code { padding: 0.8em 0.8em 0.65em 0.8em; -webkit-border-radius: 0; border-radius: 0; }
+
+.listingblock > .content { position: relative; }
+
+.listingblock code[data-lang]:before { display: none; content: attr(data-lang); position: absolute; font-size: 0.75em; top: 0.425rem; right: 0.5rem; line-height: 1; text-transform: uppercase; color: #999; }
+
+.listingblock:hover code[data-lang]:before { display: block; }
+
+.listingblock.terminal pre .command:before { content: attr(data-prompt); padding-right: 0.5em; color: #999; }
+
+.listingblock.terminal pre .command:not([data-prompt]):before { content: "$"; }
+
+table.pyhltable { border-collapse: separate; border: 0; margin-bottom: 0; background: none; }
+
+table.pyhltable td { vertical-align: top; padding-top: 0; padding-bottom: 0; line-height: 1.4; }
+
+table.pyhltable td.code { padding-left: .75em; padding-right: 0; }
+
+pre.pygments .lineno, table.pyhltable td:not(.code) { color: #999; padding-left: 0; padding-right: .5em; border-right: 1px solid #dddddd; }
+
+pre.pygments .lineno { display: inline-block; margin-right: .25em; }
+
+table.pyhltable .linenodiv { background: none !important; padding-right: 0 !important; }
+
+.quoteblock { margin: 0 1em 1.25em 1.5em; display: table; }
+.quoteblock > .title { margin-left: -1.5em; margin-bottom: 0.75em; }
+.quoteblock blockquote, .quoteblock blockquote p { color: #6f6f6f; font-size: 1.15rem; line-height: 1.75; word-spacing: 0.1em; letter-spacing: 0; font-style: italic; text-align: justify; }
+.quoteblock blockquote { margin: 0; padding: 0; border: 0; }
+.quoteblock blockquote:before { content: "\201c"; float: left; font-size: 2.75em; font-weight: bold; line-height: 0.6em; margin-left: -0.6em; color: #6f6f6f; text-shadow: 0 1px 2px rgba(0, 0, 0, 0.1); }
+.quoteblock blockquote > .paragraph:last-child p { margin-bottom: 0; }
+.quoteblock .attribution { margin-top: 0.5em; margin-right: 0.5ex; text-align: right; }
+.quoteblock .quoteblock { margin-left: 0; margin-right: 0; padding: 0.5em 0; border-left: 3px solid #555555; }
+.quoteblock .quoteblock blockquote { padding: 0 0 0 0.75em; }
+.quoteblock .quoteblock blockquote:before { display: none; }
+
+.verseblock { margin: 0 1em 1.25em 1em; }
+.verseblock pre { font-family: "Open Sans", "DejaVu Sans", sans; font-size: 1.15rem; color: #6f6f6f; font-weight: 300; text-rendering: optimizeLegibility; }
+.verseblock pre strong { font-weight: 400; }
+.verseblock .attribution { margin-top: 1.25rem; margin-left: 0.5ex; }
+
+.quoteblock .attribution, .verseblock .attribution { font-size: 0.8125em; line-height: 1.45; font-style: italic; }
+.quoteblock .attribution br, .verseblock .attribution br { display: none; }
+.quoteblock .attribution cite, .verseblock .attribution cite { display: block; letter-spacing: -0.025em; color: #555555; }
+
+.quoteblock.abstract { margin: 0 0 1.25em 0; display: block; }
+.quoteblock.abstract blockquote, .quoteblock.abstract blockquote p { text-align: left; word-spacing: 0; }
+.quoteblock.abstract blockquote:before, .quoteblock.abstract blockquote p:first-of-type:before { display: none; }
+
+table.tableblock { max-width: 100%; border-collapse: separate; }
+table.tableblock td > .paragraph:last-child p > p:last-child, table.tableblock th > p:last-child, table.tableblock td > p:last-child { margin-bottom: 0; }
+
+table.tableblock, th.tableblock, td.tableblock { border: 0 solid #dddddd; }
+
+table.grid-all th.tableblock, table.grid-all td.tableblock { border-width: 0 1px 1px 0; }
+
+table.grid-all tfoot > tr > th.tableblock, table.grid-all tfoot > tr > td.tableblock { border-width: 1px 1px 0 0; }
+
+table.grid-cols th.tableblock, table.grid-cols td.tableblock { border-width: 0 1px 0 0; }
+
+table.grid-all * > tr > .tableblock:last-child, table.grid-cols * > tr > .tableblock:last-child { border-right-width: 0; }
+
+table.grid-rows th.tableblock, table.grid-rows td.tableblock { border-width: 0 0 1px 0; }
+
+table.grid-all tbody > tr:last-child > th.tableblock, table.grid-all tbody > tr:last-child > td.tableblock, table.grid-all thead:last-child > tr > th.tableblock, table.grid-rows tbody > tr:last-child > th.tableblock, table.grid-rows tbody > tr:last-child > td.tableblock, table.grid-rows thead:last-child > tr > th.tableblock { border-bottom-width: 0; }
+
+table.grid-rows tfoot > tr > th.tableblock, table.grid-rows tfoot > tr > td.tableblock { border-width: 1px 0 0 0; }
+
+table.frame-all { border-width: 1px; }
+
+table.frame-sides { border-width: 0 1px; }
+
+table.frame-topbot { border-width: 1px 0; }
+
+th.halign-left, td.halign-left { text-align: left; }
+
+th.halign-right, td.halign-right { text-align: right; }
+
+th.halign-center, td.halign-center { text-align: center; }
+
+th.valign-top, td.valign-top { vertical-align: top; }
+
+th.valign-bottom, td.valign-bottom { vertical-align: bottom; }
+
+th.valign-middle, td.valign-middle { vertical-align: middle; }
+
+table thead th, table tfoot th { font-weight: bold; }
+
+tbody tr th { display: table-cell; line-height: 1.4; background: whitesmoke; }
+
+tbody tr th, tbody tr th p, tfoot tr th, tfoot tr th p { color: #222222; font-weight: bold; }
+
+p.tableblock > code:only-child { background: none; padding: 0; }
+
+p.tableblock { font-size: 1em; }
+
+td > div.verse { white-space: pre; }
+
+ol { margin-left: 1.75em; }
+
+ul li ol { margin-left: 1.5em; }
+
+dl dd { margin-left: 1.125em; }
+
+dl dd:last-child, dl dd:last-child > :last-child { margin-bottom: 0; }
+
+ol > li p, ul > li p, ul dd, ol dd, .olist .olist, .ulist .ulist, .ulist .olist, .olist .ulist { margin-bottom: 0.625em; }
+
+ul.unstyled, ol.unnumbered, ul.checklist, ul.none { list-style-type: none; }
+
+ul.unstyled, ol.unnumbered, ul.checklist { margin-left: 0.625em; }
+
+ul.checklist li > p:first-child > .fa-square-o:first-child, ul.checklist li > p:first-child > .fa-check-square-o:first-child { width: 1em; font-size: 0.85em; }
+
+ul.checklist li > p:first-child > input[type="checkbox"]:first-child { width: 1em; position: relative; top: 1px; }
+
+ul.inline { margin: 0 auto 0.625em auto; margin-left: -1.375em; margin-right: 0; padding: 0; list-style: none; overflow: hidden; }
+ul.inline > li { list-style: none; float: left; margin-left: 1.375em; display: block; }
+ul.inline > li > * { display: block; }
+
+.unstyled dl dt { font-weight: normal; font-style: normal; }
+
+ol.arabic { list-style-type: decimal; }
+
+ol.decimal { list-style-type: decimal-leading-zero; }
+
+ol.loweralpha { list-style-type: lower-alpha; }
+
+ol.upperalpha { list-style-type: upper-alpha; }
+
+ol.lowerroman { list-style-type: lower-roman; }
+
+ol.upperroman { list-style-type: upper-roman; }
+
+ol.lowergreek { list-style-type: lower-greek; }
+
+.hdlist > table, .colist > table { border: 0; background: none; }
+.hdlist > table > tbody > tr, .colist > table > tbody > tr { background: none; }
+
+td.hdlist1, td.hdlist2 { vertical-align: top; padding: 0 0.625em; }
+
+td.hdlist1 { font-weight: bold; padding-bottom: 1.25em; }
+
+.literalblock + .colist, .listingblock + .colist { margin-top: -0.5em; }
+
+.colist > table tr > td:first-of-type { padding: 0 0.75em; line-height: 1; }
+.colist > table tr > td:first-of-type img { max-width: initial; }
+.colist > table tr > td:last-of-type { padding: 0.25em 0; }
+
+.thumb, .th { line-height: 0; display: inline-block; border: solid 4px white; -webkit-box-shadow: 0 0 0 1px #dddddd; box-shadow: 0 0 0 1px #dddddd; }
+
+.imageblock.left, .imageblock[style*="float: left"] { margin: 0.25em 0.625em 1.25em 0; }
+.imageblock.right, .imageblock[style*="float: right"] { margin: 0.25em 0 1.25em 0.625em; }
+.imageblock > .title { margin-bottom: 0; }
+.imageblock.thumb, .imageblock.th { border-width: 6px; }
+.imageblock.thumb > .title, .imageblock.th > .title { padding: 0 0.125em; }
+
+.image.left, .image.right { margin-top: 0.25em; margin-bottom: 0.25em; display: inline-block; line-height: 0; }
+.image.left { margin-right: 0.625em; }
+.image.right { margin-left: 0.625em; }
+
+a.image { text-decoration: none; display: inline-block; }
+a.image object { pointer-events: none; }
+
+sup.footnote, sup.footnoteref { font-size: 0.875em; position: static; vertical-align: super; }
+sup.footnote a, sup.footnoteref a { text-decoration: none; }
+sup.footnote a:active, sup.footnoteref a:active { text-decoration: underline; }
+
+#footnotes { padding-top: 0.75em; padding-bottom: 0.75em; margin-bottom: 0.625em; }
+#footnotes hr { width: 20%; min-width: 6.25em; margin: -0.25em 0 0.75em 0; border-width: 1px 0 0 0; }
+#footnotes .footnote { padding: 0 0.375em 0 0.225em; line-height: 1.3334; font-size: 0.875em; margin-left: 1.2em; text-indent: -1.05em; margin-bottom: 0.2em; }
+#footnotes .footnote a:first-of-type { font-weight: bold; text-decoration: none; }
+#footnotes .footnote:last-of-type { margin-bottom: 0; }
+#content #footnotes { margin-top: -0.625em; margin-bottom: 0; padding: 0.75em 0; }
+
+.gist .file-data > table { border: 0; background: #fff; width: 100%; margin-bottom: 0; }
+.gist .file-data > table td.line-data { width: 99%; }
+
+div.unbreakable { page-break-inside: avoid; }
+
+.big { font-size: larger; }
+
+.small { font-size: smaller; }
+
+.underline { text-decoration: underline; }
+
+.overline { text-decoration: overline; }
+
+.line-through { text-decoration: line-through; }
+
+.aqua { color: #00bfbf; }
+
+.aqua-background { background-color: #00fafa; }
+
+.black { color: black; }
+
+.black-background { background-color: black; }
+
+.blue { color: #0000bf; }
+
+.blue-background { background-color: #0000fa; }
+
+.fuchsia { color: #bf00bf; }
+
+.fuchsia-background { background-color: #fa00fa; }
+
+.gray { color: #606060; }
+
+.gray-background { background-color: #7d7d7d; }
+
+.green { color: #006000; }
+
+.green-background { background-color: #007d00; }
+
+.lime { color: #00bf00; }
+
+.lime-background { background-color: #00fa00; }
+
+.maroon { color: #600000; }
+
+.maroon-background { background-color: #7d0000; }
+
+.navy { color: #000060; }
+
+.navy-background { background-color: #00007d; }
+
+.olive { color: #606000; }
+
+.olive-background { background-color: #7d7d00; }
+
+.purple { color: #600060; }
+
+.purple-background { background-color: #7d007d; }
+
+.red { color: #bf0000; }
+
+.red-background { background-color: #fa0000; }
+
+.silver { color: #909090; }
+
+.silver-background { background-color: #bcbcbc; }
+
+.teal { color: #006060; }
+
+.teal-background { background-color: #007d7d; }
+
+.white { color: #bfbfbf; }
+
+.white-background { background-color: #fafafa; }
+
+.yellow { color: #bfbf00; }
+
+.yellow-background { background-color: #fafa00; }
+
+span.icon > .fa { cursor: default; }
+
+.admonitionblock td.icon [class^="fa icon-"] { font-size: 2.5em; text-shadow: 1px 1px 2px rgba(0, 0, 0, 0.5); cursor: default; }
+.admonitionblock td.icon .icon-note:before { content: "\f05a"; color: #207c98; }
+.admonitionblock td.icon .icon-tip:before { content: "\f0eb"; text-shadow: 1px 1px 2px rgba(155, 155, 0, 0.8); color: #111; }
+.admonitionblock td.icon .icon-warning:before { content: "\f071"; color: #bf6900; }
+.admonitionblock td.icon .icon-caution:before { content: "\f06d"; color: #bf3400; }
+.admonitionblock td.icon .icon-important:before { content: "\f06a"; color: #bf0000; }
+
+.conum[data-value] { display: inline-block; color: #fff !important; background-color: #222222; -webkit-border-radius: 100px; border-radius: 100px; text-align: center; font-size: 0.75em; width: 1.67em; height: 1.67em; line-height: 1.67em; font-family: "Open Sans", "DejaVu Sans", sans-serif; font-style: normal; font-weight: bold; }
+.conum[data-value] * { color: #fff !important; }
+.conum[data-value] + b { display: none; }
+.conum[data-value]:after { content: attr(data-value); }
+pre .conum[data-value] { position: relative; top: -0.125em; }
+
+b.conum * { color: inherit !important; }
+
+.conum:not([data-value]):empty { display: none; }
+
+.literalblock pre, .listingblock pre { background: #eeeeee; }
diff --git a/http/src/docs/asciidoclet/overview.adoc b/http/src/docs/asciidoclet/overview.adoc
new file mode 100644
index 0000000..7947331
--- /dev/null
+++ b/http/src/docs/asciidoclet/overview.adoc
@@ -0,0 +1,4 @@
+= Elasticsearch Java client
+Jörg Prante
+Version 5.4.0.0
+
diff --git a/http/src/main/java/org/elasticsearch/action/admin/cluster/node/info/HttpNodesInfoAction.java b/http/src/main/java/org/elasticsearch/action/admin/cluster/node/info/HttpNodesInfoAction.java
new file mode 100644
index 0000000..e50358b
--- /dev/null
+++ b/http/src/main/java/org/elasticsearch/action/admin/cluster/node/info/HttpNodesInfoAction.java
@@ -0,0 +1,167 @@
+package org.elasticsearch.action.admin.cluster.node.info;
+
+import org.elasticsearch.Build;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.FailedNodeException;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.http.HttpInfo;
+import org.elasticsearch.ingest.IngestInfo;
+import org.elasticsearch.monitor.jvm.JvmInfo;
+import org.elasticsearch.monitor.os.OsInfo;
+import org.elasticsearch.monitor.process.ProcessInfo;
+import org.elasticsearch.threadpool.ThreadPoolInfo;
+import org.elasticsearch.transport.TransportInfo;
+import org.xbib.elasticsearch.client.http.HttpAction;
+import org.xbib.elasticsearch.client.http.HttpActionContext;
+import org.xbib.netty.http.client.RequestBuilder;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.InetAddress;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ *
+ */
+public class HttpNodesInfoAction extends HttpAction {
+
+ @Override
+ public NodesInfoAction getActionInstance() {
+ return NodesInfoAction.INSTANCE;
+ }
+
+ /**
+ * Endpoint "/_nodes/{nodeId}/{metrics}"
+ *
+ * @param url url
+ * @param request request
+ * @return HTTP request
+ */
+ @Override
+ protected RequestBuilder createHttpRequest(String url, NodesInfoRequest request) {
+ StringBuilder path = new StringBuilder("/_nodes");
+ if (request.nodesIds() != null) {
+ String nodeIds = String.join(",", request.nodesIds());
+ if (nodeIds.length() > 0) {
+ path.append("/").append(nodeIds);
+ }
+ } else {
+ path.append("/_all");
+ }
+ List metrics = new LinkedList<>();
+ if (request.http()) {
+ metrics.add("http");
+ }
+ if (request.jvm()) {
+ metrics.add("jvm");
+ }
+ if (request.os()) {
+ metrics.add("os");
+ }
+ if (request.plugins()) {
+ metrics.add("plugins");
+ }
+ if (request.process()) {
+ metrics.add("process");
+ }
+ if (request.settings()) {
+ metrics.add("settings");
+ }
+ if (request.threadPool()) {
+ metrics.add("thread_pool");
+ }
+ if (request.transport()) {
+ metrics.add("transport");
+ }
+ if (!metrics.isEmpty()) {
+ path.append("/").append(String.join(",", metrics));
+ }
+ return newGetRequest(url, path.toString());
+ }
+
+ @Override
+ protected CheckedFunction entityParser() {
+ throw new UnsupportedOperationException();
+ }
+
+ @SuppressWarnings("unchecked")
+ protected NodesInfoResponse createResponse(HttpActionContext httpContext) {
+ Map map = null;
+
+ String string = (String)map.get("cluster_name");
+ ClusterName clusterName = new ClusterName(string);
+ List nodeInfoList = new LinkedList<>();
+ map = (Map)map.get("nodes");
+ for (Map.Entry entry : map.entrySet()) {
+ String nodeId = entry.getKey();
+ String ephemeralId = null;
+ Map map2 = (Map) entry.getValue();
+ String nodeName = (String)map2.get("name");
+ String hostName = (String)map2.get("host");
+ String hostAddress = (String)map2.get("ip");
+ // [/][:]
+ String transportAddressString = (String)map2.get("transport_address");
+ int pos = transportAddressString.indexOf(':');
+ String host = pos > 0 ? transportAddressString.substring(0, pos) : transportAddressString;
+ int port = Integer.parseInt(pos > 0 ? transportAddressString.substring(pos + 1) : "0");
+ pos = host.indexOf('/');
+ host = pos > 0 ? host.substring(0, pos) : host;
+ try {
+ InetAddress[] inetAddresses = InetAddress.getAllByName(host);
+ TransportAddress transportAddress = new TransportAddress(inetAddresses[0], port);
+ Build build = new Build(Build.Flavor.OSS, Build.Type.TAR,
+ (String) map2.get("build"),
+ (String)map2.get("date"),
+ (Boolean)map2.get("snapshot"));
+ Map attributes = Collections.emptyMap();
+ Set roles = new HashSet<>();
+ Version version = Version.fromString((String) map2.get("version"));
+ DiscoveryNode discoveryNode = new DiscoveryNode(nodeName, nodeId, ephemeralId, hostName, hostAddress,
+ transportAddress,
+ attributes, roles, version);
+ /*Map settingsMap = map2.containsKey("settings") ?
+ XContentHelper.
+ SettingsLoader.Helper.loadNestedFromMap((Map) map2.get("settings")) :
+ Collections.emptyMap();
+
+ Settings settings = Settings.builder()
+
+ .put(settingsMap)
+ .build();*/
+ OsInfo os = null;
+ ProcessInfo processInfo = null;
+ JvmInfo jvmInfo = null;
+ ThreadPoolInfo threadPoolInfo = null;
+ TransportInfo transportInfo = null;
+ HttpInfo httpInfo = null;
+ PluginsAndModules pluginsAndModules = null;
+ IngestInfo ingestInfo = null;
+ ByteSizeValue totalIndexingBuffer = null;
+ NodeInfo nodeInfo = new NodeInfo(version,
+ build,
+ discoveryNode,
+ //serviceAttributes,
+ //settings,
+ null,
+ os, processInfo, jvmInfo, threadPoolInfo, transportInfo, httpInfo, pluginsAndModules,
+ ingestInfo,
+ totalIndexingBuffer);
+ nodeInfoList.add(nodeInfo);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+ List failures = null;
+ return new NodesInfoResponse(clusterName, nodeInfoList, failures);
+ }
+}
diff --git a/http/src/main/java/org/elasticsearch/action/admin/cluster/settings/HttpClusterUpdateSettingsAction.java b/http/src/main/java/org/elasticsearch/action/admin/cluster/settings/HttpClusterUpdateSettingsAction.java
new file mode 100644
index 0000000..b66675c
--- /dev/null
+++ b/http/src/main/java/org/elasticsearch/action/admin/cluster/settings/HttpClusterUpdateSettingsAction.java
@@ -0,0 +1,49 @@
+package org.elasticsearch.action.admin.cluster.settings;
+
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.xbib.elasticsearch.client.http.HttpAction;
+import org.xbib.netty.http.client.RequestBuilder;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
+/**
+ *
+ */
+public class HttpClusterUpdateSettingsAction extends HttpAction {
+
+ @Override
+ public ClusterUpdateSettingsAction getActionInstance() {
+ return ClusterUpdateSettingsAction.INSTANCE;
+ }
+
+ @Override
+ protected RequestBuilder createHttpRequest(String url, ClusterUpdateSettingsRequest request) {
+ try {
+ XContentBuilder builder = jsonBuilder();
+ builder.startObject().startObject("persistent");
+ request.persistentSettings().toXContent(builder, ToXContent.EMPTY_PARAMS);
+ builder.endObject();
+ builder.startObject("transient");
+ request.transientSettings().toXContent(builder, ToXContent.EMPTY_PARAMS);
+ builder.endObject().endObject();
+ return newPutRequest(url, "/_cluster/settings", BytesReference.bytes(builder));
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ @Override
+ protected CheckedFunction entityParser() {
+ return parser -> {
+ // TODO(jprante)
+ return new ClusterUpdateSettingsResponse();
+ };
+ }
+}
diff --git a/http/src/main/java/org/elasticsearch/action/admin/indices/create/HttpCreateIndexAction.java b/http/src/main/java/org/elasticsearch/action/admin/indices/create/HttpCreateIndexAction.java
new file mode 100644
index 0000000..da64f8b
--- /dev/null
+++ b/http/src/main/java/org/elasticsearch/action/admin/indices/create/HttpCreateIndexAction.java
@@ -0,0 +1,35 @@
+package org.elasticsearch.action.admin.indices.create;
+
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.xbib.elasticsearch.client.http.HttpAction;
+import org.xbib.netty.http.client.RequestBuilder;
+
+import java.io.IOException;
+
+public class HttpCreateIndexAction extends HttpAction {
+
+ @Override
+ public CreateIndexAction getActionInstance() {
+ return CreateIndexAction.INSTANCE;
+ }
+
+ @Override
+ protected RequestBuilder createHttpRequest(String url, CreateIndexRequest createIndexRequest) throws IOException {
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder = createIndexRequest.toXContent(builder, ToXContent.EMPTY_PARAMS);
+ return newPutRequest(url, "/" + createIndexRequest.index(), BytesReference.bytes(builder));
+ }
+
+ @Override
+ protected CheckedFunction entityParser() {
+ return parser -> {
+ // TODO(jprante) build real create index response
+ return new CreateIndexResponse();
+ };
+ }
+}
diff --git a/http/src/main/java/org/elasticsearch/action/admin/indices/refresh/HttpRefreshIndexAction.java b/http/src/main/java/org/elasticsearch/action/admin/indices/refresh/HttpRefreshIndexAction.java
new file mode 100644
index 0000000..88f76ea
--- /dev/null
+++ b/http/src/main/java/org/elasticsearch/action/admin/indices/refresh/HttpRefreshIndexAction.java
@@ -0,0 +1,30 @@
+package org.elasticsearch.action.admin.indices.refresh;
+
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.xbib.elasticsearch.client.http.HttpAction;
+import org.xbib.netty.http.client.RequestBuilder;
+
+import java.io.IOException;
+
+/**
+ *
+ */
+public class HttpRefreshIndexAction extends HttpAction {
+
+ @Override
+ public RefreshAction getActionInstance() {
+ return RefreshAction.INSTANCE;
+ }
+
+ @Override
+ protected RequestBuilder createHttpRequest(String url, RefreshRequest request) {
+ String index = request.indices() != null ? "/" + String.join(",", request.indices()) : "";
+ return newPostRequest(url, index + "/_refresh");
+ }
+
+ @Override
+ protected CheckedFunction entityParser() {
+ return parser -> new RefreshResponse();
+ }
+}
diff --git a/http/src/main/java/org/elasticsearch/action/admin/indices/settings/put/HttpUpdateSettingsAction.java b/http/src/main/java/org/elasticsearch/action/admin/indices/settings/put/HttpUpdateSettingsAction.java
new file mode 100644
index 0000000..b8facce
--- /dev/null
+++ b/http/src/main/java/org/elasticsearch/action/admin/indices/settings/put/HttpUpdateSettingsAction.java
@@ -0,0 +1,44 @@
+package org.elasticsearch.action.admin.indices.settings.put;
+
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.xbib.elasticsearch.client.http.HttpAction;
+import org.xbib.netty.http.client.RequestBuilder;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
+/**
+ *
+ */
+public class HttpUpdateSettingsAction extends HttpAction {
+
+ @Override
+ public UpdateSettingsAction getActionInstance() {
+ return UpdateSettingsAction.INSTANCE;
+ }
+
+ @Override
+ protected RequestBuilder createHttpRequest(String url, UpdateSettingsRequest request) {
+ try {
+ XContentBuilder builder = jsonBuilder();
+ builder.startObject();
+ request.settings().toXContent(builder, ToXContent.EMPTY_PARAMS);
+ builder.endObject();
+ String index = request.indices() != null ? "/" + String.join(",", request.indices()) : "";
+ return newPutRequest(url, index + "/_settings", BytesReference.bytes(builder));
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ @Override
+ protected CheckedFunction entityParser() {
+ return parser -> new UpdateSettingsResponse();
+ }
+}
diff --git a/http/src/main/java/org/elasticsearch/action/bulk/HttpBulkAction.java b/http/src/main/java/org/elasticsearch/action/bulk/HttpBulkAction.java
new file mode 100644
index 0000000..050d608
--- /dev/null
+++ b/http/src/main/java/org/elasticsearch/action/bulk/HttpBulkAction.java
@@ -0,0 +1,69 @@
+package org.elasticsearch.action.bulk;
+
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.xbib.elasticsearch.client.http.HttpAction;
+import org.xbib.netty.http.client.RequestBuilder;
+
+import java.io.IOException;
+
+/**
+ *
+ */
+public class HttpBulkAction extends HttpAction {
+
+ @Override
+ public BulkAction getActionInstance() {
+ return BulkAction.INSTANCE;
+ }
+
+ @Override
+ protected RequestBuilder createHttpRequest(String url, BulkRequest request) {
+ StringBuilder bulkContent = new StringBuilder();
+ for (DocWriteRequest actionRequest : request.requests()) {
+ if (actionRequest instanceof IndexRequest) {
+ IndexRequest indexRequest = (IndexRequest) actionRequest;
+ bulkContent.append("{\"").append(indexRequest.opType()).append("\":{");
+ bulkContent.append("\"_index\":\"").append(indexRequest.index()).append("\"");
+ bulkContent.append(",\"_type\":\"").append(indexRequest.type()).append("\"");
+ if (indexRequest.id() != null) {
+ bulkContent.append(",\"_id\":\"").append(indexRequest.id()).append("\"");
+ }
+ if (indexRequest.routing() != null) {
+ bulkContent.append(",\"_routing\":\"").append(indexRequest.routing()).append("\"");
+ }
+ if (indexRequest.parent() != null) {
+ bulkContent.append(",\"_parent\":\"").append(indexRequest.parent()).append("\"");
+ }
+ if (indexRequest.version() > 0) {
+ bulkContent.append(",\"_version\":\"").append(indexRequest.version()).append("\"");
+ if (indexRequest.versionType() != null) {
+ bulkContent.append(",\"_version_type\":\"").append(indexRequest.versionType().name()).append("\"");
+ }
+ }
+ bulkContent.append("}}\n");
+ bulkContent.append(indexRequest.source().utf8ToString());
+ bulkContent.append("\n");
+ } else if (actionRequest instanceof DeleteRequest) {
+ DeleteRequest deleteRequest = (DeleteRequest) actionRequest;
+ bulkContent.append("{\"delete\":{");
+ bulkContent.append("\"_index\":\"").append(deleteRequest.index()).append("\"");
+ bulkContent.append(",\"_type\":\"").append(deleteRequest.type()).append("\"");
+ bulkContent.append(",\"_id\":\"").append(deleteRequest.id()).append("\"");
+ if (deleteRequest.routing() != null) {
+ bulkContent.append(",\"_routing\":\"").append(deleteRequest.routing()).append("\""); // _routing
+ }
+ bulkContent.append("}}\n");
+ }
+ }
+ return newPostRequest(url, "/_bulk", bulkContent.toString());
+ }
+
+ @Override
+ protected CheckedFunction entityParser() {
+ return BulkResponse::fromXContent;
+ }
+}
diff --git a/http/src/main/java/org/elasticsearch/action/get/HttpExistsAction.java b/http/src/main/java/org/elasticsearch/action/get/HttpExistsAction.java
new file mode 100644
index 0000000..fd2443e
--- /dev/null
+++ b/http/src/main/java/org/elasticsearch/action/get/HttpExistsAction.java
@@ -0,0 +1,29 @@
+package org.elasticsearch.action.get;
+
+import org.elasticsearch.action.GenericAction;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.xbib.elasticsearch.client.http.HttpAction;
+import org.xbib.netty.http.client.RequestBuilder;
+
+import java.io.IOException;
+
+/**
+ */
+public class HttpExistsAction extends HttpAction {
+
+ @Override
+ public GenericAction getActionInstance() {
+ return GetAction.INSTANCE;
+ }
+
+ @Override
+ protected RequestBuilder createHttpRequest(String url, GetRequest request) {
+ return newHeadRequest(url, request.index() + "/" + request.type() + "/" + request.id());
+ }
+
+ @Override
+ protected CheckedFunction entityParser() {
+ return GetResponse::fromXContent;
+ }
+}
diff --git a/http/src/main/java/org/elasticsearch/action/get/HttpGetAction.java b/http/src/main/java/org/elasticsearch/action/get/HttpGetAction.java
new file mode 100644
index 0000000..3a72116
--- /dev/null
+++ b/http/src/main/java/org/elasticsearch/action/get/HttpGetAction.java
@@ -0,0 +1,29 @@
+package org.elasticsearch.action.get;
+
+import org.elasticsearch.action.GenericAction;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.xbib.elasticsearch.client.http.HttpAction;
+import org.xbib.netty.http.client.RequestBuilder;
+
+import java.io.IOException;
+
+/**
+ */
+public class HttpGetAction extends HttpAction {
+
+ @Override
+ public GenericAction getActionInstance() {
+ return GetAction.INSTANCE;
+ }
+
+ @Override
+ protected RequestBuilder createHttpRequest(String url, GetRequest request) {
+ return newGetRequest(url, request.index() + "/" + request.type() + "/" + request.id());
+ }
+
+ @Override
+ protected CheckedFunction entityParser() {
+ return GetResponse::fromXContent;
+ }
+}
diff --git a/http/src/main/java/org/elasticsearch/action/index/HttpIndexAction.java b/http/src/main/java/org/elasticsearch/action/index/HttpIndexAction.java
new file mode 100644
index 0000000..5352682
--- /dev/null
+++ b/http/src/main/java/org/elasticsearch/action/index/HttpIndexAction.java
@@ -0,0 +1,30 @@
+package org.elasticsearch.action.index;
+
+import org.elasticsearch.action.GenericAction;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.xbib.elasticsearch.client.http.HttpAction;
+import org.xbib.netty.http.client.RequestBuilder;
+
+import java.io.IOException;
+
+/**
+ */
+public class HttpIndexAction extends HttpAction {
+
+ @Override
+ public GenericAction getActionInstance() {
+ return IndexAction.INSTANCE;
+ }
+
+ @Override
+ protected RequestBuilder createHttpRequest(String url, IndexRequest request) {
+ return newPutRequest(url, request.index() + "/" + request.type() + "/" + request.id(),
+ request.source());
+ }
+
+ @Override
+ protected CheckedFunction entityParser() {
+ return IndexResponse::fromXContent;
+ }
+}
diff --git a/http/src/main/java/org/elasticsearch/action/main/HttpMainAction.java b/http/src/main/java/org/elasticsearch/action/main/HttpMainAction.java
new file mode 100644
index 0000000..ee5dc8c
--- /dev/null
+++ b/http/src/main/java/org/elasticsearch/action/main/HttpMainAction.java
@@ -0,0 +1,29 @@
+package org.elasticsearch.action.main;
+
+import org.elasticsearch.action.GenericAction;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.xbib.elasticsearch.client.http.HttpAction;
+import org.xbib.netty.http.client.RequestBuilder;
+
+import java.io.IOException;
+
+/**
+ */
+public class HttpMainAction extends HttpAction {
+
+ @Override
+ public GenericAction getActionInstance() {
+ return MainAction.INSTANCE;
+ }
+
+ @Override
+ protected RequestBuilder createHttpRequest(String url, MainRequest request) {
+ return newGetRequest(url, "/");
+ }
+
+ @Override
+ protected CheckedFunction entityParser() {
+ return MainResponse::fromXContent;
+ }
+}
diff --git a/http/src/main/java/org/elasticsearch/action/search/HttpSearchAction.java b/http/src/main/java/org/elasticsearch/action/search/HttpSearchAction.java
new file mode 100644
index 0000000..4c637b7
--- /dev/null
+++ b/http/src/main/java/org/elasticsearch/action/search/HttpSearchAction.java
@@ -0,0 +1,33 @@
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.xbib.elasticsearch.client.http.HttpAction;
+import org.xbib.netty.http.client.RequestBuilder;
+
+import java.io.IOException;
+
+/**
+ *
+ */
+public class HttpSearchAction extends HttpAction {
+
+ @Override
+ public SearchAction getActionInstance() {
+ return SearchAction.INSTANCE;
+ }
+
+ @Override
+ protected RequestBuilder createHttpRequest(String url, SearchRequest request) {
+ String index = request.indices() != null ? "/" + String.join(",", request.indices()) : "";
+ return newPostRequest(url, index + "/_search", request.source().toString() );
+ }
+
+ @Override
+ protected CheckedFunction entityParser() {
+ return parser -> {
+ // TODO(jprante) build search response
+ return new SearchResponse();
+ };
+ }
+}
diff --git a/http/src/main/java/org/elasticsearch/action/update/HttpUpdateAction.java b/http/src/main/java/org/elasticsearch/action/update/HttpUpdateAction.java
new file mode 100644
index 0000000..c703075
--- /dev/null
+++ b/http/src/main/java/org/elasticsearch/action/update/HttpUpdateAction.java
@@ -0,0 +1,61 @@
+package org.elasticsearch.action.update;
+
+import org.elasticsearch.action.GenericAction;
+import org.elasticsearch.client.Requests;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.xbib.elasticsearch.client.http.HttpAction;
+import org.xbib.netty.http.client.RequestBuilder;
+
+import java.io.IOException;
+
+/**
+ */
+public class HttpUpdateAction extends HttpAction {
+
+ @Override
+ public GenericAction getActionInstance() {
+ return UpdateAction.INSTANCE;
+ }
+
+ @Override
+ protected RequestBuilder createHttpRequest(String url, UpdateRequest updateRequest) {
+ try {
+ // The Java API allows update requests with different content types
+ // set for the partial document and the upsert document. This client
+ // only accepts update requests that have the same content types set
+ // for both doc and upsert.
+ XContentType xContentType = null;
+ if (updateRequest.doc() != null) {
+ xContentType = updateRequest.doc().getContentType();
+ }
+ if (updateRequest.upsertRequest() != null) {
+ XContentType upsertContentType = updateRequest.upsertRequest().getContentType();
+ if ((xContentType != null) && (xContentType != upsertContentType)) {
+ throw new IllegalStateException("update request cannot have different content types for doc [" + xContentType + "]" +
+ " and upsert [" + upsertContentType + "] documents");
+ } else {
+ xContentType = upsertContentType;
+ }
+ }
+ if (xContentType == null) {
+ xContentType = Requests.INDEX_CONTENT_TYPE;
+ }
+ BytesReference source = XContentHelper.toXContent(updateRequest, xContentType, false);
+ return newPostRequest(url,
+ updateRequest.index() + "/" + updateRequest.type() + "/" + updateRequest.id() + "/_update",
+ source);
+ } catch (IOException e) {
+ logger.error(e.getMessage(), e);
+ return null;
+ }
+ }
+
+ @Override
+ protected CheckedFunction entityParser() {
+ return UpdateResponse::fromXContent;
+ }
+}
diff --git a/http/src/main/java/org/xbib/elasticsearch/client/http/HttpAction.java b/http/src/main/java/org/xbib/elasticsearch/client/http/HttpAction.java
new file mode 100644
index 0000000..674ee6d
--- /dev/null
+++ b/http/src/main/java/org/xbib/elasticsearch/client/http/HttpAction.java
@@ -0,0 +1,160 @@
+package org.xbib.elasticsearch.client.http;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.HttpMethod;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.GenericAction;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.xbib.netty.http.client.Request;
+import org.xbib.netty.http.client.RequestBuilder;
+import org.xbib.netty.http.client.transport.Transport;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Base class for HTTP actions.
+ *
+ * @param the request type
+ * @param the response type
+ */
+public abstract class HttpAction {
+
+ protected final Logger logger = LogManager.getLogger(getClass().getName());
+
+ protected static final String APPLICATION_JSON = "application/json";
+
+ protected Settings settings;
+
+ protected void setSettings(Settings settings) {
+ this.settings = settings;
+ }
+
+ public abstract GenericAction getActionInstance();
+
+ /*public final ActionFuture execute(HttpActionContext httpActionContext) {
+ PlainActionFuture future = PlainActionFuture.newFuture();
+ //HttpActionFuture future = new HttpActionFuture<>();
+ execute(httpActionContext, future);
+ return future;
+ }*/
+
+ public final void execute(HttpActionContext httpActionContext, ActionListener listener) throws IOException {
+ try {
+ ActionRequestValidationException validationException = httpActionContext.getRequest().validate();
+ if (validationException != null) {
+ listener.onFailure(validationException);
+ return;
+ }
+ RequestBuilder httpRequestBuilder =
+ createHttpRequest(httpActionContext.getUrl(), httpActionContext.getRequest());
+ //httpRequestBuilder.addHeader("content-type", "application/json");
+ Request httpRequest = httpRequestBuilder.build();
+// logger.info("action = {} request = {}", this.getClass().getName(), httpRequest.toString());
+ httpRequest.setResponseListener(fullHttpResponse -> {
+ logger.info("returned response " + fullHttpResponse.status().code() +
+ " headers = " + fullHttpResponse.headers().entries() +
+ " content = " + fullHttpResponse.content().toString(StandardCharsets.UTF_8));
+ listener.onResponse(parseToResponse(httpActionContext.setHttpResponse(fullHttpResponse)));
+ });
+ Transport transport = httpActionContext.getHttpClient().internalClient().execute(httpRequest);
+ logger.info("transport = " + transport);
+ httpActionContext.setHttpClientTransport(transport);
+ if (transport.isFailed()) {
+ listener.onFailure(new Exception(transport.getFailure()));
+ }
+ logger.info("done, listener is " + listener);
+ } catch (Throwable e) {
+ listener.onFailure(new RuntimeException(e));
+ throw new IOException(e);
+ }
+ }
+
+ protected RequestBuilder newGetRequest(String url, String path) {
+ return Request.builder(HttpMethod.GET).url(url).uri(path);
+ }
+
+ protected RequestBuilder newGetRequest(String url, String path, BytesReference content) {
+ return newRequest(HttpMethod.GET, url, path, content);
+ }
+
+ protected RequestBuilder newHeadRequest(String url, String path) {
+ return newRequest(HttpMethod.HEAD, url, path);
+ }
+
+ protected RequestBuilder newPostRequest(String url, String path) {
+ return newRequest(HttpMethod.POST, url, path);
+ }
+
+ protected RequestBuilder newPostRequest(String url, String path, BytesReference content) {
+ return newRequest(HttpMethod.POST, url, path, content);
+ }
+
+ protected RequestBuilder newPostRequest(String url, String path, String content) {
+ return newRequest(HttpMethod.POST, url, path, content);
+ }
+
+ protected RequestBuilder newPutRequest(String url, String path) {
+ return newRequest(HttpMethod.PUT, url, path);
+ }
+
+ protected RequestBuilder newPutRequest(String url, String path, String content) {
+ return newRequest(HttpMethod.PUT, url, path, content);
+ }
+
+ protected RequestBuilder newPutRequest(String url, String path, BytesReference content) {
+ return newRequest(HttpMethod.PUT, url, path, content);
+ }
+
+ protected RequestBuilder newDeleteRequest(String url, String path, BytesReference content) {
+ return newRequest(HttpMethod.DELETE, url, path, content);
+ }
+
+ protected RequestBuilder newRequest(HttpMethod method, String baseUrl, String path) {
+ return Request.builder(method).url(baseUrl).uri(path);
+ }
+
+ protected RequestBuilder newRequest(HttpMethod method, String baseUrl, String path, BytesReference content) {
+ return Request.builder(method).url(baseUrl).uri(path).content(content.toBytesRef().bytes, APPLICATION_JSON);
+ }
+
+ protected RequestBuilder newRequest(HttpMethod method, String baseUrl, String path, String content) {
+ return Request.builder(method).url(baseUrl).uri(path).content(content, APPLICATION_JSON);
+ }
+
+ protected RequestBuilder newRequest(HttpMethod method, String baseUrl, String path, ByteBuf byteBuf) {
+ return Request.builder(method).url(baseUrl).uri(path).content(byteBuf, APPLICATION_JSON);
+ }
+
+ protected T parseToResponse(HttpActionContext httpActionContext) {
+ String mediaType = httpActionContext.getHttpResponse().headers().get(HttpHeaderNames.CONTENT_TYPE);
+ XContentType xContentType = XContentType.fromMediaTypeOrFormat(mediaType);
+ if (xContentType == null) {
+ throw new IllegalStateException("unsupported content-type: " + mediaType);
+ }
+ try (XContentParser parser = xContentType.xContent().createParser(httpActionContext.getHttpClient().getRegistry(),
+ DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+ httpActionContext.getHttpResponse().content().array())) {
+ return entityParser().apply(parser);
+ } catch (IOException e) {
+ logger.error(e.getMessage(), e);
+ return null;
+ }
+ }
+
+ protected abstract RequestBuilder createHttpRequest(String baseUrl, R request) throws IOException;
+
+ protected abstract CheckedFunction entityParser();
+
+}
diff --git a/http/src/main/java/org/xbib/elasticsearch/client/http/HttpActionContext.java b/http/src/main/java/org/xbib/elasticsearch/client/http/HttpActionContext.java
new file mode 100644
index 0000000..3d403db
--- /dev/null
+++ b/http/src/main/java/org/xbib/elasticsearch/client/http/HttpActionContext.java
@@ -0,0 +1,60 @@
+package org.xbib.elasticsearch.client.http;
+
+import io.netty.handler.codec.http.FullHttpResponse;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.xbib.netty.http.client.transport.Transport;
+
+/**
+ * HTTP action context.
+ *
+ * @param request type
+ * @param response type
+ */
+public class HttpActionContext {
+
+ private final HttpClient httpClient;
+
+ private final R request;
+
+ private final String url;
+
+ private Transport httpClientTransport;
+
+ private FullHttpResponse httpResponse;
+
+ HttpActionContext(HttpClient httpClient, R request, String url) {
+ this.httpClient = httpClient;
+ this.request = request;
+ this.url = url;
+ }
+
+ public HttpClient getHttpClient() {
+ return httpClient;
+ }
+
+ public R getRequest() {
+ return request;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public void setHttpClientTransport(Transport httpClientTransport) {
+ this.httpClientTransport = httpClientTransport;
+ }
+
+ public Transport getHttpClientTransport() {
+ return httpClientTransport;
+ }
+
+ public HttpActionContext setHttpResponse(FullHttpResponse fullHttpResponse) {
+ this.httpResponse = fullHttpResponse;
+ return this;
+ }
+
+ public FullHttpResponse getHttpResponse() {
+ return httpResponse;
+ }
+}
diff --git a/http/src/main/java/org/xbib/elasticsearch/client/http/HttpActionFuture.java b/http/src/main/java/org/xbib/elasticsearch/client/http/HttpActionFuture.java
new file mode 100644
index 0000000..33588c8
--- /dev/null
+++ b/http/src/main/java/org/xbib/elasticsearch/client/http/HttpActionFuture.java
@@ -0,0 +1,100 @@
+package org.xbib.elasticsearch.client.http;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchTimeoutException;
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.BaseFuture;
+import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException;
+import org.xbib.netty.http.client.transport.Transport;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ */
+public class HttpActionFuture extends BaseFuture implements ActionFuture, ActionListener {
+
+ private Transport httpClientTransport;
+
+ HttpActionFuture setHttpClientTransport(Transport httpClientTransport) {
+ this.httpClientTransport = httpClientTransport;
+ return this;
+ }
+
+ @Override
+ public T actionGet() {
+ try {
+ httpClientTransport.get();
+ return get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException("future got interrupted", e);
+ } catch (ExecutionException e) {
+ throw rethrowExecutionException(e);
+ }
+ }
+
+ @Override
+ public T actionGet(String timeout) {
+ return actionGet(TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".actionGet.timeout"));
+ }
+
+ @Override
+ public T actionGet(long timeoutMillis) {
+ return actionGet(timeoutMillis, TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public T actionGet(TimeValue timeout) {
+ return actionGet(timeout.millis(), TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public T actionGet(long timeout, TimeUnit unit) {
+ try {
+ return get(timeout, unit);
+ } catch (TimeoutException e) {
+ throw new ElasticsearchTimeoutException(e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException("Future got interrupted", e);
+ } catch (ExecutionException e) {
+ throw rethrowExecutionException(e);
+ }
+ }
+
+ private static RuntimeException rethrowExecutionException(ExecutionException e) {
+ if (e.getCause() instanceof ElasticsearchException) {
+ ElasticsearchException esEx = (ElasticsearchException) e.getCause();
+ Throwable root = esEx.unwrapCause();
+ if (root instanceof ElasticsearchException) {
+ return (ElasticsearchException) root;
+ } else if (root instanceof RuntimeException) {
+ return (RuntimeException) root;
+ }
+ return new UncategorizedExecutionException("Failed execution", root);
+ } else if (e.getCause() instanceof RuntimeException) {
+ return (RuntimeException) e.getCause();
+ } else {
+ return new UncategorizedExecutionException("Failed execution", e);
+ }
+ }
+
+ @Override
+ public void onResponse(L result) {
+ set(convert(result));
+ }
+
+ @Override
+ public void onFailure(Exception e) {
+ setException(e);
+ }
+
+ @SuppressWarnings("unchecked")
+ private T convert(L listenerResponse) {
+ return (T) listenerResponse;
+ }
+}
diff --git a/http/src/main/java/org/xbib/elasticsearch/client/http/HttpClient.java b/http/src/main/java/org/xbib/elasticsearch/client/http/HttpClient.java
new file mode 100644
index 0000000..31e5d95
--- /dev/null
+++ b/http/src/main/java/org/xbib/elasticsearch/client/http/HttpClient.java
@@ -0,0 +1,209 @@
+package org.xbib.elasticsearch.client.http;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.GenericAction;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.xbib.elasticsearch.client.AbstractClient;
+import org.xbib.elasticsearch.client.BulkControl;
+import org.xbib.elasticsearch.client.BulkMetric;
+import org.xbib.netty.http.client.Client;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.ServiceLoader;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Elasticsearch HTTP client.
+ */
+public class HttpClient extends AbstractClient implements ElasticsearchClient {
+
+ private static final Logger logger = LogManager.getLogger(HttpClient.class);
+
+ private Client client;
+
+ private NamedXContentRegistry registry;
+
+ @SuppressWarnings("rawtypes")
+ private Map actionMap;
+
+ private List urls;
+
+ //private ThreadPool threadPool;
+
+ @Override
+ public HttpClient init(ElasticsearchClient client, Settings settings, BulkMetric metric, BulkControl control) {
+ init(client, settings, metric, control, null, Collections.emptyList());
+ return this;
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private void init(ElasticsearchClient client, Settings settings, BulkMetric metric, BulkControl control,
+ ClassLoader classLoader, List namedXContentEntries) {
+ //super.init(client, settings, metric, control);
+ this.urls = settings.getAsList("urls");
+ if (urls.isEmpty()) {
+ throw new IllegalArgumentException("no urls given");
+ }
+ this.registry = new NamedXContentRegistry(Stream.of(getNamedXContents().stream(),
+ namedXContentEntries.stream()
+ ).flatMap(Function.identity()).collect(Collectors.toList()));
+ this.actionMap = new HashMap<>();
+ ServiceLoader httpActionServiceLoader = ServiceLoader.load(HttpAction.class,
+ classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader());
+ for (HttpAction httpAction : httpActionServiceLoader) {
+ httpAction.setSettings(settings);
+ actionMap.put(httpAction.getActionInstance(), httpAction);
+ }
+ this.client = Client.builder().enableDebug().build();
+ Settings threadPoolsettings = Settings.builder()
+ .put(settings)
+ .put(Node.NODE_NAME_SETTING.getKey(), "httpclient")
+ .build();
+ //this.threadPool = threadPool != null ? threadPool : new ThreadPool(threadPoolsettings);
+ logger.info("HTTP client initialized with {} actions", actionMap.size());
+ }
+
+ private static List getNamedXContents() {
+ return new ArrayList<>();
+ }
+
+ public NamedXContentRegistry getRegistry() {
+ return registry;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public Client internalClient() {
+ return client;
+ }
+
+ @Override
+ public ElasticsearchClient client() {
+ return this;
+ }
+
+ @Override
+ protected ElasticsearchClient createClient(Settings settings) throws IOException {
+ return this;
+ }
+
+ @Override
+ public void shutdown() throws IOException {
+ client.shutdownGracefully();
+ //threadPool.close();
+ }
+
+ @Override
+ public > ActionFuture
+ execute(Action action, Request request) {
+ PlainActionFuture actionFuture = PlainActionFuture.newFuture();
+ logger.info("plain action future = " + actionFuture);
+ execute(action, request, actionFuture);
+ return actionFuture;
+ }
+
+ @Override
+ public > void
+ execute(Action action, Request request, ActionListener listener) {
+ doExecute(action, request, listener);
+ }
+
+ @Override
+ public > RequestBuilder
+ prepareExecute(Action action) {
+ return action.newRequestBuilder(this);
+ }
+
+ @Override
+ public ThreadPool threadPool() {
+ logger.info("returning null for threadPool() request");
+ return null; //threadPool;
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public >
+ void doExecute(Action action, R request, ActionListener listener) {
+ HttpAction httpAction = actionMap.get(action);
+ if (httpAction == null) {
+ throw new IllegalStateException("failed to find http action [" + action + "] to execute");
+ }
+ logger.info("http action = " + httpAction);
+ String url = urls.get(0); // TODO
+ try {
+ logger.info("submitting to URL {}", url);
+ HttpActionContext httpActionContext = new HttpActionContext(this, request, url);
+ httpAction.execute(httpActionContext, listener);
+ logger.info("submitted to URL {}", url);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+
+ /**
+ * The Builder for HTTP client.
+ */
+ public static class Builder {
+
+ private final Settings.Builder settingsBuilder = Settings.builder();
+
+ private ClassLoader classLoader;
+
+ private List namedXContentEntries;
+
+ private ThreadPool threadPool = null;
+
+ public Builder settings(Settings settings) {
+ this.settingsBuilder.put(settings);
+ return this;
+ }
+
+ public Builder classLoader(ClassLoader classLoader) {
+ this.classLoader = classLoader;
+ return this;
+ }
+
+ public Builder namedXContentEntries(List namedXContentEntries) {
+ this.namedXContentEntries = namedXContentEntries;
+ return this;
+ }
+
+ public Builder threadPool(ThreadPool threadPool) {
+ this.threadPool = threadPool;
+ return this;
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public HttpClient build() {
+ Settings settings = settingsBuilder.build();
+ HttpClient httpClient = new HttpClient();
+ httpClient.init(null, settings, null, null,
+ classLoader, namedXContentEntries);
+ return httpClient;
+ }
+ }
+}
diff --git a/http/src/main/java/org/xbib/elasticsearch/client/http/package-info.java b/http/src/main/java/org/xbib/elasticsearch/client/http/package-info.java
new file mode 100644
index 0000000..a9c3ded
--- /dev/null
+++ b/http/src/main/java/org/xbib/elasticsearch/client/http/package-info.java
@@ -0,0 +1,4 @@
+/**
+ * Classes for Elasticsearch HTTP client.
+ */
+package org.xbib.elasticsearch.client.http;
diff --git a/http/src/main/resources/META-INF/services/org.xbib.elasticsearch.client.ClientMethods b/http/src/main/resources/META-INF/services/org.xbib.elasticsearch.client.ClientMethods
new file mode 100644
index 0000000..18f7ab4
--- /dev/null
+++ b/http/src/main/resources/META-INF/services/org.xbib.elasticsearch.client.ClientMethods
@@ -0,0 +1 @@
+org.xbib.elasticsearch.client.http.HttpClient
\ No newline at end of file
diff --git a/http/src/main/resources/META-INF/services/org.xbib.elasticsearch.client.http.HttpAction b/http/src/main/resources/META-INF/services/org.xbib.elasticsearch.client.http.HttpAction
new file mode 100644
index 0000000..cce80e6
--- /dev/null
+++ b/http/src/main/resources/META-INF/services/org.xbib.elasticsearch.client.http.HttpAction
@@ -0,0 +1,11 @@
+org.elasticsearch.action.admin.cluster.node.info.HttpNodesInfoAction
+org.elasticsearch.action.admin.cluster.settings.HttpClusterUpdateSettingsAction
+org.elasticsearch.action.admin.indices.create.HttpCreateIndexAction
+org.elasticsearch.action.admin.indices.refresh.HttpRefreshIndexAction
+org.elasticsearch.action.admin.indices.settings.put.HttpUpdateSettingsAction
+org.elasticsearch.action.bulk.HttpBulkAction
+org.elasticsearch.action.index.HttpIndexAction
+org.elasticsearch.action.search.HttpSearchAction
+org.elasticsearch.action.main.HttpMainAction
+org.elasticsearch.action.get.HttpExistsAction
+org.elasticsearch.action.get.HttpGetAction
diff --git a/http/src/main/resources/extra-security.policy b/http/src/main/resources/extra-security.policy
new file mode 100644
index 0000000..a1e19dd
--- /dev/null
+++ b/http/src/main/resources/extra-security.policy
@@ -0,0 +1,20 @@
+
+grant codeBase "${codebase.netty-common}" {
+ // for reading the system-wide configuration for the backlog of established sockets
+ permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read";
+ // netty makes and accepts socket connections
+ permission java.net.SocketPermission "*", "accept,connect,resolve";
+ // 4.1.24 io.netty.util.concurrent.GlobalEventExecutor$2.run(GlobalEventExecutor.java:228)
+ permission java.lang.RuntimePermission "setContextClassLoader";
+};
+
+grant codeBase "${codebase.netty-transport}" {
+ // Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854
+ // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely!
+ permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write";
+};
+
+grant codeBase "${codebase.netty-http-client}" {
+ // org.xbib.netty.http.client.Client.(Client.java:67)
+ permission java.util.PropertyPermission "io.netty.noUnsafe", "write";
+;}
diff --git a/http/src/test/java/org/xbib/elasticsearch/client/http/HttpClientAliasTests.java b/http/src/test/java/org/xbib/elasticsearch/client/http/HttpClientAliasTests.java
new file mode 100644
index 0000000..09bff1b
--- /dev/null
+++ b/http/src/test/java/org/xbib/elasticsearch/client/http/HttpClientAliasTests.java
@@ -0,0 +1,109 @@
+package org.xbib.elasticsearch.client.http;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.transport.Netty4Plugin;
+import org.junit.Before;
+import org.xbib.elasticsearch.client.ClientBuilder;
+import org.xbib.elasticsearch.client.IndexAliasAdder;
+import org.xbib.elasticsearch.client.SimpleBulkControl;
+import org.xbib.elasticsearch.client.SimpleBulkMetric;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+@ThreadLeakFilters(defaultFilters = true, filters = {TestRunnerThreadsFilter.class})
+public class HttpClientAliasTests extends ESSingleNodeTestCase {
+
+ private static final Logger logger = LogManager.getLogger(HttpClientAliasTests.class.getName());
+
+ private TransportAddress httpAddress;
+
+ @Before
+ public void fetchTransportAddress() {
+ NodeInfo nodeInfo = client().admin().cluster().prepareNodesInfo().get().getNodes().get(0);
+ httpAddress = nodeInfo.getHttp().getAddress().publishAddress();
+ }
+
+ @Override
+ protected Collection> getPlugins() {
+ return Collections.singletonList(Netty4Plugin.class);
+ }
+
+ @Override
+ public Settings nodeSettings() {
+ return Settings.builder()
+ .put(super.nodeSettings())
+ .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME)
+ .put(NetworkModule.HTTP_TYPE_DEFAULT_KEY, Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME)
+ .put(NetworkModule.HTTP_ENABLED.getKey(), true)
+ .build();
+ }
+
+ private String findHttpAddress() {
+ return "http://" + httpAddress.address().getHostName() + ":" + httpAddress.address().getPort();
+ }
+
+ public void testIndexAlias() throws Exception {
+ final HttpClient client = ClientBuilder.builder()
+ .put("urls", findHttpAddress())
+ .setMetric(new SimpleBulkMetric())
+ .setControl(new SimpleBulkControl())
+ .getClient(HttpClient.class);
+ try {
+ client.newIndex("test1234");
+ for (int i = 0; i < 1; i++) {
+ client.index("test1234", "test", randomAlphaOfLength(1), false, "{ \"name\" : \"" + randomAlphaOfLength(32) + "\"}");
+ }
+ client.flushIngest();
+ client.refreshIndex("test1234");
+
+ List simpleAliases = Arrays.asList("a", "b", "c");
+ client.switchAliases("test", "test1234", simpleAliases);
+
+ client.newIndex("test5678");
+ for (int i = 0; i < 1; i++) {
+ client.index("test5678", "test", randomAlphaOfLength(1), false, "{ \"name\" : \"" + randomAlphaOfLength(32) + "\"}");
+ }
+ client.flushIngest();
+ client.refreshIndex("test5678");
+
+ simpleAliases = Arrays.asList("d", "e", "f");
+ client.switchAliases("test", "test5678", simpleAliases, new IndexAliasAdder() {
+ @Override
+ public void addIndexAlias(IndicesAliasesRequestBuilder builder, String index, String alias) {
+ builder.addAlias(index, alias, QueryBuilders.termQuery("my_key", alias));
+ }
+ });
+ Map aliases = client.getIndexFilters("test5678");
+ logger.info("aliases of index test5678 = {}", aliases);
+
+ aliases = client.getAliasFilters("test");
+ logger.info("aliases of alias test = {}", aliases);
+
+ } catch (NoNodeAvailableException e) {
+ logger.warn("skipping, no node available");
+ } finally {
+ client.waitForResponses(TimeValue.timeValueSeconds(30));
+ client.shutdown();
+ if (client.hasThrowable()) {
+ logger.error("error", client.getThrowable());
+ }
+ assertFalse(client.hasThrowable());
+ }
+ }
+}
diff --git a/http/src/test/java/org/xbib/elasticsearch/client/http/HttpClientDuplicateIDTests.java b/http/src/test/java/org/xbib/elasticsearch/client/http/HttpClientDuplicateIDTests.java
new file mode 100644
index 0000000..456926a
--- /dev/null
+++ b/http/src/test/java/org/xbib/elasticsearch/client/http/HttpClientDuplicateIDTests.java
@@ -0,0 +1,101 @@
+package org.xbib.elasticsearch.client.http;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.transport.Netty4Plugin;
+import org.junit.Before;
+import org.xbib.elasticsearch.client.ClientBuilder;
+import org.xbib.elasticsearch.client.SimpleBulkControl;
+import org.xbib.elasticsearch.client.SimpleBulkMetric;
+
+import java.util.Collection;
+import java.util.Collections;
+
+@ThreadLeakFilters(defaultFilters = true, filters = {TestRunnerThreadsFilter.class})
+public class HttpClientDuplicateIDTests extends ESSingleNodeTestCase {
+
+ private static final Logger logger = LogManager.getLogger(HttpClientDuplicateIDTests.class.getName());
+
+ private static final long MAX_ACTIONS = 10L;
+
+ private static final long NUM_ACTIONS = 12345L;
+
+ private TransportAddress httpAddress;
+
+ @Before
+ public void fetchTransportAddress() {
+ NodeInfo nodeInfo = client().admin().cluster().prepareNodesInfo().get().getNodes().get(0);
+ httpAddress = nodeInfo.getHttp().getAddress().publishAddress();
+ }
+
+ @Override
+ protected Collection> getPlugins() {
+ return Collections.singletonList(Netty4Plugin.class);
+ }
+
+ @Override
+ public Settings nodeSettings() {
+ return Settings.builder()
+ .put(super.nodeSettings())
+ .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME)
+ .put(NetworkModule.HTTP_TYPE_DEFAULT_KEY, Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME)
+ .put(NetworkModule.HTTP_ENABLED.getKey(), true)
+ .build();
+ }
+
+ private String findHttpAddress() {
+ return "http://" + httpAddress.address().getHostName() + ":" + httpAddress.address().getPort();
+ }
+
+ public void testDuplicateDocIDs() throws Exception {
+ final HttpClient client = ClientBuilder.builder()
+ //.put(ClientBuilder.MAX_CONCURRENT_REQUESTS, 2) // avoid EsRejectedExecutionException
+ .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS)
+ .put("urls", findHttpAddress())
+ .setMetric(new SimpleBulkMetric())
+ .setControl(new SimpleBulkControl())
+ .getClient(HttpClient.class);
+ try {
+ client.newIndex("test");
+ for (int i = 0; i < NUM_ACTIONS; i++) {
+ client.index("test", "test", randomAlphaOfLength(1), false, "{ \"name\" : \"" + randomAlphaOfLength(32) + "\"}");
+ }
+ client.flushIngest();
+ client.waitForResponses(TimeValue.timeValueSeconds(30));
+ client.refreshIndex("test");
+ SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE)
+ .setIndices("test")
+ .setTypes("test")
+ .setQuery(QueryBuilders.matchAllQuery());
+ long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
+ logger.info("hits = {}", hits);
+ assertTrue(hits < NUM_ACTIONS);
+ } catch (NoNodeAvailableException e) {
+ logger.warn("skipping, no node available");
+ } finally {
+ client.shutdown();
+ if (client.hasThrowable()) {
+ logger.error("error", client.getThrowable());
+ }
+ assertFalse(client.hasThrowable());
+ logger.info("numactions = {}, submitted = {}, succeeded= {}, failed = {}", NUM_ACTIONS,
+ client.getMetric().getSubmitted().getCount(),
+ client.getMetric().getSucceeded().getCount(),
+ client.getMetric().getFailed().getCount());
+ assertEquals(NUM_ACTIONS, client.getMetric().getSubmitted().getCount());
+ assertEquals(NUM_ACTIONS, client.getMetric().getSucceeded().getCount());
+ }
+ }
+}
diff --git a/http/src/test/java/org/xbib/elasticsearch/client/http/HttpClientReplicaTests.java b/http/src/test/java/org/xbib/elasticsearch/client/http/HttpClientReplicaTests.java
new file mode 100644
index 0000000..fc036af
--- /dev/null
+++ b/http/src/test/java/org/xbib/elasticsearch/client/http/HttpClientReplicaTests.java
@@ -0,0 +1,142 @@
+package org.xbib.elasticsearch.client.http;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
+import org.elasticsearch.action.admin.indices.stats.CommonStats;
+import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
+import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.shard.IndexingStats;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.transport.Netty4Plugin;
+import org.junit.Before;
+import org.xbib.elasticsearch.client.ClientBuilder;
+import org.xbib.elasticsearch.client.SimpleBulkControl;
+import org.xbib.elasticsearch.client.SimpleBulkMetric;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+@ThreadLeakFilters(defaultFilters = true, filters = {TestRunnerThreadsFilter.class})
+@ESIntegTestCase.ClusterScope(scope=ESIntegTestCase.Scope.SUITE, numDataNodes=3)
+public class HttpClientReplicaTests extends ESIntegTestCase {
+
+ private static final Logger logger = LogManager.getLogger(HttpClientReplicaTests.class.getName());
+
+ private TransportAddress httpAddress;
+
+ @Before
+ public void fetchTransportAddress() {
+ NodeInfo nodeInfo = client().admin().cluster().prepareNodesInfo().get().getNodes().get(0);
+ httpAddress = nodeInfo.getHttp().getAddress().publishAddress();
+ }
+
+ @Override
+ protected Collection> nodePlugins() {
+ return Collections.singletonList(Netty4Plugin.class);
+ }
+
+ @Override
+ public Settings nodeSettings(int nodeNumber) {
+ return Settings.builder()
+ .put(super.nodeSettings(nodeNumber))
+ .put(EsExecutors.PROCESSORS_SETTING.getKey(), 1)
+ .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME)
+ .put(NetworkModule.HTTP_TYPE_DEFAULT_KEY, Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME)
+ .put(NetworkModule.HTTP_ENABLED.getKey(), true)
+ .build();
+ }
+
+ private String findHttpAddress() {
+ return "http://" + httpAddress.address().getHostName() + ":" + httpAddress.address().getPort();
+ }
+
+ public void testReplicaLevel() throws Exception {
+
+ Settings settingsTest1 = Settings.builder()
+ .put("index.number_of_shards", 1)
+ .put("index.number_of_replicas", 2)
+ .build();
+
+ Settings settingsTest2 = Settings.builder()
+ .put("index.number_of_shards", 1)
+ .put("index.number_of_replicas", 1)
+ .build();
+
+ final HttpClient client = ClientBuilder.builder()
+ .put("urls", findHttpAddress())
+ .setMetric(new SimpleBulkMetric())
+ .setControl(new SimpleBulkControl())
+ .getClient(HttpClient.class);
+
+ try {
+ client.newIndex("test1", settingsTest1, null)
+ .newIndex("test2", settingsTest2, null);
+ client.waitForCluster("GREEN", TimeValue.timeValueSeconds(30));
+ for (int i = 0; i < 1234; i++) {
+ client.index("test1", "test", null, false, "{ \"name\" : \"" + randomAlphaOfLength(32) + "\"}");
+ }
+ for (int i = 0; i < 1234; i++) {
+ client.index("test2", "test", null, false, "{ \"name\" : \"" + randomAlphaOfLength(32) + "\"}");
+ }
+ client.flushIngest();
+ client.waitForResponses(TimeValue.timeValueSeconds(60));
+ } catch (NoNodeAvailableException e) {
+ logger.warn("skipping, no node available");
+ } finally {
+ logger.info("refreshing");
+ client.refreshIndex("test1");
+ client.refreshIndex("test2");
+ SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE)
+ .setIndices("test1", "test2")
+ .setQuery(QueryBuilders.matchAllQuery());
+ long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
+ logger.info("query total hits={}", hits);
+ assertEquals(2468, hits);
+ IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.client(),
+ IndicesStatsAction.INSTANCE)
+ .all();
+ IndicesStatsResponse response = indicesStatsRequestBuilder.execute().actionGet();
+ for (Map.Entry m : response.getIndices().entrySet()) {
+ IndexStats indexStats = m.getValue();
+ CommonStats commonStats = indexStats.getTotal();
+ IndexingStats indexingStats = commonStats.getIndexing();
+ IndexingStats.Stats stats = indexingStats.getTotal();
+ logger.info("index {}: count = {}", m.getKey(), stats.getIndexCount());
+ for (Map.Entry me : indexStats.getIndexShards().entrySet()) {
+ IndexShardStats indexShardStats = me.getValue();
+ CommonStats commonShardStats = indexShardStats.getTotal();
+ logger.info("shard {} count = {}", me.getKey(),
+ commonShardStats.getIndexing().getTotal().getIndexCount());
+ }
+ }
+ try {
+ client.deleteIndex("test1")
+ .deleteIndex("test2");
+ } catch (Exception e) {
+ logger.error("delete index failed, ignored. Reason:", e);
+ }
+ client.shutdown();
+ if (client.hasThrowable()) {
+ logger.error("error", client.getThrowable());
+ }
+ assertFalse(client.hasThrowable());
+ }
+ }
+
+}
diff --git a/http/src/test/java/org/xbib/elasticsearch/client/http/HttpClientTests.java b/http/src/test/java/org/xbib/elasticsearch/client/http/HttpClientTests.java
new file mode 100644
index 0000000..f64999d
--- /dev/null
+++ b/http/src/test/java/org/xbib/elasticsearch/client/http/HttpClientTests.java
@@ -0,0 +1,204 @@
+package org.xbib.elasticsearch.client.http;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.transport.Netty4Plugin;
+import org.junit.Before;
+import org.xbib.elasticsearch.client.ClientBuilder;
+import org.xbib.elasticsearch.client.SimpleBulkControl;
+import org.xbib.elasticsearch.client.SimpleBulkMetric;
+
+import java.util.Collection;
+import java.util.Collections;
+
+@ThreadLeakFilters(defaultFilters = true, filters = {TestRunnerThreadsFilter.class})
+public class HttpClientTests extends ESSingleNodeTestCase {
+
+ private static final Logger logger = LogManager.getLogger(HttpClientTests.class.getName());
+
+ private static final Long MAX_ACTIONS = 10L;
+
+ private static final Long NUM_ACTIONS = 1234L;
+
+ private TransportAddress httpAddress;
+
+ @Before
+ public void fetchTransportAddress() {
+ NodeInfo nodeInfo = client().admin().cluster().prepareNodesInfo().get().getNodes().get(0);
+ httpAddress = nodeInfo.getHttp().getAddress().publishAddress();
+ }
+
+ @Override
+ protected Collection> getPlugins() {
+ return Collections.singletonList(Netty4Plugin.class);
+ }
+
+ @Override
+ public Settings nodeSettings() {
+ return Settings.builder()
+ .put(super.nodeSettings())
+ .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME)
+ .put(NetworkModule.HTTP_TYPE_DEFAULT_KEY, Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME)
+ .put(NetworkModule.HTTP_ENABLED.getKey(), true)
+ .build();
+ }
+
+ private String findHttpAddress() {
+ return "http://" + httpAddress.address().getHostName() + ":" + httpAddress.address().getPort();
+ }
+
+ public void testNewIndex() throws Exception {
+ final HttpClient client = ClientBuilder.builder()
+ .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5))
+ .put("urls", findHttpAddress())
+ .setMetric(new SimpleBulkMetric())
+ .setControl(new SimpleBulkControl())
+ .getClient(HttpClient.class);
+ client.newIndex("test");
+ if (client.hasThrowable()) {
+ logger.error("error", client.getThrowable());
+ }
+ assertFalse(client.hasThrowable());
+ client.shutdown();
+ }
+
+ /*public void testMapping() throws Exception {
+ final HttpClient client = ClientBuilder.builder()
+ .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5))
+ .put("urls", findHttpAddress())
+ .setMetric(new SimpleBulkMetric())
+ .setControl(new SimpleBulkControl())
+ .getClient(HttpClient.class);
+ XContentBuilder builder = XContentFactory.jsonBuilder()
+ .startObject()
+ .startObject("test")
+ .startObject("properties")
+ .startObject("location")
+ .field("type", "geo_point")
+ .endObject()
+ .endObject()
+ .endObject()
+ .endObject();
+ client.mapping("test", builder.string());
+ client.newIndex("test");
+ GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test");
+ GetMappingsResponse getMappingsResponse =
+ client.client().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
+ logger.info("mappings={}", getMappingsResponse.getMappings());
+ if (client.hasThrowable()) {
+ logger.error("error", client.getThrowable());
+ }
+ assertFalse(client.hasThrowable());
+ client.shutdown();
+ }
+
+ public void testSingleDoc() throws Exception {
+ final HttpClient client = ClientBuilder.builder()
+ .put("urls", findHttpAddress())
+ .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS)
+ .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(30))
+ .setMetric(new SimpleBulkMetric())
+ .setControl(new SimpleBulkControl())
+ .getClient(HttpClient.class);
+ client.newIndex("test");
+ client.index("test", "test", "1", false,"{ \"name\" : \"Hello World\"}"); // single doc ingest
+ client.flushIngest();
+ client.waitForResponses(TimeValue.timeValueSeconds(30));
+ assertEquals(1, client.getMetric().getSucceeded().getCount());
+ if (client.hasThrowable()) {
+ logger.error("error", client.getThrowable());
+ }
+ assertFalse(client.hasThrowable());
+ client.shutdown();
+ }
+
+ public void testRandomDocs() throws Exception {
+ long numactions = NUM_ACTIONS;
+ final HttpClient client = ClientBuilder.builder()
+ .put("urls", findHttpAddress())
+ .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS)
+ .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60))
+ .setMetric(new SimpleBulkMetric())
+ .setControl(new SimpleBulkControl())
+ .getClient(HttpClient.class);
+ try {
+ client.newIndex("test");
+ for (int i = 0; i < NUM_ACTIONS; i++) {
+ client.index("test", "test", null, false, "{ \"name\" : \"" + randomAlphaOfLength(32) + "\"}");
+ }
+ client.flushIngest();
+ client.waitForResponses(TimeValue.timeValueSeconds(30));
+ } catch (NoNodeAvailableException e) {
+ logger.warn("skipping, no node available");
+ } finally {
+ if (client.hasThrowable()) {
+ logger.error("error", client.getThrowable());
+ }
+ logger.info("assuring {} == {}", numactions, client.getMetric().getSucceeded().getCount());
+ assertEquals(numactions, client.getMetric().getSucceeded().getCount());
+ assertFalse(client.hasThrowable());
+ client.shutdown();
+ }
+ }
+
+ public void testThreadedRandomDocs() throws Exception {
+ int maxthreads = Runtime.getRuntime().availableProcessors();
+ Long maxactions = MAX_ACTIONS;
+ final Long maxloop = NUM_ACTIONS;
+ logger.info("max={} maxactions={} maxloop={}", maxthreads, maxactions, maxloop);
+ final HttpClient client = ClientBuilder.builder()
+ .put("urls", findHttpAddress())
+ .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, maxactions)
+ .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60))// disable auto flush for this test
+ .setMetric(new SimpleBulkMetric())
+ .setControl(new SimpleBulkControl())
+ .getClient(HttpClient.class);
+ try {
+ client.newIndex("test").startBulk("test", 30 * 1000, 1000);
+ ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
+ final CountDownLatch latch = new CountDownLatch(maxthreads);
+ for (int i = 0; i < maxthreads; i++) {
+ executorService.execute(() -> {
+ for (int i1 = 0; i1 < maxloop; i1++) {
+ client.index("test", "test", null, false, "{ \"name\" : \"" + randomAlphaOfLength(32) + "\"}");
+ }
+ latch.countDown();
+ });
+ }
+ logger.info("waiting for max 30 seconds...");
+ latch.await(30, TimeUnit.SECONDS);
+ logger.info("flush...");
+ client.flushIngest();
+ client.waitForResponses(TimeValue.timeValueSeconds(30));
+ logger.info("got all responses, executor service shutdown...");
+ executorService.shutdown();
+ logger.info("executor service is shut down");
+ client.stopBulk("test");
+ } catch (NoNodeAvailableException e) {
+ logger.warn("skipping, no node available");
+ } finally {
+ logger.info("assuring {} == {}", maxthreads * maxloop, client.getMetric().getSucceeded().getCount());
+ assertEquals(maxthreads * maxloop, client.getMetric().getSucceeded().getCount());
+ if (client.hasThrowable()) {
+ logger.error("error", client.getThrowable());
+ }
+ assertFalse(client.hasThrowable());
+ client.refreshIndex("test");
+ SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE)
+ .setIndices("test")
+ .setQuery(QueryBuilders.matchAllQuery())
+ .setSize(0);
+ assertEquals(maxthreads * maxloop,
+ searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
+ client.shutdown();
+ }
+ }*/
+}
diff --git a/http/src/test/java/org/xbib/elasticsearch/client/http/HttpClientUpdateReplicaLevelTests.java b/http/src/test/java/org/xbib/elasticsearch/client/http/HttpClientUpdateReplicaLevelTests.java
new file mode 100644
index 0000000..db0f894
--- /dev/null
+++ b/http/src/test/java/org/xbib/elasticsearch/client/http/HttpClientUpdateReplicaLevelTests.java
@@ -0,0 +1,97 @@
+package org.xbib.elasticsearch.client.http;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.transport.Netty4Plugin;
+import org.junit.Before;
+import org.xbib.elasticsearch.client.ClientBuilder;
+import org.xbib.elasticsearch.client.SimpleBulkControl;
+import org.xbib.elasticsearch.client.SimpleBulkMetric;
+
+import java.util.Collection;
+import java.util.Collections;
+
+@ThreadLeakFilters(defaultFilters = true, filters = {TestRunnerThreadsFilter.class})
+@ESIntegTestCase.ClusterScope(scope=ESIntegTestCase.Scope.SUITE, numDataNodes=3)
+public class HttpClientUpdateReplicaLevelTests extends ESIntegTestCase {
+
+ private static final Logger logger = LogManager.getLogger(HttpClientUpdateReplicaLevelTests.class.getName());
+
+ private TransportAddress httpAddress;
+
+ @Before
+ public void fetchTransportAddress() {
+ NodeInfo nodeInfo = client().admin().cluster().prepareNodesInfo().get().getNodes().get(0);
+ httpAddress = nodeInfo.getHttp().getAddress().publishAddress();
+ }
+
+ @Override
+ protected Collection> nodePlugins() {
+ return Collections.singletonList(Netty4Plugin.class);
+ }
+
+ @Override
+ public Settings nodeSettings(int nodeNumber) {
+ return Settings.builder()
+ .put(super.nodeSettings(nodeNumber))
+ .put(EsExecutors.PROCESSORS_SETTING.getKey(), 1)
+ .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME)
+ .put(NetworkModule.HTTP_TYPE_DEFAULT_KEY, Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME)
+ .put(NetworkModule.HTTP_ENABLED.getKey(), true)
+ .build();
+ }
+
+ private String findHttpAddress() {
+ return "http://" + httpAddress.address().getHostName() + ":" + httpAddress.address().getPort();
+ }
+
+ public void testUpdateReplicaLevel() throws Exception {
+
+ int numberOfShards = 1;
+ int replicaLevel = 2;
+
+ int shardsAfterReplica;
+
+ Settings settings = Settings.builder()
+ .put("index.number_of_shards", numberOfShards)
+ .put("index.number_of_replicas", 0)
+ .build();
+
+ final HttpClient client = ClientBuilder.builder()
+ .put("urls", findHttpAddress())
+ .setMetric(new SimpleBulkMetric())
+ .setControl(new SimpleBulkControl())
+ .getClient(client(), HttpClient.class);
+
+ try {
+ client.newIndex("replicatest", settings, null);
+ client.waitForCluster("GREEN", TimeValue.timeValueSeconds(30));
+ for (int i = 0; i < 12345; i++) {
+ client.index("replicatest", "replicatest", null, false, "{ \"name\" : \"" + randomAlphaOfLength(32) + "\"}");
+ }
+ client.flushIngest();
+ client.waitForResponses(TimeValue.timeValueSeconds(30));
+ shardsAfterReplica = client.updateReplicaLevel("replicatest", replicaLevel);
+ assertEquals(shardsAfterReplica, numberOfShards * (replicaLevel + 1));
+ } catch (NoNodeAvailableException e) {
+ logger.warn("skipping, no node available");
+ } finally {
+ client.shutdown();
+ if (client.hasThrowable()) {
+ logger.error("error", client.getThrowable());
+ }
+ assertFalse(client.hasThrowable());
+ }
+ }
+
+}
diff --git a/http/src/test/java/org/xbib/elasticsearch/client/http/IndexCreationTest.java b/http/src/test/java/org/xbib/elasticsearch/client/http/IndexCreationTest.java
new file mode 100644
index 0000000..db768be
--- /dev/null
+++ b/http/src/test/java/org/xbib/elasticsearch/client/http/IndexCreationTest.java
@@ -0,0 +1,50 @@
+package org.xbib.elasticsearch.client.http;
+
+import org.junit.Test;
+import org.xbib.elasticsearch.client.ClientBuilder;
+
+import java.util.logging.ConsoleHandler;
+import java.util.logging.Handler;
+import java.util.logging.Level;
+import java.util.logging.LogManager;
+import java.util.logging.Logger;
+import java.util.logging.SimpleFormatter;
+
+public class IndexCreationTest {
+
+ private static final Logger logger = Logger.getLogger(IndexCreationTest.class.getName());
+ static {
+ //System.setProperty("io.netty.leakDetection.level", "paranoid");
+ System.setProperty("io.netty.noKeySetOptimization", Boolean.toString(true));
+ System.setProperty("log4j2.disable.jmx", Boolean.toString(true));
+
+ System.setProperty("java.util.logging.SimpleFormatter.format",
+ "%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS.%1$tL %4$-7s [%3$s] %5$s %6$s%n");
+ LogManager.getLogManager().reset();
+ Logger rootLogger = LogManager.getLogManager().getLogger("");
+ Handler handler = new ConsoleHandler();
+ handler.setFormatter(new SimpleFormatter());
+ rootLogger.addHandler(handler);
+ rootLogger.setLevel(Level.ALL);
+ for (Handler h : rootLogger.getHandlers()) {
+ handler.setFormatter(new SimpleFormatter());
+ h.setLevel(Level.ALL);
+ }
+ }
+
+ @Test
+ public void testNewIndex() throws Exception {
+ HttpClient client = ClientBuilder.builder()
+ //.put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5))
+ .put("urls", "http://localhost:9200")
+ //.setMetric(new SimpleBulkMetric())
+ //.setControl(new SimpleBulkControl())
+ .getClient(HttpClient.class);
+ try {
+ client.newIndex("demo");
+ Thread.sleep(3000L);
+ } finally {
+ client.shutdown();
+ }
+ }
+}
diff --git a/http/src/test/java/org/xbib/elasticsearch/client/http/TestRunnerThreadsFilter.java b/http/src/test/java/org/xbib/elasticsearch/client/http/TestRunnerThreadsFilter.java
new file mode 100644
index 0000000..15e845e
--- /dev/null
+++ b/http/src/test/java/org/xbib/elasticsearch/client/http/TestRunnerThreadsFilter.java
@@ -0,0 +1,11 @@
+package org.xbib.elasticsearch.client.http;
+
+import com.carrotsearch.randomizedtesting.ThreadFilter;
+
+public class TestRunnerThreadsFilter implements ThreadFilter {
+
+ @Override
+ public boolean reject(Thread thread) {
+ return thread.getName().startsWith("ObjectCleanerThread");
+ }
+}
diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/package-info.java b/http/src/test/java/org/xbib/elasticsearch/client/http/package-info.java
similarity index 56%
rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/node/package-info.java
rename to http/src/test/java/org/xbib/elasticsearch/client/http/package-info.java
index 873ebae..aea79de 100644
--- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/package-info.java
+++ b/http/src/test/java/org/xbib/elasticsearch/client/http/package-info.java
@@ -1,4 +1,4 @@
/**
* Classes for testing Elasticsearch node client extras.
*/
-package org.xbib.elasticsearch.extras.client.node;
+package org.xbib.elasticsearch.client.http;
diff --git a/node/build.gradle b/node/build.gradle
new file mode 100644
index 0000000..9f401d8
--- /dev/null
+++ b/node/build.gradle
@@ -0,0 +1,65 @@
+buildscript {
+ repositories {
+ jcenter()
+ maven {
+ url 'http://xbib.org/repository'
+ }
+ }
+ dependencies {
+ classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.3.2.4"
+ }
+}
+
+apply plugin: 'org.xbib.gradle.plugin.elasticsearch.build'
+
+configurations {
+ main
+ tests
+}
+
+dependencies {
+ compile project(':common')
+ testCompile "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}"
+ testRuntime "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}"
+}
+
+jar {
+ baseName "${rootProject.name}-node"
+}
+
+/*
+task testJar(type: Jar, dependsOn: testClasses) {
+ baseName = "${project.archivesBaseName}-tests"
+ from sourceSets.test.output
+}
+*/
+
+artifacts {
+ main jar
+ tests testJar
+ archives sourcesJar, javadocJar
+}
+
+test {
+ enabled = false
+ jvmArgs "-javaagent:" + configurations.alpnagent.asPath
+ systemProperty 'path.home', projectDir.absolutePath
+ testLogging {
+ showStandardStreams = true
+ exceptionFormat = 'full'
+ }
+}
+
+randomizedTest {
+ enabled = false
+}
+
+
+esTest {
+ // test with the jars, not the classes, for security manager
+ // classpath = files(configurations.testRuntime) + configurations.main.artifacts.files + configurations.tests.artifacts.files
+ systemProperty 'tests.security.manager', 'true'
+ // maybe we like some extra security policy for our code
+ systemProperty 'tests.security.policy', '/extra-security.policy'
+}
+esTest.dependsOn jar, testJar
diff --git a/node/config/checkstyle/checkstyle.xml b/node/config/checkstyle/checkstyle.xml
new file mode 100644
index 0000000..52fe33c
--- /dev/null
+++ b/node/config/checkstyle/checkstyle.xml
@@ -0,0 +1,323 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/node/src/docs/asciidoc/css/foundation.css b/node/src/docs/asciidoc/css/foundation.css
new file mode 100644
index 0000000..27be611
--- /dev/null
+++ b/node/src/docs/asciidoc/css/foundation.css
@@ -0,0 +1,684 @@
+/*! normalize.css v2.1.2 | MIT License | git.io/normalize */
+/* ========================================================================== HTML5 display definitions ========================================================================== */
+/** Correct `block` display not defined in IE 8/9. */
+article, aside, details, figcaption, figure, footer, header, hgroup, main, nav, section, summary { display: block; }
+
+/** Correct `inline-block` display not defined in IE 8/9. */
+audio, canvas, video { display: inline-block; }
+
+/** Prevent modern browsers from displaying `audio` without controls. Remove excess height in iOS 5 devices. */
+audio:not([controls]) { display: none; height: 0; }
+
+/** Address `[hidden]` styling not present in IE 8/9. Hide the `template` element in IE, Safari, and Firefox < 22. */
+[hidden], template { display: none; }
+
+script { display: none !important; }
+
+/* ========================================================================== Base ========================================================================== */
+/** 1. Set default font family to sans-serif. 2. Prevent iOS text size adjust after orientation change, without disabling user zoom. */
+html { font-family: sans-serif; /* 1 */ -ms-text-size-adjust: 100%; /* 2 */ -webkit-text-size-adjust: 100%; /* 2 */ }
+
+/** Remove default margin. */
+body { margin: 0; }
+
+/* ========================================================================== Links ========================================================================== */
+/** Remove the gray background color from active links in IE 10. */
+a { background: transparent; }
+
+/** Address `outline` inconsistency between Chrome and other browsers. */
+a:focus { outline: thin dotted; }
+
+/** Improve readability when focused and also mouse hovered in all browsers. */
+a:active, a:hover { outline: 0; }
+
+/* ========================================================================== Typography ========================================================================== */
+/** Address variable `h1` font-size and margin within `section` and `article` contexts in Firefox 4+, Safari 5, and Chrome. */
+h1 { font-size: 2em; margin: 0.67em 0; }
+
+/** Address styling not present in IE 8/9, Safari 5, and Chrome. */
+abbr[title] { border-bottom: 1px dotted; }
+
+/** Address style set to `bolder` in Firefox 4+, Safari 5, and Chrome. */
+b, strong { font-weight: bold; }
+
+/** Address styling not present in Safari 5 and Chrome. */
+dfn { font-style: italic; }
+
+/** Address differences between Firefox and other browsers. */
+hr { -moz-box-sizing: content-box; box-sizing: content-box; height: 0; }
+
+/** Address styling not present in IE 8/9. */
+mark { background: #ff0; color: #000; }
+
+/** Correct font family set oddly in Safari 5 and Chrome. */
+code, kbd, pre, samp { font-family: monospace, serif; font-size: 1em; }
+
+/** Improve readability of pre-formatted text in all browsers. */
+pre { white-space: pre-wrap; }
+
+/** Set consistent quote types. */
+q { quotes: "\201C" "\201D" "\2018" "\2019"; }
+
+/** Address inconsistent and variable font size in all browsers. */
+small { font-size: 80%; }
+
+/** Prevent `sub` and `sup` affecting `line-height` in all browsers. */
+sub, sup { font-size: 75%; line-height: 0; position: relative; vertical-align: baseline; }
+
+sup { top: -0.5em; }
+
+sub { bottom: -0.25em; }
+
+/* ========================================================================== Embedded content ========================================================================== */
+/** Remove border when inside `a` element in IE 8/9. */
+img { border: 0; }
+
+/** Correct overflow displayed oddly in IE 9. */
+svg:not(:root) { overflow: hidden; }
+
+/* ========================================================================== Figures ========================================================================== */
+/** Address margin not present in IE 8/9 and Safari 5. */
+figure { margin: 0; }
+
+/* ========================================================================== Forms ========================================================================== */
+/** Define consistent border, margin, and padding. */
+fieldset { border: 1px solid #c0c0c0; margin: 0 2px; padding: 0.35em 0.625em 0.75em; }
+
+/** 1. Correct `color` not being inherited in IE 8/9. 2. Remove padding so people aren't caught out if they zero out fieldsets. */
+legend { border: 0; /* 1 */ padding: 0; /* 2 */ }
+
+/** 1. Correct font family not being inherited in all browsers. 2. Correct font size not being inherited in all browsers. 3. Address margins set differently in Firefox 4+, Safari 5, and Chrome. */
+button, input, select, textarea { font-family: inherit; /* 1 */ font-size: 100%; /* 2 */ margin: 0; /* 3 */ }
+
+/** Address Firefox 4+ setting `line-height` on `input` using `!important` in the UA stylesheet. */
+button, input { line-height: normal; }
+
+/** Address inconsistent `text-transform` inheritance for `button` and `select`. All other form control elements do not inherit `text-transform` values. Correct `button` style inheritance in Chrome, Safari 5+, and IE 8+. Correct `select` style inheritance in Firefox 4+ and Opera. */
+button, select { text-transform: none; }
+
+/** 1. Avoid the WebKit bug in Android 4.0.* where (2) destroys native `audio` and `video` controls. 2. Correct inability to style clickable `input` types in iOS. 3. Improve usability and consistency of cursor style between image-type `input` and others. */
+button, html input[type="button"], input[type="reset"], input[type="submit"] { -webkit-appearance: button; /* 2 */ cursor: pointer; /* 3 */ }
+
+/** Re-set default cursor for disabled elements. */
+button[disabled], html input[disabled] { cursor: default; }
+
+/** 1. Address box sizing set to `content-box` in IE 8/9. 2. Remove excess padding in IE 8/9. */
+input[type="checkbox"], input[type="radio"] { box-sizing: border-box; /* 1 */ padding: 0; /* 2 */ }
+
+/** 1. Address `appearance` set to `searchfield` in Safari 5 and Chrome. 2. Address `box-sizing` set to `border-box` in Safari 5 and Chrome (include `-moz` to future-proof). */
+input[type="search"] { -webkit-appearance: textfield; /* 1 */ -moz-box-sizing: content-box; -webkit-box-sizing: content-box; /* 2 */ box-sizing: content-box; }
+
+/** Remove inner padding and search cancel button in Safari 5 and Chrome on OS X. */
+input[type="search"]::-webkit-search-cancel-button, input[type="search"]::-webkit-search-decoration { -webkit-appearance: none; }
+
+/** Remove inner padding and border in Firefox 4+. */
+button::-moz-focus-inner, input::-moz-focus-inner { border: 0; padding: 0; }
+
+/** 1. Remove default vertical scrollbar in IE 8/9. 2. Improve readability and alignment in all browsers. */
+textarea { overflow: auto; /* 1 */ vertical-align: top; /* 2 */ }
+
+/* ========================================================================== Tables ========================================================================== */
+/** Remove most spacing between table cells. */
+table { border-collapse: collapse; border-spacing: 0; }
+
+meta.foundation-mq-small { font-family: "only screen and (min-width: 768px)"; width: 768px; }
+
+meta.foundation-mq-medium { font-family: "only screen and (min-width:1280px)"; width: 1280px; }
+
+meta.foundation-mq-large { font-family: "only screen and (min-width:1440px)"; width: 1440px; }
+
+*, *:before, *:after { -moz-box-sizing: border-box; -webkit-box-sizing: border-box; box-sizing: border-box; }
+
+html, body { font-size: 100%; }
+
+body { background: white; color: #222222; padding: 0; margin: 0; font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; font-weight: normal; font-style: normal; line-height: 1; position: relative; cursor: auto; }
+
+a:hover { cursor: pointer; }
+
+img, object, embed { max-width: 100%; height: auto; }
+
+object, embed { height: 100%; }
+
+img { -ms-interpolation-mode: bicubic; }
+
+#map_canvas img, #map_canvas embed, #map_canvas object, .map_canvas img, .map_canvas embed, .map_canvas object { max-width: none !important; }
+
+.left { float: left !important; }
+
+.right { float: right !important; }
+
+.text-left { text-align: left !important; }
+
+.text-right { text-align: right !important; }
+
+.text-center { text-align: center !important; }
+
+.text-justify { text-align: justify !important; }
+
+.hide { display: none; }
+
+.antialiased { -webkit-font-smoothing: antialiased; }
+
+img { display: inline-block; vertical-align: middle; }
+
+textarea { height: auto; min-height: 50px; }
+
+select { width: 100%; }
+
+object, svg { display: inline-block; vertical-align: middle; }
+
+.center { margin-left: auto; margin-right: auto; }
+
+.spread { width: 100%; }
+
+p.lead, .paragraph.lead > p, #preamble > .sectionbody > .paragraph:first-of-type p { font-size: 1.21875em; line-height: 1.6; }
+
+.subheader, .admonitionblock td.content > .title, .audioblock > .title, .exampleblock > .title, .imageblock > .title, .listingblock > .title, .literalblock > .title, .stemblock > .title, .openblock > .title, .paragraph > .title, .quoteblock > .title, table.tableblock > .title, .verseblock > .title, .videoblock > .title, .dlist > .title, .olist > .title, .ulist > .title, .qlist > .title, .hdlist > .title { line-height: 1.4; color: #6f6f6f; font-weight: 300; margin-top: 0.2em; margin-bottom: 0.5em; }
+
+/* Typography resets */
+div, dl, dt, dd, ul, ol, li, h1, h2, h3, #toctitle, .sidebarblock > .content > .title, h4, h5, h6, pre, form, p, blockquote, th, td { margin: 0; padding: 0; direction: ltr; }
+
+/* Default Link Styles */
+a { color: #2ba6cb; text-decoration: none; line-height: inherit; }
+a:hover, a:focus { color: #2795b6; }
+a img { border: none; }
+
+/* Default paragraph styles */
+p { font-family: inherit; font-weight: normal; font-size: 1em; line-height: 1.6; margin-bottom: 1.25em; text-rendering: optimizeLegibility; }
+p aside { font-size: 0.875em; line-height: 1.35; font-style: italic; }
+
+/* Default header styles */
+h1, h2, h3, #toctitle, .sidebarblock > .content > .title, h4, h5, h6 { font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; font-weight: bold; font-style: normal; color: #222222; text-rendering: optimizeLegibility; margin-top: 1em; margin-bottom: 0.5em; line-height: 1.2125em; }
+h1 small, h2 small, h3 small, #toctitle small, .sidebarblock > .content > .title small, h4 small, h5 small, h6 small { font-size: 60%; color: #6f6f6f; line-height: 0; }
+
+h1 { font-size: 2.125em; }
+
+h2 { font-size: 1.6875em; }
+
+h3, #toctitle, .sidebarblock > .content > .title { font-size: 1.375em; }
+
+h4 { font-size: 1.125em; }
+
+h5 { font-size: 1.125em; }
+
+h6 { font-size: 1em; }
+
+hr { border: solid #dddddd; border-width: 1px 0 0; clear: both; margin: 1.25em 0 1.1875em; height: 0; }
+
+/* Helpful Typography Defaults */
+em, i { font-style: italic; line-height: inherit; }
+
+strong, b { font-weight: bold; line-height: inherit; }
+
+small { font-size: 60%; line-height: inherit; }
+
+code { font-family: Consolas, "Liberation Mono", Courier, monospace; font-weight: bold; color: #7f0a0c; }
+
+/* Lists */
+ul, ol, dl { font-size: 1em; line-height: 1.6; margin-bottom: 1.25em; list-style-position: outside; font-family: inherit; }
+
+ul, ol { margin-left: 1.5em; }
+ul.no-bullet, ol.no-bullet { margin-left: 1.5em; }
+
+/* Unordered Lists */
+ul li ul, ul li ol { margin-left: 1.25em; margin-bottom: 0; font-size: 1em; /* Override nested font-size change */ }
+ul.square li ul, ul.circle li ul, ul.disc li ul { list-style: inherit; }
+ul.square { list-style-type: square; }
+ul.circle { list-style-type: circle; }
+ul.disc { list-style-type: disc; }
+ul.no-bullet { list-style: none; }
+
+/* Ordered Lists */
+ol li ul, ol li ol { margin-left: 1.25em; margin-bottom: 0; }
+
+/* Definition Lists */
+dl dt { margin-bottom: 0.3125em; font-weight: bold; }
+dl dd { margin-bottom: 1.25em; }
+
+/* Abbreviations */
+abbr, acronym { text-transform: uppercase; font-size: 90%; color: #222222; border-bottom: 1px dotted #dddddd; cursor: help; }
+
+abbr { text-transform: none; }
+
+/* Blockquotes */
+blockquote { margin: 0 0 1.25em; padding: 0.5625em 1.25em 0 1.1875em; border-left: 1px solid #dddddd; }
+blockquote cite { display: block; font-size: 0.8125em; color: #555555; }
+blockquote cite:before { content: "\2014 \0020"; }
+blockquote cite a, blockquote cite a:visited { color: #555555; }
+
+blockquote, blockquote p { line-height: 1.6; color: #6f6f6f; }
+
+/* Microformats */
+.vcard { display: inline-block; margin: 0 0 1.25em 0; border: 1px solid #dddddd; padding: 0.625em 0.75em; }
+.vcard li { margin: 0; display: block; }
+.vcard .fn { font-weight: bold; font-size: 0.9375em; }
+
+.vevent .summary { font-weight: bold; }
+.vevent abbr { cursor: auto; text-decoration: none; font-weight: bold; border: none; padding: 0 0.0625em; }
+
+@media only screen and (min-width: 768px) { h1, h2, h3, #toctitle, .sidebarblock > .content > .title, h4, h5, h6 { line-height: 1.4; }
+ h1 { font-size: 2.75em; }
+ h2 { font-size: 2.3125em; }
+ h3, #toctitle, .sidebarblock > .content > .title { font-size: 1.6875em; }
+ h4 { font-size: 1.4375em; } }
+/* Tables */
+table { background: white; margin-bottom: 1.25em; border: solid 1px #dddddd; }
+table thead, table tfoot { background: whitesmoke; font-weight: bold; }
+table thead tr th, table thead tr td, table tfoot tr th, table tfoot tr td { padding: 0.5em 0.625em 0.625em; font-size: inherit; color: #222222; text-align: left; }
+table tr th, table tr td { padding: 0.5625em 0.625em; font-size: inherit; color: #222222; }
+table tr.even, table tr.alt, table tr:nth-of-type(even) { background: #f9f9f9; }
+table thead tr th, table tfoot tr th, table tbody tr td, table tr td, table tfoot tr td { display: table-cell; line-height: 1.4; }
+
+body { -moz-osx-font-smoothing: grayscale; -webkit-font-smoothing: antialiased; tab-size: 4; }
+
+h1, h2, h3, #toctitle, .sidebarblock > .content > .title, h4, h5, h6 { line-height: 1.4; }
+
+.clearfix:before, .clearfix:after, .float-group:before, .float-group:after { content: " "; display: table; }
+.clearfix:after, .float-group:after { clear: both; }
+
+*:not(pre) > code { font-size: inherit; font-style: normal !important; letter-spacing: 0; padding: 0; line-height: inherit; word-wrap: break-word; }
+*:not(pre) > code.nobreak { word-wrap: normal; }
+*:not(pre) > code.nowrap { white-space: nowrap; }
+
+pre, pre > code { line-height: 1.4; color: black; font-family: monospace, serif; font-weight: normal; }
+
+em em { font-style: normal; }
+
+strong strong { font-weight: normal; }
+
+.keyseq { color: #555555; }
+
+kbd { font-family: Consolas, "Liberation Mono", Courier, monospace; display: inline-block; color: #222222; font-size: 0.65em; line-height: 1.45; background-color: #f7f7f7; border: 1px solid #ccc; -webkit-border-radius: 3px; border-radius: 3px; -webkit-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.2), 0 0 0 0.1em white inset; box-shadow: 0 1px 0 rgba(0, 0, 0, 0.2), 0 0 0 0.1em white inset; margin: 0 0.15em; padding: 0.2em 0.5em; vertical-align: middle; position: relative; top: -0.1em; white-space: nowrap; }
+
+.keyseq kbd:first-child { margin-left: 0; }
+
+.keyseq kbd:last-child { margin-right: 0; }
+
+.menuseq, .menu { color: #090909; }
+
+b.button:before, b.button:after { position: relative; top: -1px; font-weight: normal; }
+
+b.button:before { content: "["; padding: 0 3px 0 2px; }
+
+b.button:after { content: "]"; padding: 0 2px 0 3px; }
+
+#header, #content, #footnotes, #footer { width: 100%; margin-left: auto; margin-right: auto; margin-top: 0; margin-bottom: 0; max-width: 62.5em; *zoom: 1; position: relative; padding-left: 0.9375em; padding-right: 0.9375em; }
+#header:before, #header:after, #content:before, #content:after, #footnotes:before, #footnotes:after, #footer:before, #footer:after { content: " "; display: table; }
+#header:after, #content:after, #footnotes:after, #footer:after { clear: both; }
+
+#content { margin-top: 1.25em; }
+
+#content:before { content: none; }
+
+#header > h1:first-child { color: black; margin-top: 2.25rem; margin-bottom: 0; }
+#header > h1:first-child + #toc { margin-top: 8px; border-top: 1px solid #dddddd; }
+#header > h1:only-child, body.toc2 #header > h1:nth-last-child(2) { border-bottom: 1px solid #dddddd; padding-bottom: 8px; }
+#header .details { border-bottom: 1px solid #dddddd; line-height: 1.45; padding-top: 0.25em; padding-bottom: 0.25em; padding-left: 0.25em; color: #555555; display: -ms-flexbox; display: -webkit-flex; display: flex; -ms-flex-flow: row wrap; -webkit-flex-flow: row wrap; flex-flow: row wrap; }
+#header .details span:first-child { margin-left: -0.125em; }
+#header .details span.email a { color: #6f6f6f; }
+#header .details br { display: none; }
+#header .details br + span:before { content: "\00a0\2013\00a0"; }
+#header .details br + span.author:before { content: "\00a0\22c5\00a0"; color: #6f6f6f; }
+#header .details br + span#revremark:before { content: "\00a0|\00a0"; }
+#header #revnumber { text-transform: capitalize; }
+#header #revnumber:after { content: "\00a0"; }
+
+#content > h1:first-child:not([class]) { color: black; border-bottom: 1px solid #dddddd; padding-bottom: 8px; margin-top: 0; padding-top: 1rem; margin-bottom: 1.25rem; }
+
+#toc { border-bottom: 1px solid #dddddd; padding-bottom: 0.5em; }
+#toc > ul { margin-left: 0.125em; }
+#toc ul.sectlevel0 > li > a { font-style: italic; }
+#toc ul.sectlevel0 ul.sectlevel1 { margin: 0.5em 0; }
+#toc ul { font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; list-style-type: none; }
+#toc li { line-height: 1.3334; margin-top: 0.3334em; }
+#toc a { text-decoration: none; }
+#toc a:active { text-decoration: underline; }
+
+#toctitle { color: #6f6f6f; font-size: 1.2em; }
+
+@media only screen and (min-width: 768px) { #toctitle { font-size: 1.375em; }
+ body.toc2 { padding-left: 15em; padding-right: 0; }
+ #toc.toc2 { margin-top: 0 !important; background-color: #f2f2f2; position: fixed; width: 15em; left: 0; top: 0; border-right: 1px solid #dddddd; border-top-width: 0 !important; border-bottom-width: 0 !important; z-index: 1000; padding: 1.25em 1em; height: 100%; overflow: auto; }
+ #toc.toc2 #toctitle { margin-top: 0; margin-bottom: 0.8rem; font-size: 1.2em; }
+ #toc.toc2 > ul { font-size: 0.9em; margin-bottom: 0; }
+ #toc.toc2 ul ul { margin-left: 0; padding-left: 1em; }
+ #toc.toc2 ul.sectlevel0 ul.sectlevel1 { padding-left: 0; margin-top: 0.5em; margin-bottom: 0.5em; }
+ body.toc2.toc-right { padding-left: 0; padding-right: 15em; }
+ body.toc2.toc-right #toc.toc2 { border-right-width: 0; border-left: 1px solid #dddddd; left: auto; right: 0; } }
+@media only screen and (min-width: 1280px) { body.toc2 { padding-left: 20em; padding-right: 0; }
+ #toc.toc2 { width: 20em; }
+ #toc.toc2 #toctitle { font-size: 1.375em; }
+ #toc.toc2 > ul { font-size: 0.95em; }
+ #toc.toc2 ul ul { padding-left: 1.25em; }
+ body.toc2.toc-right { padding-left: 0; padding-right: 20em; } }
+#content #toc { border-style: solid; border-width: 1px; border-color: #d9d9d9; margin-bottom: 1.25em; padding: 1.25em; background: #f2f2f2; -webkit-border-radius: 0; border-radius: 0; }
+#content #toc > :first-child { margin-top: 0; }
+#content #toc > :last-child { margin-bottom: 0; }
+
+#footer { max-width: 100%; background-color: #222222; padding: 1.25em; }
+
+#footer-text { color: #dddddd; line-height: 1.44; }
+
+.sect1 { padding-bottom: 0.625em; }
+
+@media only screen and (min-width: 768px) { .sect1 { padding-bottom: 1.25em; } }
+.sect1 + .sect1 { border-top: 1px solid #dddddd; }
+
+#content h1 > a.anchor, h2 > a.anchor, h3 > a.anchor, #toctitle > a.anchor, .sidebarblock > .content > .title > a.anchor, h4 > a.anchor, h5 > a.anchor, h6 > a.anchor { position: absolute; z-index: 1001; width: 1.5ex; margin-left: -1.5ex; display: block; text-decoration: none !important; visibility: hidden; text-align: center; font-weight: normal; }
+#content h1 > a.anchor:before, h2 > a.anchor:before, h3 > a.anchor:before, #toctitle > a.anchor:before, .sidebarblock > .content > .title > a.anchor:before, h4 > a.anchor:before, h5 > a.anchor:before, h6 > a.anchor:before { content: "\00A7"; font-size: 0.85em; display: block; padding-top: 0.1em; }
+#content h1:hover > a.anchor, #content h1 > a.anchor:hover, h2:hover > a.anchor, h2 > a.anchor:hover, h3:hover > a.anchor, #toctitle:hover > a.anchor, .sidebarblock > .content > .title:hover > a.anchor, h3 > a.anchor:hover, #toctitle > a.anchor:hover, .sidebarblock > .content > .title > a.anchor:hover, h4:hover > a.anchor, h4 > a.anchor:hover, h5:hover > a.anchor, h5 > a.anchor:hover, h6:hover > a.anchor, h6 > a.anchor:hover { visibility: visible; }
+#content h1 > a.link, h2 > a.link, h3 > a.link, #toctitle > a.link, .sidebarblock > .content > .title > a.link, h4 > a.link, h5 > a.link, h6 > a.link { color: #222222; text-decoration: none; }
+#content h1 > a.link:hover, h2 > a.link:hover, h3 > a.link:hover, #toctitle > a.link:hover, .sidebarblock > .content > .title > a.link:hover, h4 > a.link:hover, h5 > a.link:hover, h6 > a.link:hover { color: #151515; }
+
+.audioblock, .imageblock, .literalblock, .listingblock, .stemblock, .videoblock { margin-bottom: 1.25em; }
+
+.admonitionblock td.content > .title, .audioblock > .title, .exampleblock > .title, .imageblock > .title, .listingblock > .title, .literalblock > .title, .stemblock > .title, .openblock > .title, .paragraph > .title, .quoteblock > .title, table.tableblock > .title, .verseblock > .title, .videoblock > .title, .dlist > .title, .olist > .title, .ulist > .title, .qlist > .title, .hdlist > .title { text-rendering: optimizeLegibility; text-align: left; }
+
+table.tableblock > caption.title { white-space: nowrap; overflow: visible; max-width: 0; }
+
+.paragraph.lead > p, #preamble > .sectionbody > .paragraph:first-of-type p { color: black; }
+
+table.tableblock #preamble > .sectionbody > .paragraph:first-of-type p { font-size: inherit; }
+
+.admonitionblock > table { border-collapse: separate; border: 0; background: none; width: 100%; }
+.admonitionblock > table td.icon { text-align: center; width: 80px; }
+.admonitionblock > table td.icon img { max-width: initial; }
+.admonitionblock > table td.icon .title { font-weight: bold; font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; text-transform: uppercase; }
+.admonitionblock > table td.content { padding-left: 1.125em; padding-right: 1.25em; border-left: 1px solid #dddddd; color: #555555; }
+.admonitionblock > table td.content > :last-child > :last-child { margin-bottom: 0; }
+
+.exampleblock > .content { border-style: solid; border-width: 1px; border-color: #e6e6e6; margin-bottom: 1.25em; padding: 1.25em; background: white; -webkit-border-radius: 0; border-radius: 0; }
+.exampleblock > .content > :first-child { margin-top: 0; }
+.exampleblock > .content > :last-child { margin-bottom: 0; }
+
+.sidebarblock { border-style: solid; border-width: 1px; border-color: #d9d9d9; margin-bottom: 1.25em; padding: 1.25em; background: #f2f2f2; -webkit-border-radius: 0; border-radius: 0; }
+.sidebarblock > :first-child { margin-top: 0; }
+.sidebarblock > :last-child { margin-bottom: 0; }
+.sidebarblock > .content > .title { color: #6f6f6f; margin-top: 0; }
+
+.exampleblock > .content > :last-child > :last-child, .exampleblock > .content .olist > ol > li:last-child > :last-child, .exampleblock > .content .ulist > ul > li:last-child > :last-child, .exampleblock > .content .qlist > ol > li:last-child > :last-child, .sidebarblock > .content > :last-child > :last-child, .sidebarblock > .content .olist > ol > li:last-child > :last-child, .sidebarblock > .content .ulist > ul > li:last-child > :last-child, .sidebarblock > .content .qlist > ol > li:last-child > :last-child { margin-bottom: 0; }
+
+.literalblock pre, .listingblock pre:not(.highlight), .listingblock pre[class="highlight"], .listingblock pre[class^="highlight "], .listingblock pre.CodeRay, .listingblock pre.prettyprint { background: #eeeeee; }
+.sidebarblock .literalblock pre, .sidebarblock .listingblock pre:not(.highlight), .sidebarblock .listingblock pre[class="highlight"], .sidebarblock .listingblock pre[class^="highlight "], .sidebarblock .listingblock pre.CodeRay, .sidebarblock .listingblock pre.prettyprint { background: #f2f1f1; }
+
+.literalblock pre, .literalblock pre[class], .listingblock pre, .listingblock pre[class] { border: 1px solid #cccccc; -webkit-border-radius: 0; border-radius: 0; word-wrap: break-word; padding: 0.8em 0.8em 0.65em 0.8em; font-size: 0.8125em; }
+.literalblock pre.nowrap, .literalblock pre[class].nowrap, .listingblock pre.nowrap, .listingblock pre[class].nowrap { overflow-x: auto; white-space: pre; word-wrap: normal; }
+@media only screen and (min-width: 768px) { .literalblock pre, .literalblock pre[class], .listingblock pre, .listingblock pre[class] { font-size: 0.90625em; } }
+@media only screen and (min-width: 1280px) { .literalblock pre, .literalblock pre[class], .listingblock pre, .listingblock pre[class] { font-size: 1em; } }
+
+.literalblock.output pre { color: #eeeeee; background-color: black; }
+
+.listingblock pre.highlightjs { padding: 0; }
+.listingblock pre.highlightjs > code { padding: 0.8em 0.8em 0.65em 0.8em; -webkit-border-radius: 0; border-radius: 0; }
+
+.listingblock > .content { position: relative; }
+
+.listingblock code[data-lang]:before { display: none; content: attr(data-lang); position: absolute; font-size: 0.75em; top: 0.425rem; right: 0.5rem; line-height: 1; text-transform: uppercase; color: #999; }
+
+.listingblock:hover code[data-lang]:before { display: block; }
+
+.listingblock.terminal pre .command:before { content: attr(data-prompt); padding-right: 0.5em; color: #999; }
+
+.listingblock.terminal pre .command:not([data-prompt]):before { content: "$"; }
+
+table.pyhltable { border-collapse: separate; border: 0; margin-bottom: 0; background: none; }
+
+table.pyhltable td { vertical-align: top; padding-top: 0; padding-bottom: 0; line-height: 1.4; }
+
+table.pyhltable td.code { padding-left: .75em; padding-right: 0; }
+
+pre.pygments .lineno, table.pyhltable td:not(.code) { color: #999; padding-left: 0; padding-right: .5em; border-right: 1px solid #dddddd; }
+
+pre.pygments .lineno { display: inline-block; margin-right: .25em; }
+
+table.pyhltable .linenodiv { background: none !important; padding-right: 0 !important; }
+
+.quoteblock { margin: 0 1em 1.25em 1.5em; display: table; }
+.quoteblock > .title { margin-left: -1.5em; margin-bottom: 0.75em; }
+.quoteblock blockquote, .quoteblock blockquote p { color: #6f6f6f; font-size: 1.15rem; line-height: 1.75; word-spacing: 0.1em; letter-spacing: 0; font-style: italic; text-align: justify; }
+.quoteblock blockquote { margin: 0; padding: 0; border: 0; }
+.quoteblock blockquote:before { content: "\201c"; float: left; font-size: 2.75em; font-weight: bold; line-height: 0.6em; margin-left: -0.6em; color: #6f6f6f; text-shadow: 0 1px 2px rgba(0, 0, 0, 0.1); }
+.quoteblock blockquote > .paragraph:last-child p { margin-bottom: 0; }
+.quoteblock .attribution { margin-top: 0.5em; margin-right: 0.5ex; text-align: right; }
+.quoteblock .quoteblock { margin-left: 0; margin-right: 0; padding: 0.5em 0; border-left: 3px solid #555555; }
+.quoteblock .quoteblock blockquote { padding: 0 0 0 0.75em; }
+.quoteblock .quoteblock blockquote:before { display: none; }
+
+.verseblock { margin: 0 1em 1.25em 1em; }
+.verseblock pre { font-family: "Open Sans", "DejaVu Sans", sans; font-size: 1.15rem; color: #6f6f6f; font-weight: 300; text-rendering: optimizeLegibility; }
+.verseblock pre strong { font-weight: 400; }
+.verseblock .attribution { margin-top: 1.25rem; margin-left: 0.5ex; }
+
+.quoteblock .attribution, .verseblock .attribution { font-size: 0.8125em; line-height: 1.45; font-style: italic; }
+.quoteblock .attribution br, .verseblock .attribution br { display: none; }
+.quoteblock .attribution cite, .verseblock .attribution cite { display: block; letter-spacing: -0.025em; color: #555555; }
+
+.quoteblock.abstract { margin: 0 0 1.25em 0; display: block; }
+.quoteblock.abstract blockquote, .quoteblock.abstract blockquote p { text-align: left; word-spacing: 0; }
+.quoteblock.abstract blockquote:before, .quoteblock.abstract blockquote p:first-of-type:before { display: none; }
+
+table.tableblock { max-width: 100%; border-collapse: separate; }
+table.tableblock td > .paragraph:last-child p > p:last-child, table.tableblock th > p:last-child, table.tableblock td > p:last-child { margin-bottom: 0; }
+
+table.tableblock, th.tableblock, td.tableblock { border: 0 solid #dddddd; }
+
+table.grid-all th.tableblock, table.grid-all td.tableblock { border-width: 0 1px 1px 0; }
+
+table.grid-all tfoot > tr > th.tableblock, table.grid-all tfoot > tr > td.tableblock { border-width: 1px 1px 0 0; }
+
+table.grid-cols th.tableblock, table.grid-cols td.tableblock { border-width: 0 1px 0 0; }
+
+table.grid-all * > tr > .tableblock:last-child, table.grid-cols * > tr > .tableblock:last-child { border-right-width: 0; }
+
+table.grid-rows th.tableblock, table.grid-rows td.tableblock { border-width: 0 0 1px 0; }
+
+table.grid-all tbody > tr:last-child > th.tableblock, table.grid-all tbody > tr:last-child > td.tableblock, table.grid-all thead:last-child > tr > th.tableblock, table.grid-rows tbody > tr:last-child > th.tableblock, table.grid-rows tbody > tr:last-child > td.tableblock, table.grid-rows thead:last-child > tr > th.tableblock { border-bottom-width: 0; }
+
+table.grid-rows tfoot > tr > th.tableblock, table.grid-rows tfoot > tr > td.tableblock { border-width: 1px 0 0 0; }
+
+table.frame-all { border-width: 1px; }
+
+table.frame-sides { border-width: 0 1px; }
+
+table.frame-topbot { border-width: 1px 0; }
+
+th.halign-left, td.halign-left { text-align: left; }
+
+th.halign-right, td.halign-right { text-align: right; }
+
+th.halign-center, td.halign-center { text-align: center; }
+
+th.valign-top, td.valign-top { vertical-align: top; }
+
+th.valign-bottom, td.valign-bottom { vertical-align: bottom; }
+
+th.valign-middle, td.valign-middle { vertical-align: middle; }
+
+table thead th, table tfoot th { font-weight: bold; }
+
+tbody tr th { display: table-cell; line-height: 1.4; background: whitesmoke; }
+
+tbody tr th, tbody tr th p, tfoot tr th, tfoot tr th p { color: #222222; font-weight: bold; }
+
+p.tableblock > code:only-child { background: none; padding: 0; }
+
+p.tableblock { font-size: 1em; }
+
+td > div.verse { white-space: pre; }
+
+ol { margin-left: 1.75em; }
+
+ul li ol { margin-left: 1.5em; }
+
+dl dd { margin-left: 1.125em; }
+
+dl dd:last-child, dl dd:last-child > :last-child { margin-bottom: 0; }
+
+ol > li p, ul > li p, ul dd, ol dd, .olist .olist, .ulist .ulist, .ulist .olist, .olist .ulist { margin-bottom: 0.625em; }
+
+ul.unstyled, ol.unnumbered, ul.checklist, ul.none { list-style-type: none; }
+
+ul.unstyled, ol.unnumbered, ul.checklist { margin-left: 0.625em; }
+
+ul.checklist li > p:first-child > .fa-square-o:first-child, ul.checklist li > p:first-child > .fa-check-square-o:first-child { width: 1em; font-size: 0.85em; }
+
+ul.checklist li > p:first-child > input[type="checkbox"]:first-child { width: 1em; position: relative; top: 1px; }
+
+ul.inline { margin: 0 auto 0.625em auto; margin-left: -1.375em; margin-right: 0; padding: 0; list-style: none; overflow: hidden; }
+ul.inline > li { list-style: none; float: left; margin-left: 1.375em; display: block; }
+ul.inline > li > * { display: block; }
+
+.unstyled dl dt { font-weight: normal; font-style: normal; }
+
+ol.arabic { list-style-type: decimal; }
+
+ol.decimal { list-style-type: decimal-leading-zero; }
+
+ol.loweralpha { list-style-type: lower-alpha; }
+
+ol.upperalpha { list-style-type: upper-alpha; }
+
+ol.lowerroman { list-style-type: lower-roman; }
+
+ol.upperroman { list-style-type: upper-roman; }
+
+ol.lowergreek { list-style-type: lower-greek; }
+
+.hdlist > table, .colist > table { border: 0; background: none; }
+.hdlist > table > tbody > tr, .colist > table > tbody > tr { background: none; }
+
+td.hdlist1, td.hdlist2 { vertical-align: top; padding: 0 0.625em; }
+
+td.hdlist1 { font-weight: bold; padding-bottom: 1.25em; }
+
+.literalblock + .colist, .listingblock + .colist { margin-top: -0.5em; }
+
+.colist > table tr > td:first-of-type { padding: 0 0.75em; line-height: 1; }
+.colist > table tr > td:first-of-type img { max-width: initial; }
+.colist > table tr > td:last-of-type { padding: 0.25em 0; }
+
+.thumb, .th { line-height: 0; display: inline-block; border: solid 4px white; -webkit-box-shadow: 0 0 0 1px #dddddd; box-shadow: 0 0 0 1px #dddddd; }
+
+.imageblock.left, .imageblock[style*="float: left"] { margin: 0.25em 0.625em 1.25em 0; }
+.imageblock.right, .imageblock[style*="float: right"] { margin: 0.25em 0 1.25em 0.625em; }
+.imageblock > .title { margin-bottom: 0; }
+.imageblock.thumb, .imageblock.th { border-width: 6px; }
+.imageblock.thumb > .title, .imageblock.th > .title { padding: 0 0.125em; }
+
+.image.left, .image.right { margin-top: 0.25em; margin-bottom: 0.25em; display: inline-block; line-height: 0; }
+.image.left { margin-right: 0.625em; }
+.image.right { margin-left: 0.625em; }
+
+a.image { text-decoration: none; display: inline-block; }
+a.image object { pointer-events: none; }
+
+sup.footnote, sup.footnoteref { font-size: 0.875em; position: static; vertical-align: super; }
+sup.footnote a, sup.footnoteref a { text-decoration: none; }
+sup.footnote a:active, sup.footnoteref a:active { text-decoration: underline; }
+
+#footnotes { padding-top: 0.75em; padding-bottom: 0.75em; margin-bottom: 0.625em; }
+#footnotes hr { width: 20%; min-width: 6.25em; margin: -0.25em 0 0.75em 0; border-width: 1px 0 0 0; }
+#footnotes .footnote { padding: 0 0.375em 0 0.225em; line-height: 1.3334; font-size: 0.875em; margin-left: 1.2em; text-indent: -1.05em; margin-bottom: 0.2em; }
+#footnotes .footnote a:first-of-type { font-weight: bold; text-decoration: none; }
+#footnotes .footnote:last-of-type { margin-bottom: 0; }
+#content #footnotes { margin-top: -0.625em; margin-bottom: 0; padding: 0.75em 0; }
+
+.gist .file-data > table { border: 0; background: #fff; width: 100%; margin-bottom: 0; }
+.gist .file-data > table td.line-data { width: 99%; }
+
+div.unbreakable { page-break-inside: avoid; }
+
+.big { font-size: larger; }
+
+.small { font-size: smaller; }
+
+.underline { text-decoration: underline; }
+
+.overline { text-decoration: overline; }
+
+.line-through { text-decoration: line-through; }
+
+.aqua { color: #00bfbf; }
+
+.aqua-background { background-color: #00fafa; }
+
+.black { color: black; }
+
+.black-background { background-color: black; }
+
+.blue { color: #0000bf; }
+
+.blue-background { background-color: #0000fa; }
+
+.fuchsia { color: #bf00bf; }
+
+.fuchsia-background { background-color: #fa00fa; }
+
+.gray { color: #606060; }
+
+.gray-background { background-color: #7d7d7d; }
+
+.green { color: #006000; }
+
+.green-background { background-color: #007d00; }
+
+.lime { color: #00bf00; }
+
+.lime-background { background-color: #00fa00; }
+
+.maroon { color: #600000; }
+
+.maroon-background { background-color: #7d0000; }
+
+.navy { color: #000060; }
+
+.navy-background { background-color: #00007d; }
+
+.olive { color: #606000; }
+
+.olive-background { background-color: #7d7d00; }
+
+.purple { color: #600060; }
+
+.purple-background { background-color: #7d007d; }
+
+.red { color: #bf0000; }
+
+.red-background { background-color: #fa0000; }
+
+.silver { color: #909090; }
+
+.silver-background { background-color: #bcbcbc; }
+
+.teal { color: #006060; }
+
+.teal-background { background-color: #007d7d; }
+
+.white { color: #bfbfbf; }
+
+.white-background { background-color: #fafafa; }
+
+.yellow { color: #bfbf00; }
+
+.yellow-background { background-color: #fafa00; }
+
+span.icon > .fa { cursor: default; }
+
+.admonitionblock td.icon [class^="fa icon-"] { font-size: 2.5em; text-shadow: 1px 1px 2px rgba(0, 0, 0, 0.5); cursor: default; }
+.admonitionblock td.icon .icon-note:before { content: "\f05a"; color: #207c98; }
+.admonitionblock td.icon .icon-tip:before { content: "\f0eb"; text-shadow: 1px 1px 2px rgba(155, 155, 0, 0.8); color: #111; }
+.admonitionblock td.icon .icon-warning:before { content: "\f071"; color: #bf6900; }
+.admonitionblock td.icon .icon-caution:before { content: "\f06d"; color: #bf3400; }
+.admonitionblock td.icon .icon-important:before { content: "\f06a"; color: #bf0000; }
+
+.conum[data-value] { display: inline-block; color: #fff !important; background-color: #222222; -webkit-border-radius: 100px; border-radius: 100px; text-align: center; font-size: 0.75em; width: 1.67em; height: 1.67em; line-height: 1.67em; font-family: "Open Sans", "DejaVu Sans", sans-serif; font-style: normal; font-weight: bold; }
+.conum[data-value] * { color: #fff !important; }
+.conum[data-value] + b { display: none; }
+.conum[data-value]:after { content: attr(data-value); }
+pre .conum[data-value] { position: relative; top: -0.125em; }
+
+b.conum * { color: inherit !important; }
+
+.conum:not([data-value]):empty { display: none; }
+
+.literalblock pre, .listingblock pre { background: #eeeeee; }
diff --git a/node/src/docs/asciidoclet/overview.adoc b/node/src/docs/asciidoclet/overview.adoc
new file mode 100644
index 0000000..7947331
--- /dev/null
+++ b/node/src/docs/asciidoclet/overview.adoc
@@ -0,0 +1,4 @@
+= Elasticsearch Java client
+Jörg Prante
+Version 5.4.0.0
+
diff --git a/node/src/main/java/org/xbib/elasticsearch/client/node/NodeBulkClient.java b/node/src/main/java/org/xbib/elasticsearch/client/node/NodeBulkClient.java
new file mode 100644
index 0000000..103c1be
--- /dev/null
+++ b/node/src/main/java/org/xbib/elasticsearch/client/node/NodeBulkClient.java
@@ -0,0 +1,79 @@
+package org.xbib.elasticsearch.client.node;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.node.NodeValidationException;
+import org.elasticsearch.plugins.Plugin;
+import org.xbib.elasticsearch.client.AbstractClient;
+import org.xbib.elasticsearch.client.BulkControl;
+import org.xbib.elasticsearch.client.BulkMetric;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ *
+ */
+public class NodeBulkClient extends AbstractClient {
+
+ private static final Logger logger = LogManager.getLogger(NodeBulkClient.class.getName());
+
+ private Node node;
+
+ public NodeBulkClient init(ElasticsearchClient client, Settings settings, BulkMetric metric, BulkControl control) {
+ super.init(client, settings, metric, control);
+ return this;
+ }
+
+ @Override
+ protected ElasticsearchClient createClient(Settings settings) throws IOException {
+ if (settings != null) {
+ String version = System.getProperty("os.name")
+ + " " + System.getProperty("java.vm.name")
+ + " " + System.getProperty("java.vm.vendor")
+ + " " + System.getProperty("java.runtime.version")
+ + " " + System.getProperty("java.vm.version");
+ Settings effectiveSettings = Settings.builder().put(settings)
+ .put("node.client", true)
+ .put("node.master", false)
+ .put("node.data", false)
+ .build();
+ logger.info("creating node client on {} with effective settings {}",
+ version, effectiveSettings.toString());
+ Collection> plugins = Collections.emptyList();
+ this.node = new BulkNode(new Environment(effectiveSettings, null), plugins);
+ try {
+ node.start();
+ } catch (NodeValidationException e) {
+ throw new IOException(e);
+ }
+ return node.client();
+ }
+ return null;
+ }
+
+ @Override
+ public synchronized void shutdown() throws IOException {
+ super.shutdown();
+ try {
+ if (node != null) {
+ logger.debug("closing node...");
+ node.close();
+ }
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+
+ private static class BulkNode extends Node {
+
+ BulkNode(Environment env, Collection> classpathPlugins) {
+ super(env, classpathPlugins);
+ }
+ }
+}
diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/node/package-info.java b/node/src/main/java/org/xbib/elasticsearch/client/node/package-info.java
similarity index 52%
rename from src/main/java/org/xbib/elasticsearch/extras/client/node/package-info.java
rename to node/src/main/java/org/xbib/elasticsearch/client/node/package-info.java
index c5c0895..08795e8 100644
--- a/src/main/java/org/xbib/elasticsearch/extras/client/node/package-info.java
+++ b/node/src/main/java/org/xbib/elasticsearch/client/node/package-info.java
@@ -1,4 +1,4 @@
/**
* Classes for Elasticsearch node client extras.
*/
-package org.xbib.elasticsearch.extras.client.node;
+package org.xbib.elasticsearch.client.node;
diff --git a/node/src/main/resources/META-INF/services/org.xbib.elasticsearch.client.ClientMethods b/node/src/main/resources/META-INF/services/org.xbib.elasticsearch.client.ClientMethods
new file mode 100644
index 0000000..631ddb7
--- /dev/null
+++ b/node/src/main/resources/META-INF/services/org.xbib.elasticsearch.client.ClientMethods
@@ -0,0 +1 @@
+org.xbib.elasticsearch.client.node.NodeBulkClient
\ No newline at end of file
diff --git a/node/src/test/java/org/xbib/elasticsearch/client/node/NodeBulkClientDuplicateIDTests.java b/node/src/test/java/org/xbib/elasticsearch/client/node/NodeBulkClientDuplicateIDTests.java
new file mode 100644
index 0000000..5681d63
--- /dev/null
+++ b/node/src/test/java/org/xbib/elasticsearch/client/node/NodeBulkClientDuplicateIDTests.java
@@ -0,0 +1,64 @@
+package org.xbib.elasticsearch.client.node;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.xbib.elasticsearch.client.ClientBuilder;
+import org.xbib.elasticsearch.client.SimpleBulkControl;
+import org.xbib.elasticsearch.client.SimpleBulkMetric;
+
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+
+@ThreadLeakFilters(defaultFilters = true, filters = {TestRunnerThreadsFilter.class})
+public class NodeBulkClientDuplicateIDTests extends ESSingleNodeTestCase {
+
+ private static final Logger logger = LogManager.getLogger(NodeBulkClientDuplicateIDTests.class.getName());
+
+ private static final long MAX_ACTIONS = 100L;
+
+ private static final long NUM_ACTIONS = 12345L;
+
+ public void testDuplicateDocIDs() throws Exception {
+ final NodeBulkClient client = ClientBuilder.builder()
+ //.put(ClientBuilder.MAX_CONCURRENT_REQUESTS, 2) // avoid EsRejectedExecutionException
+ .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS)
+ .setMetric(new SimpleBulkMetric())
+ .setControl(new SimpleBulkControl())
+ .getClient(client(), NodeBulkClient.class);
+ try {
+ client.newIndex("test");
+ for (int i = 0; i < NUM_ACTIONS; i++) {
+ client.index("test", "test", randomAlphaOfLength(1), false, "{ \"name\" : \"" + randomAlphaOfLength(32) + "\"}");
+ }
+ client.flushIngest();
+ client.waitForResponses(TimeValue.timeValueSeconds(30));
+ client.refreshIndex("test");
+ SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE)
+ .setIndices("test")
+ .setTypes("test")
+ .setQuery(matchAllQuery());
+ long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
+ logger.info("hits = {}", hits);
+ assertTrue(hits < NUM_ACTIONS);
+ } catch (NoNodeAvailableException e) {
+ logger.warn("skipping, no node available");
+ } finally {
+ client.shutdown();
+ if (client.hasThrowable()) {
+ logger.error("error", client.getThrowable());
+ }
+ assertFalse(client.hasThrowable());
+ logger.info("numactions = {}, submitted = {}, succeeded= {}, failed = {}", NUM_ACTIONS,
+ client.getMetric().getSubmitted().getCount(),
+ client.getMetric().getSucceeded().getCount(),
+ client.getMetric().getFailed().getCount());
+ assertEquals(NUM_ACTIONS, client.getMetric().getSubmitted().getCount());
+ assertEquals(NUM_ACTIONS, client.getMetric().getSucceeded().getCount());
+ }
+ }
+}
diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeIndexAliasTest.java b/node/src/test/java/org/xbib/elasticsearch/client/node/NodeBulkClientIndexAliasTests.java
similarity index 62%
rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeIndexAliasTest.java
rename to node/src/test/java/org/xbib/elasticsearch/client/node/NodeBulkClientIndexAliasTests.java
index eb5256c..7f36794 100644
--- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeIndexAliasTest.java
+++ b/node/src/test/java/org/xbib/elasticsearch/client/node/NodeBulkClientIndexAliasTests.java
@@ -1,41 +1,36 @@
-package org.xbib.elasticsearch.extras.client.node;
+package org.xbib.elasticsearch.client.node;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.ESLoggerFactory;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.QueryBuilders;
-import org.junit.Test;
-import org.xbib.elasticsearch.NodeTestUtils;
-import org.xbib.elasticsearch.extras.client.Clients;
-import org.xbib.elasticsearch.extras.client.IndexAliasAdder;
-import org.xbib.elasticsearch.extras.client.SimpleBulkControl;
-import org.xbib.elasticsearch.extras.client.SimpleBulkMetric;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.xbib.elasticsearch.client.ClientBuilder;
+import org.xbib.elasticsearch.client.IndexAliasAdder;
+import org.xbib.elasticsearch.client.SimpleBulkControl;
+import org.xbib.elasticsearch.client.SimpleBulkMetric;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import static org.junit.Assert.assertFalse;
+@ThreadLeakFilters(defaultFilters = true, filters = {TestRunnerThreadsFilter.class})
+public class NodeBulkClientIndexAliasTests extends ESSingleNodeTestCase {
-/**
- *
- */
-public class BulkNodeIndexAliasTest extends NodeTestUtils {
+ private static final Logger logger = LogManager.getLogger(NodeBulkClientIndexAliasTests.class.getName());
- private static final ESLogger logger = ESLoggerFactory.getLogger(BulkNodeIndexAliasTest.class.getSimpleName());
-
- @Test
public void testIndexAlias() throws Exception {
- final BulkNodeClient client = Clients.builder()
+ final NodeBulkClient client = ClientBuilder.builder()
.setMetric(new SimpleBulkMetric())
.setControl(new SimpleBulkControl())
- .toBulkNodeClient(client("1"));
+ .getClient(client(), NodeBulkClient.class);
try {
client.newIndex("test1234");
for (int i = 0; i < 1; i++) {
- client.index("test1234", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}");
+ client.index("test1234", "test", randomAlphaOfLength(1), false, "{ \"name\" : \"" + randomAlphaOfLength(32) + "\"}");
}
client.flushIngest();
client.refreshIndex("test1234");
@@ -45,7 +40,7 @@ public class BulkNodeIndexAliasTest extends NodeTestUtils {
client.newIndex("test5678");
for (int i = 0; i < 1; i++) {
- client.index("test5678", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}");
+ client.index("test5678", "test", randomAlphaOfLength(1), false, "{ \"name\" : \"" + randomAlphaOfLength(32) + "\"}");
}
client.flushIngest();
client.refreshIndex("test5678");
@@ -66,7 +61,7 @@ public class BulkNodeIndexAliasTest extends NodeTestUtils {
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
- client.waitForResponses("30s");
+ client.waitForResponses(TimeValue.timeValueSeconds(30));
client.shutdown();
if (client.hasThrowable()) {
logger.error("error", client.getThrowable());
diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportReplicaTest.java b/node/src/test/java/org/xbib/elasticsearch/client/node/NodeBulkClientReplicaTests.java
similarity index 65%
rename from src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportReplicaTest.java
rename to node/src/test/java/org/xbib/elasticsearch/client/node/NodeBulkClientReplicaTests.java
index bc8f449..c8e6186 100644
--- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportReplicaTest.java
+++ b/node/src/test/java/org/xbib/elasticsearch/client/node/NodeBulkClientReplicaTests.java
@@ -1,5 +1,8 @@
-package org.xbib.elasticsearch.extras.client.transport;
+package org.xbib.elasticsearch.client.node;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
@@ -9,64 +12,52 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.ESLoggerFactory;
import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.indexing.IndexingStats;
-import org.junit.Test;
-import org.xbib.elasticsearch.NodeTestUtils;
-import org.xbib.elasticsearch.extras.client.Clients;
-import org.xbib.elasticsearch.extras.client.SimpleBulkControl;
-import org.xbib.elasticsearch.extras.client.SimpleBulkMetric;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.shard.IndexingStats;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.xbib.elasticsearch.client.ClientBuilder;
+import org.xbib.elasticsearch.client.SimpleBulkControl;
+import org.xbib.elasticsearch.client.SimpleBulkMetric;
import java.util.Map;
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
+@ThreadLeakFilters(defaultFilters = true, filters = {TestRunnerThreadsFilter.class})
+@ESIntegTestCase.ClusterScope(scope=ESIntegTestCase.Scope.SUITE, numDataNodes=3)
+public class NodeBulkClientReplicaTests extends ESIntegTestCase {
-/**
- *
- */
-public class BulkTransportReplicaTest extends NodeTestUtils {
+ private static final Logger logger = LogManager.getLogger(NodeBulkClientReplicaTests.class.getName());
- private static final ESLogger logger = ESLoggerFactory.getLogger(BulkTransportReplicaTest.class.getSimpleName());
-
- @Test
public void testReplicaLevel() throws Exception {
- // we need nodes for replica levels
- startNode("2");
- startNode("3");
- startNode("4");
-
- Settings settingsTest1 = Settings.settingsBuilder()
- .put("index.number_of_shards", 2)
- .put("index.number_of_replicas", 3)
+ Settings settingsTest1 = Settings.builder()
+ .put("index.number_of_shards", 1)
+ .put("index.number_of_replicas", 2)
.build();
- Settings settingsTest2 = Settings.settingsBuilder()
- .put("index.number_of_shards", 2)
+ Settings settingsTest2 = Settings.builder()
+ .put("index.number_of_shards", 1)
.put("index.number_of_replicas", 1)
.build();
- final BulkTransportClient client = Clients.builder()
- .put(getSettings())
+ final NodeBulkClient client = ClientBuilder.builder()
.setMetric(new SimpleBulkMetric())
.setControl(new SimpleBulkControl())
- .toBulkTransportClient();
+ .getClient(client(), NodeBulkClient.class);
+
try {
client.newIndex("test1", settingsTest1, null)
.newIndex("test2", settingsTest2, null);
- client.waitForCluster("GREEN", "30s");
+ client.waitForCluster("GREEN", TimeValue.timeValueSeconds(30));
for (int i = 0; i < 1234; i++) {
- client.index("test1", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}");
+ client.index("test1", "test", null, false, "{ \"name\" : \"" + randomAlphaOfLength(32) + "\"}");
}
for (int i = 0; i < 1234; i++) {
- client.index("test2", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}");
+ client.index("test2", "test", null, false, "{ \"name\" : \"" + randomAlphaOfLength(32) + "\"}");
}
client.flushIngest();
- client.waitForResponses("30s");
+ client.waitForResponses(TimeValue.timeValueSeconds(60));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
@@ -75,12 +66,13 @@ public class BulkTransportReplicaTest extends NodeTestUtils {
client.refreshIndex("test2");
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE)
.setIndices("test1", "test2")
- .setQuery(matchAllQuery());
+ .setQuery(QueryBuilders.matchAllQuery());
long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
logger.info("query total hits={}", hits);
assertEquals(2468, hits);
IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.client(),
- IndicesStatsAction.INSTANCE).all();
+ IndicesStatsAction.INSTANCE)
+ .all();
IndicesStatsResponse response = indicesStatsRequestBuilder.execute().actionGet();
for (Map.Entry m : response.getIndices().entrySet()) {
IndexStats indexStats = m.getValue();
diff --git a/node/src/test/java/org/xbib/elasticsearch/client/node/NodeBulkClientTests.java b/node/src/test/java/org/xbib/elasticsearch/client/node/NodeBulkClientTests.java
new file mode 100644
index 0000000..6b261df
--- /dev/null
+++ b/node/src/test/java/org/xbib/elasticsearch/client/node/NodeBulkClientTests.java
@@ -0,0 +1,176 @@
+package org.xbib.elasticsearch.client.node;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.xbib.elasticsearch.client.ClientBuilder;
+import org.xbib.elasticsearch.client.SimpleBulkControl;
+import org.xbib.elasticsearch.client.SimpleBulkMetric;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+@ThreadLeakFilters(defaultFilters = true, filters = {TestRunnerThreadsFilter.class})
+public class NodeBulkClientTests extends ESSingleNodeTestCase {
+
+ private static final Logger logger = LogManager.getLogger(NodeBulkClientTests.class.getName());
+
+ private static final Long MAX_ACTIONS = 10L;
+
+ private static final Long NUM_ACTIONS = 1234L;
+
+ public void testNewIndexNodeClient() throws Exception {
+ final NodeBulkClient client = ClientBuilder.builder()
+ .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5))
+ .setMetric(new SimpleBulkMetric())
+ .setControl(new SimpleBulkControl())
+ .getClient(client(), NodeBulkClient.class);
+ client.newIndex("test");
+ if (client.hasThrowable()) {
+ logger.error("error", client.getThrowable());
+ }
+ assertFalse(client.hasThrowable());
+ client.shutdown();
+ }
+
+ public void testBulkNodeClientMapping() throws Exception {
+ final NodeBulkClient client = ClientBuilder.builder()
+ .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5))
+ .setMetric(new SimpleBulkMetric())
+ .setControl(new SimpleBulkControl())
+ .getClient(client(), NodeBulkClient.class);
+ XContentBuilder builder = XContentFactory.jsonBuilder()
+ .startObject()
+ .startObject("test")
+ .startObject("properties")
+ .startObject("location")
+ .field("type", "geo_point")
+ .endObject()
+ .endObject()
+ .endObject()
+ .endObject();
+ client.mapping("test", builder.string());
+ client.newIndex("test");
+ GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test");
+ GetMappingsResponse getMappingsResponse =
+ client.client().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
+ logger.info("mappings={}", getMappingsResponse.getMappings());
+ if (client.hasThrowable()) {
+ logger.error("error", client.getThrowable());
+ }
+ assertFalse(client.hasThrowable());
+ client.shutdown();
+ }
+
+ public void testBulkNodeClientSingleDoc() throws Exception {
+ final NodeBulkClient client = ClientBuilder.builder()
+ .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS)
+ .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(30))
+ .setMetric(new SimpleBulkMetric())
+ .setControl(new SimpleBulkControl())
+ .getClient(client(), NodeBulkClient.class);
+ client.newIndex("test");
+ client.index("test", "test", "1", false, "{ \"name\" : \"Hello World\"}"); // single doc ingest
+ client.flushIngest();
+ client.waitForResponses(TimeValue.timeValueSeconds(30));
+ assertEquals(1, client.getMetric().getSucceeded().getCount());
+ if (client.hasThrowable()) {
+ logger.error("error", client.getThrowable());
+ }
+ assertFalse(client.hasThrowable());
+ client.shutdown();
+ }
+
+ public void testBulkNodeClientRandomDocs() throws Exception {
+ long numactions = NUM_ACTIONS;
+ final NodeBulkClient client = ClientBuilder.builder()
+ .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS)
+ .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60))
+ .setMetric(new SimpleBulkMetric())
+ .setControl(new SimpleBulkControl())
+ .getClient(client(), NodeBulkClient.class);
+ try {
+ client.newIndex("test");
+ for (int i = 0; i < NUM_ACTIONS; i++) {
+ client.index("test", "test", null, false, "{ \"name\" : \"" + randomAlphaOfLength(32) + "\"}");
+ }
+ client.flushIngest();
+ client.waitForResponses(TimeValue.timeValueSeconds(30));
+ } catch (NoNodeAvailableException e) {
+ logger.warn("skipping, no node available");
+ } finally {
+ if (client.hasThrowable()) {
+ logger.error("error", client.getThrowable());
+ }
+ logger.info("assuring {} == {}", numactions, client.getMetric().getSucceeded().getCount());
+ assertEquals(numactions, client.getMetric().getSucceeded().getCount());
+ assertFalse(client.hasThrowable());
+ client.shutdown();
+ }
+ }
+
+ public void testBulkNodeClientThreadedRandomDocs() throws Exception {
+ int maxthreads = Runtime.getRuntime().availableProcessors();
+ Long maxactions = MAX_ACTIONS;
+ final Long maxloop = NUM_ACTIONS;
+ logger.info("NodeClient max={} maxactions={} maxloop={}", maxthreads, maxactions, maxloop);
+ final NodeBulkClient client = ClientBuilder.builder()
+ .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, maxactions)
+ .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60))// disable auto flush for this test
+ .setMetric(new SimpleBulkMetric())
+ .setControl(new SimpleBulkControl())
+ .getClient(client(), NodeBulkClient.class);
+ try {
+ client.newIndex("test").startBulk("test", 30 * 1000, 1000);
+ ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
+ final CountDownLatch latch = new CountDownLatch(maxthreads);
+ for (int i = 0; i < maxthreads; i++) {
+ executorService.execute(() -> {
+ for (int i1 = 0; i1 < maxloop; i1++) {
+ client.index("test", "test", null, false, "{ \"name\" : \"" + randomAlphaOfLength(32) + "\"}");
+ }
+ latch.countDown();
+ });
+ }
+ logger.info("waiting for max 30 seconds...");
+ latch.await(30, TimeUnit.SECONDS);
+ logger.info("flush...");
+ client.flushIngest();
+ client.waitForResponses(TimeValue.timeValueSeconds(30));
+ logger.info("got all responses, executor service shutdown...");
+ executorService.shutdown();
+ logger.info("executor service is shut down");
+ client.stopBulk("test");
+ } catch (NoNodeAvailableException e) {
+ logger.warn("skipping, no node available");
+ } finally {
+ logger.info("assuring {} == {}", maxthreads * maxloop, client.getMetric().getSucceeded().getCount());
+ assertEquals(maxthreads * maxloop, client.getMetric().getSucceeded().getCount());
+ if (client.hasThrowable()) {
+ logger.error("error", client.getThrowable());
+ }
+ assertFalse(client.hasThrowable());
+ client.refreshIndex("test");
+ SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE)
+ .setIndices("test")
+ .setQuery(QueryBuilders.matchAllQuery())
+ .setSize(0);
+ assertEquals(maxthreads * maxloop,
+ searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
+ client.shutdown();
+ }
+ }
+}
diff --git a/node/src/test/java/org/xbib/elasticsearch/client/node/NodeBulkClientUpdateReplicaLevelTests.java b/node/src/test/java/org/xbib/elasticsearch/client/node/NodeBulkClientUpdateReplicaLevelTests.java
new file mode 100644
index 0000000..97ab80d
--- /dev/null
+++ b/node/src/test/java/org/xbib/elasticsearch/client/node/NodeBulkClientUpdateReplicaLevelTests.java
@@ -0,0 +1,57 @@
+package org.xbib.elasticsearch.client.node;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.xbib.elasticsearch.client.ClientBuilder;
+import org.xbib.elasticsearch.client.SimpleBulkControl;
+import org.xbib.elasticsearch.client.SimpleBulkMetric;
+
+@ThreadLeakFilters(defaultFilters = true, filters = {TestRunnerThreadsFilter.class})
+@ESIntegTestCase.ClusterScope(scope=ESIntegTestCase.Scope.SUITE, numDataNodes=3)
+public class NodeBulkClientUpdateReplicaLevelTests extends ESIntegTestCase {
+
+ private static final Logger logger = LogManager.getLogger(NodeBulkClientUpdateReplicaLevelTests.class.getName());
+
+ public void testUpdateReplicaLevel() throws Exception {
+
+ int numberOfShards = 1;
+ int replicaLevel = 2;
+
+ int shardsAfterReplica;
+
+ Settings settings = Settings.builder()
+ .put("index.number_of_shards", numberOfShards)
+ .put("index.number_of_replicas", 0)
+ .build();
+
+ final NodeBulkClient client = ClientBuilder.builder()
+ .setMetric(new SimpleBulkMetric())
+ .setControl(new SimpleBulkControl())
+ .getClient(client(), NodeBulkClient.class);
+
+ try {
+ client.newIndex("replicatest", settings, null);
+ client.waitForCluster("GREEN", TimeValue.timeValueSeconds(30));
+ for (int i = 0; i < 12345; i++) {
+ client.index("replicatest", "replicatest", null, false, "{ \"name\" : \"" + randomAlphaOfLength(32) + "\"}");
+ }
+ client.flushIngest();
+ client.waitForResponses(TimeValue.timeValueSeconds(30));
+ shardsAfterReplica = client.updateReplicaLevel("replicatest", replicaLevel);
+ assertEquals(shardsAfterReplica, numberOfShards * (replicaLevel + 1));
+ } catch (NoNodeAvailableException e) {
+ logger.warn("skipping, no node available");
+ } finally {
+ client.shutdown();
+ if (client.hasThrowable()) {
+ logger.error("error", client.getThrowable());
+ }
+ assertFalse(client.hasThrowable());
+ }
+ }
+}
diff --git a/node/src/test/java/org/xbib/elasticsearch/client/node/TestRunnerThreadsFilter.java b/node/src/test/java/org/xbib/elasticsearch/client/node/TestRunnerThreadsFilter.java
new file mode 100644
index 0000000..6d0252d
--- /dev/null
+++ b/node/src/test/java/org/xbib/elasticsearch/client/node/TestRunnerThreadsFilter.java
@@ -0,0 +1,11 @@
+package org.xbib.elasticsearch.client.node;
+
+import com.carrotsearch.randomizedtesting.ThreadFilter;
+
+public class TestRunnerThreadsFilter implements ThreadFilter {
+
+ @Override
+ public boolean reject(Thread thread) {
+ return thread.getName().startsWith("ObjectCleanerThread");
+ }
+}
diff --git a/node/src/test/java/org/xbib/elasticsearch/client/node/package-info.java b/node/src/test/java/org/xbib/elasticsearch/client/node/package-info.java
new file mode 100644
index 0000000..f0ef244
--- /dev/null
+++ b/node/src/test/java/org/xbib/elasticsearch/client/node/package-info.java
@@ -0,0 +1,4 @@
+/**
+ * Classes for testing Elasticsearch node client extras.
+ */
+package org.xbib.elasticsearch.client.node;
diff --git a/node/src/test/resources/log4j2.xml b/node/src/test/resources/log4j2.xml
new file mode 100644
index 0000000..b175dfc
--- /dev/null
+++ b/node/src/test/resources/log4j2.xml
@@ -0,0 +1,13 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/src/integration-test/resources/org/xbib/elasticsearch/extras/client/settings.json b/node/src/test/resources/org/xbib/elasticsearch/client/node/settings.json
similarity index 100%
rename from src/integration-test/resources/org/xbib/elasticsearch/extras/client/settings.json
rename to node/src/test/resources/org/xbib/elasticsearch/client/node/settings.json
diff --git a/settings.gradle b/settings.gradle
index ef50653..cc4c55d 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -1 +1,5 @@
-rootProject.name = 'elasticsearch-extras-client'
+include 'api'
+include 'common'
+include 'node'
+include 'transport'
+include 'http'
diff --git a/src/integration-test/java/org/elasticsearch/node/MockNode.java b/src/integration-test/java/org/elasticsearch/node/MockNode.java
deleted file mode 100644
index b0c02eb..0000000
--- a/src/integration-test/java/org/elasticsearch/node/MockNode.java
+++ /dev/null
@@ -1,38 +0,0 @@
-package org.elasticsearch.node;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
-import org.elasticsearch.plugins.Plugin;
-
-import java.util.ArrayList;
-import java.util.Collection;
-
-/**
- *
- */
-public class MockNode extends Node {
-
- public MockNode() {
- super(Settings.EMPTY);
- }
-
- public MockNode(Settings settings) {
- super(settings);
- }
-
- public MockNode(Settings settings, Collection> classpathPlugins) {
- super(InternalSettingsPreparer.prepareEnvironment(settings, null), Version.CURRENT, classpathPlugins);
- }
-
- public MockNode(Settings settings, Class classpathPlugin) {
- this(settings, list(classpathPlugin));
- }
-
- private static Collection> list(Class classpathPlugin) {
- Collection> list = new ArrayList<>();
- list.add(classpathPlugin);
- return list;
- }
-
-}
diff --git a/src/integration-test/java/org/elasticsearch/node/package-info.java b/src/integration-test/java/org/elasticsearch/node/package-info.java
deleted file mode 100644
index f299cbc..0000000
--- a/src/integration-test/java/org/elasticsearch/node/package-info.java
+++ /dev/null
@@ -1,4 +0,0 @@
-/**
- * Classes to support Elasticsearch node creation.
- */
-package org.elasticsearch.node;
diff --git a/src/integration-test/java/org/xbib/elasticsearch/NodeTestUtils.java b/src/integration-test/java/org/xbib/elasticsearch/NodeTestUtils.java
deleted file mode 100644
index d098332..0000000
--- a/src/integration-test/java/org/xbib/elasticsearch/NodeTestUtils.java
+++ /dev/null
@@ -1,213 +0,0 @@
-package org.xbib.elasticsearch;
-
-import static org.elasticsearch.common.settings.Settings.settingsBuilder;
-
-import org.elasticsearch.ElasticsearchTimeoutException;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
-import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
-import org.elasticsearch.client.support.AbstractClient;
-import org.elasticsearch.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.node.MockNode;
-import org.elasticsearch.node.Node;
-import org.junit.After;
-import org.junit.Before;
-import org.xbib.elasticsearch.extras.client.NetworkUtils;
-
-import java.io.IOException;
-import java.nio.file.*;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- *
- */
-public class NodeTestUtils {
-
- private static final ESLogger logger = ESLoggerFactory.getLogger("test");
-
- private static Random random = new Random();
-
- private static char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray();
-
- private Map nodes = new HashMap<>();
-
- private Map clients = new HashMap<>();
-
- private AtomicInteger counter = new AtomicInteger();
-
- private String cluster;
-
- private String host;
-
- private int port;
-
- private static void deleteFiles() throws IOException {
- Path directory = Paths.get(System.getProperty("path.home") + "/data");
- Files.walkFileTree(directory, new SimpleFileVisitor() {
- @Override
- public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
- Files.delete(file);
- return FileVisitResult.CONTINUE;
- }
-
- @Override
- public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
- Files.delete(dir);
- return FileVisitResult.CONTINUE;
- }
-
- });
-
- }
-
- @Before
- public void startNodes() {
- try {
- logger.info("starting");
- setClusterName();
- startNode("1");
- findNodeAddress();
- try {
- ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE,
- new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)
- .timeout(TimeValue.timeValueSeconds(30))).actionGet();
- if (healthResponse != null && healthResponse.isTimedOut()) {
- throw new IOException("cluster state is " + healthResponse.getStatus().name()
- + ", from here on, everything will fail!");
- }
- } catch (ElasticsearchTimeoutException e) {
- throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
- }
- } catch (Throwable t) {
- logger.error("startNodes failed", t);
- }
- }
-
- @After
- public void stopNodes() {
- try {
- closeNodes();
- } catch (Exception e) {
- logger.error("can not close nodes", e);
- } finally {
- try {
- deleteFiles();
- logger.info("data files wiped");
- Thread.sleep(2000L); // let OS commit changes
- } catch (IOException e) {
- logger.error(e.getMessage(), e);
- } catch (InterruptedException e) {
- // ignore
- }
- }
- }
-
- protected void setClusterName() {
- this.cluster = "test-helper-cluster-"
- + NetworkUtils.getLocalAddress().getHostName()
- + "-" + System.getProperty("user.name")
- + "-" + counter.incrementAndGet();
- }
-
- protected String getClusterName() {
- return cluster;
- }
-
- protected Settings getSettings() {
- return settingsBuilder()
- .put("host", host)
- .put("port", port)
- .put("cluster.name", cluster)
- .put("path.home", getHome())
- .build();
- }
-
- protected Settings getNodeSettings() {
- return settingsBuilder()
- .put("cluster.name", cluster)
- .put("cluster.routing.schedule", "50ms")
- .put("cluster.routing.allocation.disk.threshold_enabled", false)
- .put("discovery.zen.multicast.enabled", true)
- .put("discovery.zen.multicast.ping_timeout", "5s")
- .put("http.enabled", true)
- .put("threadpool.bulk.size", Runtime.getRuntime().availableProcessors())
- .put("threadpool.bulk.queue_size", 16 * Runtime.getRuntime().availableProcessors()) // default is 50, too low
- .put("index.number_of_replicas", 0)
- .put("path.home", getHome())
- .build();
- }
-
- protected String getHome() {
- return System.getProperty("path.home");
- }
-
- public void startNode(String id) throws IOException {
- buildNode(id).start();
- }
-
- public AbstractClient client(String id) {
- return clients.get(id);
- }
-
- private void closeNodes() throws IOException {
- logger.info("closing all clients");
- for (AbstractClient client : clients.values()) {
- client.close();
- }
- clients.clear();
- logger.info("closing all nodes");
- for (Node node : nodes.values()) {
- if (node != null) {
- node.close();
- }
- }
- nodes.clear();
- logger.info("all nodes closed");
- }
-
- protected void findNodeAddress() {
- NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true);
- NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet();
- Object obj = response.iterator().next().getTransport().getAddress()
- .publishAddress();
- if (obj instanceof InetSocketTransportAddress) {
- InetSocketTransportAddress address = (InetSocketTransportAddress) obj;
- host = address.address().getHostName();
- port = address.address().getPort();
- }
- }
-
- private Node buildNode(String id) throws IOException {
- Settings nodeSettings = settingsBuilder()
- .put(getNodeSettings())
- .put("name", id)
- .build();
- logger.info("settings={}", nodeSettings.getAsMap());
- Node node = new MockNode(nodeSettings);
- AbstractClient client = (AbstractClient) node.client();
- nodes.put(id, node);
- clients.put(id, client);
- logger.info("clients={}", clients);
- return node;
- }
-
- protected String randomString(int len) {
- final char[] buf = new char[len];
- final int n = numbersAndLetters.length - 1;
- for (int i = 0; i < buf.length; i++) {
- buf[i] = numbersAndLetters[random.nextInt(n)];
- }
- return new String(buf);
- }
-}
diff --git a/src/integration-test/java/org/xbib/elasticsearch/SimpleTest.java b/src/integration-test/java/org/xbib/elasticsearch/SimpleTest.java
deleted file mode 100644
index 0af13df..0000000
--- a/src/integration-test/java/org/xbib/elasticsearch/SimpleTest.java
+++ /dev/null
@@ -1,59 +0,0 @@
-package org.xbib.elasticsearch;
-
-import static org.elasticsearch.common.settings.Settings.settingsBuilder;
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
-import static org.junit.Assert.assertEquals;
-
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder;
-import org.elasticsearch.action.index.IndexAction;
-import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.common.settings.Settings;
-import org.junit.Test;
-
-/**
- *
- */
-public class SimpleTest extends NodeTestUtils {
-
- protected Settings getNodeSettings() {
- return settingsBuilder()
- .put("path.home", System.getProperty("path.home"))
- .put("index.analysis.analyzer.default.filter.0", "lowercase")
- .put("index.analysis.analyzer.default.filter.1", "trim")
- .put("index.analysis.analyzer.default.tokenizer", "keyword")
- .build();
- }
-
- @Test
- public void test() throws Exception {
- try {
- DeleteIndexRequestBuilder deleteIndexRequestBuilder =
- new DeleteIndexRequestBuilder(client("1"), DeleteIndexAction.INSTANCE, "test");
- deleteIndexRequestBuilder.execute().actionGet();
- } catch (Exception e) {
- // ignore
- }
- IndexRequestBuilder indexRequestBuilder = new IndexRequestBuilder(client("1"), IndexAction.INSTANCE);
- indexRequestBuilder
- .setIndex("test")
- .setType("test")
- .setId("1")
- .setSource(jsonBuilder().startObject().field("field",
- "1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8").endObject())
- .setRefresh(true)
- .execute()
- .actionGet();
- String doc = client("1").prepareSearch("test")
- .setTypes("test")
- .setQuery(matchQuery("field",
- "1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8"))
- .execute()
- .actionGet()
- .getHits().getAt(0).getSourceAsString();
-
- assertEquals(doc,
- "{\"field\":\"1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8\"}");
- }
-}
diff --git a/src/integration-test/java/org/xbib/elasticsearch/WildcardTest.java b/src/integration-test/java/org/xbib/elasticsearch/WildcardTest.java
deleted file mode 100644
index 6e252d1..0000000
--- a/src/integration-test/java/org/xbib/elasticsearch/WildcardTest.java
+++ /dev/null
@@ -1,69 +0,0 @@
-package org.xbib.elasticsearch;
-
-import static org.elasticsearch.client.Requests.indexRequest;
-import static org.elasticsearch.common.settings.Settings.settingsBuilder;
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
-
-import org.elasticsearch.client.Client;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.junit.Test;
-
-import java.io.IOException;
-
-/**
- *
- */
-public class WildcardTest extends NodeTestUtils {
-
- protected Settings getNodeSettings() {
- return settingsBuilder()
- .put("cluster.name", getClusterName())
- .put("cluster.routing.allocation.disk.threshold_enabled", false)
- .put("discovery.zen.multicast.enabled", false)
- .put("http.enabled", false)
- .put("path.home", System.getProperty("path.home"))
- .put("index.number_of_shards", 1)
- .put("index.number_of_replicas", 0)
- .build();
- }
-
- @Test
- public void testWildcard() throws Exception {
- index(client("1"), "1", "010");
- index(client("1"), "2", "0*0");
- // exact
- validateCount(client("1"), queryStringQuery("010").defaultField("field"), 1);
- validateCount(client("1"), queryStringQuery("0\\*0").defaultField("field"), 1);
- // pattern
- validateCount(client("1"), queryStringQuery("0*0").defaultField("field"), 1); // 2?
- validateCount(client("1"), queryStringQuery("0?0").defaultField("field"), 1); // 2?
- validateCount(client("1"), queryStringQuery("0**0").defaultField("field"), 1); // 2?
- validateCount(client("1"), queryStringQuery("0??0").defaultField("field"), 0);
- validateCount(client("1"), queryStringQuery("*10").defaultField("field"), 1);
- validateCount(client("1"), queryStringQuery("*1*").defaultField("field"), 1);
- validateCount(client("1"), queryStringQuery("*\\*0").defaultField("field"), 0); // 1?
- validateCount(client("1"), queryStringQuery("*\\**").defaultField("field"), 0); // 1?
- }
-
- private void index(Client client, String id, String fieldValue) throws IOException {
- client.index(indexRequest()
- .index("index").type("type").id(id)
- .source(jsonBuilder().startObject().field("field", fieldValue).endObject())
- .refresh(true)).actionGet();
- }
-
- private long count(Client client, QueryBuilder queryBuilder) {
- return client.prepareSearch("index").setTypes("type")
- .setQuery(queryBuilder)
- .execute().actionGet().getHits().getTotalHits();
- }
-
- private void validateCount(Client client, QueryBuilder queryBuilder, long expectedHits) {
- final long actualHits = count(client, queryBuilder);
- if (actualHits != expectedHits) {
- throw new RuntimeException("actualHits=" + actualHits + ", expectedHits=" + expectedHits);
- }
- }
-}
diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClientTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClientTest.java
deleted file mode 100644
index 77b004f..0000000
--- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClientTest.java
+++ /dev/null
@@ -1,208 +0,0 @@
-package org.xbib.elasticsearch.extras.client.node;
-
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
-import org.elasticsearch.action.search.SearchAction;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.junit.Before;
-import org.junit.Test;
-import org.xbib.elasticsearch.NodeTestUtils;
-import org.xbib.elasticsearch.extras.client.Clients;
-import org.xbib.elasticsearch.extras.client.SimpleBulkControl;
-import org.xbib.elasticsearch.extras.client.SimpleBulkMetric;
-
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-/**
- *
- */
-public class BulkNodeClientTest extends NodeTestUtils {
-
- private static final ESLogger logger = ESLoggerFactory.getLogger(BulkNodeClientTest.class.getSimpleName());
-
- private static final Long MAX_ACTIONS = 1000L;
-
- private static final Long NUM_ACTIONS = 1234L;
-
- @Before
- public void startNodes() {
- try {
- super.startNodes();
- startNode("2");
- } catch (Throwable t) {
- logger.error("startNodes failed", t);
- }
- }
-
- @Test
- public void testNewIndexNodeClient() throws Exception {
- final BulkNodeClient client = Clients.builder()
- .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5))
- .setMetric(new SimpleBulkMetric())
- .setControl(new SimpleBulkControl())
- .toBulkNodeClient(client("1"));
- client.newIndex("test");
- if (client.hasThrowable()) {
- logger.error("error", client.getThrowable());
- }
- assertFalse(client.hasThrowable());
- client.shutdown();
- }
-
- @Test
- public void testMappingNodeClient() throws Exception {
- final BulkNodeClient client = Clients.builder()
- .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5))
- .setMetric(new SimpleBulkMetric())
- .setControl(new SimpleBulkControl())
- .toBulkNodeClient(client("1"));
- XContentBuilder builder = jsonBuilder()
- .startObject()
- .startObject("test")
- .startObject("properties")
- .startObject("location")
- .field("type", "geo_point")
- .endObject()
- .endObject()
- .endObject()
- .endObject();
- client.mapping("test", builder.string());
- client.newIndex("test");
- GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test");
- GetMappingsResponse getMappingsResponse =
- client.client().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
- logger.info("mappings={}", getMappingsResponse.getMappings());
- if (client.hasThrowable()) {
- logger.error("error", client.getThrowable());
- }
- assertFalse(client.hasThrowable());
- client.shutdown();
- }
-
- @Test
- public void testSingleDocNodeClient() {
- final BulkNodeClient client = Clients.builder()
- .put(Clients.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS)
- .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(30))
- .setMetric(new SimpleBulkMetric())
- .setControl(new SimpleBulkControl())
- .toBulkNodeClient(client("1"));
- try {
- client.newIndex("test");
- client.index("test", "test", "1", "{ \"name\" : \"Hello World\"}"); // single doc ingest
- client.flushIngest();
- client.waitForResponses("30s");
- } catch (InterruptedException e) {
- // ignore
- } catch (NoNodeAvailableException e) {
- logger.warn("skipping, no node available");
- } catch (ExecutionException e) {
- logger.error(e.getMessage(), e);
- } finally {
- assertEquals(1, client.getMetric().getSucceeded().getCount());
- if (client.hasThrowable()) {
- logger.error("error", client.getThrowable());
- }
- assertFalse(client.hasThrowable());
- client.shutdown();
- }
- }
-
- @Test
- public void testRandomDocsNodeClient() throws Exception {
- long numactions = NUM_ACTIONS;
- final BulkNodeClient client = Clients.builder()
- .put(Clients.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS)
- .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60))
- .setMetric(new SimpleBulkMetric())
- .setControl(new SimpleBulkControl())
- .toBulkNodeClient(client("1"));
- try {
- client.newIndex("test");
- for (int i = 0; i < NUM_ACTIONS; i++) {
- client.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}");
- }
- client.flushIngest();
- client.waitForResponses("30s");
- } catch (NoNodeAvailableException e) {
- logger.warn("skipping, no node available");
- } finally {
- assertEquals(numactions, client.getMetric().getSucceeded().getCount());
- if (client.hasThrowable()) {
- logger.error("error", client.getThrowable());
- }
- assertFalse(client.hasThrowable());
- client.shutdown();
- }
- }
-
- @Test
- public void testThreadedRandomDocsNodeClient() throws Exception {
- int maxthreads = Runtime.getRuntime().availableProcessors();
- Long maxactions = MAX_ACTIONS;
- final Long maxloop = NUM_ACTIONS;
- logger.info("NodeClient max={} maxactions={} maxloop={}", maxthreads, maxactions, maxloop);
- final BulkNodeClient client = Clients.builder()
- .put(Clients.MAX_ACTIONS_PER_REQUEST, maxactions)
- .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60))// disable auto flush for this test
- .setMetric(new SimpleBulkMetric())
- .setControl(new SimpleBulkControl())
- .toBulkNodeClient(client("1"));
- try {
- client.newIndex("test")
- .startBulk("test", -1, 1000);
- ThreadPoolExecutor pool = EsExecutors.newFixed("bulk-nodeclient-test", maxthreads, 30,
- EsExecutors.daemonThreadFactory("bulk-nodeclient-test"));
- final CountDownLatch latch = new CountDownLatch(maxthreads);
- for (int i = 0; i < maxthreads; i++) {
- pool.execute(new Runnable() {
- public void run() {
- for (int i = 0; i < maxloop; i++) {
- client.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}");
- }
- latch.countDown();
- }
- });
- }
- logger.info("waiting for max 30 seconds...");
- latch.await(30, TimeUnit.SECONDS);
- logger.info("flush...");
- client.flushIngest();
- client.waitForResponses("30s");
- logger.info("got all responses, thread pool shutdown...");
- pool.shutdown();
- logger.info("pool is shut down");
- } catch (NoNodeAvailableException e) {
- logger.warn("skipping, no node available");
- } finally {
- client.stopBulk("test");
- assertEquals(maxthreads * maxloop, client.getMetric().getSucceeded().getCount());
- if (client.hasThrowable()) {
- logger.error("error", client.getThrowable());
- }
- assertFalse(client.hasThrowable());
- client.refreshIndex("test");
- SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE)
- .setQuery(QueryBuilders.matchAllQuery()).setSize(0);
- assertEquals(maxthreads * maxloop,
- searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
- client.shutdown();
- }
- }
-
-}
diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClusterBlockTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClusterBlockTest.java
deleted file mode 100644
index 09c628d..0000000
--- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClusterBlockTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-package org.xbib.elasticsearch.extras.client.node;
-
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
-import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.cluster.block.ClusterBlockException;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.junit.Before;
-import org.junit.Test;
-import org.xbib.elasticsearch.NodeTestUtils;
-
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-
-/**
- *
- */
-public class BulkNodeClusterBlockTest extends NodeTestUtils {
-
- private static final ESLogger logger = ESLoggerFactory.getLogger("test");
-
- @Before
- public void startNodes() {
- try {
- setClusterName();
- startNode("1");
- findNodeAddress();
- // do not wait for green health state
- logger.info("ready");
- } catch (Throwable t) {
- logger.error("startNodes failed", t);
- }
- }
-
- protected Settings getNodeSettings() {
- return Settings.settingsBuilder()
- .put(super.getNodeSettings())
- .put("discovery.zen.minimum_master_nodes", 2) // block until we have two nodes
- .build();
- }
-
- @Test(expected = ClusterBlockException.class)
- public void testClusterBlock() throws Exception {
- BulkRequestBuilder brb = client("1").prepareBulk();
- XContentBuilder builder = jsonBuilder().startObject().field("field1", "value1").endObject();
- String jsonString = builder.string();
- IndexRequestBuilder irb = client("1").prepareIndex("test", "test", "1").setSource(jsonString);
- brb.add(irb);
- brb.execute().actionGet();
- }
-
-}
diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeDuplicateIDTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeDuplicateIDTest.java
deleted file mode 100644
index 7c11526..0000000
--- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeDuplicateIDTest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-package org.xbib.elasticsearch.extras.client.node;
-
-import org.elasticsearch.action.search.SearchAction;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.junit.Test;
-import org.xbib.elasticsearch.NodeTestUtils;
-import org.xbib.elasticsearch.extras.client.Clients;
-import org.xbib.elasticsearch.extras.client.SimpleBulkControl;
-import org.xbib.elasticsearch.extras.client.SimpleBulkMetric;
-
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.junit.Assert.*;
-
-/**
- *
- */
-public class BulkNodeDuplicateIDTest extends NodeTestUtils {
-
- private static final ESLogger logger = ESLoggerFactory.getLogger(BulkNodeDuplicateIDTest.class.getSimpleName());
-
- private static final Long MAX_ACTIONS = 1000L;
-
- private static final Long NUM_ACTIONS = 12345L;
-
- @Test
- public void testDuplicateDocIDs() throws Exception {
- long numactions = NUM_ACTIONS;
- final BulkNodeClient client = Clients.builder()
- .put(Clients.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS)
- .setMetric(new SimpleBulkMetric())
- .setControl(new SimpleBulkControl())
- .toBulkNodeClient(client("1"));
- try {
- client.newIndex("test");
- for (int i = 0; i < NUM_ACTIONS; i++) {
- client.index("test", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}");
- }
- client.flushIngest();
- client.waitForResponses("30s");
- client.refreshIndex("test");
- SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE)
- .setIndices("test")
- .setTypes("test")
- .setQuery(matchAllQuery());
- long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
- logger.info("hits = {}", hits);
- assertTrue(hits < NUM_ACTIONS);
- } catch (NoNodeAvailableException e) {
- logger.warn("skipping, no node available");
- } finally {
- client.shutdown();
- assertEquals(numactions, client.getMetric().getSucceeded().getCount());
- if (client.hasThrowable()) {
- logger.error("error", client.getThrowable());
- }
- assertFalse(client.hasThrowable());
- }
- }
-}
diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeUpdateReplicaLevelTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeUpdateReplicaLevelTest.java
deleted file mode 100644
index 5dc9202..0000000
--- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/node/BulkNodeUpdateReplicaLevelTest.java
+++ /dev/null
@@ -1,66 +0,0 @@
-package org.xbib.elasticsearch.extras.client.node;
-
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.elasticsearch.common.settings.Settings;
-import org.junit.Test;
-import org.xbib.elasticsearch.NodeTestUtils;
-import org.xbib.elasticsearch.extras.client.Clients;
-import org.xbib.elasticsearch.extras.client.SimpleBulkControl;
-import org.xbib.elasticsearch.extras.client.SimpleBulkMetric;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-
-/**
- *
- */
-public class BulkNodeUpdateReplicaLevelTest extends NodeTestUtils {
-
- private static final ESLogger logger = ESLoggerFactory.getLogger(BulkNodeUpdateReplicaLevelTest.class.getSimpleName());
-
- @Test
- public void testUpdateReplicaLevel() throws Exception {
-
- int numberOfShards = 2;
- int replicaLevel = 3;
-
- // we need 3 nodes for replica level 3
- startNode("2");
- startNode("3");
-
- int shardsAfterReplica;
-
- Settings settings = Settings.settingsBuilder()
- .put("index.number_of_shards", numberOfShards)
- .put("index.number_of_replicas", 0)
- .build();
-
- final BulkNodeClient client = Clients.builder()
- .setMetric(new SimpleBulkMetric())
- .setControl(new SimpleBulkControl())
- .toBulkNodeClient(client("1"));
-
- try {
- client.newIndex("replicatest", settings, null);
- client.waitForCluster("GREEN", "30s");
- for (int i = 0; i < 12345; i++) {
- client.index("replicatest", "replicatest", null, "{ \"name\" : \"" + randomString(32) + "\"}");
- }
- client.flushIngest();
- client.waitForResponses("30s");
- shardsAfterReplica = client.updateReplicaLevel("replicatest", replicaLevel);
- assertEquals(shardsAfterReplica, numberOfShards * (replicaLevel + 1));
- } catch (NoNodeAvailableException e) {
- logger.warn("skipping, no node available");
- } finally {
- client.shutdown();
- if (client.hasThrowable()) {
- logger.error("error", client.getThrowable());
- }
- assertFalse(client.hasThrowable());
- }
- }
-
-}
diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClientTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClientTest.java
deleted file mode 100644
index c7c82e0..0000000
--- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClientTest.java
+++ /dev/null
@@ -1,201 +0,0 @@
-package org.xbib.elasticsearch.extras.client.transport;
-
-import org.elasticsearch.action.search.SearchAction;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.junit.Before;
-import org.junit.Test;
-import org.xbib.elasticsearch.NodeTestUtils;
-import org.xbib.elasticsearch.extras.client.Clients;
-import org.xbib.elasticsearch.extras.client.SimpleBulkControl;
-import org.xbib.elasticsearch.extras.client.SimpleBulkMetric;
-
-import java.io.IOException;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-
-/**
- *
- */
-public class BulkTransportClientTest extends NodeTestUtils {
-
- private static final ESLogger logger = ESLoggerFactory.getLogger(BulkTransportClientTest.class.getSimpleName());
-
- private static final Long MAX_ACTIONS = 1000L;
-
- private static final Long NUM_ACTIONS = 1234L;
-
- @Before
- public void startNodes() {
- try {
- super.startNodes();
- startNode("2");
- } catch (Throwable t) {
- logger.error("startNodes failed", t);
- }
- }
-
- @Test
- public void testBulkClient() throws IOException {
- final BulkTransportClient client = Clients.builder()
- .put(getSettings())
- .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60))
- .setMetric(new SimpleBulkMetric())
- .setControl(new SimpleBulkControl())
- .toBulkTransportClient();
- client.newIndex("test");
- if (client.hasThrowable()) {
- logger.error("error", client.getThrowable());
- }
- assertFalse(client.hasThrowable());
- try {
- client.deleteIndex("test")
- .newIndex("test")
- .deleteIndex("test");
- } catch (NoNodeAvailableException e) {
- logger.error("no node available");
- } finally {
- if (client.hasThrowable()) {
- logger.error("error", client.getThrowable());
- }
- assertFalse(client.hasThrowable());
- client.shutdown();
- }
- }
-
- @Test
- public void testSingleDocBulkClient() throws IOException {
- final BulkTransportClient client = Clients.builder()
- .put(getSettings())
- .put(Clients.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS)
- .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60))
- .setMetric(new SimpleBulkMetric())
- .setControl(new SimpleBulkControl())
- .toBulkTransportClient();
- try {
- client.newIndex("test");
- client.index("test", "test", "1", "{ \"name\" : \"Hello World\"}"); // single doc ingest
- client.flushIngest();
- client.waitForResponses("30s");
- } catch (InterruptedException e) {
- // ignore
- } catch (ExecutionException e) {
- logger.error(e.getMessage(), e);
- } catch (NoNodeAvailableException e) {
- logger.warn("skipping, no node available");
- } finally {
- assertEquals(1, client.getMetric().getSucceeded().getCount());
- if (client.hasThrowable()) {
- logger.error("error", client.getThrowable());
- }
- assertFalse(client.hasThrowable());
- client.shutdown();
- }
- }
-
- @Test
- public void testRandomDocsBulkClient() throws IOException {
- long numactions = NUM_ACTIONS;
- final BulkTransportClient client = Clients.builder()
- .put(getSettings())
- .put(Clients.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS)
- .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60))
- .setMetric(new SimpleBulkMetric())
- .setControl(new SimpleBulkControl())
- .toBulkTransportClient();
- try {
- client.newIndex("test");
- for (int i = 0; i < NUM_ACTIONS; i++) {
- client.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}");
- }
- client.flushIngest();
- client.waitForResponses("30s");
- } catch (InterruptedException e) {
- // ignore
- } catch (ExecutionException e) {
- logger.error(e.getMessage(), e);
- } catch (NoNodeAvailableException e) {
- logger.warn("skipping, no node available");
- } finally {
- assertEquals(numactions, client.getMetric().getSucceeded().getCount());
- if (client.hasThrowable()) {
- logger.error("error", client.getThrowable());
- }
- assertFalse(client.hasThrowable());
- client.shutdown();
- }
- }
-
- @Test
- public void testThreadedRandomDocsBulkClient() throws Exception {
- int maxthreads = Runtime.getRuntime().availableProcessors();
- long maxactions = MAX_ACTIONS;
- final long maxloop = NUM_ACTIONS;
-
- Settings settingsForIndex = Settings.settingsBuilder()
- .put("index.number_of_shards", 2)
- .put("index.number_of_replicas", 1)
- .build();
-
- final BulkTransportClient client = Clients.builder()
- .put(getSettings())
- .put(Clients.MAX_ACTIONS_PER_REQUEST, maxactions)
- .put(Clients.FLUSH_INTERVAL, TimeValue.timeValueSeconds(60)) // = disable autoflush for this test
- .setMetric(new SimpleBulkMetric())
- .setControl(new SimpleBulkControl())
- .toBulkTransportClient();
- try {
- client.newIndex("test", settingsForIndex, null)
- .startBulk("test", -1, 1000);
- ThreadPoolExecutor pool =
- EsExecutors.newFixed("bulkclient-test", maxthreads, 30, EsExecutors.daemonThreadFactory("bulkclient-test"));
- final CountDownLatch latch = new CountDownLatch(maxthreads);
- for (int i = 0; i < maxthreads; i++) {
- pool.execute(() -> {
- for (int i1 = 0; i1 < maxloop; i1++) {
- client.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}");
- }
- latch.countDown();
- });
- }
- logger.info("waiting for max 30 seconds...");
- latch.await(30, TimeUnit.SECONDS);
- logger.info("client flush ...");
- client.flushIngest();
- client.waitForResponses("30s");
- logger.info("thread pool to be shut down ...");
- pool.shutdown();
- logger.info("poot shut down");
- } catch (NoNodeAvailableException e) {
- logger.warn("skipping, no node available");
- } finally {
- client.stopBulk("test");
- assertEquals(maxthreads * maxloop, client.getMetric().getSucceeded().getCount());
- if (client.hasThrowable()) {
- logger.error("error", client.getThrowable());
- }
- assertFalse(client.hasThrowable());
- client.refreshIndex("test");
- SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE)
- // to avoid NPE at org.elasticsearch.action.search.SearchRequest.writeTo(SearchRequest.java:580)
- .setIndices("_all")
- .setQuery(QueryBuilders.matchAllQuery())
- .setSize(0);
- assertEquals(maxthreads * maxloop,
- searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
- client.shutdown();
- }
- }
-
-}
diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportDuplicateIDTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportDuplicateIDTest.java
deleted file mode 100644
index c087601..0000000
--- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportDuplicateIDTest.java
+++ /dev/null
@@ -1,60 +0,0 @@
-package org.xbib.elasticsearch.extras.client.transport;
-
-import org.elasticsearch.action.search.SearchAction;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.junit.Test;
-import org.xbib.elasticsearch.NodeTestUtils;
-import org.xbib.elasticsearch.extras.client.Clients;
-import org.xbib.elasticsearch.extras.client.SimpleBulkControl;
-import org.xbib.elasticsearch.extras.client.SimpleBulkMetric;
-
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.junit.Assert.*;
-
-public class BulkTransportDuplicateIDTest extends NodeTestUtils {
-
- private final static ESLogger logger = ESLoggerFactory.getLogger(BulkTransportDuplicateIDTest.class.getSimpleName());
-
- private final static Long MAX_ACTIONS = 1000L;
-
- private final static Long NUM_ACTIONS = 12345L;
-
- @Test
- public void testDuplicateDocIDs() throws Exception {
- long numactions = NUM_ACTIONS;
- final BulkTransportClient client = Clients.builder()
- .put(getSettings())
- .put(Clients.MAX_ACTIONS_PER_REQUEST, MAX_ACTIONS)
- .setMetric(new SimpleBulkMetric())
- .setControl(new SimpleBulkControl())
- .toBulkTransportClient();
- try {
- client.newIndex("test");
- for (int i = 0; i < NUM_ACTIONS; i++) {
- client.index("test", "test", randomString(1), "{ \"name\" : \"" + randomString(32) + "\"}");
- }
- client.flushIngest();
- client.waitForResponses("30s");
- client.refreshIndex("test");
- SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.client(), SearchAction.INSTANCE)
- .setIndices("test")
- .setTypes("test")
- .setQuery(matchAllQuery());
- long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
- logger.info("hits = {}", hits);
- assertTrue(hits < NUM_ACTIONS);
- } catch (NoNodeAvailableException e) {
- logger.warn("skipping, no node available");
- } finally {
- client.shutdown();
- assertEquals(numactions, client.getMetric().getSucceeded().getCount());
- if (client.hasThrowable()) {
- logger.error("error", client.getThrowable());
- }
- assertFalse(client.hasThrowable());
- }
- }
-}
diff --git a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportUpdateReplicaLevelTest.java b/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportUpdateReplicaLevelTest.java
deleted file mode 100644
index 1f56df8..0000000
--- a/src/integration-test/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportUpdateReplicaLevelTest.java
+++ /dev/null
@@ -1,68 +0,0 @@
-package org.xbib.elasticsearch.extras.client.transport;
-
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.elasticsearch.common.settings.Settings;
-import org.junit.Test;
-import org.xbib.elasticsearch.NodeTestUtils;
-import org.xbib.elasticsearch.extras.client.Clients;
-import org.xbib.elasticsearch.extras.client.SimpleBulkControl;
-import org.xbib.elasticsearch.extras.client.SimpleBulkMetric;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-
-/**
- *
- */
-public class BulkTransportUpdateReplicaLevelTest extends NodeTestUtils {
-
- private static final ESLogger logger =
- ESLoggerFactory.getLogger(BulkTransportUpdateReplicaLevelTest.class.getSimpleName());
-
- @Test
- public void testUpdateReplicaLevel() throws Exception {
-
- int numberOfShards = 2;
- int replicaLevel = 3;
-
- // we need 3 nodes for replica level 3
- startNode("2");
- startNode("3");
-
- int shardsAfterReplica;
-
- Settings settings = Settings.settingsBuilder()
- .put("index.number_of_shards", numberOfShards)
- .put("index.number_of_replicas", 0)
- .build();
-
- final BulkTransportClient client = Clients.builder()
- .put(getSettings())
- .setMetric(new SimpleBulkMetric())
- .setControl(new SimpleBulkControl())
- .toBulkTransportClient();
-
- try {
- client.newIndex("replicatest", settings, null);
- client.waitForCluster("GREEN", "30s");
- for (int i = 0; i < 12345; i++) {
- client.index("replicatest", "replicatest", null, "{ \"name\" : \"" + randomString(32) + "\"}");
- }
- client.flushIngest();
- client.waitForResponses("30s");
- shardsAfterReplica = client.updateReplicaLevel("replicatest", replicaLevel);
- assertEquals(shardsAfterReplica, numberOfShards * (replicaLevel + 1));
- } catch (NoNodeAvailableException e) {
- logger.warn("skipping, no node available");
- } finally {
- client.shutdown();
- if (client.hasThrowable()) {
- logger.error("error", client.getThrowable());
- }
- assertFalse(client.hasThrowable());
- }
- }
-
-}
diff --git a/src/integration-test/java/org/xbib/elasticsearch/package-info.java b/src/integration-test/java/org/xbib/elasticsearch/package-info.java
deleted file mode 100644
index 2958ce1..0000000
--- a/src/integration-test/java/org/xbib/elasticsearch/package-info.java
+++ /dev/null
@@ -1,4 +0,0 @@
-/**
- * Test classes for testing Elasticsearch.
- */
-package org.xbib.elasticsearch;
\ No newline at end of file
diff --git a/src/integration-test/java/suites/BulkNodeTestSuite.java b/src/integration-test/java/suites/BulkNodeTestSuite.java
deleted file mode 100644
index caac820..0000000
--- a/src/integration-test/java/suites/BulkNodeTestSuite.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package suites;
-
-import org.junit.runner.RunWith;
-import org.junit.runners.Suite;
-import org.xbib.elasticsearch.extras.client.node.BulkNodeClientTest;
-import org.xbib.elasticsearch.extras.client.node.BulkNodeDuplicateIDTest;
-import org.xbib.elasticsearch.extras.client.node.BulkNodeIndexAliasTest;
-import org.xbib.elasticsearch.extras.client.node.BulkNodeReplicaTest;
-import org.xbib.elasticsearch.extras.client.node.BulkNodeUpdateReplicaLevelTest;
-
-/**
- *
- */
-@RunWith(ListenerSuite.class)
-@Suite.SuiteClasses({
- BulkNodeClientTest.class,
- BulkNodeDuplicateIDTest.class,
- BulkNodeReplicaTest.class,
- BulkNodeUpdateReplicaLevelTest.class,
- BulkNodeIndexAliasTest.class
-})
-public class BulkNodeTestSuite {
-}
diff --git a/src/integration-test/java/suites/BulkTransportTestSuite.java b/src/integration-test/java/suites/BulkTransportTestSuite.java
deleted file mode 100644
index f429dfc..0000000
--- a/src/integration-test/java/suites/BulkTransportTestSuite.java
+++ /dev/null
@@ -1,22 +0,0 @@
-package suites;
-
-import org.junit.runner.RunWith;
-import org.junit.runners.Suite;
-import org.xbib.elasticsearch.extras.client.transport.BulkTransportClientTest;
-import org.xbib.elasticsearch.extras.client.transport.BulkTransportDuplicateIDTest;
-import org.xbib.elasticsearch.extras.client.transport.BulkTransportReplicaTest;
-import org.xbib.elasticsearch.extras.client.transport.BulkTransportUpdateReplicaLevelTest;
-
-/**
- *
- */
-@RunWith(ListenerSuite.class)
-@Suite.SuiteClasses({
- BulkTransportClientTest.class,
- BulkTransportDuplicateIDTest.class,
- BulkTransportReplicaTest.class,
- BulkTransportUpdateReplicaLevelTest.class
-})
-public class BulkTransportTestSuite {
-
-}
diff --git a/src/integration-test/java/suites/ListenerSuite.java b/src/integration-test/java/suites/ListenerSuite.java
deleted file mode 100644
index c02d371..0000000
--- a/src/integration-test/java/suites/ListenerSuite.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package suites;
-
-import org.junit.runner.Runner;
-import org.junit.runner.notification.RunNotifier;
-import org.junit.runners.Suite;
-import org.junit.runners.model.InitializationError;
-import org.junit.runners.model.RunnerBuilder;
-
-public class ListenerSuite extends Suite {
-
- private final TestListener listener = new TestListener();
-
- public ListenerSuite(Class klass, RunnerBuilder builder) throws InitializationError {
- super(klass, builder);
- }
-
- @Override
- protected void runChild(Runner runner, RunNotifier notifier) {
- notifier.addListener(listener);
- runner.run(notifier);
- notifier.removeListener(listener);
- }
-}
diff --git a/src/integration-test/java/suites/MiscTestSuite.java b/src/integration-test/java/suites/MiscTestSuite.java
deleted file mode 100644
index ea23630..0000000
--- a/src/integration-test/java/suites/MiscTestSuite.java
+++ /dev/null
@@ -1,21 +0,0 @@
-package suites;
-
-import org.junit.runner.RunWith;
-import org.junit.runners.Suite;
-import org.xbib.elasticsearch.AliasTest;
-import org.xbib.elasticsearch.SearchTest;
-import org.xbib.elasticsearch.SimpleTest;
-import org.xbib.elasticsearch.WildcardTest;
-
-/**
- *
- */
-@RunWith(ListenerSuite.class)
-@Suite.SuiteClasses({
- SimpleTest.class,
- AliasTest.class,
- SearchTest.class,
- WildcardTest.class
-})
-public class MiscTestSuite {
-}
diff --git a/src/integration-test/java/suites/TestListener.java b/src/integration-test/java/suites/TestListener.java
deleted file mode 100644
index 7e24527..0000000
--- a/src/integration-test/java/suites/TestListener.java
+++ /dev/null
@@ -1,44 +0,0 @@
-package suites;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.junit.runner.Description;
-import org.junit.runner.Result;
-import org.junit.runner.notification.Failure;
-import org.junit.runner.notification.RunListener;
-
-/**
- *
- */
-public class TestListener extends RunListener {
-
- private static final Logger logger = LogManager.getLogger("test.listener");
-
- public void testRunStarted(Description description) throws java.lang.Exception {
- logger.info("number of tests to execute: {}", description.testCount());
- }
-
- public void testRunFinished(Result result) throws java.lang.Exception {
- logger.info("number of tests executed: {}", result.getRunCount());
- }
-
- public void testStarted(Description description) throws java.lang.Exception {
- logger.info("starting execution of {} {}",
- description.getClassName(), description.getMethodName());
- }
-
- public void testFinished(Description description) throws java.lang.Exception {
- logger.info("finished execution of {} {}",
- description.getClassName(), description.getMethodName());
- }
-
- public void testFailure(Failure failure) throws java.lang.Exception {
- logger.info("failed execution of tests: {}",
- failure.getMessage());
- }
-
- public void testIgnored(Description description) throws java.lang.Exception {
- logger.info("execution of test ignored: {}",
- description.getClassName(), description.getMethodName());
- }
-}
diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/Clients.java b/src/main/java/org/xbib/elasticsearch/extras/client/Clients.java
deleted file mode 100644
index daa4981..0000000
--- a/src/main/java/org/xbib/elasticsearch/extras/client/Clients.java
+++ /dev/null
@@ -1,105 +0,0 @@
-package org.xbib.elasticsearch.extras.client;
-
-import org.elasticsearch.client.Client;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.unit.TimeValue;
-import org.xbib.elasticsearch.extras.client.node.BulkNodeClient;
-import org.xbib.elasticsearch.extras.client.transport.BulkTransportClient;
-import org.xbib.elasticsearch.extras.client.transport.MockTransportClient;
-
-/**
- *
- */
-public final class Clients implements Parameters {
-
- private final Settings.Builder settingsBuilder;
-
- private BulkMetric metric;
-
- private BulkControl control;
-
- public Clients() {
- settingsBuilder = Settings.builder();
- }
-
- public static Clients builder() {
- return new Clients();
- }
-
- public Clients put(String key, String value) {
- settingsBuilder.put(key, value);
- return this;
- }
-
- public Clients put(String key, Integer value) {
- settingsBuilder.put(key, value);
- return this;
- }
-
- public Clients put(String key, Long value) {
- settingsBuilder.put(key, value);
- return this;
- }
-
- public Clients put(String key, Double value) {
- settingsBuilder.put(key, value);
- return this;
- }
-
- public Clients put(String key, ByteSizeValue value) {
- settingsBuilder.put(key, value);
- return this;
- }
-
- public Clients put(String key, TimeValue value) {
- settingsBuilder.put(key, value);
- return this;
- }
-
- public Clients put(Settings settings) {
- settingsBuilder.put(settings);
- return this;
- }
-
- public Clients setMetric(BulkMetric metric) {
- this.metric = metric;
- return this;
- }
-
- public Clients setControl(BulkControl control) {
- this.control = control;
- return this;
- }
-
- public BulkNodeClient toBulkNodeClient(Client client) {
- Settings settings = settingsBuilder.build();
- return new BulkNodeClient()
- .maxActionsPerRequest(settings.getAsInt(MAX_ACTIONS_PER_REQUEST, DEFAULT_MAX_ACTIONS_PER_REQUEST))
- .maxConcurrentRequests(settings.getAsInt(MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_REQUESTS))
- .maxVolumePerRequest(settings.get(MAX_VOLUME_PER_REQUEST, DEFAULT_MAX_VOLUME_PER_REQUEST))
- .flushIngestInterval(settings.get(FLUSH_INTERVAL, DEFAULT_FLUSH_INTERVAL))
- .init(client, metric, control);
- }
-
- public BulkTransportClient toBulkTransportClient() {
- Settings settings = settingsBuilder.build();
- return new BulkTransportClient()
- .maxActionsPerRequest(settings.getAsInt(MAX_ACTIONS_PER_REQUEST, DEFAULT_MAX_ACTIONS_PER_REQUEST))
- .maxConcurrentRequests(settings.getAsInt(MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_REQUESTS))
- .maxVolumePerRequest(settings.get(MAX_VOLUME_PER_REQUEST, DEFAULT_MAX_VOLUME_PER_REQUEST))
- .flushIngestInterval(settings.get(FLUSH_INTERVAL, DEFAULT_FLUSH_INTERVAL))
- .init(settings, metric, control);
- }
-
- public MockTransportClient toMockTransportClient() {
- Settings settings = settingsBuilder.build();
- return new MockTransportClient()
- .maxActionsPerRequest(settings.getAsInt(MAX_ACTIONS_PER_REQUEST, DEFAULT_MAX_ACTIONS_PER_REQUEST))
- .maxConcurrentRequests(settings.getAsInt(MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_REQUESTS))
- .maxVolumePerRequest(settings.get(MAX_VOLUME_PER_REQUEST, DEFAULT_MAX_VOLUME_PER_REQUEST))
- .flushIngestInterval(settings.get(FLUSH_INTERVAL, DEFAULT_FLUSH_INTERVAL))
- .init(settings, metric, control);
- }
-
-}
diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClient.java b/src/main/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClient.java
deleted file mode 100644
index 0f387b6..0000000
--- a/src/main/java/org/xbib/elasticsearch/extras/client/node/BulkNodeClient.java
+++ /dev/null
@@ -1,513 +0,0 @@
-package org.xbib.elasticsearch.extras.client.node;
-
-import com.google.common.collect.ImmutableSet;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder;
-import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.action.bulk.BulkProcessor;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.Node;
-import org.elasticsearch.plugins.Plugin;
-import org.xbib.elasticsearch.extras.client.AbstractClient;
-import org.xbib.elasticsearch.extras.client.BulkControl;
-import org.xbib.elasticsearch.extras.client.BulkMetric;
-import org.xbib.elasticsearch.extras.client.ClientMethods;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
-/**
- *
- */
-public class BulkNodeClient extends AbstractClient implements ClientMethods {
-
- private static final ESLogger logger = ESLoggerFactory.getLogger(BulkNodeClient.class.getName());
-
- private int maxActionsPerRequest = DEFAULT_MAX_ACTIONS_PER_REQUEST;
-
- private int maxConcurrentRequests = DEFAULT_MAX_CONCURRENT_REQUESTS;
-
- private ByteSizeValue maxVolume;
-
- private TimeValue flushInterval;
-
- private Node node;
-
- private ElasticsearchClient client;
-
- private BulkProcessor bulkProcessor;
-
- private BulkMetric metric;
-
- private BulkControl control;
-
- private Throwable throwable;
-
- private boolean closed;
-
- @Override
- public BulkNodeClient maxActionsPerRequest(int maxActionsPerRequest) {
- this.maxActionsPerRequest = maxActionsPerRequest;
- return this;
- }
-
- @Override
- public BulkNodeClient maxConcurrentRequests(int maxConcurrentRequests) {
- this.maxConcurrentRequests = maxConcurrentRequests;
- return this;
- }
-
- @Override
- public BulkNodeClient maxVolumePerRequest(String maxVolume) {
- this.maxVolume = ByteSizeValue.parseBytesSizeValue(maxVolume, "maxVolumePerRequest");
- return this;
- }
-
- @Override
- public BulkNodeClient flushIngestInterval(String flushInterval) {
- this.flushInterval = TimeValue.parseTimeValue(flushInterval, TimeValue.timeValueSeconds(5), "flushIngestInterval");
- return this;
- }
-
- @Override
- public BulkNodeClient init(ElasticsearchClient client,
- final BulkMetric metric, final BulkControl control) {
- this.client = client;
- this.metric = metric;
- this.control = control;
- if (metric != null) {
- metric.start();
- }
- BulkProcessor.Listener listener = new BulkProcessor.Listener() {
- @Override
- public void beforeBulk(long executionId, BulkRequest request) {
- long l = -1;
- if (metric != null) {
- metric.getCurrentIngest().inc();
- l = metric.getCurrentIngest().getCount();
- int n = request.numberOfActions();
- metric.getSubmitted().inc(n);
- metric.getCurrentIngestNumDocs().inc(n);
- metric.getTotalIngestSizeInBytes().inc(request.estimatedSizeInBytes());
- }
- logger.debug("before bulk [{}] [actions={}] [bytes={}] [concurrent requests={}]",
- executionId,
- request.numberOfActions(),
- request.estimatedSizeInBytes(),
- l);
- }
-
- @Override
- public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
- long l = -1;
- if (metric != null) {
- metric.getCurrentIngest().dec();
- l = metric.getCurrentIngest().getCount();
- metric.getSucceeded().inc(response.getItems().length);
- }
- int n = 0;
- for (BulkItemResponse itemResponse : response.getItems()) {
- if (metric != null) {
- metric.getCurrentIngest().dec(itemResponse.getIndex(), itemResponse.getType(), itemResponse.getId());
- }
- if (itemResponse.isFailed()) {
- n++;
- if (metric != null) {
- metric.getSucceeded().dec(1);
- metric.getFailed().inc(1);
- }
- }
- }
- if (metric != null) {
- logger.debug("after bulk [{}] [succeeded={}] [failed={}] [{}ms] {} concurrent requests",
- executionId,
- metric.getSucceeded().getCount(),
- metric.getFailed().getCount(),
- response.getTook().millis(),
- l);
- }
- if (n > 0) {
- logger.error("bulk [{}] failed with {} failed items, failure message = {}",
- executionId, n, response.buildFailureMessage());
- } else {
- if (metric != null) {
- metric.getCurrentIngestNumDocs().dec(response.getItems().length);
- }
- }
- }
-
- @Override
- public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
- if (metric != null) {
- metric.getCurrentIngest().dec();
- }
- throwable = failure;
- closed = true;
- logger.error("after bulk [" + executionId + "] error", failure);
- }
- };
- BulkProcessor.Builder builder = BulkProcessor.builder((Client) client, listener)
- .setBulkActions(maxActionsPerRequest)
- .setConcurrentRequests(maxConcurrentRequests)
- .setFlushInterval(flushInterval);
- if (maxVolume != null) {
- builder.setBulkSize(maxVolume);
- }
- this.bulkProcessor = builder.build();
- this.closed = false;
- return this;
- }
-
- @Override
- public BulkNodeClient init(Settings settings, BulkMetric metric, BulkControl control) throws IOException {
- createClient(settings);
- this.metric = metric;
- this.control = control;
- return this;
- }
-
- @Override
- public ElasticsearchClient client() {
- return client;
- }
-
- @Override
- protected synchronized void createClient(Settings settings) throws IOException {
- if (client != null) {
- logger.warn("client is open, closing...");
- client.threadPool().shutdown();
- client = null;
- node.close();
- }
- if (settings != null) {
- String version = System.getProperty("os.name")
- + " " + System.getProperty("java.vm.name")
- + " " + System.getProperty("java.vm.vendor")
- + " " + System.getProperty("java.runtime.version")
- + " " + System.getProperty("java.vm.version");
- Settings effectiveSettings = Settings.builder().put(settings)
- .put("node.client", true)
- .put("node.master", false)
- .put("node.data", false).build();
- logger.info("creating node client on {} with effective settings {}",
- version, effectiveSettings.getAsMap());
- Collection> plugins = Collections.emptyList();
- this.node = new BulkNode(new Environment(effectiveSettings), plugins);
- node.start();
- this.client = node.client();
- }
- }
-
- @Override
- public BulkMetric getMetric() {
- return metric;
- }
-
- @Override
- public BulkNodeClient index(String index, String type, String id, String source) {
- if (closed) {
- throwClose();
- }
- try {
- if (metric != null) {
- metric.getCurrentIngest().inc(index, type, id);
- }
- bulkProcessor.add(new IndexRequest(index).type(type).id(id).create(false).source(source));
- } catch (Exception e) {
- throwable = e;
- closed = true;
- logger.error("bulk add of index request failed: " + e.getMessage(), e);
- }
- return this;
- }
-
- @Override
- public BulkNodeClient bulkIndex(IndexRequest indexRequest) {
- if (closed) {
- throwClose();
- }
- try {
- if (metric != null) {
- metric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id());
- }
- bulkProcessor.add(indexRequest);
- } catch (Exception e) {
- throwable = e;
- closed = true;
- logger.error("bulk add of index request failed: " + e.getMessage(), e);
- }
- return this;
- }
-
- @Override
- public BulkNodeClient delete(String index, String type, String id) {
- if (closed) {
- throwClose();
- }
- try {
- if (metric != null) {
- metric.getCurrentIngest().inc(index, type, id);
- }
- bulkProcessor.add(new DeleteRequest(index).type(type).id(id));
- } catch (Exception e) {
- throwable = e;
- closed = true;
- logger.error("bulk add of delete failed: " + e.getMessage(), e);
- }
- return this;
- }
-
- @Override
- public BulkNodeClient bulkDelete(DeleteRequest deleteRequest) {
- if (closed) {
- throwClose();
- }
- try {
- if (metric != null) {
- metric.getCurrentIngest().inc(deleteRequest.index(), deleteRequest.type(), deleteRequest.id());
- }
- bulkProcessor.add(deleteRequest);
- } catch (Exception e) {
- throwable = e;
- closed = true;
- logger.error("bulk add of delete failed: " + e.getMessage(), e);
- }
- return this;
- }
-
- @Override
- public BulkNodeClient update(String index, String type, String id, String source) {
- if (closed) {
- throwClose();
- }
- try {
- if (metric != null) {
- metric.getCurrentIngest().inc(index, type, id);
- }
- bulkProcessor.add(new UpdateRequest().index(index).type(type).id(id).upsert(source));
- } catch (Exception e) {
- throwable = e;
- closed = true;
- logger.error("bulk add of update request failed: " + e.getMessage(), e);
- }
- return this;
- }
-
- @Override
- public BulkNodeClient bulkUpdate(UpdateRequest updateRequest) {
- if (closed) {
- throwClose();
- }
- try {
- if (metric != null) {
- metric.getCurrentIngest().inc(updateRequest.index(), updateRequest.type(), updateRequest.id());
- }
- bulkProcessor.add(updateRequest);
- } catch (Exception e) {
- throwable = e;
- closed = true;
- logger.error("bulk add of update request failed: " + e.getMessage(), e);
- }
- return this;
- }
-
- @Override
- public BulkNodeClient flushIngest() {
- if (closed) {
- throwClose();
- }
- logger.debug("flushing bulk processor");
- bulkProcessor.flush();
- return this;
- }
-
- @Override
- public BulkNodeClient waitForResponses(String maxWaitTime) throws InterruptedException, ExecutionException {
- if (closed) {
- throwClose();
- }
- while (!bulkProcessor.awaitClose(TimeValue.parseTimeValue(maxWaitTime, TimeValue.timeValueSeconds(30),
- "maxWaitTime").getMillis(), TimeUnit.MILLISECONDS)) {
- logger.warn("still waiting for responses");
- }
- return this;
- }
-
- @Override
- public BulkNodeClient startBulk(String index, long startRefreshIntervalMillis, long stopRefreshItervalMillis)
- throws IOException {
- if (control == null) {
- return this;
- }
- if (!control.isBulk(index)) {
- control.startBulk(index, startRefreshIntervalMillis, stopRefreshItervalMillis);
- updateIndexSetting(index, "refresh_interval", startRefreshIntervalMillis + "ms");
- }
- return this;
- }
-
- @Override
- public BulkNodeClient stopBulk(String index) throws IOException {
- if (control == null) {
- return this;
- }
- if (control.isBulk(index)) {
- updateIndexSetting(index, "refresh_interval", control.getStopBulkRefreshIntervals().get(index) + "ms");
- control.finishBulk(index);
- }
- return this;
- }
-
- @Override
- public synchronized void shutdown() {
- try {
- if (bulkProcessor != null) {
- logger.debug("closing bulk processor...");
- bulkProcessor.close();
- }
- if (control != null && control.indices() != null && !control.indices().isEmpty()) {
- logger.debug("stopping bulk mode for indices {}...", control.indices());
- for (String index : ImmutableSet.copyOf(control.indices())) {
- stopBulk(index);
- }
- metric.stop();
- }
- if (node != null) {
- logger.debug("closing node...");
- node.close();
- }
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- }
- }
-
- @Override
- public BulkNodeClient newIndex(String index) {
- return newIndex(index, null, null);
- }
-
- @Override
- public BulkNodeClient newIndex(String index, String type, InputStream settings, InputStream mappings) throws IOException {
- resetSettings();
- setting(settings);
- mapping(type, mappings);
- return newIndex(index, settings(), mappings());
- }
-
- @Override
- public BulkNodeClient newIndex(String index, Settings settings, Map mappings) {
- if (closed) {
- throwClose();
- }
- if (client == null) {
- logger.warn("no client for create index");
- return this;
- }
- if (index == null) {
- logger.warn("no index name given to create index");
- return this;
- }
- CreateIndexRequestBuilder createIndexRequestBuilder =
- new CreateIndexRequestBuilder(client(), CreateIndexAction.INSTANCE).setIndex(index);
- if (settings != null) {
- logger.info("settings = {}", settings.getAsStructuredMap());
- createIndexRequestBuilder.setSettings(settings);
- }
- if (mappings != null) {
- for (Map.Entry entry : mappings.entrySet()) {
- String type = entry.getKey();
- String mapping = entry.getValue();
- logger.info("found mapping for {}", type);
- createIndexRequestBuilder.addMapping(type, mapping);
- }
- }
- createIndexRequestBuilder.execute().actionGet();
- logger.info("index {} created", index);
- return this;
- }
-
- @Override
- public BulkNodeClient newMapping(String index, String type, Map mapping) {
- PutMappingRequestBuilder putMappingRequestBuilder =
- new PutMappingRequestBuilder(client(), PutMappingAction.INSTANCE)
- .setIndices(index)
- .setType(type)
- .setSource(mapping);
- putMappingRequestBuilder.execute().actionGet();
- logger.info("mapping created for index {} and type {}", index, type);
- return this;
- }
-
- @Override
- public BulkNodeClient deleteIndex(String index) {
- if (closed) {
- throwClose();
- }
- if (client == null) {
- logger.warn("no client");
- return this;
- }
- if (index == null) {
- logger.warn("no index name given to delete index");
- return this;
- }
- DeleteIndexRequestBuilder deleteIndexRequestBuilder =
- new DeleteIndexRequestBuilder(client(), DeleteIndexAction.INSTANCE, index);
- deleteIndexRequestBuilder.execute().actionGet();
- return this;
- }
-
- @Override
- public boolean hasThrowable() {
- return throwable != null;
- }
-
- @Override
- public Throwable getThrowable() {
- return throwable;
- }
-
- public Settings getSettings() {
- return settings();
- }
-
- @Override
- public Settings.Builder getSettingsBuilder() {
- return settingsBuilder();
- }
-
- private static void throwClose() {
- throw new ElasticsearchException("client is closed");
- }
-
- private class BulkNode extends Node {
-
- BulkNode(Environment env, Collection> classpathPlugins) {
- super(env, Version.CURRENT, classpathPlugins);
- }
- }
-
-}
diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/package-info.java b/src/main/java/org/xbib/elasticsearch/extras/client/package-info.java
deleted file mode 100644
index c231c60..0000000
--- a/src/main/java/org/xbib/elasticsearch/extras/client/package-info.java
+++ /dev/null
@@ -1,4 +0,0 @@
-/**
- * Classes for Elasticsearch client extras.
- */
-package org.xbib.elasticsearch.extras.client;
diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClient.java b/src/main/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClient.java
deleted file mode 100644
index ac37781..0000000
--- a/src/main/java/org/xbib/elasticsearch/extras/client/transport/BulkTransportClient.java
+++ /dev/null
@@ -1,564 +0,0 @@
-package org.xbib.elasticsearch.extras.client.transport;
-
-import com.google.common.collect.ImmutableSet;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
-import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder;
-import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
-import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder;
-import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.unit.TimeValue;
-import org.xbib.elasticsearch.extras.client.AbstractClient;
-import org.xbib.elasticsearch.extras.client.BulkControl;
-import org.xbib.elasticsearch.extras.client.BulkMetric;
-import org.xbib.elasticsearch.extras.client.BulkProcessor;
-import org.xbib.elasticsearch.extras.client.ClientMethods;
-import org.xbib.elasticsearch.extras.client.NetworkUtils;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Transport client with addtitional methods using the BulkProcessor.
- */
-public class BulkTransportClient extends AbstractClient implements ClientMethods {
-
- private static final ESLogger logger = ESLoggerFactory.getLogger(BulkTransportClient.class.getName());
-
- private int maxActionsPerRequest = DEFAULT_MAX_ACTIONS_PER_REQUEST;
-
- private int maxConcurrentRequests = DEFAULT_MAX_CONCURRENT_REQUESTS;
-
- private ByteSizeValue maxVolumePerRequest;
-
- private TimeValue flushInterval;
-
- private BulkProcessor bulkProcessor;
-
- private Throwable throwable;
-
- private boolean closed;
-
- private TransportClient client;
-
- private BulkMetric metric;
-
- private BulkControl control;
-
- private boolean ignoreBulkErrors;
-
- private boolean isShutdown;
-
- @Override
- public BulkTransportClient init(ElasticsearchClient client, BulkMetric metric, BulkControl control) throws IOException {
- return init(findSettings(), metric, control);
- }
-
- @Override
- public BulkTransportClient init(Settings settings, final BulkMetric metric, final BulkControl control) {
- createClient(settings);
- this.metric = metric;
- this.control = control;
- if (metric != null) {
- metric.start();
- }
- resetSettings();
- BulkProcessor.Listener listener = new BulkProcessor.Listener() {
- @Override
- public void beforeBulk(long executionId, BulkRequest request) {
- long l = -1L;
- if (metric != null) {
- metric.getCurrentIngest().inc();
- l = metric.getCurrentIngest().getCount();
- int n = request.numberOfActions();
- metric.getSubmitted().inc(n);
- metric.getCurrentIngestNumDocs().inc(n);
- metric.getTotalIngestSizeInBytes().inc(request.estimatedSizeInBytes());
- }
- logger.debug("before bulk [{}] [actions={}] [bytes={}] [concurrent requests={}]",
- executionId,
- request.numberOfActions(),
- request.estimatedSizeInBytes(),
- l);
- }
-
- @Override
- public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
- long l = -1L;
- if (metric != null) {
- metric.getCurrentIngest().dec();
- l = metric.getCurrentIngest().getCount();
- metric.getSucceeded().inc(response.getItems().length);
- }
- int n = 0;
- for (BulkItemResponse itemResponse : response.getItems()) {
- if (metric != null) {
- metric.getCurrentIngest().dec(itemResponse.getIndex(), itemResponse.getType(), itemResponse.getId());
- if (itemResponse.isFailed()) {
- n++;
- metric.getSucceeded().dec(1);
- metric.getFailed().inc(1);
- }
- }
- }
- if (metric != null) {
- logger.debug("after bulk [{}] [succeeded={}] [failed={}] [{}ms] [concurrent requests={}]",
- executionId,
- metric.getSucceeded().getCount(),
- metric.getFailed().getCount(),
- response.getTook().millis(),
- l);
- }
- if (n > 0) {
- logger.error("bulk [{}] failed with {} failed items, failure message = {}",
- executionId, n, response.buildFailureMessage());
- } else {
- if (metric != null) {
- metric.getCurrentIngestNumDocs().dec(response.getItems().length);
- }
- }
- }
-
- @Override
- public void afterBulk(long executionId, BulkRequest requst, Throwable failure) {
- if (metric != null) {
- metric.getCurrentIngest().dec();
- }
- throwable = failure;
- if (!ignoreBulkErrors) {
- closed = true;
- }
- logger.error("bulk [" + executionId + "] error", failure);
- }
- };
- BulkProcessor.Builder builder = BulkProcessor.builder(client, listener)
- .setBulkActions(maxActionsPerRequest)
- .setConcurrentRequests(maxConcurrentRequests)
- .setFlushInterval(flushInterval);
- if (maxVolumePerRequest != null) {
- builder.setBulkSize(maxVolumePerRequest);
- }
- this.bulkProcessor = builder.build();
- try {
- Collection addrs = findAddresses(settings);
- if (!connect(addrs, settings.getAsBoolean("autodiscover", false))) {
- throw new NoNodeAvailableException("no cluster nodes available, check settings "
- + settings.getAsMap());
- }
- } catch (IOException e) {
- logger.error(e.getMessage(), e);
- }
- this.closed = false;
- return this;
- }
-
- @Override
- public ClientMethods newMapping(String index, String type, Map mapping) {
- new PutMappingRequestBuilder(client(), PutMappingAction.INSTANCE)
- .setIndices(index)
- .setType(type)
- .setSource(mapping)
- .execute().actionGet();
- logger.info("mapping created for index {} and type {}", index, type);
- return this;
- }
-
- @Override
- protected void createClient(Settings settings) {
- if (client != null) {
- logger.warn("client is open, closing...");
- client.close();
- client.threadPool().shutdown();
- client = null;
- }
- if (settings != null) {
- String version = System.getProperty("os.name")
- + " " + System.getProperty("java.vm.name")
- + " " + System.getProperty("java.vm.vendor")
- + " " + System.getProperty("java.runtime.version")
- + " " + System.getProperty("java.vm.version");
- logger.info("creating transport client on {} with effective settings {}",
- version, settings.getAsMap());
- this.client = TransportClient.builder()
- .settings(settings)
- .build();
- this.ignoreBulkErrors = settings.getAsBoolean("ignoreBulkErrors", true);
- }
- }
-
- public boolean isShutdown() {
- return isShutdown;
- }
-
- @Override
- public BulkTransportClient maxActionsPerRequest(int maxActionsPerRequest) {
- this.maxActionsPerRequest = maxActionsPerRequest;
- return this;
- }
-
- @Override
- public BulkTransportClient maxConcurrentRequests(int maxConcurrentRequests) {
- this.maxConcurrentRequests = maxConcurrentRequests;
- return this;
- }
-
- @Override
- public BulkTransportClient maxVolumePerRequest(String maxVolumePerRequest) {
- this.maxVolumePerRequest = ByteSizeValue.parseBytesSizeValue(maxVolumePerRequest, "maxVolumePerRequest");
- return this;
- }
-
- @Override
- public BulkTransportClient flushIngestInterval(String flushInterval) {
- this.flushInterval = TimeValue.parseTimeValue(flushInterval, TimeValue.timeValueSeconds(5), "flushIngestInterval");
- return this;
- }
-
- @Override
- public ElasticsearchClient client() {
- return client;
- }
-
- @Override
- public BulkMetric getMetric() {
- return metric;
- }
-
- @Override
- public ClientMethods newIndex(String index) {
- if (closed) {
- throwClose();
- }
- return newIndex(index, null, null);
- }
-
- @Override
- public ClientMethods newIndex(String index, String type, InputStream settings, InputStream mappings) throws IOException {
- resetSettings();
- setting(settings);
- mapping(type, mappings);
- return newIndex(index, settings(), mappings());
- }
-
- @Override
- public ClientMethods newIndex(String index, Settings settings, Map mappings) {
- if (closed) {
- throwClose();
- }
- if (index == null) {
- logger.warn("no index name given to create index");
- return this;
- }
- CreateIndexRequestBuilder createIndexRequestBuilder =
- new CreateIndexRequestBuilder(client(), CreateIndexAction.INSTANCE).setIndex(index);
- if (settings != null) {
- logger.info("settings = {}", settings.getAsStructuredMap());
- createIndexRequestBuilder.setSettings(settings);
- }
- if (mappings != null) {
- for (Map.Entry entry : mappings.entrySet()) {
- String type = entry.getKey();
- String mapping = entry.getValue();
- logger.info("found mapping for {}", type);
- createIndexRequestBuilder.addMapping(type, mapping);
- }
- }
- createIndexRequestBuilder.execute().actionGet();
- logger.info("index {} created", index);
- return this;
- }
-
- @Override
- public ClientMethods deleteIndex(String index) {
- if (closed) {
- throwClose();
- }
- if (index == null) {
- logger.warn("no index name given to delete index");
- return this;
- }
- new DeleteIndexRequestBuilder(client(), DeleteIndexAction.INSTANCE, index).execute().actionGet();
- return this;
- }
-
- @Override
- public ClientMethods startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds)
- throws IOException {
- if (control == null) {
- return this;
- }
- if (!control.isBulk(index)) {
- control.startBulk(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds);
- updateIndexSetting(index, "refresh_interval", startRefreshIntervalSeconds + "s");
- }
- return this;
- }
-
- @Override
- public ClientMethods stopBulk(String index) throws IOException {
- if (control == null) {
- return this;
- }
- if (control.isBulk(index)) {
- updateIndexSetting(index, "refresh_interval", control.getStopBulkRefreshIntervals().get(index) + "s");
- control.finishBulk(index);
- }
- return this;
- }
-
- @Override
- public BulkTransportClient index(String index, String type, String id, String source) {
- if (closed) {
- throwClose();
- }
- try {
- metric.getCurrentIngest().inc(index, type, id);
- bulkProcessor.add(new IndexRequest().index(index).type(type).id(id).create(false).source(source));
- } catch (Exception e) {
- throwable = e;
- closed = true;
- logger.error("bulk add of index request failed: " + e.getMessage(), e);
- }
- return this;
- }
-
- @Override
- public BulkTransportClient bulkIndex(IndexRequest indexRequest) {
- if (closed) {
- throwClose();
- }
- try {
- metric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id());
- bulkProcessor.add(indexRequest);
- } catch (Exception e) {
- throwable = e;
- closed = true;
- logger.error("bulk add of index request failed: " + e.getMessage(), e);
- }
- return this;
- }
-
- @Override
- public BulkTransportClient delete(String index, String type, String id) {
- if (closed) {
- throwClose();
- }
- try {
- metric.getCurrentIngest().inc(index, type, id);
- bulkProcessor.add(new DeleteRequest().index(index).type(type).id(id));
- } catch (Exception e) {
- throwable = e;
- closed = true;
- logger.error("bulk add of delete request failed: " + e.getMessage(), e);
- }
- return this;
- }
-
- @Override
- public BulkTransportClient bulkDelete(DeleteRequest deleteRequest) {
- if (closed) {
- throwClose();
- }
- try {
- metric.getCurrentIngest().inc(deleteRequest.index(), deleteRequest.type(), deleteRequest.id());
- bulkProcessor.add(deleteRequest);
- } catch (Exception e) {
- throwable = e;
- closed = true;
- logger.error("bulk add of delete request failed: " + e.getMessage(), e);
- }
- return this;
- }
-
- @Override
- public BulkTransportClient update(String index, String type, String id, String source) {
- if (closed) {
- throwClose();
- }
- try {
- metric.getCurrentIngest().inc(index, type, id);
- bulkProcessor.add(new UpdateRequest().index(index).type(type).id(id).upsert(source));
- } catch (Exception e) {
- throwable = e;
- closed = true;
- logger.error("bulk add of update request failed: " + e.getMessage(), e);
- }
- return this;
- }
-
- @Override
- public BulkTransportClient bulkUpdate(UpdateRequest updateRequest) {
- if (closed) {
- throwClose();
- }
- try {
- metric.getCurrentIngest().inc(updateRequest.index(), updateRequest.type(), updateRequest.id());
- bulkProcessor.add(updateRequest);
- } catch (Exception e) {
- throwable = e;
- closed = true;
- logger.error("bulk add of update request failed: " + e.getMessage(), e);
- }
- return this;
- }
-
- @Override
- public synchronized BulkTransportClient flushIngest() {
- if (closed) {
- throwClose();
- }
- logger.debug("flushing bulk processor");
- bulkProcessor.flush();
- return this;
- }
-
- @Override
- public synchronized BulkTransportClient waitForResponses(String maxWaitTime)
- throws InterruptedException, ExecutionException {
- if (closed) {
- throwClose();
- }
- bulkProcessor.awaitClose(TimeValue.parseTimeValue(maxWaitTime,
- TimeValue.timeValueSeconds(30), "maxWaitTime").getMillis(), TimeUnit.MILLISECONDS);
- return this;
- }
-
- @Override
- public synchronized void shutdown() {
- if (closed) {
- shutdownClient();
- throwClose();
- }
- try {
- if (bulkProcessor != null) {
- logger.debug("closing bulk processor...");
- bulkProcessor.close();
- }
- if (control != null && control.indices() != null && !control.indices().isEmpty()) {
- logger.debug("stopping bulk mode for indices {}...", control.indices());
- for (String index : ImmutableSet.copyOf(control.indices())) {
- stopBulk(index);
- }
- metric.stop();
- }
- logger.debug("shutting down...");
- shutdownClient();
- logger.debug("shutting down completed");
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- }
- }
-
- @Override
- public boolean hasThrowable() {
- return throwable != null;
- }
-
- @Override
- public Throwable getThrowable() {
- return throwable;
- }
-
- private Settings findSettings() {
- Settings.Builder settingsBuilder = Settings.settingsBuilder();
- settingsBuilder.put("host", "localhost");
- try {
- String hostname = NetworkUtils.getLocalAddress().getHostName();
- logger.debug("the hostname is {}", hostname);
- settingsBuilder.put("host", hostname)
- .put("port", 9300);
- } catch (Exception e) {
- logger.warn(e.getMessage(), e);
- }
- return settingsBuilder.build();
- }
-
- private Collection findAddresses(Settings settings) throws IOException {
- String[] hostnames = settings.getAsArray("host", new String[]{"localhost"});
- int port = settings.getAsInt("port", 9300);
- Collection addresses = new ArrayList<>();
- for (String hostname : hostnames) {
- String[] splitHost = hostname.split(":", 2);
- if (splitHost.length == 2) {
- String host = splitHost[0];
- InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null);
- try {
- port = Integer.parseInt(splitHost[1]);
- } catch (Exception e) {
- logger.warn(e.getMessage(), e);
- }
- addresses.add(new InetSocketTransportAddress(inetAddress, port));
- }
- if (splitHost.length == 1) {
- String host = splitHost[0];
- InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null);
- addresses.add(new InetSocketTransportAddress(inetAddress, port));
- }
- }
- return addresses;
- }
-
- private static void throwClose() {
- throw new ElasticsearchException("client is closed");
- }
-
- private void shutdownClient() {
- if (client != null) {
- logger.debug("shutdown started");
- client.close();
- client.threadPool().shutdown();
- client = null;
- logger.debug("shutdown complete");
- }
- isShutdown = true;
- }
-
- private boolean connect(Collection addresses, boolean autodiscover) {
- logger.info("trying to connect to {}", addresses);
- client.addTransportAddresses(addresses);
- if (client.connectedNodes() != null) {
- List nodes = client.connectedNodes();
- if (!nodes.isEmpty()) {
- logger.info("connected to {}", nodes);
- if (autodiscover) {
- logger.info("trying to auto-discover all cluster nodes...");
- ClusterStateRequestBuilder clusterStateRequestBuilder =
- new ClusterStateRequestBuilder(client, ClusterStateAction.INSTANCE);
- ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet();
- DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes();
- client.addDiscoveryNodes(discoveryNodes);
- logger.info("after auto-discovery connected to {}", client.connectedNodes());
- }
- return true;
- }
- return false;
- }
- return false;
- }
-}
diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/transport/MockTransportClient.java b/src/main/java/org/xbib/elasticsearch/extras/client/transport/MockTransportClient.java
deleted file mode 100644
index ed0fcc7..0000000
--- a/src/main/java/org/xbib/elasticsearch/extras/client/transport/MockTransportClient.java
+++ /dev/null
@@ -1,155 +0,0 @@
-package org.xbib.elasticsearch.extras.client.transport;
-
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.common.settings.Settings;
-import org.xbib.elasticsearch.extras.client.BulkControl;
-import org.xbib.elasticsearch.extras.client.BulkMetric;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- * Mock client, it does not perform actions on a cluster.
- * Useful for testing or dry runs.
- */
-public class MockTransportClient extends BulkTransportClient {
-
- @Override
- public ElasticsearchClient client() {
- return null;
- }
-
- @Override
- public MockTransportClient init(ElasticsearchClient client, BulkMetric metric, BulkControl control) {
- return this;
- }
-
- @Override
- public MockTransportClient init(Settings settings, BulkMetric metric, BulkControl control) {
- return this;
- }
-
- @Override
- public MockTransportClient maxActionsPerRequest(int maxActions) {
- return this;
- }
-
- @Override
- public MockTransportClient maxConcurrentRequests(int maxConcurrentRequests) {
- return this;
- }
-
- @Override
- public MockTransportClient maxVolumePerRequest(String maxVolumePerRequest) {
- return this;
- }
-
- @Override
- public MockTransportClient flushIngestInterval(String interval) {
- return this;
- }
-
- @Override
- public MockTransportClient index(String index, String type, String id, String source) {
- return this;
- }
-
- @Override
- public MockTransportClient delete(String index, String type, String id) {
- return this;
- }
-
- @Override
- public MockTransportClient update(String index, String type, String id, String source) {
- return this;
- }
-
- @Override
- public MockTransportClient bulkIndex(IndexRequest indexRequest) {
- return this;
- }
-
- @Override
- public MockTransportClient bulkDelete(DeleteRequest deleteRequest) {
- return this;
- }
-
- @Override
- public MockTransportClient bulkUpdate(UpdateRequest updateRequest) {
- return this;
- }
-
- @Override
- public MockTransportClient flushIngest() {
- return this;
- }
-
- @Override
- public MockTransportClient waitForResponses(String timeValue) throws InterruptedException {
- return this;
- }
-
- @Override
- public MockTransportClient startBulk(String index, long startRefreshInterval, long stopRefreshIterval) {
- return this;
- }
-
- @Override
- public MockTransportClient stopBulk(String index) {
- return this;
- }
-
- @Override
- public MockTransportClient deleteIndex(String index) {
- return this;
- }
-
- @Override
- public MockTransportClient newIndex(String index) {
- return this;
- }
-
- @Override
- public MockTransportClient newMapping(String index, String type, Map mapping) {
- return this;
- }
-
- @Override
- public void putMapping(String index) {
- // mockup method
- }
-
- @Override
- public void refreshIndex(String index) {
- // mockup method
- }
-
- @Override
- public void flushIndex(String index) {
- // mockup method
- }
-
- @Override
- public void waitForCluster(String healthColor, String timeValue) throws IOException {
- // mockup method
- }
-
- @Override
- public int waitForRecovery(String index) throws IOException {
- return -1;
- }
-
- @Override
- public int updateReplicaLevel(String index, int level) throws IOException {
- return -1;
- }
-
- @Override
- public void shutdown() {
- // mockup method
- }
-
-}
diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/transport/TransportClient.java b/src/main/java/org/xbib/elasticsearch/extras/client/transport/TransportClient.java
deleted file mode 100644
index 3912ce7..0000000
--- a/src/main/java/org/xbib/elasticsearch/extras/client/transport/TransportClient.java
+++ /dev/null
@@ -1,445 +0,0 @@
-package org.xbib.elasticsearch.extras.client.transport;
-
-import static org.elasticsearch.common.settings.Settings.settingsBuilder;
-import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
-
-import com.google.common.collect.ImmutableMap;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.Action;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionModule;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestBuilder;
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.action.GenericAction;
-import org.elasticsearch.action.TransportActionNodeProxy;
-import org.elasticsearch.action.admin.cluster.node.liveness.LivenessRequest;
-import org.elasticsearch.action.admin.cluster.node.liveness.LivenessResponse;
-import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
-import org.elasticsearch.client.support.AbstractClient;
-import org.elasticsearch.client.support.Headers;
-import org.elasticsearch.client.transport.ClientTransportModule;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterNameModule;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.collect.MapBuilder;
-import org.elasticsearch.common.component.LifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.inject.ModulesBuilder;
-import org.elasticsearch.common.network.NetworkModule;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.indices.breaker.CircuitBreakerModule;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.PluginsModule;
-import org.elasticsearch.plugins.PluginsService;
-import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.threadpool.ThreadPoolModule;
-import org.elasticsearch.transport.FutureTransportResponseHandler;
-import org.elasticsearch.transport.TransportModule;
-import org.elasticsearch.transport.TransportRequestOptions;
-import org.elasticsearch.transport.TransportService;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * Stripped-down transport client without node sampling and without retrying.
- *
- * Merged together: original TransportClient, TransportClientNodesServce, TransportClientProxy
-
- * Configurable connect ping interval setting added.
- */
-public class TransportClient extends AbstractClient {
-
- private static final String CLIENT_TYPE = "transport";
-
- private final Injector injector;
-
- private final ProxyActionMap proxyActionMap;
-
- private final long pingTimeout;
-
- private final ClusterName clusterName;
-
- private final TransportService transportService;
-
- private final Version minCompatibilityVersion;
-
- private final Headers headers;
-
- private final AtomicInteger tempNodeId = new AtomicInteger();
-
- private final AtomicInteger nodeCounter = new AtomicInteger();
-
- private final Object mutex = new Object();
-
- private volatile List listedNodes = Collections.emptyList();
-
- private volatile List nodes = Collections.emptyList();
-
- private volatile List filteredNodes = Collections.emptyList();
-
- private volatile boolean closed;
-
- private TransportClient(Injector injector) {
- super(injector.getInstance(Settings.class), injector.getInstance(ThreadPool.class),
- injector.getInstance(Headers.class));
- this.injector = injector;
- this.clusterName = injector.getInstance(ClusterName.class);
- this.transportService = injector.getInstance(TransportService.class);
- this.minCompatibilityVersion = injector.getInstance(Version.class).minimumCompatibilityVersion();
- this.headers = injector.getInstance(Headers.class);
- this.pingTimeout = this.settings.getAsTime("client.transport.ping_timeout", timeValueSeconds(5)).millis();
- this.proxyActionMap = injector.getInstance(ProxyActionMap.class);
- }
-
- public static Builder builder() {
- return new Builder();
- }
-
- /**
- * Returns the current registered transport addresses to use.
- *
- * @return list of transport addresess
- */
- public List transportAddresses() {
- List lstBuilder = new ArrayList<>();
- for (DiscoveryNode listedNode : listedNodes) {
- lstBuilder.add(listedNode.address());
- }
- return Collections.unmodifiableList(lstBuilder);
- }
-
- /**
- * Returns the current connected transport nodes that this client will use.
- * The nodes include all the nodes that are currently alive based on the transport
- * addresses provided.
- *
- * @return list of nodes
- */
- public List connectedNodes() {
- return this.nodes;
- }
-
- /**
- * The list of filtered nodes that were not connected to, for example, due to
- * mismatch in cluster name.
- *
- * @return list of nodes
- */
- public List filteredNodes() {
- return this.filteredNodes;
- }
-
- /**
- * Returns the listed nodes in the transport client (ones added to it).
- *
- * @return list of nodes
- */
- public List listedNodes() {
- return this.listedNodes;
- }
-
- /**
- * Adds a list of transport addresses that will be used to connect to.
- * The Node this transport address represents will be used if its possible to connect to it.
- * If it is unavailable, it will be automatically connected to once it is up.
- * In order to get the list of all the current connected nodes, please see {@link #connectedNodes()}.
- *
- * @param discoveryNodes nodes
- * @return this transport client
- */
- public TransportClient addDiscoveryNodes(DiscoveryNodes discoveryNodes) {
- Collection addresses = new ArrayList<>();
- for (DiscoveryNode discoveryNode : discoveryNodes) {
- addresses.add((InetSocketTransportAddress) discoveryNode.address());
- }
- addTransportAddresses(addresses);
- return this;
- }
-
- public TransportClient addTransportAddresses(Collection transportAddresses) {
- synchronized (mutex) {
- if (closed) {
- throw new IllegalStateException("transport client is closed, can't add addresses");
- }
- List filtered = new ArrayList<>(transportAddresses.size());
- for (TransportAddress transportAddress : transportAddresses) {
- boolean found = false;
- for (DiscoveryNode otherNode : listedNodes) {
- if (otherNode.address().equals(transportAddress)) {
- found = true;
- logger.debug("address [{}] already exists with [{}], ignoring...", transportAddress, otherNode);
- break;
- }
- }
- if (!found) {
- filtered.add(transportAddress);
- }
- }
- if (filtered.isEmpty()) {
- return this;
- }
- List discoveryNodeList = new ArrayList<>();
- discoveryNodeList.addAll(listedNodes());
- for (TransportAddress transportAddress : filtered) {
- DiscoveryNode node = new DiscoveryNode("#transport#-" + tempNodeId.incrementAndGet(), transportAddress,
- minCompatibilityVersion);
- logger.debug("adding address [{}]", node);
- discoveryNodeList.add(node);
- }
- listedNodes = Collections.unmodifiableList(discoveryNodeList);
- connect();
- }
- return this;
- }
-
- /**
- * Removes a transport address from the list of transport addresses that are used to connect to.
- *
- * @param transportAddress transport address to remove
- * @return this transport client
- */
- public TransportClient removeTransportAddress(TransportAddress transportAddress) {
- synchronized (mutex) {
- if (closed) {
- throw new IllegalStateException("transport client is closed, can't remove an address");
- }
- List builder = new ArrayList<>();
- for (DiscoveryNode otherNode : listedNodes) {
- if (!otherNode.address().equals(transportAddress)) {
- builder.add(otherNode);
- } else {
- logger.debug("removing address [{}]", otherNode);
- }
- }
- listedNodes = Collections.unmodifiableList(builder);
- }
- return this;
- }
-
- @Override
- @SuppressWarnings("rawtypes")
- public void close() {
- synchronized (mutex) {
- if (closed) {
- return;
- }
- closed = true;
- for (DiscoveryNode node : nodes) {
- transportService.disconnectFromNode(node);
- }
- for (DiscoveryNode listedNode : listedNodes) {
- transportService.disconnectFromNode(listedNode);
- }
- nodes = Collections.emptyList();
- }
- injector.getInstance(TransportService.class).close();
- for (Class plugin : injector.getInstance(PluginsService.class).nodeServices()) {
- injector.getInstance(plugin).close();
- }
- try {
- ThreadPool.terminate(injector.getInstance(ThreadPool.class), 10, TimeUnit.SECONDS);
- } catch (Exception e) {
- logger.debug(e.getMessage(), e);
- }
- injector.getInstance(PageCacheRecycler.class).close();
- }
-
- private void connect() {
- Set newNodes = new HashSet<>();
- Set newFilteredNodes = new HashSet<>();
- for (DiscoveryNode listedNode : listedNodes) {
- if (!transportService.nodeConnected(listedNode)) {
- try {
- logger.trace("connecting to listed node (light) [{}]", listedNode);
- transportService.connectToNodeLight(listedNode);
- } catch (Exception e) {
- logger.debug("failed to connect to node [{}], removed from nodes list", e, listedNode);
- continue;
- }
- }
- try {
- LivenessResponse livenessResponse = transportService.submitRequest(listedNode,
- TransportLivenessAction.NAME, headers.applyTo(new LivenessRequest()),
- TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STATE)
- .withTimeout(pingTimeout).build(),
- new FutureTransportResponseHandler() {
- @Override
- public LivenessResponse newInstance() {
- return new LivenessResponse();
- }
- }).txGet();
- if (!clusterName.equals(livenessResponse.getClusterName())) {
- logger.warn("node {} not part of the cluster {}, ignoring...", listedNode, clusterName);
- newFilteredNodes.add(listedNode);
- } else if (livenessResponse.getDiscoveryNode() != null) {
- DiscoveryNode nodeWithInfo = livenessResponse.getDiscoveryNode();
- newNodes.add(new DiscoveryNode(nodeWithInfo.name(), nodeWithInfo.id(), nodeWithInfo.getHostName(),
- nodeWithInfo.getHostAddress(), listedNode.address(), nodeWithInfo.attributes(),
- nodeWithInfo.version()));
- } else {
- logger.debug("node {} didn't return any discovery info, temporarily using transport discovery node",
- listedNode);
- newNodes.add(listedNode);
- }
- } catch (Exception e) {
- logger.info("failed to get node info for {}, disconnecting...", e, listedNode);
- transportService.disconnectFromNode(listedNode);
- }
- }
- for (Iterator it = newNodes.iterator(); it.hasNext(); ) {
- DiscoveryNode node = it.next();
- if (!transportService.nodeConnected(node)) {
- try {
- logger.trace("connecting to node [{}]", node);
- transportService.connectToNode(node);
- } catch (Exception e) {
- it.remove();
- logger.debug("failed to connect to discovered node [" + node + "]", e);
- }
- }
- }
- this.nodes = Collections.unmodifiableList(new ArrayList<>(newNodes));
- this.filteredNodes = Collections.unmodifiableList(new ArrayList<>(newFilteredNodes));
- }
-
- @Override
- @SuppressWarnings({"unchecked", "rawtypes"})
- protected >
- void doExecute(Action action, final R request, final ActionListener listener) {
- final TransportActionNodeProxy proxyAction = proxyActionMap.getProxies().get(action);
- if (proxyAction == null) {
- throw new IllegalStateException("undefined action " + action);
- }
- List nodeList = this.nodes;
- if (nodeList.isEmpty()) {
- throw new NoNodeAvailableException("none of the configured nodes are available: " + this.listedNodes);
- }
- int index = nodeCounter.incrementAndGet();
- if (index < 0) {
- index = 0;
- nodeCounter.set(0);
- }
- // try once and never more
- try {
- proxyAction.execute(nodeList.get(index % nodeList.size()), request, listener);
- } catch (Exception e) {
- listener.onFailure(e);
- }
- }
-
- /**
- *
- */
- public static class Builder {
-
- private Settings settings = Settings.EMPTY;
- private List> pluginClasses = new ArrayList<>();
-
- public Builder settings(Settings.Builder settings) {
- return settings(settings.build());
- }
-
- public Builder settings(Settings settings) {
- this.settings = settings;
- return this;
- }
-
- public Builder addPlugin(Class pluginClass) {
- pluginClasses.add(pluginClass);
- return this;
- }
-
- public TransportClient build() {
- Settings transportClientSettings = settingsBuilder()
- .put("transport.ping.schedule", this.settings.get("ping.interval", "30s"))
- .put(InternalSettingsPreparer.prepareSettings(this.settings))
- .put("network.server", false)
- .put("node.client", true)
- .put(CLIENT_TYPE_SETTING, CLIENT_TYPE)
- .build();
- PluginsService pluginsService = new PluginsService(transportClientSettings, null, null, pluginClasses);
- this.settings = pluginsService.updatedSettings();
- Version version = Version.CURRENT;
- final ThreadPool threadPool = new ThreadPool(transportClientSettings);
- boolean success = false;
- try {
- ModulesBuilder modules = new ModulesBuilder();
- modules.add(new Version.Module(version));
- // plugin modules must be added here, before others or we can get crazy injection errors...
- for (Module pluginModule : pluginsService.nodeModules()) {
- modules.add(pluginModule);
- }
- modules.add(new PluginsModule(pluginsService));
- modules.add(new SettingsModule(this.settings));
- modules.add(new NetworkModule());
- modules.add(new ClusterNameModule(this.settings));
- modules.add(new ThreadPoolModule(threadPool));
- modules.add(new TransportModule(this.settings));
- modules.add(new SearchModule() {
- @Override
- protected void configure() {
- // noop
- }
- });
- modules.add(new ActionModule(true));
- modules.add(new ClientTransportModule());
- modules.add(new CircuitBreakerModule(this.settings));
- pluginsService.processModules(modules);
- Injector injector = modules.createInjector();
- injector.getInstance(TransportService.class).start();
- TransportClient transportClient = new TransportClient(injector);
- success = true;
- return transportClient;
- } finally {
- if (!success) {
- ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
- }
- }
- }
- }
-
- /**
- * The {@link ProxyActionMap} must be declared public.
- */
- @SuppressWarnings({"unchecked", "rawtypes"})
- public static class ProxyActionMap {
-
- private final ImmutableMap proxies;
-
- @Inject
- public ProxyActionMap(Settings settings, TransportService transportService, Map actions) {
- MapBuilder actionsBuilder = new MapBuilder<>();
- for (GenericAction action : actions.values()) {
- if (action instanceof Action) {
- actionsBuilder.put((Action) action, new TransportActionNodeProxy(settings, action, transportService));
- }
- }
- this.proxies = actionsBuilder.immutableMap();
- }
-
- public ImmutableMap getProxies() {
- return proxies;
- }
- }
-
-}
diff --git a/src/main/java/org/xbib/elasticsearch/extras/client/transport/package-info.java b/src/main/java/org/xbib/elasticsearch/extras/client/transport/package-info.java
deleted file mode 100644
index ac6a50d..0000000
--- a/src/main/java/org/xbib/elasticsearch/extras/client/transport/package-info.java
+++ /dev/null
@@ -1,4 +0,0 @@
-/**
- * Classes for Elasticsearch transport client extras.
- */
-package org.xbib.elasticsearch.extras.client.transport;
diff --git a/transport/build.gradle b/transport/build.gradle
new file mode 100644
index 0000000..582d3ec
--- /dev/null
+++ b/transport/build.gradle
@@ -0,0 +1,65 @@
+buildscript {
+ repositories {
+ jcenter()
+ maven {
+ url 'http://xbib.org/repository'
+ }
+ }
+ dependencies {
+ classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.3.2.4"
+ }
+}
+
+apply plugin: 'org.xbib.gradle.plugin.elasticsearch.build'
+
+configurations {
+ main
+ tests
+}
+
+dependencies {
+ compile project(':common')
+ testCompile "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}"
+ testRuntime "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}"
+}
+
+jar {
+ baseName "${rootProject.name}-transport"
+}
+
+/*
+task testJar(type: Jar, dependsOn: testClasses) {
+ baseName = "${project.archivesBaseName}-tests"
+ from sourceSets.test.output
+}
+*/
+
+artifacts {
+ main jar
+ tests testJar
+ archives sourcesJar, javadocJar
+}
+
+esTest {
+ enabled = true
+ // test with the jars, not the classes, for security manager
+ //classpath = files(configurations.testRuntime) + configurations.main.artifacts.files + configurations.tests.artifacts.files
+ systemProperty 'tests.security.manager', 'true'
+ // maybe we like some extra security policy for our code
+ systemProperty 'tests.security.policy', '/extra-security.policy'
+}
+esTest.dependsOn jar, testJar
+
+randomizedTest {
+ enabled = false
+}
+
+test {
+ enabled = false
+ jvmArgs "-javaagent:" + configurations.alpnagent.asPath
+ systemProperty 'path.home', projectDir.absolutePath
+ testLogging {
+ showStandardStreams = true
+ exceptionFormat = 'full'
+ }
+}
diff --git a/transport/config/checkstyle/checkstyle.xml b/transport/config/checkstyle/checkstyle.xml
new file mode 100644
index 0000000..52fe33c
--- /dev/null
+++ b/transport/config/checkstyle/checkstyle.xml
@@ -0,0 +1,323 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/transport/src/docs/asciidoc/css/foundation.css b/transport/src/docs/asciidoc/css/foundation.css
new file mode 100644
index 0000000..27be611
--- /dev/null
+++ b/transport/src/docs/asciidoc/css/foundation.css
@@ -0,0 +1,684 @@
+/*! normalize.css v2.1.2 | MIT License | git.io/normalize */
+/* ========================================================================== HTML5 display definitions ========================================================================== */
+/** Correct `block` display not defined in IE 8/9. */
+article, aside, details, figcaption, figure, footer, header, hgroup, main, nav, section, summary { display: block; }
+
+/** Correct `inline-block` display not defined in IE 8/9. */
+audio, canvas, video { display: inline-block; }
+
+/** Prevent modern browsers from displaying `audio` without controls. Remove excess height in iOS 5 devices. */
+audio:not([controls]) { display: none; height: 0; }
+
+/** Address `[hidden]` styling not present in IE 8/9. Hide the `template` element in IE, Safari, and Firefox < 22. */
+[hidden], template { display: none; }
+
+script { display: none !important; }
+
+/* ========================================================================== Base ========================================================================== */
+/** 1. Set default font family to sans-serif. 2. Prevent iOS text size adjust after orientation change, without disabling user zoom. */
+html { font-family: sans-serif; /* 1 */ -ms-text-size-adjust: 100%; /* 2 */ -webkit-text-size-adjust: 100%; /* 2 */ }
+
+/** Remove default margin. */
+body { margin: 0; }
+
+/* ========================================================================== Links ========================================================================== */
+/** Remove the gray background color from active links in IE 10. */
+a { background: transparent; }
+
+/** Address `outline` inconsistency between Chrome and other browsers. */
+a:focus { outline: thin dotted; }
+
+/** Improve readability when focused and also mouse hovered in all browsers. */
+a:active, a:hover { outline: 0; }
+
+/* ========================================================================== Typography ========================================================================== */
+/** Address variable `h1` font-size and margin within `section` and `article` contexts in Firefox 4+, Safari 5, and Chrome. */
+h1 { font-size: 2em; margin: 0.67em 0; }
+
+/** Address styling not present in IE 8/9, Safari 5, and Chrome. */
+abbr[title] { border-bottom: 1px dotted; }
+
+/** Address style set to `bolder` in Firefox 4+, Safari 5, and Chrome. */
+b, strong { font-weight: bold; }
+
+/** Address styling not present in Safari 5 and Chrome. */
+dfn { font-style: italic; }
+
+/** Address differences between Firefox and other browsers. */
+hr { -moz-box-sizing: content-box; box-sizing: content-box; height: 0; }
+
+/** Address styling not present in IE 8/9. */
+mark { background: #ff0; color: #000; }
+
+/** Correct font family set oddly in Safari 5 and Chrome. */
+code, kbd, pre, samp { font-family: monospace, serif; font-size: 1em; }
+
+/** Improve readability of pre-formatted text in all browsers. */
+pre { white-space: pre-wrap; }
+
+/** Set consistent quote types. */
+q { quotes: "\201C" "\201D" "\2018" "\2019"; }
+
+/** Address inconsistent and variable font size in all browsers. */
+small { font-size: 80%; }
+
+/** Prevent `sub` and `sup` affecting `line-height` in all browsers. */
+sub, sup { font-size: 75%; line-height: 0; position: relative; vertical-align: baseline; }
+
+sup { top: -0.5em; }
+
+sub { bottom: -0.25em; }
+
+/* ========================================================================== Embedded content ========================================================================== */
+/** Remove border when inside `a` element in IE 8/9. */
+img { border: 0; }
+
+/** Correct overflow displayed oddly in IE 9. */
+svg:not(:root) { overflow: hidden; }
+
+/* ========================================================================== Figures ========================================================================== */
+/** Address margin not present in IE 8/9 and Safari 5. */
+figure { margin: 0; }
+
+/* ========================================================================== Forms ========================================================================== */
+/** Define consistent border, margin, and padding. */
+fieldset { border: 1px solid #c0c0c0; margin: 0 2px; padding: 0.35em 0.625em 0.75em; }
+
+/** 1. Correct `color` not being inherited in IE 8/9. 2. Remove padding so people aren't caught out if they zero out fieldsets. */
+legend { border: 0; /* 1 */ padding: 0; /* 2 */ }
+
+/** 1. Correct font family not being inherited in all browsers. 2. Correct font size not being inherited in all browsers. 3. Address margins set differently in Firefox 4+, Safari 5, and Chrome. */
+button, input, select, textarea { font-family: inherit; /* 1 */ font-size: 100%; /* 2 */ margin: 0; /* 3 */ }
+
+/** Address Firefox 4+ setting `line-height` on `input` using `!important` in the UA stylesheet. */
+button, input { line-height: normal; }
+
+/** Address inconsistent `text-transform` inheritance for `button` and `select`. All other form control elements do not inherit `text-transform` values. Correct `button` style inheritance in Chrome, Safari 5+, and IE 8+. Correct `select` style inheritance in Firefox 4+ and Opera. */
+button, select { text-transform: none; }
+
+/** 1. Avoid the WebKit bug in Android 4.0.* where (2) destroys native `audio` and `video` controls. 2. Correct inability to style clickable `input` types in iOS. 3. Improve usability and consistency of cursor style between image-type `input` and others. */
+button, html input[type="button"], input[type="reset"], input[type="submit"] { -webkit-appearance: button; /* 2 */ cursor: pointer; /* 3 */ }
+
+/** Re-set default cursor for disabled elements. */
+button[disabled], html input[disabled] { cursor: default; }
+
+/** 1. Address box sizing set to `content-box` in IE 8/9. 2. Remove excess padding in IE 8/9. */
+input[type="checkbox"], input[type="radio"] { box-sizing: border-box; /* 1 */ padding: 0; /* 2 */ }
+
+/** 1. Address `appearance` set to `searchfield` in Safari 5 and Chrome. 2. Address `box-sizing` set to `border-box` in Safari 5 and Chrome (include `-moz` to future-proof). */
+input[type="search"] { -webkit-appearance: textfield; /* 1 */ -moz-box-sizing: content-box; -webkit-box-sizing: content-box; /* 2 */ box-sizing: content-box; }
+
+/** Remove inner padding and search cancel button in Safari 5 and Chrome on OS X. */
+input[type="search"]::-webkit-search-cancel-button, input[type="search"]::-webkit-search-decoration { -webkit-appearance: none; }
+
+/** Remove inner padding and border in Firefox 4+. */
+button::-moz-focus-inner, input::-moz-focus-inner { border: 0; padding: 0; }
+
+/** 1. Remove default vertical scrollbar in IE 8/9. 2. Improve readability and alignment in all browsers. */
+textarea { overflow: auto; /* 1 */ vertical-align: top; /* 2 */ }
+
+/* ========================================================================== Tables ========================================================================== */
+/** Remove most spacing between table cells. */
+table { border-collapse: collapse; border-spacing: 0; }
+
+meta.foundation-mq-small { font-family: "only screen and (min-width: 768px)"; width: 768px; }
+
+meta.foundation-mq-medium { font-family: "only screen and (min-width:1280px)"; width: 1280px; }
+
+meta.foundation-mq-large { font-family: "only screen and (min-width:1440px)"; width: 1440px; }
+
+*, *:before, *:after { -moz-box-sizing: border-box; -webkit-box-sizing: border-box; box-sizing: border-box; }
+
+html, body { font-size: 100%; }
+
+body { background: white; color: #222222; padding: 0; margin: 0; font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; font-weight: normal; font-style: normal; line-height: 1; position: relative; cursor: auto; }
+
+a:hover { cursor: pointer; }
+
+img, object, embed { max-width: 100%; height: auto; }
+
+object, embed { height: 100%; }
+
+img { -ms-interpolation-mode: bicubic; }
+
+#map_canvas img, #map_canvas embed, #map_canvas object, .map_canvas img, .map_canvas embed, .map_canvas object { max-width: none !important; }
+
+.left { float: left !important; }
+
+.right { float: right !important; }
+
+.text-left { text-align: left !important; }
+
+.text-right { text-align: right !important; }
+
+.text-center { text-align: center !important; }
+
+.text-justify { text-align: justify !important; }
+
+.hide { display: none; }
+
+.antialiased { -webkit-font-smoothing: antialiased; }
+
+img { display: inline-block; vertical-align: middle; }
+
+textarea { height: auto; min-height: 50px; }
+
+select { width: 100%; }
+
+object, svg { display: inline-block; vertical-align: middle; }
+
+.center { margin-left: auto; margin-right: auto; }
+
+.spread { width: 100%; }
+
+p.lead, .paragraph.lead > p, #preamble > .sectionbody > .paragraph:first-of-type p { font-size: 1.21875em; line-height: 1.6; }
+
+.subheader, .admonitionblock td.content > .title, .audioblock > .title, .exampleblock > .title, .imageblock > .title, .listingblock > .title, .literalblock > .title, .stemblock > .title, .openblock > .title, .paragraph > .title, .quoteblock > .title, table.tableblock > .title, .verseblock > .title, .videoblock > .title, .dlist > .title, .olist > .title, .ulist > .title, .qlist > .title, .hdlist > .title { line-height: 1.4; color: #6f6f6f; font-weight: 300; margin-top: 0.2em; margin-bottom: 0.5em; }
+
+/* Typography resets */
+div, dl, dt, dd, ul, ol, li, h1, h2, h3, #toctitle, .sidebarblock > .content > .title, h4, h5, h6, pre, form, p, blockquote, th, td { margin: 0; padding: 0; direction: ltr; }
+
+/* Default Link Styles */
+a { color: #2ba6cb; text-decoration: none; line-height: inherit; }
+a:hover, a:focus { color: #2795b6; }
+a img { border: none; }
+
+/* Default paragraph styles */
+p { font-family: inherit; font-weight: normal; font-size: 1em; line-height: 1.6; margin-bottom: 1.25em; text-rendering: optimizeLegibility; }
+p aside { font-size: 0.875em; line-height: 1.35; font-style: italic; }
+
+/* Default header styles */
+h1, h2, h3, #toctitle, .sidebarblock > .content > .title, h4, h5, h6 { font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; font-weight: bold; font-style: normal; color: #222222; text-rendering: optimizeLegibility; margin-top: 1em; margin-bottom: 0.5em; line-height: 1.2125em; }
+h1 small, h2 small, h3 small, #toctitle small, .sidebarblock > .content > .title small, h4 small, h5 small, h6 small { font-size: 60%; color: #6f6f6f; line-height: 0; }
+
+h1 { font-size: 2.125em; }
+
+h2 { font-size: 1.6875em; }
+
+h3, #toctitle, .sidebarblock > .content > .title { font-size: 1.375em; }
+
+h4 { font-size: 1.125em; }
+
+h5 { font-size: 1.125em; }
+
+h6 { font-size: 1em; }
+
+hr { border: solid #dddddd; border-width: 1px 0 0; clear: both; margin: 1.25em 0 1.1875em; height: 0; }
+
+/* Helpful Typography Defaults */
+em, i { font-style: italic; line-height: inherit; }
+
+strong, b { font-weight: bold; line-height: inherit; }
+
+small { font-size: 60%; line-height: inherit; }
+
+code { font-family: Consolas, "Liberation Mono", Courier, monospace; font-weight: bold; color: #7f0a0c; }
+
+/* Lists */
+ul, ol, dl { font-size: 1em; line-height: 1.6; margin-bottom: 1.25em; list-style-position: outside; font-family: inherit; }
+
+ul, ol { margin-left: 1.5em; }
+ul.no-bullet, ol.no-bullet { margin-left: 1.5em; }
+
+/* Unordered Lists */
+ul li ul, ul li ol { margin-left: 1.25em; margin-bottom: 0; font-size: 1em; /* Override nested font-size change */ }
+ul.square li ul, ul.circle li ul, ul.disc li ul { list-style: inherit; }
+ul.square { list-style-type: square; }
+ul.circle { list-style-type: circle; }
+ul.disc { list-style-type: disc; }
+ul.no-bullet { list-style: none; }
+
+/* Ordered Lists */
+ol li ul, ol li ol { margin-left: 1.25em; margin-bottom: 0; }
+
+/* Definition Lists */
+dl dt { margin-bottom: 0.3125em; font-weight: bold; }
+dl dd { margin-bottom: 1.25em; }
+
+/* Abbreviations */
+abbr, acronym { text-transform: uppercase; font-size: 90%; color: #222222; border-bottom: 1px dotted #dddddd; cursor: help; }
+
+abbr { text-transform: none; }
+
+/* Blockquotes */
+blockquote { margin: 0 0 1.25em; padding: 0.5625em 1.25em 0 1.1875em; border-left: 1px solid #dddddd; }
+blockquote cite { display: block; font-size: 0.8125em; color: #555555; }
+blockquote cite:before { content: "\2014 \0020"; }
+blockquote cite a, blockquote cite a:visited { color: #555555; }
+
+blockquote, blockquote p { line-height: 1.6; color: #6f6f6f; }
+
+/* Microformats */
+.vcard { display: inline-block; margin: 0 0 1.25em 0; border: 1px solid #dddddd; padding: 0.625em 0.75em; }
+.vcard li { margin: 0; display: block; }
+.vcard .fn { font-weight: bold; font-size: 0.9375em; }
+
+.vevent .summary { font-weight: bold; }
+.vevent abbr { cursor: auto; text-decoration: none; font-weight: bold; border: none; padding: 0 0.0625em; }
+
+@media only screen and (min-width: 768px) { h1, h2, h3, #toctitle, .sidebarblock > .content > .title, h4, h5, h6 { line-height: 1.4; }
+ h1 { font-size: 2.75em; }
+ h2 { font-size: 2.3125em; }
+ h3, #toctitle, .sidebarblock > .content > .title { font-size: 1.6875em; }
+ h4 { font-size: 1.4375em; } }
+/* Tables */
+table { background: white; margin-bottom: 1.25em; border: solid 1px #dddddd; }
+table thead, table tfoot { background: whitesmoke; font-weight: bold; }
+table thead tr th, table thead tr td, table tfoot tr th, table tfoot tr td { padding: 0.5em 0.625em 0.625em; font-size: inherit; color: #222222; text-align: left; }
+table tr th, table tr td { padding: 0.5625em 0.625em; font-size: inherit; color: #222222; }
+table tr.even, table tr.alt, table tr:nth-of-type(even) { background: #f9f9f9; }
+table thead tr th, table tfoot tr th, table tbody tr td, table tr td, table tfoot tr td { display: table-cell; line-height: 1.4; }
+
+body { -moz-osx-font-smoothing: grayscale; -webkit-font-smoothing: antialiased; tab-size: 4; }
+
+h1, h2, h3, #toctitle, .sidebarblock > .content > .title, h4, h5, h6 { line-height: 1.4; }
+
+.clearfix:before, .clearfix:after, .float-group:before, .float-group:after { content: " "; display: table; }
+.clearfix:after, .float-group:after { clear: both; }
+
+*:not(pre) > code { font-size: inherit; font-style: normal !important; letter-spacing: 0; padding: 0; line-height: inherit; word-wrap: break-word; }
+*:not(pre) > code.nobreak { word-wrap: normal; }
+*:not(pre) > code.nowrap { white-space: nowrap; }
+
+pre, pre > code { line-height: 1.4; color: black; font-family: monospace, serif; font-weight: normal; }
+
+em em { font-style: normal; }
+
+strong strong { font-weight: normal; }
+
+.keyseq { color: #555555; }
+
+kbd { font-family: Consolas, "Liberation Mono", Courier, monospace; display: inline-block; color: #222222; font-size: 0.65em; line-height: 1.45; background-color: #f7f7f7; border: 1px solid #ccc; -webkit-border-radius: 3px; border-radius: 3px; -webkit-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.2), 0 0 0 0.1em white inset; box-shadow: 0 1px 0 rgba(0, 0, 0, 0.2), 0 0 0 0.1em white inset; margin: 0 0.15em; padding: 0.2em 0.5em; vertical-align: middle; position: relative; top: -0.1em; white-space: nowrap; }
+
+.keyseq kbd:first-child { margin-left: 0; }
+
+.keyseq kbd:last-child { margin-right: 0; }
+
+.menuseq, .menu { color: #090909; }
+
+b.button:before, b.button:after { position: relative; top: -1px; font-weight: normal; }
+
+b.button:before { content: "["; padding: 0 3px 0 2px; }
+
+b.button:after { content: "]"; padding: 0 2px 0 3px; }
+
+#header, #content, #footnotes, #footer { width: 100%; margin-left: auto; margin-right: auto; margin-top: 0; margin-bottom: 0; max-width: 62.5em; *zoom: 1; position: relative; padding-left: 0.9375em; padding-right: 0.9375em; }
+#header:before, #header:after, #content:before, #content:after, #footnotes:before, #footnotes:after, #footer:before, #footer:after { content: " "; display: table; }
+#header:after, #content:after, #footnotes:after, #footer:after { clear: both; }
+
+#content { margin-top: 1.25em; }
+
+#content:before { content: none; }
+
+#header > h1:first-child { color: black; margin-top: 2.25rem; margin-bottom: 0; }
+#header > h1:first-child + #toc { margin-top: 8px; border-top: 1px solid #dddddd; }
+#header > h1:only-child, body.toc2 #header > h1:nth-last-child(2) { border-bottom: 1px solid #dddddd; padding-bottom: 8px; }
+#header .details { border-bottom: 1px solid #dddddd; line-height: 1.45; padding-top: 0.25em; padding-bottom: 0.25em; padding-left: 0.25em; color: #555555; display: -ms-flexbox; display: -webkit-flex; display: flex; -ms-flex-flow: row wrap; -webkit-flex-flow: row wrap; flex-flow: row wrap; }
+#header .details span:first-child { margin-left: -0.125em; }
+#header .details span.email a { color: #6f6f6f; }
+#header .details br { display: none; }
+#header .details br + span:before { content: "\00a0\2013\00a0"; }
+#header .details br + span.author:before { content: "\00a0\22c5\00a0"; color: #6f6f6f; }
+#header .details br + span#revremark:before { content: "\00a0|\00a0"; }
+#header #revnumber { text-transform: capitalize; }
+#header #revnumber:after { content: "\00a0"; }
+
+#content > h1:first-child:not([class]) { color: black; border-bottom: 1px solid #dddddd; padding-bottom: 8px; margin-top: 0; padding-top: 1rem; margin-bottom: 1.25rem; }
+
+#toc { border-bottom: 1px solid #dddddd; padding-bottom: 0.5em; }
+#toc > ul { margin-left: 0.125em; }
+#toc ul.sectlevel0 > li > a { font-style: italic; }
+#toc ul.sectlevel0 ul.sectlevel1 { margin: 0.5em 0; }
+#toc ul { font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; list-style-type: none; }
+#toc li { line-height: 1.3334; margin-top: 0.3334em; }
+#toc a { text-decoration: none; }
+#toc a:active { text-decoration: underline; }
+
+#toctitle { color: #6f6f6f; font-size: 1.2em; }
+
+@media only screen and (min-width: 768px) { #toctitle { font-size: 1.375em; }
+ body.toc2 { padding-left: 15em; padding-right: 0; }
+ #toc.toc2 { margin-top: 0 !important; background-color: #f2f2f2; position: fixed; width: 15em; left: 0; top: 0; border-right: 1px solid #dddddd; border-top-width: 0 !important; border-bottom-width: 0 !important; z-index: 1000; padding: 1.25em 1em; height: 100%; overflow: auto; }
+ #toc.toc2 #toctitle { margin-top: 0; margin-bottom: 0.8rem; font-size: 1.2em; }
+ #toc.toc2 > ul { font-size: 0.9em; margin-bottom: 0; }
+ #toc.toc2 ul ul { margin-left: 0; padding-left: 1em; }
+ #toc.toc2 ul.sectlevel0 ul.sectlevel1 { padding-left: 0; margin-top: 0.5em; margin-bottom: 0.5em; }
+ body.toc2.toc-right { padding-left: 0; padding-right: 15em; }
+ body.toc2.toc-right #toc.toc2 { border-right-width: 0; border-left: 1px solid #dddddd; left: auto; right: 0; } }
+@media only screen and (min-width: 1280px) { body.toc2 { padding-left: 20em; padding-right: 0; }
+ #toc.toc2 { width: 20em; }
+ #toc.toc2 #toctitle { font-size: 1.375em; }
+ #toc.toc2 > ul { font-size: 0.95em; }
+ #toc.toc2 ul ul { padding-left: 1.25em; }
+ body.toc2.toc-right { padding-left: 0; padding-right: 20em; } }
+#content #toc { border-style: solid; border-width: 1px; border-color: #d9d9d9; margin-bottom: 1.25em; padding: 1.25em; background: #f2f2f2; -webkit-border-radius: 0; border-radius: 0; }
+#content #toc > :first-child { margin-top: 0; }
+#content #toc > :last-child { margin-bottom: 0; }
+
+#footer { max-width: 100%; background-color: #222222; padding: 1.25em; }
+
+#footer-text { color: #dddddd; line-height: 1.44; }
+
+.sect1 { padding-bottom: 0.625em; }
+
+@media only screen and (min-width: 768px) { .sect1 { padding-bottom: 1.25em; } }
+.sect1 + .sect1 { border-top: 1px solid #dddddd; }
+
+#content h1 > a.anchor, h2 > a.anchor, h3 > a.anchor, #toctitle > a.anchor, .sidebarblock > .content > .title > a.anchor, h4 > a.anchor, h5 > a.anchor, h6 > a.anchor { position: absolute; z-index: 1001; width: 1.5ex; margin-left: -1.5ex; display: block; text-decoration: none !important; visibility: hidden; text-align: center; font-weight: normal; }
+#content h1 > a.anchor:before, h2 > a.anchor:before, h3 > a.anchor:before, #toctitle > a.anchor:before, .sidebarblock > .content > .title > a.anchor:before, h4 > a.anchor:before, h5 > a.anchor:before, h6 > a.anchor:before { content: "\00A7"; font-size: 0.85em; display: block; padding-top: 0.1em; }
+#content h1:hover > a.anchor, #content h1 > a.anchor:hover, h2:hover > a.anchor, h2 > a.anchor:hover, h3:hover > a.anchor, #toctitle:hover > a.anchor, .sidebarblock > .content > .title:hover > a.anchor, h3 > a.anchor:hover, #toctitle > a.anchor:hover, .sidebarblock > .content > .title > a.anchor:hover, h4:hover > a.anchor, h4 > a.anchor:hover, h5:hover > a.anchor, h5 > a.anchor:hover, h6:hover > a.anchor, h6 > a.anchor:hover { visibility: visible; }
+#content h1 > a.link, h2 > a.link, h3 > a.link, #toctitle > a.link, .sidebarblock > .content > .title > a.link, h4 > a.link, h5 > a.link, h6 > a.link { color: #222222; text-decoration: none; }
+#content h1 > a.link:hover, h2 > a.link:hover, h3 > a.link:hover, #toctitle > a.link:hover, .sidebarblock > .content > .title > a.link:hover, h4 > a.link:hover, h5 > a.link:hover, h6 > a.link:hover { color: #151515; }
+
+.audioblock, .imageblock, .literalblock, .listingblock, .stemblock, .videoblock { margin-bottom: 1.25em; }
+
+.admonitionblock td.content > .title, .audioblock > .title, .exampleblock > .title, .imageblock > .title, .listingblock > .title, .literalblock > .title, .stemblock > .title, .openblock > .title, .paragraph > .title, .quoteblock > .title, table.tableblock > .title, .verseblock > .title, .videoblock > .title, .dlist > .title, .olist > .title, .ulist > .title, .qlist > .title, .hdlist > .title { text-rendering: optimizeLegibility; text-align: left; }
+
+table.tableblock > caption.title { white-space: nowrap; overflow: visible; max-width: 0; }
+
+.paragraph.lead > p, #preamble > .sectionbody > .paragraph:first-of-type p { color: black; }
+
+table.tableblock #preamble > .sectionbody > .paragraph:first-of-type p { font-size: inherit; }
+
+.admonitionblock > table { border-collapse: separate; border: 0; background: none; width: 100%; }
+.admonitionblock > table td.icon { text-align: center; width: 80px; }
+.admonitionblock > table td.icon img { max-width: initial; }
+.admonitionblock > table td.icon .title { font-weight: bold; font-family: "Helvetica Neue", "Helvetica", Helvetica, Arial, sans-serif; text-transform: uppercase; }
+.admonitionblock > table td.content { padding-left: 1.125em; padding-right: 1.25em; border-left: 1px solid #dddddd; color: #555555; }
+.admonitionblock > table td.content > :last-child > :last-child { margin-bottom: 0; }
+
+.exampleblock > .content { border-style: solid; border-width: 1px; border-color: #e6e6e6; margin-bottom: 1.25em; padding: 1.25em; background: white; -webkit-border-radius: 0; border-radius: 0; }
+.exampleblock > .content > :first-child { margin-top: 0; }
+.exampleblock > .content > :last-child { margin-bottom: 0; }
+
+.sidebarblock { border-style: solid; border-width: 1px; border-color: #d9d9d9; margin-bottom: 1.25em; padding: 1.25em; background: #f2f2f2; -webkit-border-radius: 0; border-radius: 0; }
+.sidebarblock > :first-child { margin-top: 0; }
+.sidebarblock > :last-child { margin-bottom: 0; }
+.sidebarblock > .content > .title { color: #6f6f6f; margin-top: 0; }
+
+.exampleblock > .content > :last-child > :last-child, .exampleblock > .content .olist > ol > li:last-child > :last-child, .exampleblock > .content .ulist > ul > li:last-child > :last-child, .exampleblock > .content .qlist > ol > li:last-child > :last-child, .sidebarblock > .content > :last-child > :last-child, .sidebarblock > .content .olist > ol > li:last-child > :last-child, .sidebarblock > .content .ulist > ul > li:last-child > :last-child, .sidebarblock > .content .qlist > ol > li:last-child > :last-child { margin-bottom: 0; }
+
+.literalblock pre, .listingblock pre:not(.highlight), .listingblock pre[class="highlight"], .listingblock pre[class^="highlight "], .listingblock pre.CodeRay, .listingblock pre.prettyprint { background: #eeeeee; }
+.sidebarblock .literalblock pre, .sidebarblock .listingblock pre:not(.highlight), .sidebarblock .listingblock pre[class="highlight"], .sidebarblock .listingblock pre[class^="highlight "], .sidebarblock .listingblock pre.CodeRay, .sidebarblock .listingblock pre.prettyprint { background: #f2f1f1; }
+
+.literalblock pre, .literalblock pre[class], .listingblock pre, .listingblock pre[class] { border: 1px solid #cccccc; -webkit-border-radius: 0; border-radius: 0; word-wrap: break-word; padding: 0.8em 0.8em 0.65em 0.8em; font-size: 0.8125em; }
+.literalblock pre.nowrap, .literalblock pre[class].nowrap, .listingblock pre.nowrap, .listingblock pre[class].nowrap { overflow-x: auto; white-space: pre; word-wrap: normal; }
+@media only screen and (min-width: 768px) { .literalblock pre, .literalblock pre[class], .listingblock pre, .listingblock pre[class] { font-size: 0.90625em; } }
+@media only screen and (min-width: 1280px) { .literalblock pre, .literalblock pre[class], .listingblock pre, .listingblock pre[class] { font-size: 1em; } }
+
+.literalblock.output pre { color: #eeeeee; background-color: black; }
+
+.listingblock pre.highlightjs { padding: 0; }
+.listingblock pre.highlightjs > code { padding: 0.8em 0.8em 0.65em 0.8em; -webkit-border-radius: 0; border-radius: 0; }
+
+.listingblock > .content { position: relative; }
+
+.listingblock code[data-lang]:before { display: none; content: attr(data-lang); position: absolute; font-size: 0.75em; top: 0.425rem; right: 0.5rem; line-height: 1; text-transform: uppercase; color: #999; }
+
+.listingblock:hover code[data-lang]:before { display: block; }
+
+.listingblock.terminal pre .command:before { content: attr(data-prompt); padding-right: 0.5em; color: #999; }
+
+.listingblock.terminal pre .command:not([data-prompt]):before { content: "$"; }
+
+table.pyhltable { border-collapse: separate; border: 0; margin-bottom: 0; background: none; }
+
+table.pyhltable td { vertical-align: top; padding-top: 0; padding-bottom: 0; line-height: 1.4; }
+
+table.pyhltable td.code { padding-left: .75em; padding-right: 0; }
+
+pre.pygments .lineno, table.pyhltable td:not(.code) { color: #999; padding-left: 0; padding-right: .5em; border-right: 1px solid #dddddd; }
+
+pre.pygments .lineno { display: inline-block; margin-right: .25em; }
+
+table.pyhltable .linenodiv { background: none !important; padding-right: 0 !important; }
+
+.quoteblock { margin: 0 1em 1.25em 1.5em; display: table; }
+.quoteblock > .title { margin-left: -1.5em; margin-bottom: 0.75em; }
+.quoteblock blockquote, .quoteblock blockquote p { color: #6f6f6f; font-size: 1.15rem; line-height: 1.75; word-spacing: 0.1em; letter-spacing: 0; font-style: italic; text-align: justify; }
+.quoteblock blockquote { margin: 0; padding: 0; border: 0; }
+.quoteblock blockquote:before { content: "\201c"; float: left; font-size: 2.75em; font-weight: bold; line-height: 0.6em; margin-left: -0.6em; color: #6f6f6f; text-shadow: 0 1px 2px rgba(0, 0, 0, 0.1); }
+.quoteblock blockquote > .paragraph:last-child p { margin-bottom: 0; }
+.quoteblock .attribution { margin-top: 0.5em; margin-right: 0.5ex; text-align: right; }
+.quoteblock .quoteblock { margin-left: 0; margin-right: 0; padding: 0.5em 0; border-left: 3px solid #555555; }
+.quoteblock .quoteblock blockquote { padding: 0 0 0 0.75em; }
+.quoteblock .quoteblock blockquote:before { display: none; }
+
+.verseblock { margin: 0 1em 1.25em 1em; }
+.verseblock pre { font-family: "Open Sans", "DejaVu Sans", sans; font-size: 1.15rem; color: #6f6f6f; font-weight: 300; text-rendering: optimizeLegibility; }
+.verseblock pre strong { font-weight: 400; }
+.verseblock .attribution { margin-top: 1.25rem; margin-left: 0.5ex; }
+
+.quoteblock .attribution, .verseblock .attribution { font-size: 0.8125em; line-height: 1.45; font-style: italic; }
+.quoteblock .attribution br, .verseblock .attribution br { display: none; }
+.quoteblock .attribution cite, .verseblock .attribution cite { display: block; letter-spacing: -0.025em; color: #555555; }
+
+.quoteblock.abstract { margin: 0 0 1.25em 0; display: block; }
+.quoteblock.abstract blockquote, .quoteblock.abstract blockquote p { text-align: left; word-spacing: 0; }
+.quoteblock.abstract blockquote:before, .quoteblock.abstract blockquote p:first-of-type:before { display: none; }
+
+table.tableblock { max-width: 100%; border-collapse: separate; }
+table.tableblock td > .paragraph:last-child p > p:last-child, table.tableblock th > p:last-child, table.tableblock td > p:last-child { margin-bottom: 0; }
+
+table.tableblock, th.tableblock, td.tableblock { border: 0 solid #dddddd; }
+
+table.grid-all th.tableblock, table.grid-all td.tableblock { border-width: 0 1px 1px 0; }
+
+table.grid-all tfoot > tr > th.tableblock, table.grid-all tfoot > tr > td.tableblock { border-width: 1px 1px 0 0; }
+
+table.grid-cols th.tableblock, table.grid-cols td.tableblock { border-width: 0 1px 0 0; }
+
+table.grid-all * > tr > .tableblock:last-child, table.grid-cols * > tr > .tableblock:last-child { border-right-width: 0; }
+
+table.grid-rows th.tableblock, table.grid-rows td.tableblock { border-width: 0 0 1px 0; }
+
+table.grid-all tbody > tr:last-child > th.tableblock, table.grid-all tbody > tr:last-child > td.tableblock, table.grid-all thead:last-child > tr > th.tableblock, table.grid-rows tbody > tr:last-child > th.tableblock, table.grid-rows tbody > tr:last-child > td.tableblock, table.grid-rows thead:last-child > tr > th.tableblock { border-bottom-width: 0; }
+
+table.grid-rows tfoot > tr > th.tableblock, table.grid-rows tfoot > tr > td.tableblock { border-width: 1px 0 0 0; }
+
+table.frame-all { border-width: 1px; }
+
+table.frame-sides { border-width: 0 1px; }
+
+table.frame-topbot { border-width: 1px 0; }
+
+th.halign-left, td.halign-left { text-align: left; }
+
+th.halign-right, td.halign-right { text-align: right; }
+
+th.halign-center, td.halign-center { text-align: center; }
+
+th.valign-top, td.valign-top { vertical-align: top; }
+
+th.valign-bottom, td.valign-bottom { vertical-align: bottom; }
+
+th.valign-middle, td.valign-middle { vertical-align: middle; }
+
+table thead th, table tfoot th { font-weight: bold; }
+
+tbody tr th { display: table-cell; line-height: 1.4; background: whitesmoke; }
+
+tbody tr th, tbody tr th p, tfoot tr th, tfoot tr th p { color: #222222; font-weight: bold; }
+
+p.tableblock > code:only-child { background: none; padding: 0; }
+
+p.tableblock { font-size: 1em; }
+
+td > div.verse { white-space: pre; }
+
+ol { margin-left: 1.75em; }
+
+ul li ol { margin-left: 1.5em; }
+
+dl dd { margin-left: 1.125em; }
+
+dl dd:last-child, dl dd:last-child > :last-child { margin-bottom: 0; }
+
+ol > li p, ul > li p, ul dd, ol dd, .olist .olist, .ulist .ulist, .ulist .olist, .olist .ulist { margin-bottom: 0.625em; }
+
+ul.unstyled, ol.unnumbered, ul.checklist, ul.none { list-style-type: none; }
+
+ul.unstyled, ol.unnumbered, ul.checklist { margin-left: 0.625em; }
+
+ul.checklist li > p:first-child > .fa-square-o:first-child, ul.checklist li > p:first-child > .fa-check-square-o:first-child { width: 1em; font-size: 0.85em; }
+
+ul.checklist li > p:first-child > input[type="checkbox"]:first-child { width: 1em; position: relative; top: 1px; }
+
+ul.inline { margin: 0 auto 0.625em auto; margin-left: -1.375em; margin-right: 0; padding: 0; list-style: none; overflow: hidden; }
+ul.inline > li { list-style: none; float: left; margin-left: 1.375em; display: block; }
+ul.inline > li > * { display: block; }
+
+.unstyled dl dt { font-weight: normal; font-style: normal; }
+
+ol.arabic { list-style-type: decimal; }
+
+ol.decimal { list-style-type: decimal-leading-zero; }
+
+ol.loweralpha { list-style-type: lower-alpha; }
+
+ol.upperalpha { list-style-type: upper-alpha; }
+
+ol.lowerroman { list-style-type: lower-roman; }
+
+ol.upperroman { list-style-type: upper-roman; }
+
+ol.lowergreek { list-style-type: lower-greek; }
+
+.hdlist > table, .colist > table { border: 0; background: none; }
+.hdlist > table > tbody > tr, .colist > table > tbody > tr { background: none; }
+
+td.hdlist1, td.hdlist2 { vertical-align: top; padding: 0 0.625em; }
+
+td.hdlist1 { font-weight: bold; padding-bottom: 1.25em; }
+
+.literalblock + .colist, .listingblock + .colist { margin-top: -0.5em; }
+
+.colist > table tr > td:first-of-type { padding: 0 0.75em; line-height: 1; }
+.colist > table tr > td:first-of-type img { max-width: initial; }
+.colist > table tr > td:last-of-type { padding: 0.25em 0; }
+
+.thumb, .th { line-height: 0; display: inline-block; border: solid 4px white; -webkit-box-shadow: 0 0 0 1px #dddddd; box-shadow: 0 0 0 1px #dddddd; }
+
+.imageblock.left, .imageblock[style*="float: left"] { margin: 0.25em 0.625em 1.25em 0; }
+.imageblock.right, .imageblock[style*="float: right"] { margin: 0.25em 0 1.25em 0.625em; }
+.imageblock > .title { margin-bottom: 0; }
+.imageblock.thumb, .imageblock.th { border-width: 6px; }
+.imageblock.thumb > .title, .imageblock.th > .title { padding: 0 0.125em; }
+
+.image.left, .image.right { margin-top: 0.25em; margin-bottom: 0.25em; display: inline-block; line-height: 0; }
+.image.left { margin-right: 0.625em; }
+.image.right { margin-left: 0.625em; }
+
+a.image { text-decoration: none; display: inline-block; }
+a.image object { pointer-events: none; }
+
+sup.footnote, sup.footnoteref { font-size: 0.875em; position: static; vertical-align: super; }
+sup.footnote a, sup.footnoteref a { text-decoration: none; }
+sup.footnote a:active, sup.footnoteref a:active { text-decoration: underline; }
+
+#footnotes { padding-top: 0.75em; padding-bottom: 0.75em; margin-bottom: 0.625em; }
+#footnotes hr { width: 20%; min-width: 6.25em; margin: -0.25em 0 0.75em 0; border-width: 1px 0 0 0; }
+#footnotes .footnote { padding: 0 0.375em 0 0.225em; line-height: 1.3334; font-size: 0.875em; margin-left: 1.2em; text-indent: -1.05em; margin-bottom: 0.2em; }
+#footnotes .footnote a:first-of-type { font-weight: bold; text-decoration: none; }
+#footnotes .footnote:last-of-type { margin-bottom: 0; }
+#content #footnotes { margin-top: -0.625em; margin-bottom: 0; padding: 0.75em 0; }
+
+.gist .file-data > table { border: 0; background: #fff; width: 100%; margin-bottom: 0; }
+.gist .file-data > table td.line-data { width: 99%; }
+
+div.unbreakable { page-break-inside: avoid; }
+
+.big { font-size: larger; }
+
+.small { font-size: smaller; }
+
+.underline { text-decoration: underline; }
+
+.overline { text-decoration: overline; }
+
+.line-through { text-decoration: line-through; }
+
+.aqua { color: #00bfbf; }
+
+.aqua-background { background-color: #00fafa; }
+
+.black { color: black; }
+
+.black-background { background-color: black; }
+
+.blue { color: #0000bf; }
+
+.blue-background { background-color: #0000fa; }
+
+.fuchsia { color: #bf00bf; }
+
+.fuchsia-background { background-color: #fa00fa; }
+
+.gray { color: #606060; }
+
+.gray-background { background-color: #7d7d7d; }
+
+.green { color: #006000; }
+
+.green-background { background-color: #007d00; }
+
+.lime { color: #00bf00; }
+
+.lime-background { background-color: #00fa00; }
+
+.maroon { color: #600000; }
+
+.maroon-background { background-color: #7d0000; }
+
+.navy { color: #000060; }
+
+.navy-background { background-color: #00007d; }
+
+.olive { color: #606000; }
+
+.olive-background { background-color: #7d7d00; }
+
+.purple { color: #600060; }
+
+.purple-background { background-color: #7d007d; }
+
+.red { color: #bf0000; }
+
+.red-background { background-color: #fa0000; }
+
+.silver { color: #909090; }
+
+.silver-background { background-color: #bcbcbc; }
+
+.teal { color: #006060; }
+
+.teal-background { background-color: #007d7d; }
+
+.white { color: #bfbfbf; }
+
+.white-background { background-color: #fafafa; }
+
+.yellow { color: #bfbf00; }
+
+.yellow-background { background-color: #fafa00; }
+
+span.icon > .fa { cursor: default; }
+
+.admonitionblock td.icon [class^="fa icon-"] { font-size: 2.5em; text-shadow: 1px 1px 2px rgba(0, 0, 0, 0.5); cursor: default; }
+.admonitionblock td.icon .icon-note:before { content: "\f05a"; color: #207c98; }
+.admonitionblock td.icon .icon-tip:before { content: "\f0eb"; text-shadow: 1px 1px 2px rgba(155, 155, 0, 0.8); color: #111; }
+.admonitionblock td.icon .icon-warning:before { content: "\f071"; color: #bf6900; }
+.admonitionblock td.icon .icon-caution:before { content: "\f06d"; color: #bf3400; }
+.admonitionblock td.icon .icon-important:before { content: "\f06a"; color: #bf0000; }
+
+.conum[data-value] { display: inline-block; color: #fff !important; background-color: #222222; -webkit-border-radius: 100px; border-radius: 100px; text-align: center; font-size: 0.75em; width: 1.67em; height: 1.67em; line-height: 1.67em; font-family: "Open Sans", "DejaVu Sans", sans-serif; font-style: normal; font-weight: bold; }
+.conum[data-value] * { color: #fff !important; }
+.conum[data-value] + b { display: none; }
+.conum[data-value]:after { content: attr(data-value); }
+pre .conum[data-value] { position: relative; top: -0.125em; }
+
+b.conum * { color: inherit !important; }
+
+.conum:not([data-value]):empty { display: none; }
+
+.literalblock pre, .listingblock pre { background: #eeeeee; }
diff --git a/transport/src/docs/asciidoclet/overview.adoc b/transport/src/docs/asciidoclet/overview.adoc
new file mode 100644
index 0000000..7947331
--- /dev/null
+++ b/transport/src/docs/asciidoclet/overview.adoc
@@ -0,0 +1,4 @@
+= Elasticsearch Java client
+Jörg Prante
+Version 5.4.0.0
+
diff --git a/transport/src/main/java/org/xbib/elasticsearch/client/transport/ByteBufBytesReference.java b/transport/src/main/java/org/xbib/elasticsearch/client/transport/ByteBufBytesReference.java
new file mode 100644
index 0000000..e0b96d6
--- /dev/null
+++ b/transport/src/main/java/org/xbib/elasticsearch/client/transport/ByteBufBytesReference.java
@@ -0,0 +1,74 @@
+package org.xbib.elasticsearch.client.transport;
+
+import io.netty.buffer.ByteBuf;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+
+final class ByteBufBytesReference extends BytesReference {
+
+ private final ByteBuf buffer;
+ private final int length;
+ private final int offset;
+
+ ByteBufBytesReference(ByteBuf buffer, int length) {
+ this.buffer = buffer;
+ this.length = length;
+ this.offset = buffer.readerIndex();
+ assert length <= buffer.readableBytes() : "length[" + length +"] > " + buffer.readableBytes();
+ }
+
+ @Override
+ public byte get(int index) {
+ return buffer.getByte(offset + index);
+ }
+
+ @Override
+ public int length() {
+ return length;
+ }
+
+ @Override
+ public BytesReference slice(int from, int length) {
+ return new ByteBufBytesReference(buffer.slice(offset + from, length), length);
+ }
+
+ @Override
+ public StreamInput streamInput() {
+ return new ByteBufStreamInput(buffer.duplicate(), length);
+ }
+
+ @Override
+ public void writeTo(OutputStream os) throws IOException {
+ buffer.getBytes(offset, os, length);
+ }
+
+ ByteBuf toByteBuf() {
+ return buffer.duplicate();
+ }
+
+ @Override
+ public String utf8ToString() {
+ return buffer.toString(offset, length, StandardCharsets.UTF_8);
+ }
+
+ @Override
+ public BytesRef toBytesRef() {
+ if (buffer.hasArray()) {
+ return new BytesRef(buffer.array(), buffer.arrayOffset() + offset, length);
+ }
+ final byte[] copy = new byte[length];
+ buffer.getBytes(offset, copy);
+ return new BytesRef(copy);
+ }
+
+ @Override
+ public long ramBytesUsed() {
+ return buffer.capacity();
+ }
+
+}
diff --git a/transport/src/main/java/org/xbib/elasticsearch/client/transport/ByteBufStreamInput.java b/transport/src/main/java/org/xbib/elasticsearch/client/transport/ByteBufStreamInput.java
new file mode 100644
index 0000000..1dadaea
--- /dev/null
+++ b/transport/src/main/java/org/xbib/elasticsearch/client/transport/ByteBufStreamInput.java
@@ -0,0 +1,131 @@
+package org.xbib.elasticsearch.client.transport;
+
+import io.netty.buffer.ByteBuf;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+/**
+ * A Netty {@link io.netty.buffer.ByteBuf} based {@link org.elasticsearch.common.io.stream.StreamInput}.
+ */
+class ByteBufStreamInput extends StreamInput {
+
+ private final ByteBuf buffer;
+ private final int endIndex;
+
+ ByteBufStreamInput(ByteBuf buffer, int length) {
+ if (length > buffer.readableBytes()) {
+ throw new IndexOutOfBoundsException();
+ }
+ this.buffer = buffer;
+ int startIndex = buffer.readerIndex();
+ endIndex = startIndex + length;
+ buffer.markReaderIndex();
+ }
+
+ @Override
+ public BytesReference readBytesReference(int length) throws IOException {
+ // NOTE: It is unsafe to share a reference of the internal structure, so we
+ // use the default implementation which will copy the bytes. It is unsafe because
+ // a netty ByteBuf might be pooled which requires a manual release to prevent
+ // memory leaks.
+ return super.readBytesReference(length);
+ }
+
+ @Override
+ public BytesRef readBytesRef(int length) throws IOException {
+ // NOTE: It is unsafe to share a reference of the internal structure, so we
+ // use the default implementation which will copy the bytes. It is unsafe because
+ // a netty ByteBuf might be pooled which requires a manual release to prevent
+ // memory leaks.
+ return super.readBytesRef(length);
+ }
+
+ @Override
+ public int available() throws IOException {
+ return endIndex - buffer.readerIndex();
+ }
+
+ @Override
+ protected void ensureCanReadBytes(int length) throws EOFException {
+ int bytesAvailable = endIndex - buffer.readerIndex();
+ if (bytesAvailable < length) {
+ throw new EOFException("tried to read: " + length + " bytes but only " + bytesAvailable + " remaining");
+ }
+ }
+
+ @Override
+ public void mark(int readlimit) {
+ buffer.markReaderIndex();
+ }
+
+ @Override
+ public boolean markSupported() {
+ return true;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (available() == 0) {
+ return -1;
+ }
+ return buffer.readByte() & 0xff;
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ if (len == 0) {
+ return 0;
+ }
+ int available = available();
+ if (available == 0) {
+ return -1;
+ }
+
+ len = Math.min(available, len);
+ buffer.readBytes(b, off, len);
+ return len;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ buffer.resetReaderIndex();
+ }
+
+ @Override
+ public long skip(long n) throws IOException {
+ if (n > Integer.MAX_VALUE) {
+ return skipBytes(Integer.MAX_VALUE);
+ } else {
+ return skipBytes((int) n);
+ }
+ }
+
+ public int skipBytes(int n) throws IOException {
+ int nBytes = Math.min(available(), n);
+ buffer.skipBytes(nBytes);
+ return nBytes;
+ }
+
+
+ @Override
+ public byte readByte() throws IOException {
+ return buffer.readByte();
+ }
+
+ @Override
+ public void readBytes(byte[] b, int offset, int len) throws IOException {
+ int read = read(b, offset, len);
+ if (read < len) {
+ throw new IndexOutOfBoundsException();
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ // nothing to do here
+ }
+}
diff --git a/transport/src/main/java/org/xbib/elasticsearch/client/transport/CompressibleBytesOutputStream.java b/transport/src/main/java/org/xbib/elasticsearch/client/transport/CompressibleBytesOutputStream.java
new file mode 100644
index 0000000..3318068
--- /dev/null
+++ b/transport/src/main/java/org/xbib/elasticsearch/client/transport/CompressibleBytesOutputStream.java
@@ -0,0 +1,87 @@
+package org.xbib.elasticsearch.client.transport;
+
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.compress.CompressorFactory;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.io.stream.BytesStream;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.zip.DeflaterOutputStream;
+
+/**
+ * This class exists to provide a stream with optional compression. This is useful as using compression
+ * requires that the underlying {@link DeflaterOutputStream} be closed to write EOS bytes. However, the
+ * {@link BytesStream} should not be closed yet, as we have not used the bytes. This class handles these
+ * intricacies.
+ *
+ * {@link CompressibleBytesOutputStream#materializeBytes()} should be called when all the bytes have been
+ * written to this stream. If compression is enabled, the proper EOS bytes will be written at that point.
+ * The underlying {@link BytesReference} will be returned.
+ *
+ * {@link CompressibleBytesOutputStream#close()} should be called when the bytes are no longer needed and
+ * can be safely released.
+ */
+final class CompressibleBytesOutputStream extends StreamOutput {
+
+ private final StreamOutput stream;
+ private final BytesStream bytesStreamOutput;
+ private final boolean shouldCompress;
+
+ CompressibleBytesOutputStream(BytesStream bytesStreamOutput, boolean shouldCompress) throws IOException {
+ this.bytesStreamOutput = bytesStreamOutput;
+ this.shouldCompress = shouldCompress;
+ if (shouldCompress) {
+ this.stream = CompressorFactory.COMPRESSOR.streamOutput(Streams.flushOnCloseStream(bytesStreamOutput));
+ } else {
+ this.stream = bytesStreamOutput;
+ }
+ }
+
+ /**
+ * This method ensures that compression is complete and returns the underlying bytes.
+ *
+ * @return bytes underlying the stream
+ * @throws IOException if an exception occurs when writing or flushing
+ */
+ BytesReference materializeBytes() throws IOException {
+ // If we are using compression the stream needs to be closed to ensure that EOS marker bytes are written.
+ // The actual ReleasableBytesStreamOutput will not be closed yet as it is wrapped in flushOnCloseStream when
+ // passed to the deflater stream.
+ if (shouldCompress) {
+ stream.close();
+ }
+
+ return bytesStreamOutput.bytes();
+ }
+
+ @Override
+ public void writeByte(byte b) throws IOException {
+ stream.write(b);
+ }
+
+ @Override
+ public void writeBytes(byte[] b, int offset, int length) throws IOException {
+ stream.writeBytes(b, offset, length);
+ }
+
+ @Override
+ public void flush() throws IOException {
+ stream.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (stream == bytesStreamOutput) {
+ IOUtils.close(stream);
+ } else {
+ IOUtils.close(stream, bytesStreamOutput);
+ }
+ }
+
+ @Override
+ public void reset() {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/transport/src/main/java/org/xbib/elasticsearch/client/transport/ConnectionProfile.java b/transport/src/main/java/org/xbib/elasticsearch/client/transport/ConnectionProfile.java
new file mode 100644
index 0000000..ed36792
--- /dev/null
+++ b/transport/src/main/java/org/xbib/elasticsearch/client/transport/ConnectionProfile.java
@@ -0,0 +1,209 @@
+package org.xbib.elasticsearch.client.transport;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.transport.TransportRequestOptions;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * A connection profile describes how many connection are established to specific node for each of the available request types.
+ * ({@link org.elasticsearch.transport.TransportRequestOptions.Type}). This allows to tailor a connection towards a specific usage.
+ */
+public final class ConnectionProfile {
+
+ /**
+ * Builds a connection profile that is dedicated to a single channel type. Use this
+ * when opening single use connections
+ */
+ public static ConnectionProfile buildSingleChannelProfile(TransportRequestOptions.Type channelType,
+ @Nullable TimeValue connectTimeout,
+ @Nullable TimeValue handshakeTimeout) {
+ Builder builder = new Builder();
+ builder.addConnections(1, channelType);
+ final EnumSet otherTypes = EnumSet.allOf(TransportRequestOptions.Type.class);
+ otherTypes.remove(channelType);
+ builder.addConnections(0, otherTypes.stream().toArray(TransportRequestOptions.Type[]::new));
+ if (connectTimeout != null) {
+ builder.setConnectTimeout(connectTimeout);
+ }
+ if (handshakeTimeout != null) {
+ builder.setHandshakeTimeout(handshakeTimeout);
+ }
+ return builder.build();
+ }
+
+ private final List handles;
+ private final int numConnections;
+ private final TimeValue connectTimeout;
+ private final TimeValue handshakeTimeout;
+
+ private ConnectionProfile(List handles, int numConnections, TimeValue connectTimeout, TimeValue handshakeTimeout)
+ {
+ this.handles = handles;
+ this.numConnections = numConnections;
+ this.connectTimeout = connectTimeout;
+ this.handshakeTimeout = handshakeTimeout;
+ }
+
+ /**
+ * A builder to build a new {@link ConnectionProfile}
+ */
+ public static class Builder {
+ private final List handles = new ArrayList<>();
+ private final Set addedTypes = EnumSet.noneOf(TransportRequestOptions.Type.class);
+ private int offset = 0;
+ private TimeValue connectTimeout;
+ private TimeValue handshakeTimeout;
+
+ /** create an empty builder */
+ public Builder() {
+ }
+
+ /** copy constructor, using another profile as a base */
+ public Builder(ConnectionProfile source) {
+ handles.addAll(source.getHandles());
+ offset = source.getNumConnections();
+ handles.forEach(th -> addedTypes.addAll(th.types));
+ connectTimeout = source.getConnectTimeout();
+ handshakeTimeout = source.getHandshakeTimeout();
+ }
+ /**
+ * Sets a connect timeout for this connection profile
+ */
+ public void setConnectTimeout(TimeValue connectTimeout) {
+ if (connectTimeout.millis() < 0) {
+ throw new IllegalArgumentException("connectTimeout must be non-negative but was: " + connectTimeout);
+ }
+ this.connectTimeout = connectTimeout;
+ }
+
+ /**
+ * Sets a handshake timeout for this connection profile
+ */
+ public void setHandshakeTimeout(TimeValue handshakeTimeout) {
+ if (handshakeTimeout.millis() < 0) {
+ throw new IllegalArgumentException("handshakeTimeout must be non-negative but was: " + handshakeTimeout);
+ }
+ this.handshakeTimeout = handshakeTimeout;
+ }
+
+ /**
+ * Adds a number of connections for one or more types. Each type can only be added once.
+ * @param numConnections the number of connections to use in the pool for the given connection types
+ * @param types a set of types that should share the given number of connections
+ */
+ public void addConnections(int numConnections, TransportRequestOptions.Type... types) {
+ if (types == null || types.length == 0) {
+ throw new IllegalArgumentException("types must not be null");
+ }
+ for (TransportRequestOptions.Type type : types) {
+ if (addedTypes.contains(type)) {
+ throw new IllegalArgumentException("type [" + type + "] is already registered");
+ }
+ }
+ addedTypes.addAll(Arrays.asList(types));
+ handles.add(new ConnectionTypeHandle(offset, numConnections, EnumSet.copyOf(Arrays.asList(types))));
+ offset += numConnections;
+ }
+
+ /**
+ * Creates a new {@link ConnectionProfile} based on the added connections.
+ * @throws IllegalStateException if any of the {@link org.elasticsearch.transport.TransportRequestOptions.Type} enum is missing
+ */
+ public ConnectionProfile build() {
+ EnumSet types = EnumSet.allOf(TransportRequestOptions.Type.class);
+ types.removeAll(addedTypes);
+ if (types.isEmpty() == false) {
+ throw new IllegalStateException("not all types are added for this connection profile - missing types: " + types);
+ }
+ return new ConnectionProfile(Collections.unmodifiableList(handles), offset, connectTimeout, handshakeTimeout);
+ }
+
+ }
+
+ /**
+ * Returns the connect timeout or null if no explicit timeout is set on this profile.
+ */
+ public TimeValue getConnectTimeout() {
+ return connectTimeout;
+ }
+
+ /**
+ * Returns the handshake timeout or null if no explicit timeout is set on this profile.
+ */
+ public TimeValue getHandshakeTimeout() {
+ return handshakeTimeout;
+ }
+
+ /**
+ * Returns the total number of connections for this profile
+ */
+ public int getNumConnections() {
+ return numConnections;
+ }
+
+ /**
+ * Returns the number of connections per type for this profile. This might return a count that is shared with other types such
+ * that the sum of all connections per type might be higher than {@link #getNumConnections()}. For instance if
+ * {@link org.elasticsearch.transport.TransportRequestOptions.Type#BULK} shares connections with
+ * {@link org.elasticsearch.transport.TransportRequestOptions.Type#REG} they will return both the same number of connections from
+ * this method but the connections are not distinct.
+ */
+ public int getNumConnectionsPerType(TransportRequestOptions.Type type) {
+ for (ConnectionTypeHandle handle : handles) {
+ if (handle.getTypes().contains(type)) {
+ return handle.length;
+ }
+ }
+ throw new AssertionError("no handle found for type: " + type);
+ }
+
+ /**
+ * Returns the type handles for this connection profile
+ */
+ List getHandles() {
+ return Collections.unmodifiableList(handles);
+ }
+
+ /**
+ * Connection type handle encapsulates the logic which connection
+ */
+ static final class ConnectionTypeHandle {
+ public final int length;
+ public final int offset;
+ private final Set types;
+ private final AtomicInteger counter = new AtomicInteger();
+
+ private ConnectionTypeHandle(int offset, int length, Set types) {
+ this.length = length;
+ this.offset = offset;
+ this.types = types;
+ }
+
+ /**
+ * Returns one of the channels out configured for this handle. The channel is selected in a round-robin
+ * fashion.
+ */
+ T getChannel(List channels) {
+ if (length == 0) {
+ throw new IllegalStateException("can't select channel size is 0 for types: " + types);
+ }
+ assert channels.size() >= offset + length : "illegal size: " + channels.size() + " expected >= " + (offset + length);
+ return channels.get(offset + Math.floorMod(counter.incrementAndGet(), length));
+ }
+
+ /**
+ * Returns all types for this handle
+ */
+ Set getTypes() {
+ return types;
+ }
+ }
+}
diff --git a/transport/src/main/java/org/xbib/elasticsearch/client/transport/ESLoggingHandler.java b/transport/src/main/java/org/xbib/elasticsearch/client/transport/ESLoggingHandler.java
new file mode 100644
index 0000000..5ed6ef6
--- /dev/null
+++ b/transport/src/main/java/org/xbib/elasticsearch/client/transport/ESLoggingHandler.java
@@ -0,0 +1,108 @@
+package org.xbib.elasticsearch.client.transport;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.logging.LogLevel;
+import io.netty.handler.logging.LoggingHandler;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.compress.Compressor;
+import org.elasticsearch.common.compress.CompressorFactory;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.transport.TcpHeader;
+import org.elasticsearch.transport.TransportStatus;
+
+import java.io.IOException;
+
+final class ESLoggingHandler extends LoggingHandler {
+
+ ESLoggingHandler() {
+ super(LogLevel.TRACE);
+ }
+
+ @Override
+ protected String format(final ChannelHandlerContext ctx, final String eventName, final Object arg) {
+ if (arg instanceof ByteBuf) {
+ try {
+ return format(ctx, eventName, (ByteBuf) arg);
+ } catch (final Exception e) {
+ // we really do not want to allow a bug in the formatting handling to escape
+ logger.trace("an exception occurred formatting a trace message", e);
+ // we are going to let this be formatted via the default formatting
+ return super.format(ctx, eventName, arg);
+ }
+ } else {
+ return super.format(ctx, eventName, arg);
+ }
+ }
+
+ private static final int MESSAGE_LENGTH_OFFSET = TcpHeader.MARKER_BYTES_SIZE;
+ private static final int REQUEST_ID_OFFSET = MESSAGE_LENGTH_OFFSET + TcpHeader.MESSAGE_LENGTH_SIZE;
+ private static final int STATUS_OFFSET = REQUEST_ID_OFFSET + TcpHeader.REQUEST_ID_SIZE;
+ private static final int VERSION_ID_OFFSET = STATUS_OFFSET + TcpHeader.STATUS_SIZE;
+ private static final int ACTION_OFFSET = VERSION_ID_OFFSET + TcpHeader.VERSION_ID_SIZE;
+
+ private String format(final ChannelHandlerContext ctx, final String eventName, final ByteBuf arg) throws IOException {
+ final int readableBytes = arg.readableBytes();
+ if (readableBytes == 0) {
+ return super.format(ctx, eventName, arg);
+ } else if (readableBytes >= 2) {
+ final StringBuilder sb = new StringBuilder();
+ sb.append(ctx.channel().toString());
+ final int offset = arg.readerIndex();
+ // this might be an ES message, check the header
+ if (arg.getByte(offset) == (byte) 'E' && arg.getByte(offset + 1) == (byte) 'S') {
+ if (readableBytes == TcpHeader.MARKER_BYTES_SIZE + TcpHeader.MESSAGE_LENGTH_SIZE) {
+ final int length = arg.getInt(offset + MESSAGE_LENGTH_OFFSET);
+ if (length == TcpTransport.PING_DATA_SIZE) {
+ sb.append(" [ping]").append(' ').append(eventName).append(": ").append(readableBytes).append('B');
+ return sb.toString();
+ }
+ }
+ else if (readableBytes >= TcpHeader.HEADER_SIZE) {
+ // we are going to try to decode this as an ES message
+ final int length = arg.getInt(offset + MESSAGE_LENGTH_OFFSET);
+ final long requestId = arg.getLong(offset + REQUEST_ID_OFFSET);
+ final byte status = arg.getByte(offset + STATUS_OFFSET);
+ final boolean isRequest = TransportStatus.isRequest(status);
+ final String type = isRequest ? "request" : "response";
+ final String version = Version.fromId(arg.getInt(offset + VERSION_ID_OFFSET)).toString();
+ sb.append(" [length: ").append(length);
+ sb.append(", request id: ").append(requestId);
+ sb.append(", type: ").append(type);
+ sb.append(", version: ").append(version);
+ if (isRequest) {
+ // it looks like an ES request, try to decode the action
+ final int remaining = readableBytes - ACTION_OFFSET;
+ final ByteBuf slice = arg.slice(offset + ACTION_OFFSET, remaining);
+ // the stream might be compressed
+ try (StreamInput in = in(status, slice, remaining)) {
+ // the first bytes in the message is the context headers
+ try (ThreadContext context = new ThreadContext(Settings.EMPTY)) {
+ context.readHeaders(in);
+ }
+ // now we can decode the action name
+ sb.append(", action: ").append(in.readString());
+ }
+ }
+ sb.append(']');
+ sb.append(' ').append(eventName).append(": ").append(readableBytes).append('B');
+ return sb.toString();
+ }
+ }
+ }
+ // we could not decode this as an ES message, use the default formatting
+ return super.format(ctx, eventName, arg);
+ }
+
+ private StreamInput in(final Byte status, final ByteBuf slice, final int remaining) throws IOException {
+ final ByteBufStreamInput in = new ByteBufStreamInput(slice, remaining);
+ if (TransportStatus.isCompress(status)) {
+ final Compressor compressor = CompressorFactory.compressor(Netty4Utils.toBytesReference(slice));
+ return compressor.streamInput(in);
+ } else {
+ return in;
+ }
+ }
+}
diff --git a/transport/src/main/java/org/xbib/elasticsearch/client/transport/MockTransportBulkClient.java b/transport/src/main/java/org/xbib/elasticsearch/client/transport/MockTransportBulkClient.java
new file mode 100644
index 0000000..04321a8
--- /dev/null
+++ b/transport/src/main/java/org/xbib/elasticsearch/client/transport/MockTransportBulkClient.java
@@ -0,0 +1,151 @@
+package org.xbib.elasticsearch.client.transport;
+
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.xbib.elasticsearch.client.BulkControl;
+import org.xbib.elasticsearch.client.BulkMetric;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Mock client, it does not perform actions on a cluster.
+ * Useful for testing or dry runs.
+ */
+public class MockTransportBulkClient extends TransportBulkClient {
+
+ @Override
+ public ElasticsearchClient client() {
+ return null;
+ }
+
+ @Override
+ public MockTransportBulkClient init(ElasticsearchClient client, Settings settings, BulkMetric metric, BulkControl control) {
+ return this;
+ }
+
+ @Override
+ public MockTransportBulkClient maxActionsPerRequest(int maxActions) {
+ return this;
+ }
+
+ @Override
+ public MockTransportBulkClient maxConcurrentRequests(int maxConcurrentRequests) {
+ return this;
+ }
+
+ @Override
+ public MockTransportBulkClient maxVolumePerRequest(ByteSizeValue maxVolumePerRequest) {
+ return this;
+ }
+
+ @Override
+ public MockTransportBulkClient flushIngestInterval(TimeValue interval) {
+ return this;
+ }
+
+ @Override
+ public MockTransportBulkClient index(String index, String type, String id, boolean create, String source) {
+ return this;
+ }
+
+ @Override
+ public MockTransportBulkClient delete(String index, String type, String id) {
+ return this;
+ }
+
+ @Override
+ public MockTransportBulkClient update(String index, String type, String id, String source) {
+ return this;
+ }
+
+ @Override
+ public MockTransportBulkClient indexRequest(IndexRequest indexRequest) {
+ return this;
+ }
+
+ @Override
+ public MockTransportBulkClient deleteRequest(DeleteRequest deleteRequest) {
+ return this;
+ }
+
+ @Override
+ public MockTransportBulkClient updateRequest(UpdateRequest updateRequest) {
+ return this;
+ }
+
+ @Override
+ public MockTransportBulkClient flushIngest() {
+ return this;
+ }
+
+ @Override
+ public MockTransportBulkClient waitForResponses(TimeValue timeValue) throws InterruptedException {
+ return this;
+ }
+
+ @Override
+ public MockTransportBulkClient startBulk(String index, long startRefreshInterval, long stopRefreshIterval) {
+ return this;
+ }
+
+ @Override
+ public MockTransportBulkClient stopBulk(String index) {
+ return this;
+ }
+
+ @Override
+ public MockTransportBulkClient deleteIndex(String index) {
+ return this;
+ }
+
+ @Override
+ public MockTransportBulkClient newIndex(String index) {
+ return this;
+ }
+
+ @Override
+ public MockTransportBulkClient newMapping(String index, String type, Map mapping) {
+ return this;
+ }
+
+ @Override
+ public void putMapping(String index) {
+ // mockup method
+ }
+
+ @Override
+ public void refreshIndex(String index) {
+ // mockup method
+ }
+
+ @Override
+ public void flushIndex(String index) {
+ // mockup method
+ }
+
+ @Override
+ public void waitForCluster(String healthColor, TimeValue timeValue) throws IOException {
+ // mockup method
+ }
+
+ @Override
+ public int waitForRecovery(String index) throws IOException {
+ return -1;
+ }
+
+ @Override
+ public int updateReplicaLevel(String index, int level) throws IOException {
+ return -1;
+ }
+
+ @Override
+ public void shutdown() {
+ // mockup method
+ }
+}
diff --git a/transport/src/main/java/org/xbib/elasticsearch/client/transport/Netty4InternalESLogger.java b/transport/src/main/java/org/xbib/elasticsearch/client/transport/Netty4InternalESLogger.java
new file mode 100644
index 0000000..33429bf
--- /dev/null
+++ b/transport/src/main/java/org/xbib/elasticsearch/client/transport/Netty4InternalESLogger.java
@@ -0,0 +1,168 @@
+package org.xbib.elasticsearch.client.transport;
+
+import io.netty.util.internal.logging.AbstractInternalLogger;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.SuppressLoggerChecks;
+import org.elasticsearch.common.logging.Loggers;
+
+@SuppressLoggerChecks(reason = "safely delegates to logger")
+class Netty4InternalESLogger extends AbstractInternalLogger {
+
+ private final Logger logger;
+
+ Netty4InternalESLogger(final String name) {
+ super(name);
+ this.logger = Loggers.getLogger(name);
+ }
+
+ @Override
+ public boolean isTraceEnabled() {
+ return logger.isTraceEnabled();
+ }
+
+ @Override
+ public void trace(String msg) {
+ logger.trace(msg);
+ }
+
+ @Override
+ public void trace(String format, Object arg) {
+ logger.trace(format, arg);
+ }
+
+ @Override
+ public void trace(String format, Object argA, Object argB) {
+ logger.trace(format, argA, argB);
+ }
+
+ @Override
+ public void trace(String format, Object... arguments) {
+ logger.trace(format, arguments);
+ }
+
+ @Override
+ public void trace(String msg, Throwable t) {
+ logger.trace(msg, t);
+ }
+
+ @Override
+ public boolean isDebugEnabled() {
+ return logger.isDebugEnabled();
+ }
+
+ @Override
+ public void debug(String msg) {
+ logger.debug(msg);
+ }
+
+ @Override
+ public void debug(String format, Object arg) {
+ logger.debug(format, arg);
+ }
+
+ @Override
+ public void debug(String format, Object argA, Object argB) {
+ logger.debug(format, argA, argB);
+ }
+
+ @Override
+ public void debug(String format, Object... arguments) {
+ logger.debug(format, arguments);
+ }
+
+ @Override
+ public void debug(String msg, Throwable t) {
+ logger.debug(msg, t);
+ }
+
+ @Override
+ public boolean isInfoEnabled() {
+ return logger.isInfoEnabled();
+ }
+
+ @Override
+ public void info(String msg) {
+ logger.info(msg);
+ }
+
+ @Override
+ public void info(String format, Object arg) {
+ logger.info(format, arg);
+ }
+
+ @Override
+ public void info(String format, Object argA, Object argB) {
+ logger.info(format, argA, argB);
+ }
+
+ @Override
+ public void info(String format, Object... arguments) {
+ logger.info(format, arguments);
+ }
+
+ @Override
+ public void info(String msg, Throwable t) {
+ logger.info(msg, t);
+ }
+
+ @Override
+ public boolean isWarnEnabled() {
+ return logger.isWarnEnabled();
+ }
+
+ @Override
+ public void warn(String msg) {
+ logger.warn(msg);
+ }
+
+ @Override
+ public void warn(String format, Object arg) {
+ logger.warn(format, arg);
+ }
+
+ @Override
+ public void warn(String format, Object... arguments) {
+ logger.warn(format, arguments);
+ }
+
+ @Override
+ public void warn(String format, Object argA, Object argB) {
+ logger.warn(format, argA, argB);
+ }
+
+ @Override
+ public void warn(String msg, Throwable t) {
+ logger.warn(msg, t);
+ }
+
+ @Override
+ public boolean isErrorEnabled() {
+ return logger.isErrorEnabled();
+ }
+
+ @Override
+ public void error(String msg) {
+ logger.error(msg);
+ }
+
+ @Override
+ public void error(String format, Object arg) {
+ logger.error(format, arg);
+ }
+
+ @Override
+ public void error(String format, Object argA, Object argB) {
+ logger.error(format, argA, argB);
+ }
+
+ @Override
+ public void error(String format, Object... arguments) {
+ logger.error(format, arguments);
+ }
+
+ @Override
+ public void error(String msg, Throwable t) {
+ logger.error(msg, t);
+ }
+
+}
diff --git a/transport/src/main/java/org/xbib/elasticsearch/client/transport/Netty4MessageChannelHandler.java b/transport/src/main/java/org/xbib/elasticsearch/client/transport/Netty4MessageChannelHandler.java
new file mode 100644
index 0000000..4126944
--- /dev/null
+++ b/transport/src/main/java/org/xbib/elasticsearch/client/transport/Netty4MessageChannelHandler.java
@@ -0,0 +1,58 @@
+package org.xbib.elasticsearch.client.transport;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelDuplexHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.util.Attribute;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.transport.TcpHeader;
+import org.elasticsearch.transport.Transports;
+
+import java.net.InetSocketAddress;
+
+/**
+ * A handler (must be the last one!) that does size based frame decoding and forwards the actual message
+ * to the relevant action.
+ */
+final class Netty4MessageChannelHandler extends ChannelDuplexHandler {
+
+ private final Netty4Transport transport;
+ private final String profileName;
+
+ Netty4MessageChannelHandler(Netty4Transport transport, String profileName) {
+ this.transport = transport;
+ this.profileName = profileName;
+ }
+
+ @Override
+ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
+ Transports.assertTransportThread();
+ if (!(msg instanceof ByteBuf)) {
+ ctx.fireChannelRead(msg);
+ return;
+ }
+ final ByteBuf buffer = (ByteBuf) msg;
+ final int remainingMessageSize = buffer.getInt(buffer.readerIndex() - TcpHeader.MESSAGE_LENGTH_SIZE);
+ final int expectedReaderIndex = buffer.readerIndex() + remainingMessageSize;
+ try {
+ Channel channel = ctx.channel();
+ InetSocketAddress remoteAddress = (InetSocketAddress) channel.remoteAddress();
+ // netty always copies a buffer, either in NioWorker in its read handler, where it copies to a fresh
+ // buffer, or in the cumulative buffer, which is cleaned each time so it could be bigger than the actual size
+ BytesReference reference = Netty4Utils.toBytesReference(buffer, remainingMessageSize);
+ Attribute channelAttribute = channel.attr(Netty4Transport.CHANNEL_KEY);
+ transport.messageReceived(reference, channelAttribute.get(), profileName, remoteAddress, remainingMessageSize);
+ } finally {
+ // Set the expected position of the buffer, no matter what happened
+ buffer.readerIndex(expectedReaderIndex);
+ }
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
+ Netty4Utils.maybeDie(cause);
+ transport.exceptionCaught(ctx, cause);
+ }
+
+}
diff --git a/transport/src/main/java/org/xbib/elasticsearch/client/transport/Netty4Plugin.java b/transport/src/main/java/org/xbib/elasticsearch/client/transport/Netty4Plugin.java
new file mode 100644
index 0000000..cb0b450
--- /dev/null
+++ b/transport/src/main/java/org/xbib/elasticsearch/client/transport/Netty4Plugin.java
@@ -0,0 +1,77 @@
+package org.xbib.elasticsearch.client.transport;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.PageCacheRecycler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.http.netty4.Netty4HttpServerTransport;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+public class Netty4Plugin extends Plugin implements NetworkPlugin {
+
+ static {
+ Netty4Utils.setup();
+ }
+
+ public static final String NETTY_TRANSPORT_NAME = "netty4";
+ public static final String NETTY_HTTP_TRANSPORT_NAME = "netty4";
+
+ @Override
+ public List> getSettings() {
+ return Arrays.asList(
+ Netty4HttpServerTransport.SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS,
+ Netty4HttpServerTransport.SETTING_HTTP_WORKER_COUNT,
+ Netty4HttpServerTransport.SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE,
+ Netty4HttpServerTransport.SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MIN,
+ Netty4HttpServerTransport.SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MAX,
+ Netty4Transport.WORKER_COUNT,
+ Netty4Transport.NETTY_RECEIVE_PREDICTOR_SIZE,
+ Netty4Transport.NETTY_RECEIVE_PREDICTOR_MIN,
+ Netty4Transport.NETTY_RECEIVE_PREDICTOR_MAX,
+ Netty4Transport.NETTY_BOSS_COUNT
+ );
+ }
+
+ @Override
+ public Settings additionalSettings() {
+ return Settings.builder()
+ // here we set the netty4 transport and http transport as the default. This is a set once setting
+ // ie. if another plugin does that as well the server will fail - only one default network can exist!
+ .put(NetworkModule.HTTP_DEFAULT_TYPE_SETTING.getKey(), NETTY_HTTP_TRANSPORT_NAME)
+ .put(NetworkModule.TRANSPORT_DEFAULT_TYPE_SETTING.getKey(), NETTY_TRANSPORT_NAME)
+ .build();
+ }
+
+ @Override
+ public Map> getTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+ PageCacheRecycler pageCacheRecycler,
+ CircuitBreakerService circuitBreakerService,
+ NamedWriteableRegistry namedWriteableRegistry,
+ NetworkService networkService) {
+ return Collections.singletonMap(NETTY_TRANSPORT_NAME, () -> new Netty4Transport(settings, threadPool, networkService, bigArrays,
+ namedWriteableRegistry, circuitBreakerService));
+ }
+
+ @Override
+ public Map> getHttpTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+ CircuitBreakerService circuitBreakerService,
+ NamedWriteableRegistry namedWriteableRegistry,
+ NamedXContentRegistry xContentRegistry,
+ NetworkService networkService,
+ HttpServerTransport.Dispatcher dispatcher) {
+ return Collections.singletonMap(NETTY_HTTP_TRANSPORT_NAME,
+ () -> new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher));
+ }
+}
diff --git a/transport/src/main/java/org/xbib/elasticsearch/client/transport/Netty4SizeHeaderFrameDecoder.java b/transport/src/main/java/org/xbib/elasticsearch/client/transport/Netty4SizeHeaderFrameDecoder.java
new file mode 100644
index 0000000..bf18d0f
--- /dev/null
+++ b/transport/src/main/java/org/xbib/elasticsearch/client/transport/Netty4SizeHeaderFrameDecoder.java
@@ -0,0 +1,30 @@
+package org.xbib.elasticsearch.client.transport;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.codec.ByteToMessageDecoder;
+import io.netty.handler.codec.TooLongFrameException;
+import org.elasticsearch.transport.TcpHeader;
+
+import java.util.List;
+
+final class Netty4SizeHeaderFrameDecoder extends ByteToMessageDecoder {
+
+ @Override
+ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List