From a5c8dc8b5f7a916c28dff96a8a2e9b17d2bf41da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Sun, 10 May 2020 17:40:58 +0200 Subject: [PATCH] new elx implementation for Elasticsearch 7.6.1 --- build.gradle | 17 +- elx-api/build.gradle | 2 +- .../java/org/xbib/elx/api/BulkProcessor.java | 2 - .../java/org/xbib/elx/api/ExtendedClient.java | 17 +- elx-common/build.gradle | 5 +- .../elx/common/AbstractExtendedClient.java | 189 +++++++++++------- .../elx/common/DefaultBulkController.java | 6 +- .../xbib/elx/common/DefaultBulkProcessor.java | 22 +- .../org/xbib/elx/common/test/MockNode.java | 4 +- .../org/xbib/elx/common/test/SearchTest.java | 5 +- .../org/xbib/elx/common/test/SimpleTest.java | 6 +- .../xbib/elx/common/test/TestExtension.java | 11 +- .../xbib/elx/common/test/WildcardTest.java | 16 +- elx-http/build.gradle | 3 +- .../org/xbib/elx/http/ExtendedHttpClient.java | 26 +-- .../java/org/xbib/elx/http/HttpAction.java | 17 +- .../health/HttpClusterHealthAction.java | 8 +- .../health/HttpClusterHealthResponse.java | 68 ------- .../node/info/HttpNodesInfoAction.java | 103 +--------- .../HttpClusterUpdateSettingsAction.java | 8 +- .../cluster/state/HttpClusterStateAction.java | 11 +- .../alias/HttpIndicesAliasesAction.java | 14 +- .../indices/alias/get/HttpGetAliasAction.java | 13 +- .../indices/create/HttpCreateIndexAction.java | 47 ++++- .../indices/delete/HttpDeleteIndexAction.java | 14 +- .../indices/HttpIndicesExistsAction.java | 30 +-- .../mapping/get/HttpGetMappingsAction.java | 40 ++-- .../refresh/HttpRefreshIndexAction.java | 8 +- .../settings/get/HttpGetSettingsAction.java | 65 +----- .../put/HttpUpdateSettingsAction.java | 14 +- .../elx/http/action/bulk/HttpBulkAction.java | 22 +- .../elx/http/action/get/HttpExistsAction.java | 14 +- .../elx/http/action/get/HttpGetAction.java | 14 +- .../http/action/index/HttpIndexAction.java | 14 +- .../elx/http/action/main/HttpMainAction.java | 12 +- .../http/action/search/HttpSearchAction.java | 8 +- .../http/action/update/HttpUpdateAction.java | 14 +- .../services/org.xbib.elx.http.HttpAction | 2 +- .../org/xbib/elx/http/test/ClientTest.java | 68 +++++-- .../xbib/elx/http/test/DuplicateIDTest.java | 23 ++- .../xbib/elx/http/test/IndexPruneTest.java | 2 +- .../java/org/xbib/elx/http/test/MockNode.java | 4 +- .../org/xbib/elx/http/test/TestExtension.java | 11 +- elx-node/build.gradle | 5 +- .../org/xbib/elx/node/ExtendedNodeClient.java | 2 +- .../org/xbib/elx/node/test/ClientTest.java | 61 ++++-- .../xbib/elx/node/test/DuplicateIDTest.java | 20 +- .../java/org/xbib/elx/node/test/MockNode.java | 4 +- .../org/xbib/elx/node/test/TestExtension.java | 11 +- elx-transport/build.gradle | 5 +- .../transport/ExtendedTransportClient.java | 2 +- .../xbib/elx/transport/test/ClientTest.java | 51 +++-- .../elx/transport/test/DuplicateIDTest.java | 18 +- .../org/xbib/elx/transport/test/MockNode.java | 4 +- .../elx/transport/test/TestExtension.java | 10 +- gradle.properties | 15 +- 56 files changed, 546 insertions(+), 661 deletions(-) rename elx-http/src/main/java/org/{elasticsearch => xbib/elx/http}/action/admin/indices/mapping/get/HttpGetMappingsAction.java (58%) diff --git a/build.gradle b/build.gradle index e07c67a..16649b5 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ plugins { } if (JavaVersion.current() < JavaVersion.VERSION_11) { - throw new GradleException("This build must be run with java 11") + throw new GradleException("This build must be run with Java/OpenJDK 11+") } subprojects { @@ -50,16 +50,15 @@ subprojects { test { enabled = true useJUnitPlatform() - doFirst { - jvmArgs = [ - '--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED', - '--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED', - '--add-opens=java.base/java.nio=ALL-UNNAMED' - ] - } + // for Lucene to access jdk.internal.ref and jdk.internal.misc in Java 11+ + jvmArgs = [ + '--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED', + '--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED', + '--add-opens=java.base/java.nio=ALL-UNNAMED' + ] systemProperty 'java.util.logging.manager', 'org.apache.logging.log4j.jul.LogManager' - systemProperty 'path.home', "${project.buildDir}/" systemProperty 'jna.debug_load', 'true' + systemProperty 'path.home', "${project.buildDir}/" failFast = true testLogging { events 'PASSED', 'FAILED', 'SKIPPED' diff --git a/elx-api/build.gradle b/elx-api/build.gradle index 42f4078..81fc153 100644 --- a/elx-api/build.gradle +++ b/elx-api/build.gradle @@ -1,4 +1,4 @@ dependencies { compile "org.xbib:metrics-common:${project.property('xbib-metrics.version')}" - compile "org.xbib.elasticsearch:elasticsearch:${rootProject.property('elasticsearch-server.version')}" + compile "org.elasticsearch:elasticsearch:${rootProject.property('elasticsearch.version')}" } \ No newline at end of file diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java b/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java index 0ab7b79..2e16b03 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java @@ -12,8 +12,6 @@ public interface BulkProcessor extends Closeable, Flushable { BulkProcessor add(ActionRequest request); - BulkProcessor add(ActionRequest request, Object payload); - boolean awaitFlush(long timeout, TimeUnit unit) throws InterruptedException; boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException; diff --git a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java index a6325f6..63fab80 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java +++ b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java @@ -6,6 +6,7 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.Closeable; import java.io.Flushable; @@ -126,8 +127,9 @@ public interface ExtendedClient extends Flushable, Closeable { * @param id the id * @param source the source * @return this + * @throws IOException if update fails */ - ExtendedClient update(String index, String id, BytesReference source); + ExtendedClient update(String index, String id, BytesReference source) throws IOException; /** * Update document. Use with precaution! Does not work in all cases. @@ -199,7 +201,18 @@ public interface ExtendedClient extends Flushable, Closeable { * @return this * @throws IOException if settings/mapping is invalid or index creation fails */ - ExtendedClient newIndex(String index, Settings settings, Map mapping) throws IOException; + ExtendedClient newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException; + + /** + * Create a new index. + * + * @param index index + * @param settings settings + * @param mapping mapping + * @return this + * @throws IOException if settings/mapping is invalid or index creation fails + */ + ExtendedClient newIndex(String index, Settings settings, Map mapping) throws IOException; /** * Create a new index. diff --git a/elx-common/build.gradle b/elx-common/build.gradle index 4336a23..4238e95 100644 --- a/elx-common/build.gradle +++ b/elx-common/build.gradle @@ -1,5 +1,6 @@ dependencies{ compile project(':elx-api') - testCompile "org.xbib.elasticsearch:elasticsearch-analysis-common:${rootProject.property('elasticsearch-server.version')}" - testCompile "org.xbib.elasticsearch:transport-netty4:${rootProject.property('elasticsearch-server.version')}" + testCompile "org.elasticsearch.plugin:transport-netty4-client:${rootProject.property('elasticsearch.version')}" + testCompile "io.netty:netty-codec-http:${project.property('netty.version')}" + testCompile "io.netty:netty-transport:${project.property('netty.version')}" } diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java index 417bc1e..4a337ca 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java @@ -5,6 +5,7 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.search.TotalHits; import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; @@ -14,7 +15,6 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; @@ -23,7 +23,6 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; import org.elasticsearch.action.admin.indices.flush.FlushAction; import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction; @@ -46,6 +45,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.client.transport.NoNodeAvailableException; @@ -53,7 +53,6 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.AliasOrIndex; import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; @@ -89,8 +88,8 @@ import java.time.LocalDate; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -113,22 +112,13 @@ public abstract class AbstractExtendedClient implements ExtendedClient { private static final Logger logger = LogManager.getLogger(AbstractExtendedClient.class.getName()); - /** - * The one and only index type name used in the extended client. - * Notr that all Elasticsearch version < 6.2.0 do not allow a prepending "_". - */ - private static final String TYPE_NAME = "doc"; - - /** - * The Elasticsearch client. - */ private ElasticsearchClient client; private BulkMetric bulkMetric; private BulkController bulkController; - private AtomicBoolean closed; + private final AtomicBoolean closed; private static final IndexShiftResult EMPTY_INDEX_SHIFT_RESULT = new IndexShiftResult() { @Override @@ -195,7 +185,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public AbstractExtendedClient init(Settings settings) throws IOException { - logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(',')); + logger.info("initializing with settings = " + settings.toDelimitedString(',')); if (client == null) { client = createClient(settings); } @@ -303,7 +293,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient newIndex(String index, Settings settings) throws IOException { - return newIndex(index, settings, (Map) null); + return newIndex(index, settings, (Map) null); } @Override @@ -314,7 +304,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public ExtendedClient newIndex(String index, Settings settings, Map mapping) throws IOException { + public ExtendedClient newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException { ensureActive(); if (index == null) { logger.warn("no index name given to create index"); @@ -325,13 +315,37 @@ public abstract class AbstractExtendedClient implements ExtendedClient { createIndexRequest.settings(settings); } if (mapping != null) { - createIndexRequest.mapping(TYPE_NAME, mapping); + createIndexRequest.mapping("_doc", mapping); } CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); - XContentBuilder builder = XContentFactory.jsonBuilder(); - logger.info("index {} created: {}", index, - Strings.toString(createIndexResponse.toXContent(builder, ToXContent.EMPTY_PARAMS))); - return this; + if (createIndexResponse.isAcknowledged()) { + return this; + } + throw new IllegalStateException("index creation not acknowledged"); + } + + @Override + public ExtendedClient newIndex(String index, Settings settings, Map mapping) throws IOException { + ensureActive(); + if (index == null) { + logger.warn("no index name given to create index"); + return this; + } + CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index); + if (settings != null) { + createIndexRequest.settings(settings); + } + if (mapping != null) { + if (mapping.size() != 1) { + throw new IllegalArgumentException("mapping invalid, just use 'doc' for mapping"); + } + createIndexRequest.mapping("_doc", mapping); + } + CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); + if (createIndexResponse.isAcknowledged()) { + return this; + } + throw new IllegalStateException("index creation not acknowledged"); } @Override @@ -387,13 +401,13 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient index(String index, String id, boolean create, String source) { - return index(new IndexRequest(index, TYPE_NAME, id).create(create) + return index(new IndexRequest().index(index).id(id).create(create) .source(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON)); } @Override public ExtendedClient index(String index, String id, boolean create, BytesReference source) { - return index(new IndexRequest(index, TYPE_NAME, id).create(create) + return index(new IndexRequest().index(index).id(id).create(create) .source(source, XContentType.JSON)); } @@ -406,7 +420,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient delete(String index, String id) { - return delete(new DeleteRequest(index, TYPE_NAME, id)); + return delete(new DeleteRequest().index(index).id(id)); } @Override @@ -417,14 +431,14 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public ExtendedClient update(String index, String id, BytesReference source) { - return update(new UpdateRequest(index, TYPE_NAME, id) + public ExtendedClient update(String index, String id, BytesReference source) throws IOException { + return update(new UpdateRequest().index(index).id(id) .doc(source, XContentType.JSON)); } @Override public ExtendedClient update(String index, String id, String source) { - return update(new UpdateRequest(index, TYPE_NAME, id) + return update(new UpdateRequest().index(index).id(id) .doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON)); } @@ -458,7 +472,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, clusterHealthRequest).actionGet(); if (healthResponse != null && healthResponse.isTimedOut()) { - logger.error("timeout waiting for recovery"); + logger.warn("timeout waiting for recovery"); return false; } } @@ -473,9 +487,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet(); if (healthResponse != null && healthResponse.isTimedOut()) { - if (logger.isErrorEnabled()) { - logger.error("timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name()); - } + logger.warn("timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name()); return false; } return true; @@ -693,7 +705,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } if (!indicesAliasesRequest.getAliasActions().isEmpty()) { logger.debug("indices alias request = {}", indicesAliasesRequest.getAliasActions().toString()); - IndicesAliasesResponse indicesAliasesResponse = + AcknowledgedResponse indicesAliasesResponse = client.execute(IndicesAliasesAction.INSTANCE, indicesAliasesRequest).actionGet(); logger.debug("response isAcknowledged = {} isFragment = {}", indicesAliasesResponse.isAcknowledged(), indicesAliasesResponse.isFragment()); @@ -719,7 +731,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE); GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet(); Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); - logger.info("{} indices", getIndexResponse.getIndices().length); + logger.info("pruneIndex: total of {} indices", getIndexResponse.getIndices().length); List candidateIndices = new ArrayList<>(); for (String s : getIndexResponse.getIndices()) { Matcher m = pattern.matcher(s); @@ -754,21 +766,30 @@ public abstract class AbstractExtendedClient implements ExtendedClient { String[] s = new String[indicesToDelete.size()]; DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest() .indices(indicesToDelete.toArray(s)); - DeleteIndexResponse response = client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet(); - return new SuccessPruneResult(candidateIndices, indicesToDelete, response); + AcknowledgedResponse response = client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet(); + if( response.isAcknowledged()) { + logger.log(Level.INFO, "deletion of {} acknowledged, waiting for GREEN", Arrays.asList(s)); + waitForCluster("GREEN", 30L, TimeUnit.SECONDS); + return new SuccessPruneResult(candidateIndices, indicesToDelete, response); + } else { + logger.log(Level.WARN, "deletion of {} not acknowledged", Arrays.asList(s)); + return new FailPruneResult(candidateIndices, indicesToDelete, response); + } } @Override public Long mostRecentDocument(String index, String timestampfieldname) { ensureActive(); - SortBuilder sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC); - SearchSourceBuilder builder = new SearchSourceBuilder(); - builder.sort(sort); - builder.storedField(timestampfieldname); - builder.size(1); - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices(index); - searchRequest.source(builder); + SortBuilder sort = SortBuilders + .fieldSort(timestampfieldname) + .order(SortOrder.DESC); + SearchSourceBuilder builder = new SearchSourceBuilder() + .sort(sort) + .storedField(timestampfieldname) + .size(1); + SearchRequest searchRequest = new SearchRequest() + .indices(index) + .source(builder); SearchResponse searchResponse = client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); if (searchResponse.getHits().getHits().length == 1) { @@ -912,17 +933,17 @@ public abstract class AbstractExtendedClient implements ExtendedClient { private void checkMapping(String index, String type, MappingMetaData mappingMetaData) { try { - SearchSourceBuilder builder = new SearchSourceBuilder(); - builder.query(QueryBuilders.matchAllQuery()); - builder.size(0); - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices(index); - searchRequest.types(type); - searchRequest.source(builder); + SearchSourceBuilder builder = new SearchSourceBuilder() + .query(QueryBuilders.matchAllQuery()) + .size(0) + .trackTotalHits(true); + SearchRequest searchRequest = new SearchRequest() + .indices(index) + .source(builder); SearchResponse searchResponse = client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); - long total = searchResponse.getHits().getTotalHits(); - if (total > 0L) { + TotalHits total = searchResponse.getHits().getTotalHits(); + if (total.value > 0L) { Map fields = new TreeMap<>(); Map root = mappingMetaData.getSourceAsMap(); checkMapping(index, type, "", "", root, fields); @@ -930,9 +951,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { Map map = sortByValue(fields); map.forEach((key, value) -> { logger.info("{} {} {}", - key, - value, - (double) value * 100 / total); + key, value, (double) value * 100 / total.value); if (value == 0) { empty.incrementAndGet(); } @@ -976,23 +995,23 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } else if ("type".equals(key)) { QueryBuilder filterBuilder = QueryBuilders.existsQuery(path); QueryBuilder queryBuilder = QueryBuilders.constantScoreQuery(filterBuilder); - SearchSourceBuilder builder = new SearchSourceBuilder(); - builder.query(queryBuilder); - builder.size(0); - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices(index); - searchRequest.types(type); - searchRequest.source(builder); + SearchSourceBuilder builder = new SearchSourceBuilder() + .query(queryBuilder) + .size(0) + .trackTotalHits(true); + SearchRequest searchRequest = new SearchRequest() + .indices(index) + .source(builder); SearchResponse searchResponse = client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); - fields.put(path, searchResponse.getHits().getTotalHits()); + fields.put(path, searchResponse.getHits().getTotalHits().value); } } } private static > Map sortByValue(Map map) { Map result = new LinkedHashMap<>(); - map.entrySet().stream().sorted(Comparator.comparing(Map.Entry::getValue)) + map.entrySet().stream().sorted(Map.Entry.comparingByValue()) .forEachOrdered(e -> result.put(e.getKey(), e.getValue())); return result; } @@ -1046,10 +1065,46 @@ public abstract class AbstractExtendedClient implements ExtendedClient { List indicesToDelete; - DeleteIndexResponse response; + AcknowledgedResponse response; SuccessPruneResult(List candidateIndices, List indicesToDelete, - DeleteIndexResponse response) { + AcknowledgedResponse response) { + this.candidateIndices = candidateIndices; + this.indicesToDelete = indicesToDelete; + this.response = response; + } + + @Override + public IndexPruneResult.State getState() { + return IndexPruneResult.State.SUCCESS; + } + + @Override + public List getCandidateIndices() { + return candidateIndices; + } + + @Override + public List getDeletedIndices() { + return indicesToDelete; + } + + @Override + public boolean isAcknowledged() { + return response.isAcknowledged(); + } + } + + private static class FailPruneResult implements IndexPruneResult { + + List candidateIndices; + + List indicesToDelete; + + AcknowledgedResponse response; + + FailPruneResult(List candidateIndices, List indicesToDelete, + AcknowledgedResponse response) { this.candidateIndices = candidateIndices; this.indicesToDelete = indicesToDelete; this.response = response; diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java index 432cae4..53970a3 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java @@ -125,7 +125,7 @@ public class DefaultBulkController implements BulkController { } try { if (bulkMetric != null) { - bulkMetric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id()); + bulkMetric.getCurrentIngest().inc(indexRequest.index(), "_doc", indexRequest.id()); } bulkProcessor.add(indexRequest); } catch (Exception e) { @@ -144,7 +144,7 @@ public class DefaultBulkController implements BulkController { } try { if (bulkMetric != null) { - bulkMetric.getCurrentIngest().inc(deleteRequest.index(), deleteRequest.type(), deleteRequest.id()); + bulkMetric.getCurrentIngest().inc(deleteRequest.index(), "_doc", deleteRequest.id()); } bulkProcessor.add(deleteRequest); } catch (Exception e) { @@ -163,7 +163,7 @@ public class DefaultBulkController implements BulkController { } try { if (bulkMetric != null) { - bulkMetric.getCurrentIngest().inc(updateRequest.index(), updateRequest.type(), updateRequest.id()); + bulkMetric.getCurrentIngest().inc(updateRequest.index(), "_doc", updateRequest.id()); } bulkProcessor.add(updateRequest); } catch (Exception e) { diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java index f80742a..811f7de 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java @@ -140,19 +140,7 @@ public class DefaultBulkProcessor implements BulkProcessor { */ @Override public DefaultBulkProcessor add(ActionRequest request) { - return add(request, null); - } - - /** - * Adds either a delete or an index request with a payload. - * - * @param request request - * @param payload payload - * @return his bulk processor - */ - @Override - public DefaultBulkProcessor add(ActionRequest request, Object payload) { - internalAdd(request, payload); + internalAdd(request); return this; } @@ -186,14 +174,14 @@ public class DefaultBulkProcessor implements BulkProcessor { } } - private synchronized void internalAdd(ActionRequest request, Object payload) { + private synchronized void internalAdd(ActionRequest request) { ensureOpen(); if (request instanceof IndexRequest) { - bulkRequest.add((IndexRequest) request, payload); + bulkRequest.add((IndexRequest) request); } else if (request instanceof DeleteRequest) { - bulkRequest.add((DeleteRequest) request, payload); + bulkRequest.add((DeleteRequest) request); } else if (request instanceof UpdateRequest) { - bulkRequest.add((UpdateRequest) request, payload); + bulkRequest.add((UpdateRequest) request); } else { throw new UnsupportedOperationException(); } diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/MockNode.java b/elx-common/src/test/java/org/xbib/elx/common/test/MockNode.java index 48df8a0..941c2de 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/MockNode.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/MockNode.java @@ -6,10 +6,12 @@ import org.elasticsearch.node.Node; import org.elasticsearch.plugins.Plugin; import java.util.List; +import java.util.Map; class MockNode extends Node { MockNode(Settings settings, List> classpathPlugins) { - super(InternalSettingsPreparer.prepareEnvironment(settings, null), classpathPlugins); + super(InternalSettingsPreparer.prepareEnvironment(settings, Map.of(), null, () -> "mock"), + classpathPlugins, false); } } diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java index d1a4412..eccdb08 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java @@ -33,7 +33,7 @@ class SearchTest { ElasticsearchClient client = helper.client("1"); BulkRequestBuilder builder = new BulkRequestBuilder(client, BulkAction.INSTANCE); for (int i = 0; i < 1000; i++) { - IndexRequest indexRequest = new IndexRequest("pages", "row") + IndexRequest indexRequest = new IndexRequest().index("pages") .source(XContentFactory.jsonBuilder() .startObject() .field("user1", "joerg") @@ -61,10 +61,9 @@ class SearchTest { searchSource.size(10); SearchRequest searchRequest = new SearchRequest(); searchRequest.indices("pages"); - searchRequest.types("row"); searchRequest.source(searchSource); SearchResponse searchResponse = client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); - assertTrue(searchResponse.getHits().getTotalHits() > 0); + assertTrue(searchResponse.getHits().getTotalHits().value > 0); } } } diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java index 4c2e7ba..6bd38e5 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java @@ -41,13 +41,13 @@ class SimpleTest { Settings indexSettings = Settings.builder() .put("index.analysis.analyzer.default.filter.0", "lowercase") .put("index.analysis.analyzer.default.filter.1", "trim") - .put("index.analysis.analyzer.default.tokenizer", "keyword") + .put("index.analysis.analyzer.default.type", "keyword") .build(); CreateIndexRequest createIndexRequest = new CreateIndexRequest(); createIndexRequest.index("test").settings(indexSettings); helper.client("1").execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); IndexRequest indexRequest = new IndexRequest(); - indexRequest.index("test").type("test").id("1") + indexRequest.index("test").id("1") .source(XContentFactory.jsonBuilder().startObject().field("field", "1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8").endObject()); helper.client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet(); @@ -58,7 +58,7 @@ class SimpleTest { builder.query(QueryBuilders.matchQuery("field", "1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8")); SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices("test").types("test"); + searchRequest.indices("test"); searchRequest.source(builder); String doc = helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet() .getHits().getAt(0).getSourceAsString(); diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java b/elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java index 7a4b476..e8a3b56 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.analysis.common.CommonAnalysisPlugin; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.cluster.health.ClusterHealthStatus; @@ -22,7 +21,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeValidationException; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.transport.netty4.Netty4Plugin; +import org.elasticsearch.transport.Netty4Plugin; import org.junit.jupiter.api.extension.AfterEachCallback; import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; @@ -37,7 +36,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; -import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -188,8 +187,8 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft return Settings.builder() .put("cluster.name", getClusterName()) .put("path.home", getHome()) - .put("discovery.zen.master_election.ignore_non_master_pings", "true") - .put("transport.netty.epoll", "false") + .put("cluster.initial_master_nodes", "1") + .put("discovery.seed_hosts", "127.0.0.1:9300") .build(); } @@ -215,7 +214,7 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft .put(getNodeSettings()) .put("node.name", id) .build(); - List> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class); + List> plugins = Collections.singletonList(Netty4Plugin.class); Node node = new MockNode(nodeSettings, plugins); AbstractClient client = (AbstractClient) node.client(); nodes.put(id, node); diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java index 0bc9b0f..f2c69bf 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java @@ -45,20 +45,20 @@ class WildcardTest { } private void index(ElasticsearchClient client, String id, String fieldValue) throws IOException { - client.execute(IndexAction.INSTANCE, new IndexRequest("index", "type", id) + client.execute(IndexAction.INSTANCE, new IndexRequest().index("index").id(id) .source(XContentFactory.jsonBuilder().startObject().field("field", fieldValue).endObject())) .actionGet(); client.execute(RefreshAction.INSTANCE, new RefreshRequest()).actionGet(); } private long count(ElasticsearchClient client, QueryBuilder queryBuilder) { - SearchSourceBuilder builder = new SearchSourceBuilder(); - builder.query(queryBuilder); - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices("index"); - searchRequest.types("type"); - searchRequest.source(builder); - return client.execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits(); + SearchSourceBuilder builder = new SearchSourceBuilder() + .query(queryBuilder) + .trackTotalHits(true); + SearchRequest searchRequest = new SearchRequest() + .indices("index") + .source(builder); + return client.execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits().value; } private void validateCount(ElasticsearchClient client, QueryBuilder queryBuilder, long expectedHits) { diff --git a/elx-http/build.gradle b/elx-http/build.gradle index c8966b5..05d5d69 100644 --- a/elx-http/build.gradle +++ b/elx-http/build.gradle @@ -2,7 +2,7 @@ import org.apache.tools.ant.taskdefs.condition.Os dependencies{ compile project(':elx-common') - compile "org.xbib.elasticsearch:transport-netty4:${rootProject.property('elasticsearch-server.version')}" + compile "org.elasticsearch.plugin:transport-netty4-client:${rootProject.property('elasticsearch.version')}" compile "org.xbib:netty-http-client:${project.property('xbib-netty-http.version')}" runtime "org.bouncycastle:bcpkix-jdk15on:${project.property('bouncycastle.version')}" if (Os.isFamily(Os.FAMILY_MAC)) { @@ -10,5 +10,4 @@ dependencies{ } else if (Os.isFamily(Os.FAMILY_UNIX)) { runtime "io.netty:netty-tcnative-boringssl-static:${project.property('tcnative.version')}" } - testCompile "org.xbib.elasticsearch:elasticsearch-analysis-common:${rootProject.property('elasticsearch-server.version')}" } diff --git a/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java b/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java index cda7a87..bf9aa8e 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java +++ b/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java @@ -3,13 +3,12 @@ package org.xbib.elx.http; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.GenericAction; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; @@ -43,7 +42,7 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti private final NamedXContentRegistry registry; @SuppressWarnings("rawtypes") - private final Map actionMap; + private final Map actionMap; private String url; @@ -109,28 +108,19 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti } @Override - public > ActionFuture - execute(Action action, Request request) { + public + ActionFuture execute(ActionType action, Request request) { PlainActionFuture actionFuture = PlainActionFuture.newFuture(); execute(action, request, actionFuture); return actionFuture; } @Override - public > void - execute(Action action, Request request, ActionListener listener) { + public + void execute(ActionType action, Request request, ActionListener listener) { doExecute(action, request, listener); } - @Override - public > RequestBuilder - prepareExecute(Action action) { - return action.newRequestBuilder(this); - } - @Override public ThreadPool threadPool() { logger.log(Level.DEBUG, "returning null for threadPool() request"); @@ -138,8 +128,8 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti } @SuppressWarnings({"unchecked", "rawtypes"}) - private > - void doExecute(Action action, R request, ActionListener listener) { + private > + void doExecute(ActionType action, R request, ActionListener listener) { HttpAction httpAction = actionMap.get(action); if (httpAction == null) { throw new IllegalStateException("failed to find http action [" + action + "] to execute"); diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpAction.java b/elx-http/src/main/java/org/xbib/elx/http/HttpAction.java index 816f2bd..5aa54f2 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/HttpAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpAction.java @@ -12,7 +12,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.GenericAction; +import org.elasticsearch.action.ActionType; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; @@ -25,6 +25,7 @@ import org.elasticsearch.rest.RestStatus; import org.xbib.net.URL; import org.xbib.netty.http.client.api.Request; import org.xbib.netty.http.client.api.Transport; +import org.xbib.netty.http.common.HttpResponse; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -47,7 +48,7 @@ public abstract class HttpAction getActionInstance(); + public abstract ActionType getActionInstance(); final void execute(HttpActionContext httpActionContext, ActionListener listener) throws IOException { try { @@ -71,7 +72,7 @@ public abstract class HttpAction httpActionContext) { + // we assume a non-empty, valid JSON response body. If there is none, this method must be overriden. try (XContentParser parser = XContentFactory.xContent(XContentType.JSON) .createParser(httpActionContext.getExtendedHttpClient().getRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, httpActionContext.getHttpResponse().getBody().toString(StandardCharsets.UTF_8))) { return errorParser().apply(parser); - } catch (IOException e) { + } catch (Exception e) { logger.error(e.getMessage(), e); return new ElasticsearchStatusException(e.getMessage(), RestStatus.INTERNAL_SERVER_ERROR, e); } @@ -183,8 +185,5 @@ public abstract class HttpAction entityParser(); - - protected abstract T emptyResponse(); - + protected abstract CheckedFunction entityParser(HttpResponse httpResponse); } diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/admin/cluster/health/HttpClusterHealthAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/admin/cluster/health/HttpClusterHealthAction.java index 09925b7..9a2e6c8 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/action/admin/cluster/health/HttpClusterHealthAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/admin/cluster/health/HttpClusterHealthAction.java @@ -7,6 +7,7 @@ import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.xcontent.XContentParser; import org.xbib.elx.http.HttpAction; import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; import java.io.IOException; @@ -23,12 +24,7 @@ public class HttpClusterHealthAction extends HttpAction entityParser() { + protected CheckedFunction entityParser(HttpResponse httpResponse) { return HttpClusterHealthResponse::fromXContent; } - - @Override - protected ClusterHealthResponse emptyResponse() { - return new HttpClusterHealthResponse(); - } } diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/admin/cluster/health/HttpClusterHealthResponse.java b/elx-http/src/main/java/org/xbib/elx/http/action/admin/cluster/health/HttpClusterHealthResponse.java index 2a4c3a1..50a0e5f 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/action/admin/cluster/health/HttpClusterHealthResponse.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/admin/cluster/health/HttpClusterHealthResponse.java @@ -6,36 +6,13 @@ import org.elasticsearch.cluster.health.ClusterIndexHealth; import org.elasticsearch.cluster.health.ClusterStateHealth; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.rest.RestStatus; import java.io.IOException; -import java.util.Collections; import java.util.Map; -import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; - public class HttpClusterHealthResponse extends ClusterHealthResponse { - private static final String CLUSTER_NAME = "cluster_name"; - private static final String STATUS = "status"; - private static final String TIMED_OUT = "timed_out"; - private static final String NUMBER_OF_NODES = "number_of_nodes"; - private static final String NUMBER_OF_DATA_NODES = "number_of_data_nodes"; - private static final String NUMBER_OF_PENDING_TASKS = "number_of_pending_tasks"; - private static final String NUMBER_OF_IN_FLIGHT_FETCH = "number_of_in_flight_fetch"; - private static final String DELAYED_UNASSIGNED_SHARDS = "delayed_unassigned_shards"; - private static final String TASK_MAX_WAIT_TIME_IN_QUEUE = "task_max_waiting_in_queue"; - private static final String TASK_MAX_WAIT_TIME_IN_QUEUE_IN_MILLIS = "task_max_waiting_in_queue_millis"; - private static final String ACTIVE_SHARDS_PERCENT_AS_NUMBER = "active_shards_percent_as_number"; - private static final String ACTIVE_SHARDS_PERCENT = "active_shards_percent"; - private static final String ACTIVE_PRIMARY_SHARDS = "active_primary_shards"; - private static final String ACTIVE_SHARDS = "active_shards"; - private static final String RELOCATING_SHARDS = "relocating_shards"; - private static final String INITIALIZING_SHARDS = "initializing_shards"; - private static final String UNASSIGNED_SHARDS = "unassigned_shards"; - private static final String INDICES = "indices"; - private String clusterName; private ClusterStateHealth clusterStateHealth; @@ -93,11 +70,6 @@ public class HttpClusterHealthResponse extends ClusterHealthResponse { return clusterName; } - @Override - public ClusterStateHealth getClusterStateHealth() { - return clusterStateHealth; - } - @Override public boolean isTimedOut() { return this.timedOut; @@ -172,44 +144,4 @@ public class HttpClusterHealthResponse extends ClusterHealthResponse { public RestStatus status() { return isTimedOut() ? RestStatus.REQUEST_TIMEOUT : RestStatus.OK; } - - public static HttpClusterHealthResponse fromXContent(XContentParser parser) throws IOException { - XContentParser.Token token = parser.nextToken(); - ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation); - Map map = parser.map(); - String clusterName = (String) map.get(CLUSTER_NAME); - ClusterHealthStatus status = ClusterHealthStatus.fromString((String) map.get(STATUS)); - Boolean timedOut = (Boolean) map.get(TIMED_OUT); - Integer numberOfNodes = (Integer) map.get(NUMBER_OF_NODES); - Integer numberOfDataNodes = (Integer) map.get(NUMBER_OF_DATA_NODES); - Integer activePrimaryShards = (Integer) map.get(ACTIVE_PRIMARY_SHARDS); - Integer activeShards = (Integer) map.get(ACTIVE_SHARDS); - Integer relocatingShards = (Integer) map.get(RELOCATING_SHARDS); - Integer initializingShards = (Integer) map.get(INITIALIZING_SHARDS); - Integer unassignedShards = (Integer) map.get(UNASSIGNED_SHARDS); - Integer delayedUnassignedShards = (Integer) map.get(DELAYED_UNASSIGNED_SHARDS); - Integer numberOfPendingTasks = (Integer) map.get(NUMBER_OF_PENDING_TASKS); - Integer numberOfInFlightFetch = (Integer) map.get(NUMBER_OF_IN_FLIGHT_FETCH); - Integer taskMaxWaitingInQueueMillis = (Integer) map.get(TASK_MAX_WAIT_TIME_IN_QUEUE_IN_MILLIS); - Double activeShardsPercentAsNumber = (Double) map.get(ACTIVE_SHARDS_PERCENT_AS_NUMBER); - HttpClusterHealthResponse clusterHealthResponse = new HttpClusterHealthResponse(); - clusterHealthResponse.init(clusterName, - status, - timedOut, - numberOfNodes, - numberOfDataNodes, - Collections.emptyMap(), - activePrimaryShards, - activeShards, - relocatingShards, - initializingShards, - unassignedShards, - delayedUnassignedShards, - numberOfPendingTasks, - numberOfInFlightFetch, - TimeValue.timeValueMillis(taskMaxWaitingInQueueMillis), - activeShardsPercentAsNumber - ); - return clusterHealthResponse; - } } \ No newline at end of file diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/admin/cluster/node/info/HttpNodesInfoAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/admin/cluster/node/info/HttpNodesInfoAction.java index ac0c39d..d2b562d 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/action/admin/cluster/node/info/HttpNodesInfoAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/admin/cluster/node/info/HttpNodesInfoAction.java @@ -1,39 +1,17 @@ package org.xbib.elx.http.action.admin.cluster.node.info; -import org.elasticsearch.Build; -import org.elasticsearch.Version; -import org.elasticsearch.action.FailedNodeException; -import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.CheckedFunction; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.http.HttpInfo; -import org.elasticsearch.ingest.IngestInfo; -import org.elasticsearch.monitor.jvm.JvmInfo; -import org.elasticsearch.monitor.os.OsInfo; -import org.elasticsearch.monitor.process.ProcessInfo; -import org.elasticsearch.threadpool.ThreadPoolInfo; -import org.elasticsearch.transport.TransportInfo; import org.xbib.elx.http.HttpAction; -import org.xbib.elx.http.HttpActionContext; import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; import java.io.IOException; -import java.io.UncheckedIOException; -import java.net.InetAddress; -import java.util.Collections; -import java.util.HashSet; import java.util.LinkedList; import java.util.List; -import java.util.Map; -import java.util.Set; /** * @@ -95,84 +73,7 @@ public class HttpNodesInfoAction extends HttpAction entityParser() { + protected CheckedFunction entityParser(HttpResponse httpResponse) { throw new UnsupportedOperationException(); } - - @Override - protected NodesInfoResponse emptyResponse() { - return new NodesInfoResponse(); - } - - @SuppressWarnings("unchecked") - protected NodesInfoResponse createResponse(HttpActionContext httpContext) { - // BROKEN - Map map = null; - //String string = (String)map.get("cluster_name"); - ClusterName clusterName = new ClusterName(""); - List nodeInfoList = new LinkedList<>(); - //map = (Map)map.get("nodes"); - - for (Map.Entry entry : map.entrySet()) { - String nodeId = entry.getKey(); - String ephemeralId = null; - Map map2 = (Map) entry.getValue(); - String nodeName = (String) map2.get("name"); - String hostName = (String) map2.get("host"); - String hostAddress = (String) map2.get("ip"); - // [/][:] - String transportAddressString = (String) map2.get("transport_address"); - int pos = transportAddressString.indexOf(':'); - String host = pos > 0 ? transportAddressString.substring(0, pos) : transportAddressString; - int port = Integer.parseInt(pos > 0 ? transportAddressString.substring(pos + 1) : "0"); - pos = host.indexOf('/'); - host = pos > 0 ? host.substring(0, pos) : host; - try { - InetAddress[] inetAddresses = InetAddress.getAllByName(host); - TransportAddress transportAddress = new TransportAddress(inetAddresses[0], port); - Build build = new Build(Build.Flavor.OSS, Build.Type.TAR, - (String) map2.get("build"), - (String) map2.get("date"), - (Boolean) map2.get("snapshot")); - Map attributes = Collections.emptyMap(); - Set roles = new HashSet<>(); - Version version = Version.fromString((String) map2.get("version")); - DiscoveryNode discoveryNode = new DiscoveryNode(nodeName, nodeId, ephemeralId, hostName, hostAddress, - transportAddress, - attributes, roles, version); - /*Map settingsMap = map2.containsKey("settings") ? - XContentHelper. - SettingsLoader.Helper.loadNestedFromMap((Map) map2.get("settings")) : - Collections.emptyMap(); - - Settings settings = Settings.builder() - - .put(settingsMap) - .build();*/ - OsInfo os = null; - ProcessInfo processInfo = null; - JvmInfo jvmInfo = null; - ThreadPoolInfo threadPoolInfo = null; - TransportInfo transportInfo = null; - HttpInfo httpInfo = null; - PluginsAndModules pluginsAndModules = null; - IngestInfo ingestInfo = null; - ByteSizeValue totalIndexingBuffer = null; - NodeInfo nodeInfo = new NodeInfo(version, - build, - discoveryNode, - //serviceAttributes, - //settings, - null, - os, processInfo, jvmInfo, threadPoolInfo, transportInfo, httpInfo, pluginsAndModules, - ingestInfo, - totalIndexingBuffer); - nodeInfoList.add(nodeInfo); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - List failures = null; - return new NodesInfoResponse(clusterName, nodeInfoList, failures); - } } diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/admin/cluster/settings/HttpClusterUpdateSettingsAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/admin/cluster/settings/HttpClusterUpdateSettingsAction.java index 9090fe3..e882a2d 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/action/admin/cluster/settings/HttpClusterUpdateSettingsAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/admin/cluster/settings/HttpClusterUpdateSettingsAction.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.xbib.elx.http.HttpAction; import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; import java.io.IOException; import java.io.UncheckedIOException; @@ -40,12 +41,7 @@ public class HttpClusterUpdateSettingsAction extends HttpAction entityParser() { + protected CheckedFunction entityParser(HttpResponse httpResponse) { return ClusterUpdateSettingsResponse::fromXContent; } - - @Override - protected ClusterUpdateSettingsResponse emptyResponse() { - return new ClusterUpdateSettingsResponse(); - } } diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/admin/cluster/state/HttpClusterStateAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/admin/cluster/state/HttpClusterStateAction.java index 18cd26d..2f4693c 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/action/admin/cluster/state/HttpClusterStateAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/admin/cluster/state/HttpClusterStateAction.java @@ -1,7 +1,6 @@ package org.xbib.elx.http.action.admin.cluster.state; import com.carrotsearch.hppc.LongArrayList; -import org.apache.logging.log4j.Level; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; @@ -20,6 +19,7 @@ import org.elasticsearch.common.xcontent.NamedObjectNotFoundException; import org.elasticsearch.common.xcontent.XContentParser; import org.xbib.elx.http.HttpAction; import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; import java.io.IOException; import java.util.ArrayList; @@ -73,15 +73,10 @@ public class HttpClusterStateAction extends HttpAction entityParser() { + protected CheckedFunction entityParser(HttpResponse httpResponse) { return this::fromXContent; } - @Override - protected ClusterStateResponse emptyResponse() { - return new ClusterStateResponse(); - } - private ClusterStateResponse fromXContent(XContentParser parser) throws IOException { XContentParser.Token token = parser.nextToken(); ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation); @@ -109,7 +104,7 @@ public class HttpClusterStateAction extends HttpAction { +public class HttpIndicesAliasesAction extends HttpAction { @Override public IndicesAliasesAction getActionInstance() { @@ -37,12 +38,7 @@ public class HttpIndicesAliasesAction extends HttpAction entityParser() { - return IndicesAliasesResponse::fromXContent; - } - - @Override - protected IndicesAliasesResponse emptyResponse() { - return new IndicesAliasesResponse(); + protected CheckedFunction entityParser(HttpResponse httpResponse) { + return AcknowledgedResponse::fromXContent; } } diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/alias/get/HttpGetAliasAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/alias/get/HttpGetAliasAction.java index e2b37d0..bd44439 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/alias/get/HttpGetAliasAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/alias/get/HttpGetAliasAction.java @@ -1,6 +1,6 @@ package org.xbib.elx.http.action.admin.indices.alias.get; -import org.elasticsearch.action.GenericAction; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; @@ -10,6 +10,7 @@ import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.xcontent.XContentParser; import org.xbib.elx.http.HttpAction; import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; import java.io.IOException; import java.util.ArrayList; @@ -29,21 +30,15 @@ public class HttpGetAliasAction extends HttpAction getActionInstance() { + public ActionType getActionInstance() { return GetAliasesAction.INSTANCE; } @Override - protected CheckedFunction entityParser() { + protected CheckedFunction entityParser(HttpResponse httpResponse) { return this::fromXContent; } - @Override - protected GetAliasesResponse emptyResponse() { - ImmutableOpenMap.Builder> aliasesBuilder = ImmutableOpenMap.builder(); - return new GetAliasesResponse(aliasesBuilder.build()); - } - private GetAliasesResponse fromXContent(XContentParser parser) throws IOException { if (parser.currentToken() == null) { parser.nextToken(); diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/create/HttpCreateIndexAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/create/HttpCreateIndexAction.java index 2c867db..5eea544 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/create/HttpCreateIndexAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/create/HttpCreateIndexAction.java @@ -1,21 +1,32 @@ package org.xbib.elx.http.action.admin.indices.create; +import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.xbib.elx.http.HttpAction; import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; import java.io.IOException; +import java.util.Map; public class HttpCreateIndexAction extends HttpAction { + public static final ParseField MAPPINGS = new ParseField("mappings"); + public static final ParseField SETTINGS = new ParseField("settings"); + public static final ParseField ALIASES = new ParseField("aliases"); + @Override public CreateIndexAction getActionInstance() { return CreateIndexAction.INSTANCE; @@ -23,18 +34,40 @@ public class HttpCreateIndexAction extends HttpAction entityParser() { + protected CheckedFunction entityParser(HttpResponse httpResponse) { return CreateIndexResponse::fromXContent; } - @Override - protected CreateIndexResponse emptyResponse() { - return new CreateIndexResponse(); + // fixed version from CreateIndexRequest - use only one mapping + private XContentBuilder toXContent(CreateIndexRequest createIndexRequest, + XContentBuilder builder) throws IOException { + builder.startObject(); + builder.startObject(SETTINGS.getPreferredName()); + createIndexRequest.settings().toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + // there is only an empty or a single entry for mappings + if (createIndexRequest.mappings().isEmpty()) { + // ES wants a mappings element with an empty map + builder.startObject(MAPPINGS.getPreferredName()); + builder.endObject(); + } else { + Map mappingAsMap = createIndexRequest.mappings(); + String mappingString = mappingAsMap.values().iterator().next().toString(); + builder.field(MAPPINGS.getPreferredName()); + builder.map(XContentHelper.convertToMap(new BytesArray(mappingString), false, XContentType.JSON).v2()); + } + builder.startObject(ALIASES.getPreferredName()); + for (Alias alias : createIndexRequest.aliases()) { + alias.toXContent(builder, ToXContent.EMPTY_PARAMS); + } + builder.endObject(); + builder.endObject(); + return builder; } } diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/delete/HttpDeleteIndexAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/delete/HttpDeleteIndexAction.java index 0c81ff5..12fc864 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/delete/HttpDeleteIndexAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/delete/HttpDeleteIndexAction.java @@ -2,15 +2,16 @@ package org.xbib.elx.http.action.admin.indices.delete; import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.xcontent.XContentParser; import org.xbib.elx.http.HttpAction; import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; import java.io.IOException; -public class HttpDeleteIndexAction extends HttpAction { +public class HttpDeleteIndexAction extends HttpAction { @Override public DeleteIndexAction getActionInstance() { @@ -23,12 +24,7 @@ public class HttpDeleteIndexAction extends HttpAction entityParser() { - return DeleteIndexResponse::fromXContent; - } - - @Override - protected DeleteIndexResponse emptyResponse() { - return new DeleteIndexResponse(); + protected CheckedFunction entityParser(HttpResponse httpResponse) { + return AcknowledgedResponse::fromXContent; } } diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/exists/indices/HttpIndicesExistsAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/exists/indices/HttpIndicesExistsAction.java index 69a94a1..26ac3db 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/exists/indices/HttpIndicesExistsAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/exists/indices/HttpIndicesExistsAction.java @@ -10,6 +10,7 @@ import org.elasticsearch.rest.RestStatus; import org.xbib.elx.http.HttpAction; import org.xbib.elx.http.HttpActionContext; import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; import java.io.IOException; @@ -26,22 +27,27 @@ public class HttpIndicesExistsAction extends HttpAction entityParser() { - return this::fromXContent; - } - - @Override - protected IndicesExistsResponse emptyResponse() { - return new IndicesExistsResponse(false); // used for 404 Not found - } - + /** + * Override for non-body logic. + * @param httpActionContext the HTTP action context + * @return the ELasticsearch sttatus exception + */ @Override protected ElasticsearchStatusException parseToError(HttpActionContext httpActionContext) { return new ElasticsearchStatusException("not found", RestStatus.NOT_FOUND); } - private IndicesExistsResponse fromXContent(XContentParser parser) throws IOException { - return new IndicesExistsResponse(true); // used for 200 OK + @Override + protected CheckedFunction entityParser(HttpResponse httpResponse) { + return httpResponse.getStatus().getCode() == 200 ? this::found : this::notfound; } + + private IndicesExistsResponse found(XContentParser parser) { + return new IndicesExistsResponse(true); + } + + private IndicesExistsResponse notfound(XContentParser parser) { + return new IndicesExistsResponse(false); + } + } \ No newline at end of file diff --git a/elx-http/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/HttpGetMappingsAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/mapping/get/HttpGetMappingsAction.java similarity index 58% rename from elx-http/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/HttpGetMappingsAction.java rename to elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/mapping/get/HttpGetMappingsAction.java index d8f3ddd..3fdeacf 100644 --- a/elx-http/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/HttpGetMappingsAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/mapping/get/HttpGetMappingsAction.java @@ -1,5 +1,8 @@ -package org.elasticsearch.action.admin.indices.mapping.get; +package org.xbib.elx.http.action.admin.indices.mapping.get; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.ParseField; @@ -7,12 +10,10 @@ import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.xcontent.XContentParser; import org.xbib.elx.http.HttpAction; import org.xbib.netty.http.client.api.Request; - +import org.xbib.netty.http.common.HttpResponse; import java.io.IOException; import java.util.Map; -import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; - public class HttpGetMappingsAction extends HttpAction { private static final ParseField MAPPINGS = new ParseField("mappings"); @@ -29,33 +30,26 @@ public class HttpGetMappingsAction extends HttpAction entityParser() { + protected CheckedFunction entityParser(HttpResponse httpResponse) { return this::fromXContent; } - @Override - protected GetMappingsResponse emptyResponse() { - return new GetMappingsResponse(); - } - + // fixed version from GetMappingsRequest - use only one mapping per index with type "_doc" @SuppressWarnings("unchecked") private GetMappingsResponse fromXContent(XContentParser parser) throws IOException { - XContentParser.Token token = parser.nextToken(); - ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation); - Map parts = parser.map(); + if (parser.currentToken() == null) { + parser.nextToken(); + } + Map map = parser.map(); ImmutableOpenMap.Builder> builder = new ImmutableOpenMap.Builder<>(); - for (Map.Entry entry : parts.entrySet()) { - String indexName = entry.getKey(); - Map mapping = (Map) ((Map) entry.getValue()).get(MAPPINGS.getPreferredName()); + for (Map.Entry entry : map.entrySet()) { + final String indexName = entry.getKey(); + final Map mapping = (Map) ((Map) entry.getValue()).get(MAPPINGS.getPreferredName()); ImmutableOpenMap.Builder typeBuilder = new ImmutableOpenMap.Builder<>(); - for (Map.Entry typeEntry : mapping.entrySet()) { - String typeName = typeEntry.getKey(); - Map fieldMappings = (Map) typeEntry.getValue(); - MappingMetaData mmd = new MappingMetaData(typeName, fieldMappings); - typeBuilder.put(typeName, mmd); - } + MappingMetaData mmd = new MappingMetaData("_doc", mapping); + typeBuilder.put("_doc", mmd); builder.put(indexName, typeBuilder.build()); } return new GetMappingsResponse(builder.build()); } -} \ No newline at end of file +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/refresh/HttpRefreshIndexAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/refresh/HttpRefreshIndexAction.java index c6797ac..a38ef59 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/refresh/HttpRefreshIndexAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/refresh/HttpRefreshIndexAction.java @@ -7,6 +7,7 @@ import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.xcontent.XContentParser; import org.xbib.elx.http.HttpAction; import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; import java.io.IOException; @@ -24,12 +25,7 @@ public class HttpRefreshIndexAction extends HttpAction entityParser() { + protected CheckedFunction entityParser(HttpResponse httpResponse) { return RefreshResponse::fromXContent; } - - @Override - protected RefreshResponse emptyResponse() { - return new RefreshResponse(); - } } diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/settings/get/HttpGetSettingsAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/settings/get/HttpGetSettingsAction.java index 89b0503..3aee38e 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/settings/get/HttpGetSettingsAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/settings/get/HttpGetSettingsAction.java @@ -4,16 +4,12 @@ import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.common.CheckedFunction; -import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentParserUtils; import org.xbib.elx.http.HttpAction; import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; public class HttpGetSettingsAction extends HttpAction { @@ -30,62 +26,7 @@ public class HttpGetSettingsAction extends HttpAction entityParser() { - return this::fromXContent; - } - - @Override - protected GetSettingsResponse emptyResponse() { - ImmutableOpenMap settingsMap = ImmutableOpenMap.builder().build(); - return new GetSettingsResponse(settingsMap); - } - - private GetSettingsResponse fromXContent(XContentParser parser) throws IOException { - Map indexToSettings = new HashMap<>(); - Map indexToDefaultSettings = new HashMap<>(); - if (parser.currentToken() == null) { - parser.nextToken(); - } - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); - parser.nextToken(); - while (!parser.isClosed()) { - if (parser.currentToken() == XContentParser.Token.START_OBJECT) { - parseIndexEntry(parser, indexToSettings, indexToDefaultSettings); - } else if (parser.currentToken() == XContentParser.Token.START_ARRAY) { - parser.skipChildren(); - } else { - parser.nextToken(); - } - } - ImmutableOpenMap settingsMap = ImmutableOpenMap.builder().putAll(indexToSettings).build(); - return new GetSettingsResponse(settingsMap); - } - - private static void parseIndexEntry(XContentParser parser, Map indexToSettings, - Map indexToDefaultSettings) throws IOException { - String indexName = parser.currentName(); - parser.nextToken(); - while (!parser.isClosed() && parser.currentToken() != XContentParser.Token.END_OBJECT) { - parseSettingsField(parser, indexName, indexToSettings, indexToDefaultSettings); - } - } - - private static void parseSettingsField(XContentParser parser, String currentIndexName, Map indexToSettings, - Map indexToDefaultSettings) throws IOException { - if (parser.currentToken() == XContentParser.Token.START_OBJECT) { - switch (parser.currentName()) { - case "settings": - indexToSettings.put(currentIndexName, Settings.fromXContent(parser)); - break; - case "defaults": - indexToDefaultSettings.put(currentIndexName, Settings.fromXContent(parser)); - break; - default: - parser.skipChildren(); - } - } else if (parser.currentToken() == XContentParser.Token.START_ARRAY) { - parser.skipChildren(); - } - parser.nextToken(); + protected CheckedFunction entityParser(HttpResponse httpResponse) { + return GetSettingsResponse::fromXContent; } } diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/settings/put/HttpUpdateSettingsAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/settings/put/HttpUpdateSettingsAction.java index d501522..d06e7b0 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/settings/put/HttpUpdateSettingsAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/settings/put/HttpUpdateSettingsAction.java @@ -2,7 +2,7 @@ package org.xbib.elx.http.action.admin.indices.settings.put; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsResponse; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.ToXContent; @@ -11,11 +11,12 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.xbib.elx.http.HttpAction; import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; import java.io.IOException; import java.io.UncheckedIOException; -public class HttpUpdateSettingsAction extends HttpAction { +public class HttpUpdateSettingsAction extends HttpAction { @Override public UpdateSettingsAction getActionInstance() { @@ -37,12 +38,7 @@ public class HttpUpdateSettingsAction extends HttpAction entityParser() { - return UpdateSettingsResponse::fromXContent; - } - - @Override - protected UpdateSettingsResponse emptyResponse() { - return new UpdateSettingsResponse(); + protected CheckedFunction entityParser(HttpResponse httpResponse) { + return AcknowledgedResponse::fromXContent; } } diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/bulk/HttpBulkAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/bulk/HttpBulkAction.java index b2a5689..2bef62f 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/action/bulk/HttpBulkAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/bulk/HttpBulkAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.xbib.elx.http.HttpAction; import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; import java.io.IOException; @@ -32,16 +33,13 @@ public class HttpBulkAction extends HttpAction { IndexRequest indexRequest = (IndexRequest) actionRequest; bulkContent.append("{\"").append(indexRequest.opType().getLowercase()).append("\":{"); bulkContent.append("\"_index\":\"").append(indexRequest.index()).append("\""); - bulkContent.append(",\"_type\":\"").append(indexRequest.type()).append("\""); + bulkContent.append(",\"_type\":\"").append("_doc").append("\""); if (indexRequest.id() != null) { bulkContent.append(",\"_id\":\"").append(indexRequest.id()).append("\""); } if (indexRequest.routing() != null) { bulkContent.append(",\"_routing\":\"").append(indexRequest.routing()).append("\""); } - if (indexRequest.parent() != null) { - bulkContent.append(",\"_parent\":\"").append(indexRequest.parent()).append("\""); - } if (indexRequest.version() > 0) { bulkContent.append(",\"_version\":\"").append(indexRequest.version()).append("\""); if (indexRequest.versionType() != null) { @@ -55,14 +53,11 @@ public class HttpBulkAction extends HttpAction { UpdateRequest updateRequest = (UpdateRequest) actionRequest; bulkContent.append("{\"update\":{"); bulkContent.append("\"_index\":\"").append(updateRequest.index()).append("\""); - bulkContent.append(",\"_type\":\"").append(updateRequest.type()).append("\""); + bulkContent.append(",\"_type\":\"").append("_doc").append("\""); bulkContent.append(",\"_id\":\"").append(updateRequest.id()).append("\""); if (updateRequest.routing() != null) { bulkContent.append(",\"_routing\":\"").append(updateRequest.routing()).append("\""); } - if (updateRequest.parent() != null) { - bulkContent.append(",\"_parent\":\"").append(updateRequest.parent()).append("\""); - } if (updateRequest.version() > 0) { bulkContent.append(",\"_version\":\"").append(updateRequest.version()).append("\""); if (updateRequest.versionType() != null) { @@ -80,7 +75,7 @@ public class HttpBulkAction extends HttpAction { DeleteRequest deleteRequest = (DeleteRequest) actionRequest; bulkContent.append("{\"delete\":{"); bulkContent.append("\"_index\":\"").append(deleteRequest.index()).append("\""); - bulkContent.append(",\"_type\":\"").append(deleteRequest.type()).append("\""); + bulkContent.append(",\"_type\":\"").append("_doc").append("\""); bulkContent.append(",\"_id\":\"").append(deleteRequest.id()).append("\""); if (deleteRequest.routing() != null) { bulkContent.append(",\"_routing\":\"").append(deleteRequest.routing()).append("\""); // _routing @@ -92,14 +87,7 @@ public class HttpBulkAction extends HttpAction { } @Override - protected CheckedFunction entityParser() { + protected CheckedFunction entityParser(HttpResponse httpResponse) { return BulkResponse::fromXContent; } - - @Override - protected BulkResponse emptyResponse() { - BulkItemResponse[] responses = null; - long took = 0L; - return new BulkResponse(responses, took); - } } diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/get/HttpExistsAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/get/HttpExistsAction.java index 5a6e39f..4b33932 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/action/get/HttpExistsAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/get/HttpExistsAction.java @@ -1,6 +1,6 @@ package org.xbib.elx.http.action.get; -import org.elasticsearch.action.GenericAction; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.get.GetAction; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; @@ -8,28 +8,24 @@ import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.xcontent.XContentParser; import org.xbib.elx.http.HttpAction; import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; import java.io.IOException; public class HttpExistsAction extends HttpAction { @Override - public GenericAction getActionInstance() { + public ActionType getActionInstance() { return GetAction.INSTANCE; } @Override protected Request.Builder createHttpRequest(String url, GetRequest request) { - return newHeadRequest(url, "/" + request.index() + "/" + request.type() + "/" + request.id()); + return newHeadRequest(url, "/" + request.index() + "/_doc/" + request.id()); } @Override - protected CheckedFunction entityParser() { + protected CheckedFunction entityParser(HttpResponse httpResponse) { return GetResponse::fromXContent; } - - @Override - protected GetResponse emptyResponse() { - return new GetResponse(); - } } diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/get/HttpGetAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/get/HttpGetAction.java index d273548..fef8e38 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/action/get/HttpGetAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/get/HttpGetAction.java @@ -1,6 +1,6 @@ package org.xbib.elx.http.action.get; -import org.elasticsearch.action.GenericAction; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.get.GetAction; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; @@ -8,28 +8,24 @@ import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.xcontent.XContentParser; import org.xbib.elx.http.HttpAction; import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; import java.io.IOException; public class HttpGetAction extends HttpAction { @Override - public GenericAction getActionInstance() { + public ActionType getActionInstance() { return GetAction.INSTANCE; } @Override protected Request.Builder createHttpRequest(String url, GetRequest request) { - return newGetRequest(url, "/" + request.index() + "/" + request.type() + "/" + request.id()); + return newGetRequest(url, "/" + request.index() + "/_doc/" + "/" + request.id()); } @Override - protected CheckedFunction entityParser() { + protected CheckedFunction entityParser(HttpResponse httpResponse) { return GetResponse::fromXContent; } - - @Override - protected GetResponse emptyResponse() { - return new GetResponse(); - } } diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/index/HttpIndexAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/index/HttpIndexAction.java index a010872..f54de02 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/action/index/HttpIndexAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/index/HttpIndexAction.java @@ -1,6 +1,6 @@ package org.xbib.elx.http.action.index; -import org.elasticsearch.action.GenericAction; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.index.IndexAction; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; @@ -8,29 +8,25 @@ import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.xcontent.XContentParser; import org.xbib.elx.http.HttpAction; import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; import java.io.IOException; public class HttpIndexAction extends HttpAction { @Override - public GenericAction getActionInstance() { + public ActionType getActionInstance() { return IndexAction.INSTANCE; } @Override protected Request.Builder createHttpRequest(String url, IndexRequest request) { - return newPutRequest(url, "/" + request.index() + "/" + request.type() + "/" + request.id(), + return newPutRequest(url, "/" + request.index() + "/_doc/" + request.id(), request.source()); } @Override - protected CheckedFunction entityParser() { + protected CheckedFunction entityParser(HttpResponse httpResponse) { return IndexResponse::fromXContent; } - - @Override - protected IndexResponse emptyResponse() { - return new IndexResponse(); - } } diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/main/HttpMainAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/main/HttpMainAction.java index 7bc9658..416eddd 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/action/main/HttpMainAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/main/HttpMainAction.java @@ -1,6 +1,6 @@ package org.xbib.elx.http.action.main; -import org.elasticsearch.action.GenericAction; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.main.MainAction; import org.elasticsearch.action.main.MainRequest; import org.elasticsearch.action.main.MainResponse; @@ -8,13 +8,14 @@ import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.xcontent.XContentParser; import org.xbib.elx.http.HttpAction; import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; import java.io.IOException; public class HttpMainAction extends HttpAction { @Override - public GenericAction getActionInstance() { + public ActionType getActionInstance() { return MainAction.INSTANCE; } @@ -24,12 +25,7 @@ public class HttpMainAction extends HttpAction { } @Override - protected CheckedFunction entityParser() { + protected CheckedFunction entityParser(HttpResponse httpResponse) { return MainResponse::fromXContent; } - - @Override - protected MainResponse emptyResponse() { - return new MainResponse(); - } } diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/search/HttpSearchAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/search/HttpSearchAction.java index a2e84fe..4398195 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/action/search/HttpSearchAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/search/HttpSearchAction.java @@ -7,6 +7,7 @@ import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.xcontent.XContentParser; import org.xbib.elx.http.HttpAction; import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; import java.io.IOException; @@ -25,12 +26,7 @@ public class HttpSearchAction extends HttpAction } @Override - protected CheckedFunction entityParser() { + protected CheckedFunction entityParser(HttpResponse httpResponse) { return SearchResponse::fromXContent; } - - @Override - protected SearchResponse emptyResponse() { - return new SearchResponse(); - } } diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/update/HttpUpdateAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/update/HttpUpdateAction.java index 606ff53..5ef8021 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/action/update/HttpUpdateAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/update/HttpUpdateAction.java @@ -1,6 +1,6 @@ package org.xbib.elx.http.action.update; -import org.elasticsearch.action.GenericAction; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.update.UpdateAction; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; @@ -12,13 +12,14 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.xbib.elx.http.HttpAction; import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; import java.io.IOException; public class HttpUpdateAction extends HttpAction { @Override - public GenericAction getActionInstance() { + public ActionType getActionInstance() { return UpdateAction.INSTANCE; } @@ -47,7 +48,7 @@ public class HttpUpdateAction extends HttpAction } BytesReference source = XContentHelper.toXContent(updateRequest, xContentType, false); return newPostRequest(url, - "/" + updateRequest.index() + "/" + updateRequest.type() + "/" + updateRequest.id() + "/_update", + "/" + updateRequest.index() + "/_doc/" + updateRequest.id() + "/_update", source); } catch (IOException e) { logger.error(e.getMessage(), e); @@ -56,12 +57,7 @@ public class HttpUpdateAction extends HttpAction } @Override - protected CheckedFunction entityParser() { + protected CheckedFunction entityParser(HttpResponse httpResponse) { return UpdateResponse::fromXContent; } - - @Override - protected UpdateResponse emptyResponse() { - return new UpdateResponse(); - } } diff --git a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction index eb3a014..6f8771f 100644 --- a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction +++ b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction @@ -1,4 +1,4 @@ -org.elasticsearch.action.admin.indices.mapping.get.HttpGetMappingsAction +org.xbib.elx.http.action.admin.indices.mapping.get.HttpGetMappingsAction org.xbib.elx.http.action.admin.cluster.health.HttpClusterHealthAction org.xbib.elx.http.action.admin.cluster.node.info.HttpNodesInfoAction org.xbib.elx.http.action.admin.cluster.settings.HttpClusterUpdateSettingsAction diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java index 57a388b..b68486c 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java @@ -1,19 +1,25 @@ package org.xbib.elx.http.test; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; @@ -21,6 +27,7 @@ import org.xbib.elx.common.Parameters; import org.xbib.elx.http.ExtendedHttpClient; import org.xbib.elx.http.ExtendedHttpClientProvider; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -82,28 +89,50 @@ class ClientTest { } @Test - void testMapping() throws Exception { + void testNewIndexWithSettings() throws Exception { final ExtendedHttpClient client = ClientBuilder.builder() .provider(ExtendedHttpClientProvider.class) .put(helper.getHttpSettings()) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) .build(); + Settings settings = Settings.builder().put("index.number_of_shards", "1").build(); + client.newIndex("test", settings); + GetSettingsRequest getSettingsRequest = new GetSettingsRequest() + .indices("test"); + GetSettingsResponse getSettingsResponse = + client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet(); + logger.log(Level.INFO, "settings=" + getSettingsResponse.getSetting("test", "index.number_of_shards")); + assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards")); + client.close(); + } + + @Test + void testNewIndexWithSettingsAndMappings() throws Exception { + final ExtendedHttpClient client = ClientBuilder.builder() + .provider(ExtendedHttpClientProvider.class) + .put(helper.getHttpSettings()) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) + .build(); + Settings settings = Settings.builder().put("index.number_of_shards", "1").build(); XContentBuilder builder = JsonXContent.contentBuilder() .startObject() - .startObject("doc") .startObject("properties") - .startObject("location") + .startObject("pos") .field("type", "geo_point") .endObject() .endObject() - .endObject() .endObject(); - client.newIndex("test", Settings.EMPTY, Strings.toString(builder)); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test"); + client.newIndex("test", settings, builder); + GetSettingsRequest getSettingsRequest = new GetSettingsRequest() + .indices("test"); + GetSettingsResponse getSettingsResponse = + client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet(); + assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards")); + GetMappingsRequest getMappingsRequest = new GetMappingsRequest() + .indices("test"); GetMappingsResponse getMappingsResponse = client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); - logger.info("mappings={}", getMappingsResponse.getMappings()); - assertTrue(getMappingsResponse.getMappings().get("test").containsKey("doc")); + assertTrue(getMappingsResponse.getMappings().get("test").containsKey("_doc")); client.close(); } @@ -136,8 +165,7 @@ class ClientTest { client.refreshIndex("test"); SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) .setQuery(QueryBuilders.matchAllQuery()).setSize(0); - assertEquals(numactions, - searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); + assertEquals(numactions, searchRequestBuilder.execute().actionGet().getHits().getTotalHits().value); client.close(); } } @@ -146,7 +174,7 @@ class ClientTest { void testThreadedRandomDocs() throws Exception { int maxthreads = Runtime.getRuntime().availableProcessors(); Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; - final Long actions = ACTIONS; + final long actions = ACTIONS; logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); final ExtendedHttpClient client = ClientBuilder.builder() .provider(ExtendedHttpClientProvider.class) @@ -196,11 +224,19 @@ class ClientTest { logger.error("error", client.getBulkController().getLastBulkError()); } assertNull(client.getBulkController().getLastBulkError()); + assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount()); + logger.log(Level.INFO, "refreshing index test"); client.refreshIndex("test"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setQuery(QueryBuilders.matchAllQuery()).setSize(0); - assertEquals(maxthreads * actions, - searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); + SearchSourceBuilder builder = new SearchSourceBuilder() + .query(QueryBuilders.matchAllQuery()) + .size(0) + .trackTotalHits(true); + SearchRequest searchRequest = new SearchRequest() + .indices("test") + .source(builder); + SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet(); + assertEquals(maxthreads * actions, searchResponse.getHits().getTotalHits().value); + client.close(); client.close(); } } diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java index 5e966c0..692cc76 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java @@ -3,8 +3,11 @@ package org.xbib.elx.http.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; @@ -14,7 +17,6 @@ import org.xbib.elx.http.ExtendedHttpClientProvider; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -26,7 +28,7 @@ class DuplicateIDTest { private static final Long MAX_ACTIONS_PER_REQUEST = 10L; - private static final Long ACTIONS = 50L; + private static final Long ACTIONS = 100L; private final TestExtension.Helper helper; @@ -51,11 +53,16 @@ class DuplicateIDTest { client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); client.refreshIndex("test"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setIndices("test") - .setTypes("test") - .setQuery(matchAllQuery()); - long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); + SearchSourceBuilder builder = new SearchSourceBuilder() + .query(QueryBuilders.matchAllQuery()) + .size(0) + .trackTotalHits(true); + SearchRequest searchRequest = new SearchRequest() + .indices("test") + .source(builder); + SearchResponse searchResponse = + helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet(); + long hits = searchResponse.getHits().getTotalHits().value; logger.info("hits = {}", hits); assertTrue(hits < ACTIONS); } catch (NoNodeAvailableException e) { diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java index 54c3d6f..3068771 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java @@ -66,9 +66,9 @@ class IndexPruneTest { indicesExistsRequest.indices(index); IndicesExistsResponse indicesExistsResponse = client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet(); + logger.info("indices exists response for {} is {}", index, indicesExistsResponse.isExists()); list.add(indicesExistsResponse.isExists()); } - logger.info(list); assertFalse(list.get(0)); assertFalse(list.get(1)); assertTrue(list.get(2)); diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/MockNode.java b/elx-http/src/test/java/org/xbib/elx/http/test/MockNode.java index fc62993..7d83b63 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/MockNode.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/MockNode.java @@ -6,10 +6,12 @@ import org.elasticsearch.node.Node; import org.elasticsearch.plugins.Plugin; import java.util.List; +import java.util.Map; public class MockNode extends Node { public MockNode(Settings settings, List> classpathPlugins) { - super(InternalSettingsPreparer.prepareEnvironment(settings, null), classpathPlugins); + super(InternalSettingsPreparer.prepareEnvironment(settings, Map.of(), null, () -> "mock"), + classpathPlugins, false); } } diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/TestExtension.java b/elx-http/src/test/java/org/xbib/elx/http/test/TestExtension.java index 3788433..5e05fae 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/TestExtension.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/TestExtension.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.analysis.common.CommonAnalysisPlugin; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.cluster.health.ClusterHealthStatus; @@ -22,7 +21,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeValidationException; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.transport.netty4.Netty4Plugin; +import org.elasticsearch.transport.Netty4Plugin; import org.junit.jupiter.api.extension.AfterEachCallback; import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; @@ -37,7 +36,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; -import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -192,8 +191,8 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft return Settings.builder() .put("cluster.name", getClusterName()) .put("path.home", getHome()) - .put("discovery.zen.master_election.ignore_non_master_pings", "true") - .put("transport.netty.epoll", "false") + .put("cluster.initial_master_nodes", "1") + .put("discovery.seed_hosts", "127.0.0.1:9300") .build(); } @@ -228,7 +227,7 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft .put(getNodeSettings()) .put("node.name", id) .build(); - List> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class); + List> plugins = Collections.singletonList(Netty4Plugin.class); Node node = new MockNode(nodeSettings, plugins); AbstractClient client = (AbstractClient) node.client(); nodes.put(id, node); diff --git a/elx-node/build.gradle b/elx-node/build.gradle index 6f6191c..6d92165 100644 --- a/elx-node/build.gradle +++ b/elx-node/build.gradle @@ -1,5 +1,6 @@ dependencies { compile project(':elx-common') - compile "org.xbib.elasticsearch:transport-netty4:${rootProject.property('elasticsearch-server.version')}" - testCompile "org.xbib.elasticsearch:elasticsearch-analysis-common:${rootProject.property('elasticsearch-server.version')}" + compile "org.elasticsearch.plugin:transport-netty4-client:${project.property('elasticsearch.version')}" + compile "io.netty:netty-codec-http:${project.property('netty.version')}" + compile "io.netty:netty-transport:${project.property('netty.version')}" } diff --git a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java b/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java index fab3714..9d35b9b 100644 --- a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java +++ b/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java @@ -63,7 +63,7 @@ public class ExtendedNodeClient extends AbstractExtendedClient { private static class BulkNode extends Node { BulkNode(Environment env, Collection> classpathPlugins) { - super(env, classpathPlugins); + super(env, classpathPlugins, false); } } } diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java index d071e40..ed26bd0 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java @@ -1,19 +1,25 @@ package org.xbib.elx.node.test; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; @@ -80,27 +86,50 @@ class ClientTest { } @Test - void testMapping() throws Exception { + void testNewIndexWithSettings() throws Exception { final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) .provider(ExtendedNodeClientProvider.class) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) .build(); + Settings settings = Settings.builder().put("index.number_of_shards", "1").build(); + client.newIndex("test", settings); + GetSettingsRequest getSettingsRequest = new GetSettingsRequest() + .indices("test"); + GetSettingsResponse getSettingsResponse = + client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet(); + logger.log(Level.INFO, "settings=" + getSettingsResponse.getSetting("test", "index.number_of_shards")); + assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards")); + client.close(); + } + + @Test + void testNewIndexWithSettingsAndMappings() throws Exception { + final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) + .provider(ExtendedNodeClientProvider.class) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) + .build(); + Settings settings = Settings.builder().put("index.number_of_shards", "1").build(); XContentBuilder builder = JsonXContent.contentBuilder() .startObject() - .startObject("doc") .startObject("properties") .startObject("location") .field("type", "geo_point") .endObject() .endObject() - .endObject() .endObject(); - client.newIndex("test", Settings.EMPTY, Strings.toString(builder)); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test"); + client.newIndex("test", settings, builder); + GetSettingsRequest getSettingsRequest = new GetSettingsRequest() + .indices("test"); + GetSettingsResponse getSettingsResponse = + client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet(); + logger.log(Level.INFO, "settings=" + getSettingsResponse.getSetting("test", "index.number_of_shards")); + assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards")); + GetMappingsRequest getMappingsRequest = new GetMappingsRequest() + .indices("test"); GetMappingsResponse getMappingsResponse = client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); logger.info("mappings={}", getMappingsResponse.getMappings()); - assertTrue(getMappingsResponse.getMappings().get("test").containsKey("doc")); + assertTrue(getMappingsResponse.getMappings().get("test").containsKey("_doc")); client.close(); } @@ -131,8 +160,7 @@ class ClientTest { client.refreshIndex("test"); SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) .setQuery(QueryBuilders.matchAllQuery()).setSize(0); - assertEquals(numactions, - searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); + assertEquals(numactions, searchRequestBuilder.execute().actionGet().getHits().getTotalHits().value); client.close(); } } @@ -189,11 +217,18 @@ class ClientTest { logger.error("error", client.getBulkController().getLastBulkError()); } assertNull(client.getBulkController().getLastBulkError()); + assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount()); + logger.log(Level.INFO, "refreshing index test"); client.refreshIndex("test"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setQuery(QueryBuilders.matchAllQuery()).setSize(0); - assertEquals(maxthreads * actions, - searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); + SearchSourceBuilder builder = new SearchSourceBuilder() + .query(QueryBuilders.matchAllQuery()) + .size(0) + .trackTotalHits(true); + SearchRequest searchRequest = new SearchRequest() + .indices("test") + .source(builder); + SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet(); + assertEquals(maxthreads * actions, searchResponse.getHits().getTotalHits().value); client.close(); } } diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java index 16b782f..0eb1547 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java @@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -27,7 +28,7 @@ class DuplicateIDTest { private static final Long MAX_ACTIONS_PER_REQUEST = 10L; - private static final Long ACTIONS = 50L; + private static final Long ACTIONS = 100L; private final TestExtension.Helper helper; @@ -51,13 +52,16 @@ class DuplicateIDTest { client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); client.refreshIndex("test"); - SearchSourceBuilder builder = new SearchSourceBuilder(); - builder.query(QueryBuilders.matchAllQuery()); - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices("test"); - searchRequest.types("test"); - searchRequest.source(builder); - long hits = helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits(); + SearchSourceBuilder builder = new SearchSourceBuilder() + .query(QueryBuilders.matchAllQuery()) + .size(0) + .trackTotalHits(true); + SearchRequest searchRequest = new SearchRequest() + .indices("test") + .source(builder); + SearchResponse searchResponse = + helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet(); + long hits = searchResponse.getHits().getTotalHits().value; logger.info("hits = {}", hits); assertTrue(hits < ACTIONS); } catch (NoNodeAvailableException e) { diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/MockNode.java b/elx-node/src/test/java/org/xbib/elx/node/test/MockNode.java index 0d0568a..4ad917c 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/MockNode.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/MockNode.java @@ -6,10 +6,12 @@ import org.elasticsearch.node.Node; import org.elasticsearch.plugins.Plugin; import java.util.List; +import java.util.Map; public class MockNode extends Node { public MockNode(Settings settings, List> classpathPlugins) { - super(InternalSettingsPreparer.prepareEnvironment(settings, null), classpathPlugins); + super(InternalSettingsPreparer.prepareEnvironment(settings, Map.of(), null, () -> "mock"), + classpathPlugins, false); } } diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java b/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java index 7436c4b..1c5b2e5 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.analysis.common.CommonAnalysisPlugin; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.cluster.health.ClusterHealthStatus; @@ -22,7 +21,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeValidationException; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.transport.netty4.Netty4Plugin; +import org.elasticsearch.transport.Netty4Plugin; import org.junit.jupiter.api.extension.AfterEachCallback; import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; @@ -37,7 +36,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; -import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -192,8 +191,8 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft return Settings.builder() .put("cluster.name", getClusterName()) .put("path.home", getHome()) - .put("discovery.zen.master_election.ignore_non_master_pings", "true") - .put("transport.netty.epoll", "false") + .put("cluster.initial_master_nodes", "1") + .put("discovery.seed_hosts", "127.0.0.1:9300") .build(); } @@ -219,7 +218,7 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft .put(getNodeSettings()) .put("node.name", id) .build(); - List> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class); + List> plugins = Collections.singletonList(Netty4Plugin.class); Node node = new MockNode(nodeSettings, plugins); AbstractClient client = (AbstractClient) node.client(); nodes.put(id, node); diff --git a/elx-transport/build.gradle b/elx-transport/build.gradle index 6f6191c..fa59fd8 100644 --- a/elx-transport/build.gradle +++ b/elx-transport/build.gradle @@ -1,5 +1,6 @@ dependencies { compile project(':elx-common') - compile "org.xbib.elasticsearch:transport-netty4:${rootProject.property('elasticsearch-server.version')}" - testCompile "org.xbib.elasticsearch:elasticsearch-analysis-common:${rootProject.property('elasticsearch-server.version')}" + compile "org.elasticsearch.plugin:transport-netty4-client:${rootProject.property('elasticsearch.version')}" + compile "io.netty:netty-codec-http:${project.property('netty.version')}" + compile "io.netty:netty-transport:${project.property('netty.version')}" } diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java index b5b3a51..cefe71f 100644 --- a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java +++ b/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java @@ -22,7 +22,7 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.transport.netty4.Netty4Plugin; +import org.elasticsearch.transport.Netty4Plugin; import org.xbib.elx.common.AbstractExtendedClient; import org.xbib.elx.common.util.NetworkUtils; diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java index 76e6bf5..5f2ae9a 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java @@ -6,11 +6,13 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -85,28 +87,45 @@ class ClientTest { } @Test - void testMapping() throws Exception { + void testNewIndexWithSettings() throws Exception { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) .put(helper.getTransportSettings()) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) .build(); + Settings settings = Settings.builder().put("index.number_of_shards", "1").build(); + client.newIndex("test", settings); + GetSettingsRequest getSettingsRequest = new GetSettingsRequest() + .indices("test"); + GetSettingsResponse getSettingsResponse = + client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet(); + logger.log(Level.INFO, "settings=" + getSettingsResponse.getSetting("test", "index.number_of_shards")); + assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards")); + client.close(); + } + + @Test + void testNewIndexWithSettingsAndMapping() throws Exception { + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) + .put(helper.getTransportSettings()) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) + .build(); + Settings settings = Settings.builder().put("index.number_of_shards", "1").build(); XContentBuilder builder = JsonXContent.contentBuilder() .startObject() - .startObject("doc") .startObject("properties") .startObject("location") .field("type", "geo_point") .endObject() .endObject() - .endObject() .endObject(); - client.newIndex("test", Settings.EMPTY, Strings.toString(builder)); + client.newIndex("test", settings, builder); GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test"); GetMappingsResponse getMappingsResponse = client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); logger.info("mappings={}", getMappingsResponse.getMappings()); - assertTrue(getMappingsResponse.getMappings().get("test").containsKey("doc")); + assertTrue(getMappingsResponse.getMappings().get("test").containsKey("_doc")); client.close(); } @@ -146,7 +165,7 @@ class ClientTest { searchRequest.source(builder); SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet(); logger.log(Level.INFO, searchResponse.toString()); - assertEquals(numactions, searchResponse.getHits().getTotalHits()); + assertEquals(numactions, searchResponse.getHits().getTotalHits().value); client.close(); } } @@ -207,16 +226,18 @@ class ClientTest { logger.error("error", client.getBulkController().getLastBulkError()); } assertNull(client.getBulkController().getLastBulkError()); + assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount()); + logger.log(Level.INFO, "refreshing index test"); client.refreshIndex("test"); - SearchSourceBuilder builder = new SearchSourceBuilder(); - builder.query(QueryBuilders.matchAllQuery()); - builder.size(0); - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices("test"); - searchRequest.source(builder); + SearchSourceBuilder builder = new SearchSourceBuilder() + .query(QueryBuilders.matchAllQuery()) + .size(0) + .trackTotalHits(true); + SearchRequest searchRequest = new SearchRequest() + .indices("test") + .source(builder); SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet(); - logger.log(Level.INFO, searchResponse.toString()); - assertEquals(maxthreads * actions, searchResponse.getHits().getTotalHits()); + assertEquals(maxthreads * actions, searchResponse.getHits().getTotalHits().value); client.close(); } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java index c1afc40..d2c2773 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java @@ -28,7 +28,7 @@ class DuplicateIDTest { private static final Long MAX_ACTIONS_PER_REQUEST = 10L; - private static final Long ACTIONS = 5L; + private static final Long ACTIONS = 100L; private final TestExtension.Helper helper; @@ -53,16 +53,16 @@ class DuplicateIDTest { client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); client.refreshIndex("test_dup"); - SearchSourceBuilder builder = new SearchSourceBuilder(); - builder.query(QueryBuilders.matchAllQuery()); - builder.size(0); - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices("test_dup"); - searchRequest.types("test_dup"); - searchRequest.source(builder); + SearchSourceBuilder builder = new SearchSourceBuilder() + .query(QueryBuilders.matchAllQuery()) + .size(0) + .trackTotalHits(true); + SearchRequest searchRequest = new SearchRequest() + .indices("test_dup") + .source(builder); SearchResponse searchResponse = helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet(); - long hits = searchResponse.getHits().getTotalHits(); + long hits = searchResponse.getHits().getTotalHits().value; logger.info("hits = {}", hits); assertTrue(hits < ACTIONS); } catch (NoNodeAvailableException e) { diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/MockNode.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/MockNode.java index f586691..3e5f3da 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/MockNode.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/MockNode.java @@ -6,10 +6,12 @@ import org.elasticsearch.node.Node; import org.elasticsearch.plugins.Plugin; import java.util.List; +import java.util.Map; public class MockNode extends Node { public MockNode(Settings settings, List> classpathPlugins) { - super(InternalSettingsPreparer.prepareEnvironment(settings, null), classpathPlugins); + super(InternalSettingsPreparer.prepareEnvironment(settings, Map.of(), null, () -> "mock"), + classpathPlugins, false); } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java index 95d2db5..754f5c4 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.analysis.common.CommonAnalysisPlugin; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.cluster.health.ClusterHealthStatus; @@ -22,7 +21,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeValidationException; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.transport.netty4.Netty4Plugin; +import org.elasticsearch.transport.Netty4Plugin; import org.junit.jupiter.api.extension.AfterEachCallback; import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; @@ -37,7 +36,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; -import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -192,7 +191,8 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft return Settings.builder() .put("cluster.name", getClusterName()) .put("path.home", getHome()) - .put("discovery.zen.master_election.ignore_non_master_pings", "true") + .put("cluster.initial_master_nodes", "1") + .put("discovery.seed_hosts", "127.0.0.1:9300") .build(); } @@ -227,7 +227,7 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft .put(getNodeSettings()) .put("node.name", id) .build(); - List> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class); + List> plugins = Collections.singletonList(Netty4Plugin.class); Node node = new MockNode(nodeSettings, plugins); AbstractClient client = (AbstractClient) node.client(); nodes.put(id, node); diff --git a/gradle.properties b/gradle.properties index d8892d7..4f0ebb6 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,19 +1,16 @@ group = org.xbib name = elx -version = 6.3.2.5 -profile = default -release = 0 +version = 7.6.1.0 -elasticsearch-server.version = 6.3.2.4 -log4j.version = 2.12.1 +elasticsearch.version = 7.6.1 +netty.version = 4.1.49.Final +xbib-netty-http.version = 4.1.49.0 tcnative.version = 2.0.29.Final tcnative-legacy-macosx.version = 2.0.26.Final bouncycastle.version = 1.64 xbib-metrics.version = 2.0.0 -xbib-netty-http.version = 4.1.49.0 # test -junit.version = 5.4.2 +log4j.version = 2.13.1 +junit.version = 5.6.2 asciidoclet.version = 1.6.0.0 - -org.gradle.warning.mode = all