joerg/elx
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix mapping / date detection, adapt to API changes

Browse source
This commit is contained in:
Jörg Prante 2020-05-11 14:11:59 +02:00
parent c03e47cf26
commit fb0dfbb683
11 changed files with 38 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java
View file

@ -68,7 +68,7 @@ class ClientTest {
} catch (NoNodeAvailableException e) { } catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available"); logger.warn("skipping, no node available");
} finally { } finally {
assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) { if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError()); logger.error("error", client.getBulkController().getLastBulkError());
} }
@ -116,6 +116,7 @@ class ClientTest {
Settings settings = Settings.builder().put("index.number_of_shards", "1").build(); Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
XContentBuilder builder = JsonXContent.contentBuilder() XContentBuilder builder = JsonXContent.contentBuilder()
.startObject() .startObject()
.field("date_detection", false)
.startObject("properties") .startObject("properties")
.startObject("pos") .startObject("pos")
.field("type", "geo_point") .field("type", "geo_point")
@ -157,7 +158,7 @@ class ClientTest {
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
} finally { } finally {
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) { if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError()); logger.error("error", client.getBulkController().getLastBulkError());
} }
@ -214,7 +215,7 @@ class ClientTest {
logger.warn("latch timeout"); logger.warn("latch timeout");
} }
client.stopBulk("test", 30L, TimeUnit.SECONDS); client.stopBulk("test", 30L, TimeUnit.SECONDS);
assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount()); assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
} catch (NoNodeAvailableException e) { } catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available"); logger.warn("skipping, no node available");
} catch (Exception e) { } catch (Exception e) {
@ -224,7 +225,7 @@ class ClientTest {
logger.error("error", client.getBulkController().getLastBulkError()); logger.error("error", client.getBulkController().getLastBulkError());
} }
assertNull(client.getBulkController().getLastBulkError()); assertNull(client.getBulkController().getLastBulkError());
assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount()); assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
logger.log(Level.INFO, "refreshing index test"); logger.log(Level.INFO, "refreshing index test");
client.refreshIndex("test"); client.refreshIndex("test");
SearchSourceBuilder builder = new SearchSourceBuilder() SearchSourceBuilder builder = new SearchSourceBuilder()

2
elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java
View file

@ -69,7 +69,7 @@ class DuplicateIDTest {
logger.warn("skipping, no node available"); logger.warn("skipping, no node available");
} finally { } finally {
client.close(); client.close();
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) { if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError()); logger.error("error", client.getBulkController().getLastBulkError());
} }

4
elx-http/src/test/java/org/xbib/elx/http/test/SmokeTest.java
View file

@ -60,8 +60,8 @@ class SmokeTest {
int replica = client.getReplicaLevel(indexDefinition); int replica = client.getReplicaLevel(indexDefinition);
assertEquals(2, replica); assertEquals(2, replica);
client.deleteIndex(indexDefinition); client.deleteIndex(indexDefinition);
assertEquals(0, client.getBulkMetric().getFailed().getCount()); assertEquals(0, client.getBulkController().getBulkMetric().getFailed().getCount());
assertEquals(6, client.getBulkMetric().getSucceeded().getCount()); assertEquals(6, client.getBulkController().getBulkMetric().getSucceeded().getCount());
} catch (NoNodeAvailableException e) { } catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available"); logger.warn("skipping, no node available");
} finally { } finally {

9
elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java
View file

@ -66,7 +66,7 @@ class ClientTest {
} catch (NoNodeAvailableException e) { } catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available"); logger.warn("skipping, no node available");
} finally { } finally {
assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) { if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError()); logger.error("error", client.getBulkController().getLastBulkError());
} }
@ -111,6 +111,7 @@ class ClientTest {
Settings settings = Settings.builder().put("index.number_of_shards", "1").build(); Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
XContentBuilder builder = JsonXContent.contentBuilder() XContentBuilder builder = JsonXContent.contentBuilder()
.startObject() .startObject()
.field("date_detection", false)
.startObject("properties") .startObject("properties")
.startObject("location") .startObject("location")
.field("type", "geo_point") .field("type", "geo_point")
@ -152,7 +153,7 @@ class ClientTest {
} catch (NoNodeAvailableException e) { } catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available"); logger.warn("skipping, no node available");
} finally { } finally {
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) { if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError()); logger.error("error", client.getBulkController().getLastBulkError());
} }
@ -209,7 +210,7 @@ class ClientTest {
logger.warn("latch timeout"); logger.warn("latch timeout");
} }
client.stopBulk("test", 30L, TimeUnit.SECONDS); client.stopBulk("test", 30L, TimeUnit.SECONDS);
assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount()); assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
} catch (NoNodeAvailableException e) { } catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available"); logger.warn("skipping, no node available");
} finally { } finally {
@ -217,7 +218,7 @@ class ClientTest {
logger.error("error", client.getBulkController().getLastBulkError()); logger.error("error", client.getBulkController().getLastBulkError());
} }
assertNull(client.getBulkController().getLastBulkError()); assertNull(client.getBulkController().getLastBulkError());
assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount()); assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
logger.log(Level.INFO, "refreshing index test"); logger.log(Level.INFO, "refreshing index test");
client.refreshIndex("test"); client.refreshIndex("test");
SearchSourceBuilder builder = new SearchSourceBuilder() SearchSourceBuilder builder = new SearchSourceBuilder()

2
elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java
View file

@ -68,7 +68,7 @@ class DuplicateIDTest {
logger.warn("skipping, no node available"); logger.warn("skipping, no node available");
} finally { } finally {
client.close(); client.close();
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) { if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError()); logger.error("error", client.getBulkController().getLastBulkError());
} }

4
elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java
View file

@ -59,8 +59,8 @@ class SmokeTest {
int replica = client.getReplicaLevel(indexDefinition); int replica = client.getReplicaLevel(indexDefinition);
assertEquals(2, replica); assertEquals(2, replica);
client.deleteIndex(indexDefinition); client.deleteIndex(indexDefinition);
//assertEquals(0, client.getBulkMetric().getFailed().getCount()); assertEquals(0, client.getBulkController().getBulkMetric().getFailed().getCount());
//assertEquals(6, client.getBulkMetric().getSucceeded().getCount()); assertEquals(6, client.getBulkController().getBulkMetric().getSucceeded().getCount());
} catch (NoNodeAvailableException e) { } catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available"); logger.warn("skipping, no node available");
} finally { } finally {

20
elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java
View file

@ -124,13 +124,15 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
logger.info("connected to nodes = {}", nodes); logger.info("connected to nodes = {}", nodes);
if (nodes != null && !nodes.isEmpty()) { if (nodes != null && !nodes.isEmpty()) {
if (autodiscover) { if (autodiscover) {
logger.debug("trying to auto-discover all nodes..."); logger.debug("trying to discover all nodes...");
ClusterStateRequestBuilder clusterStateRequestBuilder = ClusterStateRequestBuilder clusterStateRequestBuilder =
new ClusterStateRequestBuilder(getClient(), ClusterStateAction.INSTANCE); new ClusterStateRequestBuilder(getClient(), ClusterStateAction.INSTANCE);
ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet();
DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes(); DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes();
addDiscoveryNodes(transportClient, discoveryNodes); for (DiscoveryNode discoveryNode : discoveryNodes) {
logger.info("after auto-discovery: connected to {}", transportClient.connectedNodes()); transportClient.addTransportAddress(discoveryNode.getAddress());
}
logger.info("after discovery: connected to {}", transportClient.connectedNodes());
} }
return true; return true;
} }
@ -142,9 +144,9 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
// "cluster.name" // "cluster.name"
.put(ClusterName.CLUSTER_NAME_SETTING.getKey(), .put(ClusterName.CLUSTER_NAME_SETTING.getKey(),
settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey())) settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey()))
// "processors" // "node.processors"
.put(EsExecutors.PROCESSORS_SETTING.getKey(), .put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(),
settings.get(EsExecutors.PROCESSORS_SETTING.getKey(), settings.get(EsExecutors.NODE_PROCESSORS_SETTING.getKey(),
String.valueOf(Runtime.getRuntime().availableProcessors()))) String.valueOf(Runtime.getRuntime().availableProcessors())))
// "transport.type" // "transport.type"
.put(NetworkModule.TRANSPORT_TYPE_KEY, .put(NetworkModule.TRANSPORT_TYPE_KEY,
@ -152,12 +154,6 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
.build(); .build();
} }
private void addDiscoveryNodes(TransportClient transportClient, DiscoveryNodes discoveryNodes) {
for (DiscoveryNode discoveryNode : discoveryNodes) {
transportClient.addTransportAddress(discoveryNode.getAddress());
}
}
static class MyTransportClient extends TransportClient { static class MyTransportClient extends TransportClient {
MyTransportClient(Settings settings, Collection<Class<? extends Plugin>> plugins) { MyTransportClient(Settings settings, Collection<Class<? extends Plugin>> plugins) {

17
elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java
View file

@ -66,7 +66,7 @@ class ClientTest {
} catch (NoNodeAvailableException e) { } catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available"); logger.warn("skipping, no node available");
} finally { } finally {
assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) { if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError()); logger.error("error", client.getBulkController().getLastBulkError());
} }
@ -114,6 +114,7 @@ class ClientTest {
Settings settings = Settings.builder().put("index.number_of_shards", "1").build(); Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
XContentBuilder builder = JsonXContent.contentBuilder() XContentBuilder builder = JsonXContent.contentBuilder()
.startObject() .startObject()
.field("date_detection", false)
.startObject("properties") .startObject("properties")
.startObject("location") .startObject("location")
.field("type", "geo_point") .field("type", "geo_point")
@ -121,9 +122,11 @@ class ClientTest {
.endObject() .endObject()
.endObject(); .endObject();
client.newIndex("test", settings, builder); client.newIndex("test", settings, builder);
GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test"); GetMappingsRequest getMappingsRequest = new GetMappingsRequest()
GetMappingsResponse getMappingsResponse = .indices("test");
client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); GetMappingsResponse getMappingsResponse = client.getClient()
.execute(GetMappingsAction.INSTANCE, getMappingsRequest)
.actionGet();
logger.info("mappings={}", getMappingsResponse.getMappings()); logger.info("mappings={}", getMappingsResponse.getMappings());
assertTrue(getMappingsResponse.getMappings().get("test").containsKey("_doc")); assertTrue(getMappingsResponse.getMappings().get("test").containsKey("_doc"));
client.close(); client.close();
@ -151,7 +154,7 @@ class ClientTest {
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
} finally { } finally {
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) { if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError()); logger.error("error", client.getBulkController().getLastBulkError());
} }
@ -216,7 +219,7 @@ class ClientTest {
logger.warn("latch timeout"); logger.warn("latch timeout");
} }
client.stopBulk("test", 30L, TimeUnit.SECONDS); client.stopBulk("test", 30L, TimeUnit.SECONDS);
assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount()); assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
} catch (NoNodeAvailableException e) { } catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available"); logger.warn("skipping, no node available");
} catch (Exception e) { } catch (Exception e) {
@ -226,7 +229,7 @@ class ClientTest {
logger.error("error", client.getBulkController().getLastBulkError()); logger.error("error", client.getBulkController().getLastBulkError());
} }
assertNull(client.getBulkController().getLastBulkError()); assertNull(client.getBulkController().getLastBulkError());
assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount()); assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
logger.log(Level.INFO, "refreshing index test"); logger.log(Level.INFO, "refreshing index test");
client.refreshIndex("test"); client.refreshIndex("test");
SearchSourceBuilder builder = new SearchSourceBuilder() SearchSourceBuilder builder = new SearchSourceBuilder()

2
elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java
View file

@ -69,7 +69,7 @@ class DuplicateIDTest {
logger.warn("skipping, no node available"); logger.warn("skipping, no node available");
} finally { } finally {
client.close(); client.close();
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) { if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError()); logger.error("error", client.getBulkController().getLastBulkError());
} }

4
elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java
View file

@ -57,8 +57,8 @@ class SmokeTest {
int replica = client.getReplicaLevel(indexDefinition); int replica = client.getReplicaLevel(indexDefinition);
assertEquals(2, replica); assertEquals(2, replica);
client.deleteIndex(indexDefinition); client.deleteIndex(indexDefinition);
assertEquals(0, client.getBulkMetric().getFailed().getCount()); assertEquals(0, client.getBulkController().getBulkMetric().getFailed().getCount());
assertEquals(4, client.getBulkMetric().getSucceeded().getCount()); assertEquals(4, client.getBulkController().getBulkMetric().getSucceeded().getCount());
} catch (NoNodeAvailableException e) { } catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available"); logger.warn("skipping, no node available");
} finally { } finally {

2
gradle.properties
View file

@ -1,6 +1,6 @@
group = org.xbib group = org.xbib
name = elx name = elx
version = 7.6.1.0 version = 7.6.1.1
elasticsearch.version = 7.6.1 elasticsearch.version = 7.6.1
netty.version = 4.1.49.Final netty.version = 4.1.49.Final