|
|
|
@ -4,7 +4,7 @@ import org.apache.logging.log4j.Level;
|
|
|
|
|
import org.apache.logging.log4j.LogManager;
|
|
|
|
|
import org.apache.logging.log4j.Logger;
|
|
|
|
|
import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
|
|
|
|
|
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
|
|
|
|
|
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
|
|
|
|
|
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
|
|
|
|
|
import org.elasticsearch.action.admin.indices.flush.FlushAction;
|
|
|
|
|
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
|
|
|
|
@ -17,12 +17,9 @@ import org.elasticsearch.common.bytes.BytesArray;
|
|
|
|
|
import org.elasticsearch.common.bytes.BytesReference;
|
|
|
|
|
import org.elasticsearch.common.settings.Settings;
|
|
|
|
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
|
|
|
|
import org.elasticsearch.common.xcontent.XContentFactory;
|
|
|
|
|
import org.elasticsearch.common.xcontent.XContentType;
|
|
|
|
|
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
|
|
|
|
import org.xbib.elx.api.BulkClient;
|
|
|
|
|
import org.xbib.elx.api.BulkController;
|
|
|
|
|
import org.xbib.elx.api.BulkMetric;
|
|
|
|
|
import org.xbib.elx.api.IndexDefinition;
|
|
|
|
|
import java.io.IOException;
|
|
|
|
|
import java.nio.charset.StandardCharsets;
|
|
|
|
@ -34,8 +31,6 @@ public abstract class AbstractBulkClient extends AbstractBasicClient implements
|
|
|
|
|
|
|
|
|
|
private static final Logger logger = LogManager.getLogger(AbstractBulkClient.class.getName());
|
|
|
|
|
|
|
|
|
|
private BulkMetric bulkMetric;
|
|
|
|
|
|
|
|
|
|
private BulkController bulkController;
|
|
|
|
|
|
|
|
|
|
private final AtomicBoolean closed = new AtomicBoolean(true);
|
|
|
|
@ -44,21 +39,12 @@ public abstract class AbstractBulkClient extends AbstractBasicClient implements
|
|
|
|
|
public void init(Settings settings) throws IOException {
|
|
|
|
|
if (closed.compareAndSet(true, false)) {
|
|
|
|
|
super.init(settings);
|
|
|
|
|
logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(','));
|
|
|
|
|
bulkMetric = new DefaultBulkMetric();
|
|
|
|
|
bulkMetric.init(settings);
|
|
|
|
|
bulkController = new DefaultBulkController(this, bulkMetric);
|
|
|
|
|
bulkController = new DefaultBulkController(this);
|
|
|
|
|
logger.log(Level.INFO, "initializing bulk controller with settings = " + settings.toDelimitedString(','));
|
|
|
|
|
bulkController.init(settings);
|
|
|
|
|
} else {
|
|
|
|
|
logger.log(Level.WARN, "not initializing");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public BulkMetric getBulkMetric() {
|
|
|
|
|
return bulkMetric;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public BulkController getBulkController() {
|
|
|
|
|
return bulkController;
|
|
|
|
@ -75,10 +61,6 @@ public abstract class AbstractBulkClient extends AbstractBasicClient implements
|
|
|
|
|
public void close() throws IOException {
|
|
|
|
|
if (closed.compareAndSet(false, true)) {
|
|
|
|
|
ensureClientIsPresent();
|
|
|
|
|
if (bulkMetric != null) {
|
|
|
|
|
logger.info("closing bulk metric");
|
|
|
|
|
bulkMetric.close();
|
|
|
|
|
}
|
|
|
|
|
if (bulkController != null) {
|
|
|
|
|
logger.info("closing bulk controller");
|
|
|
|
|
bulkController.close();
|
|
|
|
@ -98,40 +80,50 @@ public abstract class AbstractBulkClient extends AbstractBasicClient implements
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public void newIndex(String index) throws IOException {
|
|
|
|
|
newIndex(index, Settings.EMPTY, (Map<String, ?>) null);
|
|
|
|
|
newIndex(index, Settings.EMPTY, (XContentBuilder) null);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public void newIndex(String index, Settings settings) throws IOException {
|
|
|
|
|
newIndex(index, settings, (Map<String, ?>) null);
|
|
|
|
|
newIndex(index, settings, (XContentBuilder) null);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public void newIndex(String index, Settings settings, XContentBuilder builder) throws IOException {
|
|
|
|
|
String mappingString = builder.string();
|
|
|
|
|
Map<String, ?> mappings = JsonXContent.jsonXContent.createParser(mappingString).mapOrdered();
|
|
|
|
|
newIndex(index, settings, mappings);
|
|
|
|
|
public void newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException {
|
|
|
|
|
if (mapping == null || mapping.isEmpty()) {
|
|
|
|
|
newIndex(index, settings, (XContentBuilder) null);
|
|
|
|
|
} else {
|
|
|
|
|
newIndex(index, settings, JsonXContent.contentBuilder().map(mapping));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public void newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException {
|
|
|
|
|
public void newIndex(String index, Settings settings, XContentBuilder builder) throws IOException {
|
|
|
|
|
if (index == null) {
|
|
|
|
|
logger.warn("no index name given to create index");
|
|
|
|
|
logger.warn("unable to create index, no index name given");
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
ensureClientIsPresent();
|
|
|
|
|
waitForCluster("YELLOW", 30L, TimeUnit.SECONDS);
|
|
|
|
|
CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index);
|
|
|
|
|
CreateIndexRequestBuilder createIndexRequestBuilder = new CreateIndexRequestBuilder(client, CreateIndexAction.INSTANCE)
|
|
|
|
|
.setIndex(index);
|
|
|
|
|
if (settings != null) {
|
|
|
|
|
createIndexRequest.settings(settings);
|
|
|
|
|
createIndexRequestBuilder.setSettings(settings);
|
|
|
|
|
}
|
|
|
|
|
if (mapping != null) {
|
|
|
|
|
createIndexRequest.mapping(TYPE_NAME, mapping);
|
|
|
|
|
if (builder != null) {
|
|
|
|
|
createIndexRequestBuilder.addMapping(TYPE_NAME, builder);
|
|
|
|
|
logger.debug("adding mapping = {}", builder.string());
|
|
|
|
|
} else {
|
|
|
|
|
// empty mapping
|
|
|
|
|
createIndexRequestBuilder.addMapping(TYPE_NAME,
|
|
|
|
|
JsonXContent.contentBuilder().startObject().startObject(TYPE_NAME).endObject().endObject());
|
|
|
|
|
logger.debug("empty mapping");
|
|
|
|
|
}
|
|
|
|
|
CreateIndexResponse createIndexResponse = createIndexRequestBuilder.execute().actionGet();
|
|
|
|
|
if (createIndexResponse.isAcknowledged()) {
|
|
|
|
|
logger.info("index {} created", index);
|
|
|
|
|
} else {
|
|
|
|
|
logger.warn("index creation of {} not acknowledged", index);
|
|
|
|
|
}
|
|
|
|
|
CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
|
|
|
|
|
XContentBuilder builder = XContentFactory.jsonBuilder();
|
|
|
|
|
logger.info("index {} created: {}", index,
|
|
|
|
|
createIndexResponse.toString());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
@ -142,36 +134,40 @@ public abstract class AbstractBulkClient extends AbstractBasicClient implements
|
|
|
|
|
@Override
|
|
|
|
|
public void startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds)
|
|
|
|
|
throws IOException {
|
|
|
|
|
if (bulkController != null) {
|
|
|
|
|
ensureClientIsPresent();
|
|
|
|
|
bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds);
|
|
|
|
|
}
|
|
|
|
|
ensureClientIsPresent();
|
|
|
|
|
bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public void stopBulk(IndexDefinition indexDefinition) throws IOException {
|
|
|
|
|
if (bulkController != null) {
|
|
|
|
|
ensureClientIsPresent();
|
|
|
|
|
bulkController.stopBulkMode(indexDefinition);
|
|
|
|
|
}
|
|
|
|
|
ensureClientIsPresent();
|
|
|
|
|
bulkController.stopBulkMode(indexDefinition);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException {
|
|
|
|
|
if (bulkController != null) {
|
|
|
|
|
ensureClientIsPresent();
|
|
|
|
|
bulkController.stopBulkMode(index, timeout, timeUnit);
|
|
|
|
|
}
|
|
|
|
|
ensureClientIsPresent();
|
|
|
|
|
bulkController.stopBulkMode(index, timeout, timeUnit);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public BulkClient index(String index, String id, boolean create, String source) {
|
|
|
|
|
return index(index, id, create, new BytesArray(source.getBytes(StandardCharsets.UTF_8)));
|
|
|
|
|
return index(new IndexRequest()
|
|
|
|
|
.index(index)
|
|
|
|
|
.type(TYPE_NAME)
|
|
|
|
|
.id(id)
|
|
|
|
|
.create(create)
|
|
|
|
|
.source(source)); // will be converted into a bytes reference
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public BulkClient index(String index, String id, boolean create, BytesReference source) {
|
|
|
|
|
return index(new IndexRequest(index, TYPE_NAME, id).create(create).source(source));
|
|
|
|
|
return index(new IndexRequest()
|
|
|
|
|
.index(index)
|
|
|
|
|
.type(TYPE_NAME)
|
|
|
|
|
.id(id)
|
|
|
|
|
.create(create)
|
|
|
|
|
.source(source));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
@ -183,7 +179,10 @@ public abstract class AbstractBulkClient extends AbstractBasicClient implements
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public BulkClient delete(String index, String id) {
|
|
|
|
|
return delete(new DeleteRequest(index, TYPE_NAME, id));
|
|
|
|
|
return delete(new DeleteRequest()
|
|
|
|
|
.index(index)
|
|
|
|
|
.type(TYPE_NAME)
|
|
|
|
|
.id(id));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
@ -194,15 +193,17 @@ public abstract class AbstractBulkClient extends AbstractBasicClient implements
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public BulkClient update(String index, String id, BytesReference source) {
|
|
|
|
|
return update(new UpdateRequest(index, TYPE_NAME, id)
|
|
|
|
|
.doc(source, XContentType.JSON));
|
|
|
|
|
public BulkClient update(String index, String id, String source) {
|
|
|
|
|
return update(index, id, new BytesArray(source.getBytes(StandardCharsets.UTF_8)));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public BulkClient update(String index, String id, String source) {
|
|
|
|
|
return update(new UpdateRequest(index, TYPE_NAME, id)
|
|
|
|
|
.doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
|
|
|
|
|
public BulkClient update(String index, String id, BytesReference source) {
|
|
|
|
|
return update(new UpdateRequest()
|
|
|
|
|
.index(index)
|
|
|
|
|
.type(TYPE_NAME)
|
|
|
|
|
.id(id)
|
|
|
|
|
.doc(source.hasArray() ? source.array() : source.toBytes()));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|