add queue length operation

This commit is contained in:
Jörg Prante 2023-07-26 17:04:55 +02:00
parent 26bbb56b35
commit b84de6995f
11 changed files with 100 additions and 22 deletions

View file

@ -187,6 +187,12 @@ public class MariaDB implements Flavor {
return false;
}
@Override
public String queueLength(String table) {
return "select count(*) from " + table
+ " where channel = ? and state is NULL";
}
@Override
public String writeNextIntoQueue(String table) {
return "insert into " + table

View file

@ -198,6 +198,12 @@ public class Oracle implements Flavor {
return false;
}
@Override
public String queueLength(String table) {
return "select count(*) from " + table
+ " where channel = ? and state is NULL";
}
@Override
public String writeNextIntoQueue(String table) {
return "insert into " + table

View file

@ -184,6 +184,12 @@ public class Postgresql implements Flavor {
return false;
}
@Override
public String queueLength(String table) {
return "select count(*) from " + table
+ " where channel = ? and state is NULL";
}
@Override
public String writeNextIntoQueue(String table) {
return "insert into " + table
@ -195,7 +201,7 @@ public class Postgresql implements Flavor {
public String readNextFromQueue(String table) {
return "select id, data from " + table +
" where channel = ? and state is NULL and pushed_at <= (CAST (? as TIMESTAMP) - (INTERVAL '1 minute' * delay))" +
" order by id asc limit 1 for update skip lock";
" order by id asc limit ? for update skip lock";
}
@Override

View file

@ -245,7 +245,16 @@ public interface Database extends Supplier<Database> {
void writeQueue(String table, String channel, String data, Consumer<Long> consumer) throws SQLException;
/**
* Read from a queue table and lock rows in a transaction until processing by a consumer
* Proble queue for length.
* @param table the queue table
* @param channel the queue table channel. A queue table can have many channels
* @return the queue length
* @throws SQLException if statement fails
*/
long probeQueue(String table, String channel) throws SQLException;
/**
* Read from a queue table until limit is reached and lock rows in a transaction until processing by a consumer
* has succeeded or failed.
* Commit only if processing was successful, otherwise roll back the transaction.
* The connection is switched into manual commit and afterwards, auto-commit is enabled again.
@ -254,5 +263,5 @@ public interface Database extends Supplier<Database> {
* @param consumer a consumer for the queue table data column or null
* @throws SQLException if rollback fails
*/
public void readQueue(String table, String channel, Consumer<String> consumer) throws SQLException;
public void consumeQueue(String table, String channel, int limit, Consumer<String> consumer) throws SQLException;
}

View file

@ -533,19 +533,32 @@ public class DatabaseImpl implements Database {
}
@Override
public void readQueue(String table, String channel, Consumer<String> consumer) throws SQLException {
public long probeQueue(String table, String channel) throws SQLException {
try (ResultSet resultSet = queueLength(connection, table, channel)) {
if (resultSet.next()) {
return resultSet.getLong(0);
} else {
return -1L;
}
}
}
@Override
public void consumeQueue(String table, String channel, int limit, Consumer<String> consumer) throws SQLException {
List<Long> consumedKeys = new ArrayList<>();
try {
connection.setAutoCommit(false);
while (true) {
try (ResultSet resultSet = readNextFromQueue(connection, table, channel)) {
if (resultSet.next()) {
try (ResultSet resultSet = readNextFromQueue(connection, table, channel, limit)) {
while (resultSet.next()) {
Long key = resultSet.getLong("key");
consumedKeys.add(key);
try {
if (consumer != null) {
consumer.accept(resultSet.getString("data"));
}
succeedInQueue(connection, table, key);
} catch (Exception e) {
} catch (QueueException e) {
connection.rollback();
failInQueue(connection, table, key);
throw e;
}
@ -553,9 +566,11 @@ public class DatabaseImpl implements Database {
} finally {
connection.commit();
}
}
} catch (Exception e) {
connection.rollback();
for (Long key : consumedKeys) {
failInQueue(connection, table, key);
}
throw e;
} finally {
connection.setAutoCommit(true);
@ -585,10 +600,17 @@ public class DatabaseImpl implements Database {
}
}
private ResultSet readNextFromQueue(Connection connection, String table, String channel) throws SQLException {
private ResultSet queueLength(Connection connection, String table, String channel) throws SQLException {
PreparedStatement preparedStatement = connection.prepareStatement(flavor().queueLength(table));
preparedStatement.setString(1, channel);
return preparedStatement.executeQuery();
}
private ResultSet readNextFromQueue(Connection connection, String table, String channel, int limit) throws SQLException {
PreparedStatement preparedStatement = connection.prepareStatement(flavor().readNextFromQueue(table));
preparedStatement.setString(1, channel);
preparedStatement.setTimestamp(2, Timestamp.valueOf(LocalDateTime.now()));
preparedStatement.setInt(3, limit);
return preparedStatement.executeQuery();
}

View file

@ -77,11 +77,12 @@ public interface Flavor {
/**
* In JDBC, all drivers are set to autocommit by default.
* Returns true if we let the JDBC driver exclusively decide about commit(). If false, it allows explicit commit() calls.
* @return
* @return Returns true if we let the JDBC driver exclusively decide about commit(). If false, it allows explicit commit() calls.
*/
boolean isAutoCommitOnly();
String queueLength(String table);
String writeNextIntoQueue(String table);
String readNextFromQueue(String table);

View file

@ -0,0 +1,8 @@
package org.xbib.jdbc.query;
public class QueueException extends DatabaseException {
public QueueException(String message, Throwable cause) {
super(message, cause);
}
}

View file

@ -180,6 +180,11 @@ public class Derby implements Flavor {
return false;
}
@Override
public String queueLength(String table) {
throw new UnsupportedOperationException();
}
@Override
public String writeNextIntoQueue(String table) {
throw new UnsupportedOperationException();

View file

@ -180,6 +180,11 @@ public class H2 implements Flavor {
return false;
}
@Override
public String queueLength(String table) {
throw new UnsupportedOperationException();
}
@Override
public String writeNextIntoQueue(String table) {
throw new UnsupportedOperationException();

View file

@ -180,6 +180,11 @@ public class Hsql implements Flavor {
return false;
}
@Override
public String queueLength(String table) {
throw new UnsupportedOperationException();
}
@Override
public String writeNextIntoQueue(String table) {
throw new UnsupportedOperationException();

View file

@ -187,6 +187,11 @@ public class SqlServer implements Flavor {
return false;
}
@Override
public String queueLength(String table) {
throw new UnsupportedOperationException();
}
@Override
public String writeNextIntoQueue(String table) {
throw new UnsupportedOperationException();