joerg/net
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

working on json file store

Browse source
This commit is contained in:
Jörg Prante 2023-08-14 17:03:46 +02:00
parent f963f42c57
commit 5c91f1c557
3 changed files with 166 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
net/src/main/java/org/xbib/net/Store.java
View file

@ -1,6 +1,7 @@
package org.xbib.net; package org.xbib.net;
import java.io.IOException; import java.io.IOException;
import java.util.Map;
import java.util.function.Consumer; import java.util.function.Consumer;
public interface Store<S> { public interface Store<S> {
@ -11,11 +12,19 @@ public interface Store<S> {
S read(String key) throws IOException; S read(String key) throws IOException;
void readAll(String key, Consumer<S> consumer) throws IOException; void readAll(String prefix, Consumer<S> consumer) throws IOException;
void write(String key, S s) throws IOException; void write(String key, S s) throws IOException;
void write(String prefix, String key, S s) throws IOException;
void update(String key, Map<String, Object> map) throws IOException;
void update(String prefix, String key, Map<String, Object> map) throws IOException;
void remove(String key) throws IOException; void remove(String key) throws IOException;
void purge(long expiredAfterSeconds) throws IOException; void purge(long expiredAfterSeconds) throws IOException;
void destroy() throws IOException;
} }

97
net/src/main/java/org/xbib/net/util/JsonFileStore.java
View file

@ -7,9 +7,12 @@ import org.xbib.net.PercentEncoders;
import java.io.IOException; import java.io.IOException;
import java.io.Writer; import java.io.Writer;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.FileVisitResult;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.attribute.FileTime; import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.time.Duration; import java.time.Duration;
import java.time.Instant; import java.time.Instant;
import java.util.Map; import java.util.Map;
@ -31,11 +34,12 @@ public class JsonFileStore implements Store<Map<String, Object>> {
private final Path path; private final Path path;
public JsonFileStore(String name, Path path) throws IOException { public JsonFileStore(String name, Path path, long expireAfterSeconds) throws IOException {
this.name = name; this.name = name;
this.path = path; this.path = path;
this.lock = new ReentrantReadWriteLock(); this.lock = new ReentrantReadWriteLock();
Files.createDirectories(path); Files.createDirectories(path);
purge(expireAfterSeconds);
} }
@Override @Override
@ -48,7 +52,7 @@ public class JsonFileStore implements Store<Map<String, Object>> {
AtomicLong size = new AtomicLong(0L); AtomicLong size = new AtomicLong(0L);
try (Stream<Path> stream = Files.walk(path)) { try (Stream<Path> stream = Files.walk(path)) {
stream.forEach(p -> { stream.forEach(p -> {
if (Files.isRegularFile(p)) { if (Files.isRegularFile(p) && Files.isReadable(p)) {
size.incrementAndGet(); size.incrementAndGet();
} }
}); });
@ -57,9 +61,9 @@ public class JsonFileStore implements Store<Map<String, Object>> {
} }
@Override @Override
public void readAll(String key, Consumer<Map<String, Object>> consumer) throws IOException { public void readAll(String prefix, Consumer<Map<String, Object>> consumer) throws IOException {
Objects.requireNonNull(consumer); Objects.requireNonNull(consumer);
try (Stream<Path> stream = Files.walk(path.resolve(key))) { try (Stream<Path> stream = Files.walk(path.resolve(prefix))) {
stream.forEach(p -> { stream.forEach(p -> {
try { try {
consumer.accept(read(p.getFileName().toString())); consumer.accept(read(p.getFileName().toString()));
@ -76,7 +80,12 @@ public class JsonFileStore implements Store<Map<String, Object>> {
try { try {
readLock.lock(); readLock.lock();
PercentEncoder percentEncoder = PercentEncoders.getUnreservedEncoder(StandardCharsets.UTF_8); PercentEncoder percentEncoder = PercentEncoders.getUnreservedEncoder(StandardCharsets.UTF_8);
return JsonUtil.toMap(Files.readString(path.resolve(percentEncoder.encode(key)))); Path p = path.resolve(percentEncoder.encode(key));
if (Files.isRegularFile(p) && Files.isReadable(p)) {
return JsonUtil.toMap(Files.readString(p));
} else {
return null;
}
} finally { } finally {
readLock.unlock(); readLock.unlock();
} }
@ -92,7 +101,61 @@ public class JsonFileStore implements Store<Map<String, Object>> {
if (!Files.exists(p.getParent())) { if (!Files.exists(p.getParent())) {
Files.createDirectories(p.getParent()); Files.createDirectories(p.getParent());
} }
try (Writer writer = Files.newBufferedWriter(p)) { try (Writer writer = Files.newBufferedWriter(p, StandardOpenOption.CREATE_NEW)) {
writer.write(JsonUtil.toString(map));
}
} finally {
writeLock.unlock();
}
}
@Override
public void write(String prefix, String key, Map<String, Object> map) throws IOException {
ReentrantReadWriteLock.WriteLock writeLock = lock.writeLock();
try {
writeLock.lock();
PercentEncoder percentEncoder = PercentEncoders.getUnreservedEncoder(StandardCharsets.UTF_8);
Path p = path.resolve(prefix).resolve(percentEncoder.encode(key));
if (!Files.exists(p.getParent())) {
Files.createDirectories(p.getParent());
}
try (Writer writer = Files.newBufferedWriter(p, StandardOpenOption.CREATE_NEW)) {
writer.write(JsonUtil.toString(map));
}
} finally {
writeLock.unlock();
}
}
@Override
public void update(String key, Map<String, Object> map) throws IOException {
ReentrantReadWriteLock.WriteLock writeLock = lock.writeLock();
try {
writeLock.lock();
PercentEncoder percentEncoder = PercentEncoders.getUnreservedEncoder(StandardCharsets.UTF_8);
Path p = path.resolve(percentEncoder.encode(key));
if (!Files.exists(p.getParent())) {
Files.createDirectories(p.getParent());
}
try (Writer writer = Files.newBufferedWriter(p, StandardOpenOption.CREATE)) {
writer.write(JsonUtil.toString(map));
}
} finally {
writeLock.unlock();
}
}
@Override
public void update(String prefix, String key, Map<String, Object> map) throws IOException {
ReentrantReadWriteLock.WriteLock writeLock = lock.writeLock();
try {
writeLock.lock();
PercentEncoder percentEncoder = PercentEncoders.getUnreservedEncoder(StandardCharsets.UTF_8);
Path p = path.resolve(prefix).resolve(percentEncoder.encode(key));
if (!Files.exists(p.getParent())) {
Files.createDirectories(p.getParent());
}
try (Writer writer = Files.newBufferedWriter(p, StandardOpenOption.CREATE)) {
writer.write(JsonUtil.toString(map)); writer.write(JsonUtil.toString(map));
} }
} finally { } finally {
@ -119,8 +182,7 @@ public class JsonFileStore implements Store<Map<String, Object>> {
try (Stream<Path> stream = Files.walk(path)) { try (Stream<Path> stream = Files.walk(path)) {
stream.forEach(p -> { stream.forEach(p -> {
try { try {
FileTime fileTime = Files.getLastModifiedTime(p); Duration duration = Duration.between(Files.getLastModifiedTime(p).toInstant(), instant);
Duration duration = Duration.between(fileTime.toInstant(), instant);
if (duration.toSeconds() > expiredAfterSeconds) { if (duration.toSeconds() > expiredAfterSeconds) {
Files.delete(p); Files.delete(p);
} }
@ -131,4 +193,21 @@ public class JsonFileStore implements Store<Map<String, Object>> {
} }
} }
} }
@Override
public void destroy() throws IOException {
Files.walkFileTree(path, new SimpleFileVisitor<>() {
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
});
}
} }

68
net/src/test/java/org/xbib/net/util/JsonFileStoreTest.java Normal file
View file

@ -0,0 +1,68 @@
package org.xbib.net.util;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class JsonFileStoreTest {
private static final Logger logger = Logger.getLogger(JsonFileStoreTest.class.getName());
private static JsonFileStore jsonFileStore;
@BeforeAll
public static void before() throws IOException {
Path path = Files.createTempDirectory("jsonstore");
jsonFileStore = new JsonFileStore("test", path, 3600L);
}
@AfterAll
public static void after() throws IOException {
if (jsonFileStore != null) {
jsonFileStore.destroy();
}
}
@Test
public void testJsonFileSTore() throws IOException {
singleReadWriteCycle(jsonFileStore, "1", Map.of("a", "b"));
}
@Test
public void consumeJsonFileStore() throws IOException {
for (int i =0; i < 100; i++) {
String key = RandomUtil.randomString(32);
singleReadWriteCycle(jsonFileStore, key, Map.of("a", "b"));
}
jsonFileStore.readAll("", map -> logger.log(Level.INFO, "map = " + map));
}
@Test
public void consumeJsonFileStoreByPrefix() throws IOException {
String prefix = "DE-38";
for (int i =0; i < 10; i++) {
String key = RandomUtil.randomString(32);
jsonFileStore.write(prefix, key, Map.of("a", "b"));
}
for (int i =0; i < 10; i++) {
String key = RandomUtil.randomString(32);
jsonFileStore.write("", key, Map.of("a", "b"));
}
jsonFileStore.readAll(prefix, map -> logger.log(Level.INFO, "map = " + map));
}
private void singleReadWriteCycle(JsonFileStore jsonFileStore, String key, Map<String, Object> map) throws IOException {
jsonFileStore.write(key, map);
Map<String, Object> readedMap = jsonFileStore.read(key);
assertEquals(map, readedMap);
}
}