joerg/files
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

implement FileService API, with streaming API

Browse source
This commit is contained in:
Jörg Prante 2021-12-12 00:05:24 +01:00
parent 2a724f1b37
commit 7992786e96
29 changed files with 975 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
files-api/src/main/java/module-info.java
View file

@ -1,8 +1,8 @@
import org.xbib.files.DefaultFilesProvider;
import org.xbib.files.Provider;
import org.xbib.files.DefaultFileServiceProvider;
import org.xbib.files.FileServiceProvider;
module org.xbib.files {
exports org.xbib.files;
uses Provider;
provides Provider with DefaultFilesProvider;
uses FileServiceProvider;
provides FileServiceProvider with DefaultFileServiceProvider;
}

29
files-api/src/main/java/org/xbib/files/DefaultFiles.java → files-api/src/main/java/org/xbib/files/DefaultFileService.java
View file

@ -10,6 +10,7 @@ import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.file.CopyOption;
import java.nio.file.DirectoryStream;
import java.nio.file.FileVisitOption;
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.Paths;
@ -26,8 +27,9 @@ import java.util.Map;
import java.util.Set;
import java.nio.file.Files;
import java.util.stream.Stream;
public class DefaultFiles implements org.xbib.files.Files {
public class DefaultFileService implements FileService {
private static final int READ_BUFFER_SIZE = 128 * 1024;
@ -39,13 +41,7 @@ public class DefaultFiles implements org.xbib.files.Files {
private static final Set<PosixFilePermission> DEFAULT_FILE_PERMISSIONS =
PosixFilePermissions.fromString("rw-r--r--");
private final URI uri;
private final Map<String, ?> env;
public DefaultFiles(URI uri, Map<String, ?> env) {
this.uri = uri;
this.env = env;
public DefaultFileService(URI uri, Map<String, ?> env) {
}
@Override
@ -209,10 +205,25 @@ public class DefaultFiles implements org.xbib.files.Files {
}
@Override
public DirectoryStream<Path> filter(String path, DirectoryStream.Filter<Path> filter) throws IOException {
public DirectoryStream<Path> stream(String path, DirectoryStream.Filter<Path> filter) throws IOException {
return Files.newDirectoryStream(Paths.get(path), filter);
}
@Override
public Stream<Path> list(String path) throws IOException {
return Files.list(Paths.get(path));
}
@Override
public Stream<Path> walk(String path, FileVisitOption... options) throws IOException {
return Files.walk(Paths.get(path), options);
}
@Override
public Stream<Path> walk(String path, int maxdepth, FileVisitOption... options) throws IOException {
return Files.walk(Paths.get(path), maxdepth, options);
}
private void upload(Path source, Path target,
Set<PosixFilePermission> dirPerms,
Set<PosixFilePermission> filePerms,

11
files-api/src/main/java/org/xbib/files/DefaultFileServiceProvider.java Normal file
View file

@ -0,0 +1,11 @@
package org.xbib.files;
import java.net.URI;
import java.util.Map;
public class DefaultFileServiceProvider implements FileServiceProvider {
@Override
public FileService provide(URI uri, Map<String, ?> env) {
return !uri.isAbsolute() || uri.getScheme().equals("file") ? new DefaultFileService(uri, env) : null;
}
}

11
files-api/src/main/java/org/xbib/files/DefaultFilesProvider.java
View file

@ -1,11 +0,0 @@
package org.xbib.files;
import java.net.URI;
import java.util.Map;
public class DefaultFilesProvider implements Provider {
@Override
public Files provide(URI uri, Map<String, ?> env) {
return !uri.isAbsolute() || uri.getScheme().equals("file") ? new DefaultFiles(uri, env) : null;
}
}

25
files-api/src/main/java/org/xbib/files/Files.java → files-api/src/main/java/org/xbib/files/FileService.java
View file

@ -6,6 +6,8 @@ import java.io.OutputStream;
import java.net.URI;
import java.nio.file.CopyOption;
import java.nio.file.DirectoryStream;
import java.nio.file.FileVisitOption;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.FileTime;
@ -16,14 +18,15 @@ import java.util.Objects;
import java.util.Optional;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.stream.Stream;
public interface Files {
public interface FileService {
class Holder {
private static Files createFiles(URI uri, Map<String, ?> env) {
ServiceLoader<Provider> serviceLoader = ServiceLoader.load(Provider.class);
Optional<Files> first = serviceLoader.stream()
private static FileService createFiles(URI uri, Map<String, ?> env) {
ServiceLoader<FileServiceProvider> serviceLoader = ServiceLoader.load(FileServiceProvider.class);
Optional<FileService> first = serviceLoader.stream()
.map(ServiceLoader.Provider::get)
.filter(Objects::nonNull)
.map(p -> p.provide(uri, env))
@ -33,7 +36,11 @@ public interface Files {
}
}
default Files newInstance(URI uri, Map<String, ?> env) {
static FileService newInstance(String uri, Map<String, ?> env) {
return Holder.createFiles(URI.create(uri), env);
}
static FileService newInstance(URI uri, Map<String, ?> env) {
return Holder.createFiles(uri, env);
}
@ -101,5 +108,11 @@ public interface Files {
DirectoryStream<Path> stream(String path, String glob) throws IOException;
DirectoryStream<Path> filter(String path, DirectoryStream.Filter<Path> filter) throws IOException;
DirectoryStream<Path> stream(String path, DirectoryStream.Filter<Path> filter) throws IOException;
Stream<Path> list(String path) throws IOException;
Stream<Path> walk(String path, FileVisitOption... options) throws IOException;
Stream<Path> walk(String path, int maxdepth, FileVisitOption... options) throws IOException;
}

9
files-api/src/main/java/org/xbib/files/FileServiceProvider.java Normal file
View file

@ -0,0 +1,9 @@
package org.xbib.files;
import java.net.URI;
import java.util.Map;
public interface FileServiceProvider {
FileService provide(URI uri, Map<String, ?> env);
}

102
files-api/src/main/java/org/xbib/files/FileTreeIterator.java Normal file
View file

@ -0,0 +1,102 @@
package org.xbib.files;
import java.io.Closeable;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.FileVisitOption;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Iterator;
import java.util.NoSuchElementException;
import org.xbib.files.FileTreeWalker.Event;
/**
* This source is a copy taken from java.nio.file.FileTreeIterator which is inaccessible.
*
* An {@code Iterator} to iterate over the nodes of a file tree.
*
* <pre>{@code
* try (FileTreeIterator iterator = new FileTreeIterator(start, maxDepth, options)) {
* while (iterator.hasNext()) {
* Event ev = iterator.next();
* Path path = ev.file();
* BasicFileAttributes attrs = ev.attributes();
* }
* }
* }</pre>
*/
class FileTreeIterator implements Iterator<Event>, Closeable {
private final FileTreeWalker walker;
private Event next;
/**
* Creates a new iterator to walk the file tree starting at the given file.
*
* @throws IllegalArgumentException
* if {@code maxDepth} is negative
* @throws IOException
* if an I/O errors occurs opening the starting file
* @throws SecurityException
* if the security manager denies access to the starting file
* @throws NullPointerException
* if {@code start} or {@code options} is {@code null} or
* the options array contains a {@code null} element
*/
FileTreeIterator(Path start, int maxDepth, FileVisitOption... options)
throws IOException
{
this.walker = new FileTreeWalker(Arrays.asList(options), maxDepth);
this.next = walker.walk(start);
assert next.type() == FileTreeWalker.EventType.ENTRY ||
next.type() == FileTreeWalker.EventType.START_DIRECTORY;
// IOException if there a problem accessing the starting file
IOException ioe = next.ioeException();
if (ioe != null)
throw ioe;
}
private void fetchNextIfNeeded() {
if (next == null) {
FileTreeWalker.Event ev = walker.next();
while (ev != null) {
IOException ioe = ev.ioeException();
if (ioe != null)
throw new UncheckedIOException(ioe);
// END_DIRECTORY events are ignored
if (ev.type() != FileTreeWalker.EventType.END_DIRECTORY) {
next = ev;
return;
}
ev = walker.next();
}
}
}
@Override
public boolean hasNext() {
if (!walker.isOpen())
throw new IllegalStateException();
fetchNextIfNeeded();
return next != null;
}
@Override
public Event next() {
if (!walker.isOpen())
throw new IllegalStateException();
fetchNextIfNeeded();
if (next == null)
throw new NoSuchElementException();
Event result = next;
next = null;
return result;
}
@Override
public void close() {
walker.close();
}
}

398
files-api/src/main/java/org/xbib/files/FileTreeWalker.java Normal file
View file

@ -0,0 +1,398 @@
package org.xbib.files;
import java.nio.file.DirectoryIteratorException;
import java.nio.file.DirectoryStream;
import java.nio.file.FileSystemLoopException;
import java.nio.file.FileVisitOption;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayDeque;
import java.util.Collection;
import java.util.Iterator;
/**
* This source is a copy taken from java.nio.file.FileTreeWalker which is inaccessible.
*
* Walks a file tree, generating a sequence of events corresponding to the files
* in the tree.
*
* <pre>{@code
* Path top = ...
* Set<FileVisitOption> options = ...
* int maxDepth = ...
*
* try (FileTreeWalker walker = new FileTreeWalker(options, maxDepth)) {
* FileTreeWalker.Event ev = walker.walk(top);
* do {
* process(ev);
* ev = walker.next();
* } while (ev != null);
* }
* }</pre>
*
* @see Files#walkFileTree
*/
class FileTreeWalker implements Closeable {
private final boolean followLinks;
private final LinkOption[] linkOptions;
private final int maxDepth;
private final ArrayDeque<DirectoryNode> stack = new ArrayDeque<>();
private boolean closed;
/**
* The element on the walking stack corresponding to a directory node.
*/
private static class DirectoryNode {
private final Path dir;
private final Object key;
private final DirectoryStream<Path> stream;
private final Iterator<Path> iterator;
private boolean skipped;
DirectoryNode(Path dir, Object key, DirectoryStream<Path> stream) {
this.dir = dir;
this.key = key;
this.stream = stream;
this.iterator = stream.iterator();
}
Path directory() {
return dir;
}
Object key() {
return key;
}
DirectoryStream<Path> stream() {
return stream;
}
Iterator<Path> iterator() {
return iterator;
}
void skip() {
skipped = true;
}
boolean skipped() {
return skipped;
}
}
/**
* The event types.
*/
static enum EventType {
/**
* Start of a directory
*/
START_DIRECTORY,
/**
* End of a directory
*/
END_DIRECTORY,
/**
* An entry in a directory
*/
ENTRY;
}
/**
* Events returned by the {@link #walk} and {@link #next} methods.
*/
static class Event {
private final EventType type;
private final Path file;
private final BasicFileAttributes attrs;
private final IOException ioe;
private Event(EventType type, Path file, BasicFileAttributes attrs, IOException ioe) {
this.type = type;
this.file = file;
this.attrs = attrs;
this.ioe = ioe;
}
Event(EventType type, Path file, BasicFileAttributes attrs) {
this(type, file, attrs, null);
}
Event(EventType type, Path file, IOException ioe) {
this(type, file, null, ioe);
}
EventType type() {
return type;
}
Path file() {
return file;
}
BasicFileAttributes attributes() {
return attrs;
}
IOException ioeException() {
return ioe;
}
}
/**
* Creates a {@code FileTreeWalker}.
*
* @throws IllegalArgumentException
* if {@code maxDepth} is negative
* @throws ClassCastException
* if {@code options} contains an element that is not a
* {@code FileVisitOption}
* @throws NullPointerException
* if {@code options} is {@code null} or the options
* array contains a {@code null} element
*/
FileTreeWalker(Collection<FileVisitOption> options, int maxDepth) {
boolean fl = false;
for (FileVisitOption option: options) {
// will throw NPE if options contains null
switch (option) {
case FOLLOW_LINKS : fl = true; break;
default:
throw new AssertionError("Should not get here");
}
}
if (maxDepth < 0)
throw new IllegalArgumentException("'maxDepth' is negative");
this.followLinks = fl;
this.linkOptions = (fl) ? new LinkOption[0] :
new LinkOption[] { LinkOption.NOFOLLOW_LINKS };
this.maxDepth = maxDepth;
}
/**
* Returns the attributes of the given file, taking into account whether
* the walk is following sym links is not. The {@code canUseCached}
* argument determines whether this method can use cached attributes.
*/
private BasicFileAttributes getAttributes(Path file, boolean canUseCached)
throws IOException
{
// attempt to get attributes of file. If fails and we are following
// links then a link target might not exist so get attributes of link
BasicFileAttributes attrs;
try {
attrs = Files.readAttributes(file, BasicFileAttributes.class, linkOptions);
} catch (IOException ioe) {
if (!followLinks)
throw ioe;
// attempt to get attrmptes without following links
attrs = Files.readAttributes(file,
BasicFileAttributes.class,
LinkOption.NOFOLLOW_LINKS);
}
return attrs;
}
/**
* Returns true if walking into the given directory would result in a
* file system loop/cycle.
*/
private boolean wouldLoop(Path dir, Object key) {
// if this directory and ancestor has a file key then we compare
// them; otherwise we use less efficient isSameFile test.
for (DirectoryNode ancestor: stack) {
Object ancestorKey = ancestor.key();
if (key != null && ancestorKey != null) {
if (key.equals(ancestorKey)) {
// cycle detected
return true;
}
} else {
try {
if (Files.isSameFile(dir, ancestor.directory())) {
// cycle detected
return true;
}
} catch (IOException | SecurityException x) {
// ignore
}
}
}
return false;
}
/**
* Visits the given file, returning the {@code Event} corresponding to that
* visit.
*
* The {@code ignoreSecurityException} parameter determines whether
* any SecurityException should be ignored or not. If a SecurityException
* is thrown, and is ignored, then this method returns {@code null} to
* mean that there is no event corresponding to a visit to the file.
*
* The {@code canUseCached} parameter determines whether cached attributes
* for the file can be used or not.
*/
private Event visit(Path entry, boolean ignoreSecurityException, boolean canUseCached) {
// need the file attributes
BasicFileAttributes attrs;
try {
attrs = getAttributes(entry, canUseCached);
} catch (IOException ioe) {
return new Event(EventType.ENTRY, entry, ioe);
} catch (SecurityException se) {
if (ignoreSecurityException)
return null;
throw se;
}
// at maximum depth or file is not a directory
int depth = stack.size();
if (depth >= maxDepth || !attrs.isDirectory()) {
return new Event(EventType.ENTRY, entry, attrs);
}
// check for cycles when following links
if (followLinks && wouldLoop(entry, attrs.fileKey())) {
return new Event(EventType.ENTRY, entry,
new FileSystemLoopException(entry.toString()));
}
// file is a directory, attempt to open it
DirectoryStream<Path> stream = null;
try {
stream = Files.newDirectoryStream(entry);
} catch (IOException ioe) {
return new Event(EventType.ENTRY, entry, ioe);
} catch (SecurityException se) {
if (ignoreSecurityException)
return null;
throw se;
}
// push a directory node to the stack and return an event
stack.push(new DirectoryNode(entry, attrs.fileKey(), stream));
return new Event(EventType.START_DIRECTORY, entry, attrs);
}
/**
* Start walking from the given file.
*/
Event walk(Path file) {
if (closed)
throw new IllegalStateException("Closed");
Event ev = visit(file,
false, // ignoreSecurityException
false); // canUseCached
assert ev != null;
return ev;
}
/**
* Returns the next Event or {@code null} if there are no more events or
* the walker is closed.
*/
Event next() {
DirectoryNode top = stack.peek();
if (top == null)
return null; // stack is empty, we are done
// continue iteration of the directory at the top of the stack
Event ev;
do {
Path entry = null;
IOException ioe = null;
// get next entry in the directory
if (!top.skipped()) {
Iterator<Path> iterator = top.iterator();
try {
if (iterator.hasNext()) {
entry = iterator.next();
}
} catch (DirectoryIteratorException x) {
ioe = x.getCause();
}
}
// no next entry so close and pop directory,
// creating corresponding event
if (entry == null) {
try {
top.stream().close();
} catch (IOException e) {
if (ioe == null) {
ioe = e;
} else {
ioe.addSuppressed(e);
}
}
stack.pop();
return new Event(EventType.END_DIRECTORY, top.directory(), ioe);
}
// visit the entry
ev = visit(entry,
true, // ignoreSecurityException
true); // canUseCached
} while (ev == null);
return ev;
}
/**
* Pops the directory node that is the current top of the stack so that
* there are no more events for the directory (including no END_DIRECTORY)
* event. This method is a no-op if the stack is empty or the walker is
* closed.
*/
void pop() {
if (!stack.isEmpty()) {
DirectoryNode node = stack.pop();
try {
node.stream().close();
} catch (IOException ignore) { }
}
}
/**
* Skips the remaining entries in the directory at the top of the stack.
* This method is a no-op if the stack is empty or the walker is closed.
*/
void skipRemainingSiblings() {
if (!stack.isEmpty()) {
stack.peek().skip();
}
}
/**
* Returns {@code true} if the walker is open.
*/
boolean isOpen() {
return !closed;
}
/**
* Closes/pops all directories on the stack.
*/
@Override
public void close() {
if (!closed) {
while (!stack.isEmpty()) {
pop();
}
closed = true;
}
}
}

228
files-api/src/main/java/org/xbib/files/FileWalker.java Normal file
View file

@ -0,0 +1,228 @@
package org.xbib.files;
import java.io.Closeable;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.DirectoryIteratorException;
import java.nio.file.DirectoryStream;
import java.nio.file.FileSystemLoopException;
import java.nio.file.FileVisitOption;
import java.nio.file.Files;
import java.nio.file.NotDirectoryException;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Iterator;
import java.util.Spliterator;
import java.util.Spliterators;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
/**
* This source is a copy taken from java.nio.file.Files#walk and modified to process a java.io.Closeable after the walk.
*/
public class FileWalker {
/**
* Return a lazily populated {@code Stream}, the elements of
* which are the entries in the directory. The listing is not recursive.
*
* <p> The elements of the stream are {@link Path} objects that are
* obtained as if by {@link Path#resolve(Path) resolving} the name of the
* directory entry against {@code dir}. Some file systems maintain special
* links to the directory itself and the directory's parent directory.
* Entries representing these links are not included.
*
* <p> The stream is <i>weakly consistent</i>. It is thread safe but does
* not freeze the directory while iterating, so it may (or may not)
* reflect updates to the directory that occur after returning from this
* method.
*
* <p> The returned stream contains a reference to an open directory.
* The directory is closed by closing the stream.
*
* <p> Operating on a closed stream behaves as if the end of stream
* has been reached. Due to read-ahead, one or more elements may be
* returned after the stream has been closed.
*
* <p> If an {@link IOException} is thrown when accessing the directory
* after this method has returned, it is wrapped in an {@link
* UncheckedIOException} which will be thrown from the method that caused
* the access to take place.
*
* @apiNote
* This method must be used within a try-with-resources statement or similar
* control structure to ensure that the stream's open directory is closed
* promptly after the stream's operations have completed.
*
* @param ds The directory stream
*
* @return The {@code Stream} describing the content of the
* directory
*
* @throws NotDirectoryException
* if the file could not otherwise be opened because it is not
* a directory <i>(optional specific exception)</i>
* @throws IOException
* if an I/O error occurs when opening the directory
* @throws SecurityException
* In the case of the default provider, and a security manager is
* installed, the {@link SecurityManager#checkRead(String) checkRead}
* method is invoked to check read access to the directory.
*/
public static Stream<Path> list(DirectoryStream<Path> ds) throws IOException {
try {
final Iterator<Path> delegate = ds.iterator();
Iterator<Path> iterator = new Iterator<>() {
@Override
public boolean hasNext() {
try {
return delegate.hasNext();
} catch (DirectoryIteratorException e) {
throw new UncheckedIOException(e.getCause());
}
}
@Override
public Path next() {
try {
return delegate.next();
} catch (DirectoryIteratorException e) {
throw new UncheckedIOException(e.getCause());
}
}
};
Spliterator<Path> spliterator =
Spliterators.spliteratorUnknownSize(iterator, Spliterator.DISTINCT);
return StreamSupport.stream(spliterator, false)
.onClose(asUncheckedRunnable(ds));
} catch (Error|RuntimeException e) {
try {
ds.close();
} catch (IOException ex) {
try {
e.addSuppressed(ex);
} catch (Throwable ignore) {}
}
throw e;
}
}
/**
* Return a {@code Stream} that is lazily populated with {@code
* Path} by walking the file tree rooted at a given starting file. The
* file tree is traversed <em>depth-first</em>, the elements in the stream
* are {@link Path} objects that are obtained as if by {@link
* Path#resolve(Path) resolving} the relative path against {@code start}.
*
* <p> The {@code stream} walks the file tree as elements are consumed.
* The {@code Stream} returned is guaranteed to have at least one
* element, the starting file itself. For each file visited, the stream
* attempts to read its {@link BasicFileAttributes}. If the file is a
* directory and can be opened successfully, entries in the directory, and
* their <em>descendants</em> will follow the directory in the stream as
* they are encountered. When all entries have been visited, then the
* directory is closed. The file tree walk then continues at the next
* <em>sibling</em> of the directory.
*
* <p> The stream is <i>weakly consistent</i>. It does not freeze the
* file tree while iterating, so it may (or may not) reflect updates to
* the file tree that occur after returned from this method.
*
* <p> By default, symbolic links are not automatically followed by this
* method. If the {@code options} parameter contains the {@link
* FileVisitOption#FOLLOW_LINKS FOLLOW_LINKS} option then symbolic links are
* followed. When following links, and the attributes of the target cannot
* be read, then this method attempts to get the {@code BasicFileAttributes}
* of the link.
*
* <p> If the {@code options} parameter contains the {@link
* FileVisitOption#FOLLOW_LINKS FOLLOW_LINKS} option then the stream keeps
* track of directories visited so that cycles can be detected. A cycle
* arises when there is an entry in a directory that is an ancestor of the
* directory. Cycle detection is done by recording the {@link
* java.nio.file.attribute.BasicFileAttributes#fileKey file-key} of directories,
* or if file keys are not available, by invoking the {@link java.nio.file.Files#isSameFile
* isSameFile} method to test if a directory is the same file as an
* ancestor. When a cycle is detected it is treated as an I/O error with
* an instance of {@link FileSystemLoopException}.
*
* <p> The {@code maxDepth} parameter is the maximum number of levels of
* directories to visit. A value of {@code 0} means that only the starting
* file is visited, unless denied by the security manager. A value of
* {@link Integer#MAX_VALUE MAX_VALUE} may be used to indicate that all
* levels should be visited.
*
* <p> When a security manager is installed and it denies access to a file
* (or directory), then it is ignored and not included in the stream.
*
* <p> The returned stream contains references to one or more open directories.
* The directories are closed by closing the stream.
*
* <p> If an {@link IOException} is thrown when accessing the directory
* after this method has returned, it is wrapped in an {@link
* UncheckedIOException} which will be thrown from the method that caused
* the access to take place.
*
* @apiNote
* This method must be used within a try-with-resources statement or similar
* control structure to ensure that the stream's open directories are closed
* promptly after the stream's operations have completed.
*
* @param closeable a closeable which must be closed after walking
* @param start
* the starting file
* @param maxDepth
* the maximum number of directory levels to visit
* @param options
* options to configure the traversal
*
* @return the {@link Stream} of {@link Path}
*
* @throws IllegalArgumentException
* if the {@code maxDepth} parameter is negative
* @throws SecurityException
* If the security manager denies access to the starting file.
* In the case of the default provider, the {@link
* SecurityManager#checkRead(String) checkRead} method is invoked
* to check read access to the directory.
* @throws IOException
* if an I/O error is thrown when accessing the starting file.
*/
public static Stream<Path> walk(Closeable closeable,
Path start,
int maxDepth,
FileVisitOption... options) throws IOException {
FileTreeIterator iterator = new FileTreeIterator(start, maxDepth, options);
try {
Spliterator<FileTreeWalker.Event> spliterator =
Spliterators.spliteratorUnknownSize(iterator, Spliterator.DISTINCT);
return StreamSupport.stream(spliterator, false)
.onClose(() -> {
iterator.close();
try {
closeable.close();
} catch (IOException e) {
throw new UncheckedIOException(e);
}
})
.map(FileTreeWalker.Event::file);
} catch (Error|RuntimeException e) {
iterator.close();
closeable.close();
throw e;
}
}
/**
* Convert a Closeable to a Runnable by converting checked IOException
* to UncheckedIOException
*/
private static Runnable asUncheckedRunnable(Closeable c) {
return () -> {
try {
c.close();
} catch (IOException e) {
throw new UncheckedIOException(e);
}
};
}
}

9
files-api/src/main/java/org/xbib/files/Provider.java
View file

@ -1,9 +0,0 @@
package org.xbib.files;
import java.net.URI;
import java.util.Map;
public interface Provider {
Files provide(URI uri, Map<String, ?> env);
}

29
files-api/src/main/java/org/xbib/files/WrappedDirectoryStream.java Normal file
View file

@ -0,0 +1,29 @@
package org.xbib.files;
import java.io.Closeable;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.util.Iterator;
public class WrappedDirectoryStream<T> implements DirectoryStream<T> {
private final Closeable closeable;
private final DirectoryStream<T> directoryStream;
public WrappedDirectoryStream(Closeable closeable, DirectoryStream<T> directoryStream) {
this.closeable = closeable;
this.directoryStream = directoryStream;
}
@Override
public Iterator<T> iterator() {
return directoryStream.iterator();
}
@Override
public void close() throws IOException {
directoryStream.close();
closeable.close();
}
}

1
files-api/src/main/resources/META-INF/services/org.xbib.files.FileServiceProvider Normal file
View file

@ -0,0 +1 @@
org.xbib.files.DefaultFileServiceProvider

1
files-api/src/main/resources/META-INF/services/org.xbib.files.Provider
View file

@ -1 +0,0 @@
org.xbib.files.DefaultFilesProvider

4
files-ftp-fs/src/main/java/module-info.java
View file

@ -1,5 +1,5 @@
import java.nio.file.spi.FileSystemProvider;
import org.xbib.files.Provider;
import org.xbib.files.FileServiceProvider;
import org.xbib.io.ftp.fs.FTPFileSystemProvider;
import org.xbib.io.ftp.fs.spi.FTPFilesProvider;
@ -7,5 +7,5 @@ module org.xbib.files.ftp.fs {
requires org.xbib.files;
requires org.xbib.files.ftp;
provides FileSystemProvider with FTPFileSystemProvider;
provides Provider with FTPFilesProvider;
provides FileServiceProvider with FTPFilesProvider;
}

6
files-ftp-fs/src/main/java/org/xbib/io/ftp/fs/spi/FTPContext.java
View file

@ -3,12 +3,13 @@ package org.xbib.io.ftp.fs.spi;
import org.xbib.io.ftp.fs.FTPEnvironment;
import org.xbib.io.ftp.fs.FTPFileSystemProvider;
import java.io.Closeable;
import java.io.IOException;
import java.net.URI;
import java.nio.file.FileSystem;
import java.util.Map;
class FTPContext {
class FTPContext implements Closeable {
final FTPFileSystemProvider provider;
@ -19,7 +20,8 @@ class FTPContext {
this.fileSystem = provider.newFileSystem(uri, env != null ? env : new FTPEnvironment());
}
void close() throws IOException {
@Override
public void close() throws IOException {
fileSystem.close();
}
}

48
files-ftp-fs/src/main/java/org/xbib/io/ftp/fs/spi/FTPFiles.java
View file

@ -1,5 +1,9 @@
package org.xbib.io.ftp.fs.spi;
import org.xbib.files.FileService;
import org.xbib.files.FileWalker;
import org.xbib.files.WrappedDirectoryStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@ -10,6 +14,7 @@ import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.file.CopyOption;
import java.nio.file.DirectoryStream;
import java.nio.file.FileVisitOption;
import java.nio.file.Files;
import java.nio.file.OpenOption;
import java.nio.file.Path;
@ -24,8 +29,9 @@ import java.nio.file.attribute.UserPrincipal;
import java.util.EnumSet;
import java.util.Map;
import java.util.Set;
import java.util.stream.Stream;
public class FTPFiles implements org.xbib.files.Files {
public class FTPFiles implements FileService {
private static final int READ_BUFFER_SIZE = 128 * 1024;
@ -146,21 +152,41 @@ public class FTPFiles implements org.xbib.files.Files {
return performWithContext(ctx -> Files.getOwner(ctx.fileSystem.getPath(path)));
}
@Override
public DirectoryStream<Path> stream(String path, String glob) throws IOException {
return performWithContext(ctx -> Files.newDirectoryStream(ctx.fileSystem.getPath(path), glob));
}
@Override
public DirectoryStream<Path> filter(String path, DirectoryStream.Filter<Path> filter) throws IOException {
return performWithContext(ctx -> Files.newDirectoryStream(ctx.fileSystem.getPath(path), filter));
}
@Override
public void upload(Path source, Path target, CopyOption... copyOptions) throws IOException {
upload(source, target, DEFAULT_DIR_PERMISSIONS, DEFAULT_FILE_PERMISSIONS, copyOptions);
}
@Override
public DirectoryStream<Path> stream(String path, String glob) throws IOException {
FTPContext ctx = new FTPContext(uri, env);
return new WrappedDirectoryStream<>(ctx, Files.newDirectoryStream(ctx.fileSystem.getPath(path), glob));
}
@Override
public DirectoryStream<Path> stream(String path, DirectoryStream.Filter<Path> filter) throws IOException {
FTPContext ctx = new FTPContext(uri, env);
return new WrappedDirectoryStream<>(ctx, Files.newDirectoryStream(ctx.fileSystem.getPath(path), filter));
}
@Override
public Stream<Path> list(String path) throws IOException {
FTPContext ctx = new FTPContext(uri, env);
return FileWalker.list(new WrappedDirectoryStream<>(ctx, Files.newDirectoryStream(ctx.fileSystem.getPath(path))));
}
@Override
public Stream<Path> walk(String path, FileVisitOption... options) throws IOException {
FTPContext ctx = new FTPContext(uri, env);
return FileWalker.walk(ctx, ctx.fileSystem.getPath(path), Integer.MAX_VALUE, options);
}
@Override
public Stream<Path> walk(String path, int maxdepth, FileVisitOption... options) throws IOException {
FTPContext ctx = new FTPContext(uri, env);
return FileWalker.walk(ctx, ctx.fileSystem.getPath(path), maxdepth, options);
}
public void upload(Path source, Path target,
Set<PosixFilePermission> dirPerms,
Set<PosixFilePermission> filePerms,

10
files-ftp-fs/src/main/java/org/xbib/io/ftp/fs/spi/FTPFilesProvider.java
View file

@ -1,14 +1,14 @@
package org.xbib.io.ftp.fs.spi;
import org.xbib.files.Files;
import org.xbib.files.Provider;
import org.xbib.files.FileService;
import org.xbib.files.FileServiceProvider;
import java.net.URI;
import java.util.Map;
public class FTPFilesProvider implements Provider {
public class FTPFilesProvider implements FileServiceProvider {
@Override
public Files provide(URI uri, Map<String, ?> env) {
return !uri.isAbsolute() || uri.getScheme().equals("file") ? new FTPFiles(uri, env) : null;
public FileService provide(URI uri, Map<String, ?> env) {
return uri.isAbsolute() && uri.getScheme().equals("ftp") ? new FTPFiles(uri, env) : null;
}
}

0
files-ftp-fs/src/main/resources/META-INF/services/org.xbib.files.Provider → files-ftp-fs/src/main/resources/META-INF/services/org.xbib.files.FileServiceProvider
View file

7
files-sftp-fs/src/main/java/module-info.java
View file

@ -1,7 +1,7 @@
import org.apache.sshd.fs.SftpFileSystemProvider;
import java.nio.file.spi.FileSystemProvider;
import org.xbib.files.Provider;
import org.apache.sshd.fs.spi.SFTPFilesProvider;
import org.xbib.files.FileServiceProvider;
import org.apache.sshd.fs.spi.SFTPFileServiceProvider;
module org.xbib.files.sftp.fs {
exports org.apache.sshd.fs;
@ -9,5 +9,6 @@ module org.xbib.files.sftp.fs {
requires org.xbib.files;
requires transitive org.xbib.files.sftp;
provides FileSystemProvider with SftpFileSystemProvider;
provides Provider with SFTPFilesProvider;
provides FileServiceProvider with SFTPFileServiceProvider;
requires java.logging;
}

2
files-sftp-fs/src/main/java/org/apache/sshd/fs/SftpFileSystem.java
View file

@ -71,7 +71,7 @@ public class SftpFileSystem
private final ThreadLocal<Wrapper> wrappers = new ThreadLocal<>();
private final int version;
private final Set<String> supportedViews;
private SftpPath defaultDir;
private final SftpPath defaultDir;
private int readBufferSize;
private int writeBufferSize;
private final List<FileStore> stores;

8
files-sftp-fs/src/main/java/org/apache/sshd/fs/spi/SFTPContext.java
View file

@ -5,11 +5,12 @@ import org.apache.sshd.client.SshClient;
import org.apache.sshd.fs.SftpFileSystem;
import org.apache.sshd.fs.SftpFileSystemProvider;
import java.io.Closeable;
import java.io.IOException;
import java.net.URI;
import java.util.Map;
class SFTPContext {
public class SFTPContext implements Closeable {
private final SshClient sshClient;
@ -17,7 +18,7 @@ class SFTPContext {
final SftpFileSystem fileSystem;
SFTPContext(URI uri, Map<String, ?> env) throws IOException {
public SFTPContext(URI uri, Map<String, ?> env) throws IOException {
this.sshClient = ClientBuilder.builder().build();
Object object = env.get("workers");
if (object instanceof Integer) {
@ -33,7 +34,8 @@ class SFTPContext {
this.fileSystem = provider.newFileSystem(uri, env);
}
void close() throws IOException {
@Override
public void close() throws IOException {
sshClient.stop();
fileSystem.close();
}

54
files-sftp-fs/src/main/java/org/apache/sshd/fs/spi/SFTPFiles.java → files-sftp-fs/src/main/java/org/apache/sshd/fs/spi/SFTPFileService.java
View file

@ -1,5 +1,9 @@
package org.apache.sshd.fs.spi;
import org.xbib.files.FileService;
import org.xbib.files.FileWalker;
import org.xbib.files.WrappedDirectoryStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@ -10,6 +14,7 @@ import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.file.CopyOption;
import java.nio.file.DirectoryStream;
import java.nio.file.FileVisitOption;
import java.nio.file.Files;
import java.nio.file.OpenOption;
import java.nio.file.Path;
@ -24,8 +29,9 @@ import java.nio.file.attribute.UserPrincipal;
import java.util.EnumSet;
import java.util.Map;
import java.util.Set;
import java.util.stream.Stream;
public class SFTPFiles implements org.xbib.files.Files {
public class SFTPFileService implements FileService {
private static final int READ_BUFFER_SIZE = 128 * 1024;
@ -41,7 +47,7 @@ public class SFTPFiles implements org.xbib.files.Files {
private final Map<String, ?> env;
public SFTPFiles(URI uri, Map<String, ?> env) {
public SFTPFileService(URI uri, Map<String, ?> env) {
this.uri = uri;
this.env = env;
}
@ -146,16 +152,6 @@ public class SFTPFiles implements org.xbib.files.Files {
return performWithContext(ctx -> Files.getOwner(ctx.fileSystem.getPath(path)));
}
@Override
public DirectoryStream<Path> stream(String path, String glob) throws IOException {
return performWithContext(ctx -> Files.newDirectoryStream(ctx.fileSystem.getPath(path), glob));
}
@Override
public DirectoryStream<Path> filter(String path, DirectoryStream.Filter<Path> filter) throws IOException {
return performWithContext(ctx -> Files.newDirectoryStream(ctx.fileSystem.getPath(path), filter));
}
@Override
public void upload(Path source, Path target, CopyOption... copyOptions) throws IOException {
upload(source, target, DEFAULT_DIR_PERMISSIONS, DEFAULT_FILE_PERMISSIONS, copyOptions);
@ -223,6 +219,36 @@ public class SFTPFiles implements org.xbib.files.Files {
performWithContext(ctx -> Files.deleteIfExists(ctx.fileSystem.getPath(source)));
}
@Override
public DirectoryStream<Path> stream(String path, String glob) throws IOException {
SFTPContext ctx = new SFTPContext(uri, env);
return new WrappedDirectoryStream<>(ctx, Files.newDirectoryStream(ctx.fileSystem.getPath(path), glob));
}
@Override
public DirectoryStream<Path> stream(String path, DirectoryStream.Filter<Path> filter) throws IOException {
SFTPContext ctx = new SFTPContext(uri, env);
return new WrappedDirectoryStream<>(ctx, Files.newDirectoryStream(ctx.fileSystem.getPath(path), filter));
}
@Override
public Stream<Path> list(String path) throws IOException {
SFTPContext ctx = new SFTPContext(uri, env);
return FileWalker.list(new WrappedDirectoryStream<>(ctx, Files.newDirectoryStream(ctx.fileSystem.getPath(path))));
}
@Override
public Stream<Path> walk(String path, FileVisitOption... options) throws IOException {
SFTPContext ctx = new SFTPContext(uri, env);
return FileWalker.walk(ctx, ctx.fileSystem.getPath(path), Integer.MAX_VALUE, options);
}
@Override
public Stream<Path> walk(String path, int maxdepth, FileVisitOption... options) throws IOException {
SFTPContext ctx = new SFTPContext(uri, env);
return FileWalker.walk(ctx, ctx.fileSystem.getPath(path), maxdepth, options);
}
public void upload(Path source, Path target,
Set<PosixFilePermission> dirPermissions,
Set<PosixFilePermission> filePermissions,
@ -389,4 +415,8 @@ public class SFTPFiles implements org.xbib.files.Files {
}
}
}
private <T> T performWithoutClose(SFTPContext ctx, WithContext<T> action) throws IOException {
return action.perform(ctx);
}
}

14
files-sftp-fs/src/main/java/org/apache/sshd/fs/spi/SFTPFileServiceProvider.java Normal file
View file

@ -0,0 +1,14 @@
package org.apache.sshd.fs.spi;
import org.xbib.files.FileService;
import org.xbib.files.FileServiceProvider;
import java.net.URI;
import java.util.Map;
public class SFTPFileServiceProvider implements FileServiceProvider {
@Override
public FileService provide(URI uri, Map<String, ?> env) {
return uri.isAbsolute() && uri.getScheme().equals("sftp") ? new SFTPFileService(uri, env) : null;
}
}

14
files-sftp-fs/src/main/java/org/apache/sshd/fs/spi/SFTPFilesProvider.java
View file

@ -1,14 +0,0 @@
package org.apache.sshd.fs.spi;
import org.xbib.files.Files;
import org.xbib.files.Provider;
import java.net.URI;
import java.util.Map;
public class SFTPFilesProvider implements Provider {
@Override
public Files provide(URI uri, Map<String, ?> env) {
return !uri.isAbsolute() || uri.getScheme().equals("file") ? new SFTPFiles(uri, env) : null;
}
}

1
files-sftp-fs/src/main/resources/META-INF/services/org.xbib.files.FileServiceProvider Normal file
View file

@ -0,0 +1 @@
org.apache.sshd.fs.spi.SFTPFileServiceProvider

1
files-sftp-fs/src/main/resources/META-INF/services/org.xbib.files.Provider
View file

@ -1 +0,0 @@
org.apache.sshd.fs.spi.SFTPFilesProvider

30
files-sftp-fs/src/test/java/org/apache/sshd/fs/test/FileServiceProviderTest.java Normal file
View file

@ -0,0 +1,30 @@
package org.apache.sshd.fs.test;
import org.junit.jupiter.api.Test;
import org.xbib.files.FileService;
import java.io.IOException;
import java.security.Provider;
import java.security.Security;
import java.util.Map;
import java.util.ServiceLoader;
import java.util.logging.Logger;
public class FileServiceProviderTest {
static {
// load bouncy castle provider (and other security providers)
for (Provider p : ServiceLoader.load(Provider.class)) {
if (Security.getProvider(p.getName()) == null) {
Security.addProvider(p);
}
}
}
@Test
public void testSFTP() throws IOException {
Map<String, ?> env = Map.of("username", "joerg");
FileService fs = FileService.newInstance("sftp://alkmene:22", env);
fs.list(".").forEach(p -> Logger.getAnonymousLogger().info(p.toString()));
}
}

11
files-sftp-fs/src/test/java/org/apache/sshd/fs/test/SFTPFileSystemTest.java
View file

@ -13,9 +13,11 @@ import java.net.URI;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.KeyPair;
import java.security.Provider;
import java.security.Security;
import java.util.HashMap;
import java.util.Map;
import java.util.ServiceLoader;
import java.util.logging.Logger;
public class SFTPFileSystemTest {
@ -23,8 +25,12 @@ public class SFTPFileSystemTest {
private static final Logger logger = Logger.getLogger(SFTPFileSystemTest.class.getName());
static {
Security.addProvider(new EdDSASecurityProvider());
Security.addProvider(new BouncyCastleProvider());
// load bouncy castle provider (and other security providers)
for (Provider p : ServiceLoader.load(Provider.class)) {
if (Security.getProvider(p.getName()) == null) {
Security.addProvider(p);
}
}
}
@Test
@ -44,6 +50,7 @@ public class SFTPFileSystemTest {
sshClient.setNioWorkers(1);
sshClient.start();
SftpFileSystem fileSystem = new SftpFileSystemProvider(sshClient).newFileSystem(uri, env);
// ...
sshClient.stop();
}
}

4
files-sftp/src/main/java/org/apache/sshd/client/impl/AbstractSftpClient.java
View file

@ -675,8 +675,6 @@ public abstract class AbstractSftpClient extends AbstractSubsystemClient impleme
if (!isOpen()) {
throw new IOException("close(" + handle + ") client is closed");
}
byte[] id = Objects.requireNonNull(handle, "No handle").getIdentifier();
Buffer buffer = new ByteArrayBuffer(id.length + Long.SIZE /* some extra fields */, false);
buffer.putBytes(id);
@ -688,8 +686,6 @@ public abstract class AbstractSftpClient extends AbstractSubsystemClient impleme
if (!isOpen()) {
throw new IOException("remove(" + path + ") client is closed");
}
Buffer buffer = new ByteArrayBuffer(path.length() + Long.SIZE /* some extra fields */, false);
buffer = putReferencedName(SftpConstants.SSH_FXP_REMOVE, buffer, path, 0);
checkCommandStatus(SftpConstants.SSH_FXP_REMOVE, buffer);