implement FileService API, with streaming API

files-api/src/main/java/module-info.java

@@ -1,8 +1,8 @@
-import org.xbib.files.DefaultFilesProvider;
+import org.xbib.files.DefaultFileServiceProvider;
-import org.xbib.files.Provider;
+import org.xbib.files.FileServiceProvider;
 
 module org.xbib.files {
     exports org.xbib.files;
-    uses Provider;
+    uses FileServiceProvider;
-    provides Provider with DefaultFilesProvider;
+    provides FileServiceProvider with DefaultFileServiceProvider;
 }

files-api/src/main/java/org/xbib/files/DefaultFileService.java

@@ -10,6 +10,7 @@ import java.nio.channels.ReadableByteChannel;
 import java.nio.channels.WritableByteChannel;
 import java.nio.file.CopyOption;
 import java.nio.file.DirectoryStream;
+import java.nio.file.FileVisitOption;
 import java.nio.file.OpenOption;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -26,8 +27,9 @@ import java.util.Map;
 import java.util.Set;
 
 import java.nio.file.Files;
+import java.util.stream.Stream;
 
-public class DefaultFiles implements org.xbib.files.Files {
+public class DefaultFileService implements FileService {
 
     private static final int READ_BUFFER_SIZE = 128 * 1024;
 
@@ -39,13 +41,7 @@ public class DefaultFiles implements org.xbib.files.Files {
     private static final Set<PosixFilePermission> DEFAULT_FILE_PERMISSIONS =
             PosixFilePermissions.fromString("rw-r--r--");
 
-    private final URI uri;
+    public DefaultFileService(URI uri, Map<String, ?> env) {
-
-    private final Map<String, ?> env;
-
-    public DefaultFiles(URI uri, Map<String, ?> env) {
-        this.uri = uri;
-        this.env = env;
     }
 
     @Override
@@ -209,10 +205,25 @@ public class DefaultFiles implements org.xbib.files.Files {
     }
 
     @Override
-    public DirectoryStream<Path> filter(String path, DirectoryStream.Filter<Path> filter) throws IOException {
+    public DirectoryStream<Path> stream(String path, DirectoryStream.Filter<Path> filter) throws IOException {
         return Files.newDirectoryStream(Paths.get(path), filter);
     }
 
+    @Override
+    public Stream<Path> list(String path) throws IOException {
+        return Files.list(Paths.get(path));
+    }
+
+    @Override
+    public Stream<Path> walk(String path, FileVisitOption... options) throws IOException {
+        return Files.walk(Paths.get(path), options);
+    }
+
+    @Override
+    public Stream<Path> walk(String path, int maxdepth, FileVisitOption... options) throws IOException {
+        return Files.walk(Paths.get(path), maxdepth, options);
+    }
+
     private void upload(Path source, Path target,
                         Set<PosixFilePermission> dirPerms,
                         Set<PosixFilePermission> filePerms,

files-api/src/main/java/org/xbib/files/DefaultFileServiceProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.files;
+
+import java.net.URI;
+import java.util.Map;
+
+public class DefaultFileServiceProvider implements FileServiceProvider {
+    @Override
+    public FileService provide(URI uri, Map<String, ?> env) {
+        return !uri.isAbsolute() || uri.getScheme().equals("file") ?  new DefaultFileService(uri, env) : null;
+    }
+}

files-api/src/main/java/org/xbib/files/DefaultFilesProvider.java

@@ -1,11 +0,0 @@
-package org.xbib.files;
-
-import java.net.URI;
-import java.util.Map;
-
-public class DefaultFilesProvider implements Provider {
-    @Override
-    public Files provide(URI uri, Map<String, ?> env) {
-        return !uri.isAbsolute() || uri.getScheme().equals("file") ?  new DefaultFiles(uri, env) : null;
-    }
-}

files-api/src/main/java/org/xbib/files/FileService.java

@@ -6,6 +6,8 @@ import java.io.OutputStream;
 import java.net.URI;
 import java.nio.file.CopyOption;
 import java.nio.file.DirectoryStream;
+import java.nio.file.FileVisitOption;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.attribute.FileAttribute;
 import java.nio.file.attribute.FileTime;
@@ -16,14 +18,15 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.ServiceLoader;
 import java.util.Set;
+import java.util.stream.Stream;
 
-public interface Files {
+public interface FileService {
 
     class Holder {
 
-        private static Files createFiles(URI uri, Map<String, ?> env) {
+        private static FileService createFiles(URI uri, Map<String, ?> env) {
-            ServiceLoader<Provider> serviceLoader = ServiceLoader.load(Provider.class);
+            ServiceLoader<FileServiceProvider> serviceLoader = ServiceLoader.load(FileServiceProvider.class);
-            Optional<Files> first = serviceLoader.stream()
+            Optional<FileService> first = serviceLoader.stream()
                     .map(ServiceLoader.Provider::get)
                     .filter(Objects::nonNull)
                     .map(p -> p.provide(uri, env))
@@ -33,7 +36,11 @@ public interface Files {
         }
     }
 
-    default Files newInstance(URI uri, Map<String, ?> env) {
+    static FileService newInstance(String uri, Map<String, ?> env) {
+        return Holder.createFiles(URI.create(uri), env);
+    }
+
+    static FileService newInstance(URI uri, Map<String, ?> env) {
         return Holder.createFiles(uri, env);
     }
 
@@ -101,5 +108,11 @@ public interface Files {
 
     DirectoryStream<Path> stream(String path, String glob) throws IOException;
 
-    DirectoryStream<Path> filter(String path, DirectoryStream.Filter<Path> filter) throws IOException;
+    DirectoryStream<Path> stream(String path, DirectoryStream.Filter<Path> filter) throws IOException;
+
+    Stream<Path> list(String path) throws IOException;
+
+    Stream<Path> walk(String path, FileVisitOption... options) throws IOException;
+
+    Stream<Path> walk(String path, int maxdepth, FileVisitOption... options) throws IOException;
 }

files-api/src/main/java/org/xbib/files/FileServiceProvider.java

@@ -0,0 +1,9 @@
+package org.xbib.files;
+
+import java.net.URI;
+import java.util.Map;
+
+public interface FileServiceProvider {
+
+    FileService provide(URI uri, Map<String, ?> env);
+}

files-api/src/main/java/org/xbib/files/FileTreeIterator.java

@@ -0,0 +1,102 @@
+package org.xbib.files;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.FileVisitOption;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import org.xbib.files.FileTreeWalker.Event;
+
+/**
+ * This source is a copy taken from java.nio.file.FileTreeIterator which is inaccessible.
+ *
+ * An {@code Iterator} to iterate over the nodes of a file tree.
+ *
+ * <pre>{@code
+ *     try (FileTreeIterator iterator = new FileTreeIterator(start, maxDepth, options)) {
+ *         while (iterator.hasNext()) {
+ *             Event ev = iterator.next();
+ *             Path path = ev.file();
+ *             BasicFileAttributes attrs = ev.attributes();
+ *         }
+ *     }
+ * }</pre>
+ */
+
+class FileTreeIterator implements Iterator<Event>, Closeable {
+    private final FileTreeWalker walker;
+    private Event next;
+
+    /**
+     * Creates a new iterator to walk the file tree starting at the given file.
+     *
+     * @throws  IllegalArgumentException
+     *          if {@code maxDepth} is negative
+     * @throws  IOException
+     *          if an I/O errors occurs opening the starting file
+     * @throws  SecurityException
+     *          if the security manager denies access to the starting file
+     * @throws  NullPointerException
+     *          if {@code start} or {@code options} is {@code null} or
+     *          the options array contains a {@code null} element
+     */
+    FileTreeIterator(Path start, int maxDepth, FileVisitOption... options)
+        throws IOException
+    {
+        this.walker = new FileTreeWalker(Arrays.asList(options), maxDepth);
+        this.next = walker.walk(start);
+        assert next.type() == FileTreeWalker.EventType.ENTRY ||
+               next.type() == FileTreeWalker.EventType.START_DIRECTORY;
+
+        // IOException if there a problem accessing the starting file
+        IOException ioe = next.ioeException();
+        if (ioe != null)
+            throw ioe;
+    }
+
+    private void fetchNextIfNeeded() {
+        if (next == null) {
+            FileTreeWalker.Event ev = walker.next();
+            while (ev != null) {
+                IOException ioe = ev.ioeException();
+                if (ioe != null)
+                    throw new UncheckedIOException(ioe);
+
+                // END_DIRECTORY events are ignored
+                if (ev.type() != FileTreeWalker.EventType.END_DIRECTORY) {
+                    next = ev;
+                    return;
+                }
+                ev = walker.next();
+            }
+        }
+    }
+
+    @Override
+    public boolean hasNext() {
+        if (!walker.isOpen())
+            throw new IllegalStateException();
+        fetchNextIfNeeded();
+        return next != null;
+    }
+
+    @Override
+    public Event next() {
+        if (!walker.isOpen())
+            throw new IllegalStateException();
+        fetchNextIfNeeded();
+        if (next == null)
+            throw new NoSuchElementException();
+        Event result = next;
+        next = null;
+        return result;
+    }
+
+    @Override
+    public void close() {
+        walker.close();
+    }
+}

files-api/src/main/java/org/xbib/files/FileTreeWalker.java

@@ -0,0 +1,398 @@
+package org.xbib.files;
+
+import java.nio.file.DirectoryIteratorException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.FileSystemLoopException;
+import java.nio.file.FileVisitOption;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.Collection;
+import java.util.Iterator;
+
+/**
+ * This source is a copy taken from java.nio.file.FileTreeWalker which is inaccessible.
+ *
+ * Walks a file tree, generating a sequence of events corresponding to the files
+ * in the tree.
+ *
+ * <pre>{@code
+ *     Path top = ...
+ *     Set<FileVisitOption> options = ...
+ *     int maxDepth = ...
+ *
+ *     try (FileTreeWalker walker = new FileTreeWalker(options, maxDepth)) {
+ *         FileTreeWalker.Event ev = walker.walk(top);
+ *         do {
+ *             process(ev);
+ *             ev = walker.next();
+ *         } while (ev != null);
+ *     }
+ * }</pre>
+ *
+ * @see Files#walkFileTree
+ */
+
+class FileTreeWalker implements Closeable {
+    private final boolean followLinks;
+    private final LinkOption[] linkOptions;
+    private final int maxDepth;
+    private final ArrayDeque<DirectoryNode> stack = new ArrayDeque<>();
+    private boolean closed;
+
+    /**
+     * The element on the walking stack corresponding to a directory node.
+     */
+    private static class DirectoryNode {
+        private final Path dir;
+        private final Object key;
+        private final DirectoryStream<Path> stream;
+        private final Iterator<Path> iterator;
+        private boolean skipped;
+
+        DirectoryNode(Path dir, Object key, DirectoryStream<Path> stream) {
+            this.dir = dir;
+            this.key = key;
+            this.stream = stream;
+            this.iterator = stream.iterator();
+        }
+
+        Path directory() {
+            return dir;
+        }
+
+        Object key() {
+            return key;
+        }
+
+        DirectoryStream<Path> stream() {
+            return stream;
+        }
+
+        Iterator<Path> iterator() {
+            return iterator;
+        }
+
+        void skip() {
+            skipped = true;
+        }
+
+        boolean skipped() {
+            return skipped;
+        }
+    }
+
+    /**
+     * The event types.
+     */
+    static enum EventType {
+        /**
+         * Start of a directory
+         */
+        START_DIRECTORY,
+        /**
+         * End of a directory
+         */
+        END_DIRECTORY,
+        /**
+         * An entry in a directory
+         */
+        ENTRY;
+    }
+
+    /**
+     * Events returned by the {@link #walk} and {@link #next} methods.
+     */
+    static class Event {
+        private final EventType type;
+        private final Path file;
+        private final BasicFileAttributes attrs;
+        private final IOException ioe;
+
+        private Event(EventType type, Path file, BasicFileAttributes attrs, IOException ioe) {
+            this.type = type;
+            this.file = file;
+            this.attrs = attrs;
+            this.ioe = ioe;
+        }
+
+        Event(EventType type, Path file, BasicFileAttributes attrs) {
+            this(type, file, attrs, null);
+        }
+
+        Event(EventType type, Path file, IOException ioe) {
+            this(type, file, null, ioe);
+        }
+
+        EventType type() {
+            return type;
+        }
+
+        Path file() {
+            return file;
+        }
+
+        BasicFileAttributes attributes() {
+            return attrs;
+        }
+
+        IOException ioeException() {
+            return ioe;
+        }
+    }
+
+    /**
+     * Creates a {@code FileTreeWalker}.
+     *
+     * @throws  IllegalArgumentException
+     *          if {@code maxDepth} is negative
+     * @throws  ClassCastException
+     *          if {@code options} contains an element that is not a
+     *          {@code FileVisitOption}
+     * @throws  NullPointerException
+     *          if {@code options} is {@code null} or the options
+     *          array contains a {@code null} element
+     */
+    FileTreeWalker(Collection<FileVisitOption> options, int maxDepth) {
+        boolean fl = false;
+        for (FileVisitOption option: options) {
+            // will throw NPE if options contains null
+            switch (option) {
+                case FOLLOW_LINKS : fl = true; break;
+                default:
+                    throw new AssertionError("Should not get here");
+            }
+        }
+        if (maxDepth < 0)
+            throw new IllegalArgumentException("'maxDepth' is negative");
+
+        this.followLinks = fl;
+        this.linkOptions = (fl) ? new LinkOption[0] :
+            new LinkOption[] { LinkOption.NOFOLLOW_LINKS };
+        this.maxDepth = maxDepth;
+    }
+
+    /**
+     * Returns the attributes of the given file, taking into account whether
+     * the walk is following sym links is not. The {@code canUseCached}
+     * argument determines whether this method can use cached attributes.
+     */
+    private BasicFileAttributes getAttributes(Path file, boolean canUseCached)
+        throws IOException
+    {
+        // attempt to get attributes of file. If fails and we are following
+        // links then a link target might not exist so get attributes of link
+        BasicFileAttributes attrs;
+        try {
+            attrs = Files.readAttributes(file, BasicFileAttributes.class, linkOptions);
+        } catch (IOException ioe) {
+            if (!followLinks)
+                throw ioe;
+
+            // attempt to get attrmptes without following links
+            attrs = Files.readAttributes(file,
+                                         BasicFileAttributes.class,
+                                         LinkOption.NOFOLLOW_LINKS);
+        }
+        return attrs;
+    }
+
+    /**
+     * Returns true if walking into the given directory would result in a
+     * file system loop/cycle.
+     */
+    private boolean wouldLoop(Path dir, Object key) {
+        // if this directory and ancestor has a file key then we compare
+        // them; otherwise we use less efficient isSameFile test.
+        for (DirectoryNode ancestor: stack) {
+            Object ancestorKey = ancestor.key();
+            if (key != null && ancestorKey != null) {
+                if (key.equals(ancestorKey)) {
+                    // cycle detected
+                    return true;
+                }
+            } else {
+                try {
+                    if (Files.isSameFile(dir, ancestor.directory())) {
+                        // cycle detected
+                        return true;
+                    }
+                } catch (IOException | SecurityException x) {
+                    // ignore
+                }
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Visits the given file, returning the {@code Event} corresponding to that
+     * visit.
+     *
+     * The {@code ignoreSecurityException} parameter determines whether
+     * any SecurityException should be ignored or not. If a SecurityException
+     * is thrown, and is ignored, then this method returns {@code null} to
+     * mean that there is no event corresponding to a visit to the file.
+     *
+     * The {@code canUseCached} parameter determines whether cached attributes
+     * for the file can be used or not.
+     */
+    private Event visit(Path entry, boolean ignoreSecurityException, boolean canUseCached) {
+        // need the file attributes
+        BasicFileAttributes attrs;
+        try {
+            attrs = getAttributes(entry, canUseCached);
+        } catch (IOException ioe) {
+            return new Event(EventType.ENTRY, entry, ioe);
+        } catch (SecurityException se) {
+            if (ignoreSecurityException)
+                return null;
+            throw se;
+        }
+
+        // at maximum depth or file is not a directory
+        int depth = stack.size();
+        if (depth >= maxDepth || !attrs.isDirectory()) {
+            return new Event(EventType.ENTRY, entry, attrs);
+        }
+
+        // check for cycles when following links
+        if (followLinks && wouldLoop(entry, attrs.fileKey())) {
+            return new Event(EventType.ENTRY, entry,
+                             new FileSystemLoopException(entry.toString()));
+        }
+
+        // file is a directory, attempt to open it
+        DirectoryStream<Path> stream = null;
+        try {
+            stream = Files.newDirectoryStream(entry);
+        } catch (IOException ioe) {
+            return new Event(EventType.ENTRY, entry, ioe);
+        } catch (SecurityException se) {
+            if (ignoreSecurityException)
+                return null;
+            throw se;
+        }
+
+        // push a directory node to the stack and return an event
+        stack.push(new DirectoryNode(entry, attrs.fileKey(), stream));
+        return new Event(EventType.START_DIRECTORY, entry, attrs);
+    }
+
+
+    /**
+     * Start walking from the given file.
+     */
+    Event walk(Path file) {
+        if (closed)
+            throw new IllegalStateException("Closed");
+
+        Event ev = visit(file,
+                         false,   // ignoreSecurityException
+                         false);  // canUseCached
+        assert ev != null;
+        return ev;
+    }
+
+    /**
+     * Returns the next Event or {@code null} if there are no more events or
+     * the walker is closed.
+     */
+    Event next() {
+        DirectoryNode top = stack.peek();
+        if (top == null)
+            return null;      // stack is empty, we are done
+
+        // continue iteration of the directory at the top of the stack
+        Event ev;
+        do {
+            Path entry = null;
+            IOException ioe = null;
+
+            // get next entry in the directory
+            if (!top.skipped()) {
+                Iterator<Path> iterator = top.iterator();
+                try {
+                    if (iterator.hasNext()) {
+                        entry = iterator.next();
+                    }
+                } catch (DirectoryIteratorException x) {
+                    ioe = x.getCause();
+                }
+            }
+
+            // no next entry so close and pop directory,
+            // creating corresponding event
+            if (entry == null) {
+                try {
+                    top.stream().close();
+                } catch (IOException e) {
+                    if (ioe == null) {
+                        ioe = e;
+                    } else {
+                        ioe.addSuppressed(e);
+                    }
+                }
+                stack.pop();
+                return new Event(EventType.END_DIRECTORY, top.directory(), ioe);
+            }
+
+            // visit the entry
+            ev = visit(entry,
+                       true,   // ignoreSecurityException
+                       true);  // canUseCached
+
+        } while (ev == null);
+
+        return ev;
+    }
+
+    /**
+     * Pops the directory node that is the current top of the stack so that
+     * there are no more events for the directory (including no END_DIRECTORY)
+     * event. This method is a no-op if the stack is empty or the walker is
+     * closed.
+     */
+    void pop() {
+        if (!stack.isEmpty()) {
+            DirectoryNode node = stack.pop();
+            try {
+                node.stream().close();
+            } catch (IOException ignore) { }
+        }
+    }
+
+    /**
+     * Skips the remaining entries in the directory at the top of the stack.
+     * This method is a no-op if the stack is empty or the walker is closed.
+     */
+    void skipRemainingSiblings() {
+        if (!stack.isEmpty()) {
+            stack.peek().skip();
+        }
+    }
+
+    /**
+     * Returns {@code true} if the walker is open.
+     */
+    boolean isOpen() {
+        return !closed;
+    }
+
+    /**
+     * Closes/pops all directories on the stack.
+     */
+    @Override
+    public void close() {
+        if (!closed) {
+            while (!stack.isEmpty()) {
+                pop();
+            }
+            closed = true;
+        }
+    }
+}

files-api/src/main/java/org/xbib/files/FileWalker.java

@@ -0,0 +1,228 @@
+package org.xbib.files;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.DirectoryIteratorException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.FileSystemLoopException;
+import java.nio.file.FileVisitOption;
+import java.nio.file.Files;
+import java.nio.file.NotDirectoryException;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Iterator;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+/**
+ * This source is a copy taken from java.nio.file.Files#walk and modified to process a java.io.Closeable after the walk.
+ */
+public class FileWalker {
+
+    /**
+     * Return a lazily populated {@code Stream}, the elements of
+     * which are the entries in the directory.  The listing is not recursive.
+     *
+     * <p> The elements of the stream are {@link Path} objects that are
+     * obtained as if by {@link Path#resolve(Path) resolving} the name of the
+     * directory entry against {@code dir}. Some file systems maintain special
+     * links to the directory itself and the directory's parent directory.
+     * Entries representing these links are not included.
+     *
+     * <p> The stream is <i>weakly consistent</i>. It is thread safe but does
+     * not freeze the directory while iterating, so it may (or may not)
+     * reflect updates to the directory that occur after returning from this
+     * method.
+     *
+     * <p> The returned stream contains a reference to an open directory.
+     * The directory is closed by closing the stream.
+     *
+     * <p> Operating on a closed stream behaves as if the end of stream
+     * has been reached. Due to read-ahead, one or more elements may be
+     * returned after the stream has been closed.
+     *
+     * <p> If an {@link IOException} is thrown when accessing the directory
+     * after this method has returned, it is wrapped in an {@link
+     * UncheckedIOException} which will be thrown from the method that caused
+     * the access to take place.
+     *
+     * @apiNote
+     * This method must be used within a try-with-resources statement or similar
+     * control structure to ensure that the stream's open directory is closed
+     * promptly after the stream's operations have completed.
+     *
+     * @param   ds  The directory stream
+     *
+     * @return  The {@code Stream} describing the content of the
+     *          directory
+     *
+     * @throws NotDirectoryException
+     *          if the file could not otherwise be opened because it is not
+     *          a directory <i>(optional specific exception)</i>
+     * @throws  IOException
+     *          if an I/O error occurs when opening the directory
+     * @throws  SecurityException
+     *          In the case of the default provider, and a security manager is
+     *          installed, the {@link SecurityManager#checkRead(String) checkRead}
+     *          method is invoked to check read access to the directory.
+     */
+    public static Stream<Path> list(DirectoryStream<Path> ds) throws IOException {
+        try {
+            final Iterator<Path> delegate = ds.iterator();
+            Iterator<Path> iterator = new Iterator<>() {
+                @Override
+                public boolean hasNext() {
+                    try {
+                        return delegate.hasNext();
+                    } catch (DirectoryIteratorException e) {
+                        throw new UncheckedIOException(e.getCause());
+                    }
+                }
+                @Override
+                public Path next() {
+                    try {
+                        return delegate.next();
+                    } catch (DirectoryIteratorException e) {
+                        throw new UncheckedIOException(e.getCause());
+                    }
+                }
+            };
+            Spliterator<Path> spliterator =
+                    Spliterators.spliteratorUnknownSize(iterator, Spliterator.DISTINCT);
+            return StreamSupport.stream(spliterator, false)
+                    .onClose(asUncheckedRunnable(ds));
+        } catch (Error|RuntimeException e) {
+            try {
+                ds.close();
+            } catch (IOException ex) {
+                try {
+                    e.addSuppressed(ex);
+                } catch (Throwable ignore) {}
+            }
+            throw e;
+        }
+    }
+
+    /**
+     * Return a {@code Stream} that is lazily populated with {@code
+     * Path} by walking the file tree rooted at a given starting file.  The
+     * file tree is traversed <em>depth-first</em>, the elements in the stream
+     * are {@link Path} objects that are obtained as if by {@link
+     * Path#resolve(Path) resolving} the relative path against {@code start}.
+     *
+     * <p> The {@code stream} walks the file tree as elements are consumed.
+     * The {@code Stream} returned is guaranteed to have at least one
+     * element, the starting file itself. For each file visited, the stream
+     * attempts to read its {@link BasicFileAttributes}. If the file is a
+     * directory and can be opened successfully, entries in the directory, and
+     * their <em>descendants</em> will follow the directory in the stream as
+     * they are encountered. When all entries have been visited, then the
+     * directory is closed. The file tree walk then continues at the next
+     * <em>sibling</em> of the directory.
+     *
+     * <p> The stream is <i>weakly consistent</i>. It does not freeze the
+     * file tree while iterating, so it may (or may not) reflect updates to
+     * the file tree that occur after returned from this method.
+     *
+     * <p> By default, symbolic links are not automatically followed by this
+     * method. If the {@code options} parameter contains the {@link
+     * FileVisitOption#FOLLOW_LINKS FOLLOW_LINKS} option then symbolic links are
+     * followed. When following links, and the attributes of the target cannot
+     * be read, then this method attempts to get the {@code BasicFileAttributes}
+     * of the link.
+     *
+     * <p> If the {@code options} parameter contains the {@link
+     * FileVisitOption#FOLLOW_LINKS FOLLOW_LINKS} option then the stream keeps
+     * track of directories visited so that cycles can be detected. A cycle
+     * arises when there is an entry in a directory that is an ancestor of the
+     * directory. Cycle detection is done by recording the {@link
+     * java.nio.file.attribute.BasicFileAttributes#fileKey file-key} of directories,
+     * or if file keys are not available, by invoking the {@link java.nio.file.Files#isSameFile
+     * isSameFile} method to test if a directory is the same file as an
+     * ancestor. When a cycle is detected it is treated as an I/O error with
+     * an instance of {@link FileSystemLoopException}.
+     *
+     * <p> The {@code maxDepth} parameter is the maximum number of levels of
+     * directories to visit. A value of {@code 0} means that only the starting
+     * file is visited, unless denied by the security manager. A value of
+     * {@link Integer#MAX_VALUE MAX_VALUE} may be used to indicate that all
+     * levels should be visited.
+     *
+     * <p> When a security manager is installed and it denies access to a file
+     * (or directory), then it is ignored and not included in the stream.
+     *
+     * <p> The returned stream contains references to one or more open directories.
+     * The directories are closed by closing the stream.
+     *
+     * <p> If an {@link IOException} is thrown when accessing the directory
+     * after this method has returned, it is wrapped in an {@link
+     * UncheckedIOException} which will be thrown from the method that caused
+     * the access to take place.
+     *
+     * @apiNote
+     * This method must be used within a try-with-resources statement or similar
+     * control structure to ensure that the stream's open directories are closed
+     * promptly after the stream's operations have completed.
+     *
+     * @param closeable a closeable which must be closed after walking
+     * @param   start
+     *          the starting file
+     * @param   maxDepth
+     *          the maximum number of directory levels to visit
+     * @param   options
+     *          options to configure the traversal
+     *
+     * @return  the {@link Stream} of {@link Path}
+     *
+     * @throws  IllegalArgumentException
+     *          if the {@code maxDepth} parameter is negative
+     * @throws  SecurityException
+     *          If the security manager denies access to the starting file.
+     *          In the case of the default provider, the {@link
+     *          SecurityManager#checkRead(String) checkRead} method is invoked
+     *          to check read access to the directory.
+     * @throws  IOException
+     *          if an I/O error is thrown when accessing the starting file.
+     */
+    public static Stream<Path> walk(Closeable closeable,
+                                    Path start,
+                                    int maxDepth,
+                                    FileVisitOption... options) throws IOException {
+        FileTreeIterator iterator = new FileTreeIterator(start, maxDepth, options);
+        try {
+            Spliterator<FileTreeWalker.Event> spliterator =
+                    Spliterators.spliteratorUnknownSize(iterator, Spliterator.DISTINCT);
+            return StreamSupport.stream(spliterator, false)
+                    .onClose(() -> {
+                        iterator.close();
+                        try {
+                            closeable.close();
+                        } catch (IOException e) {
+                            throw new UncheckedIOException(e);
+                        }
+                    })
+                    .map(FileTreeWalker.Event::file);
+        } catch (Error|RuntimeException e) {
+            iterator.close();
+            closeable.close();
+            throw e;
+        }
+    }
+
+    /**
+     * Convert a Closeable to a Runnable by converting checked IOException
+     * to UncheckedIOException
+     */
+    private static Runnable asUncheckedRunnable(Closeable c) {
+        return () -> {
+            try {
+                c.close();
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        };
+    }
+}

files-api/src/main/java/org/xbib/files/Provider.java

@@ -1,9 +0,0 @@
-package org.xbib.files;
-
-import java.net.URI;
-import java.util.Map;
-
-public interface Provider {
-
-    Files provide(URI uri, Map<String, ?> env);
-}

files-api/src/main/java/org/xbib/files/WrappedDirectoryStream.java

@@ -0,0 +1,29 @@
+package org.xbib.files;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.util.Iterator;
+
+public class WrappedDirectoryStream<T> implements DirectoryStream<T> {
+
+    private final Closeable closeable;
+
+    private final DirectoryStream<T> directoryStream;
+
+    public WrappedDirectoryStream(Closeable closeable, DirectoryStream<T> directoryStream) {
+        this.closeable = closeable;
+        this.directoryStream = directoryStream;
+    }
+
+    @Override
+    public Iterator<T> iterator() {
+        return directoryStream.iterator();
+    }
+
+    @Override
+    public void close() throws IOException {
+        directoryStream.close();
+        closeable.close();
+    }
+}

files-api/src/main/resources/META-INF/services/org.xbib.files.FileServiceProvider

@@ -0,0 +1 @@
+org.xbib.files.DefaultFileServiceProvider

files-api/src/main/resources/META-INF/services/org.xbib.files.Provider

@@ -1 +0,0 @@
-org.xbib.files.DefaultFilesProvider

files-ftp-fs/src/main/java/module-info.java

@@ -1,5 +1,5 @@
 import java.nio.file.spi.FileSystemProvider;
-import org.xbib.files.Provider;
+import org.xbib.files.FileServiceProvider;
 import org.xbib.io.ftp.fs.FTPFileSystemProvider;
 import org.xbib.io.ftp.fs.spi.FTPFilesProvider;
 
@@ -7,5 +7,5 @@ module org.xbib.files.ftp.fs {
     requires org.xbib.files;
     requires org.xbib.files.ftp;
     provides FileSystemProvider with FTPFileSystemProvider;
-    provides Provider with FTPFilesProvider;
+    provides FileServiceProvider with FTPFilesProvider;
 }

files-ftp-fs/src/main/java/org/xbib/io/ftp/fs/spi/FTPContext.java

@@ -3,12 +3,13 @@ package org.xbib.io.ftp.fs.spi;
 import org.xbib.io.ftp.fs.FTPEnvironment;
 import org.xbib.io.ftp.fs.FTPFileSystemProvider;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.net.URI;
 import java.nio.file.FileSystem;
 import java.util.Map;
 
-class FTPContext {
+class FTPContext implements Closeable {
 
     final FTPFileSystemProvider provider;
 
@@ -19,7 +20,8 @@ class FTPContext {
         this.fileSystem = provider.newFileSystem(uri, env != null ? env : new FTPEnvironment());
     }
 
-    void close() throws IOException {
+    @Override
+    public void close() throws IOException {
         fileSystem.close();
     }
 }

files-ftp-fs/src/main/java/org/xbib/io/ftp/fs/spi/FTPFiles.java

@@ -1,5 +1,9 @@
 package org.xbib.io.ftp.fs.spi;
 
+import org.xbib.files.FileService;
+import org.xbib.files.FileWalker;
+import org.xbib.files.WrappedDirectoryStream;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -10,6 +14,7 @@ import java.nio.channels.ReadableByteChannel;
 import java.nio.channels.WritableByteChannel;
 import java.nio.file.CopyOption;
 import java.nio.file.DirectoryStream;
+import java.nio.file.FileVisitOption;
 import java.nio.file.Files;
 import java.nio.file.OpenOption;
 import java.nio.file.Path;
@@ -24,8 +29,9 @@ import java.nio.file.attribute.UserPrincipal;
 import java.util.EnumSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Stream;
 
-public class FTPFiles implements org.xbib.files.Files {
+public class FTPFiles implements FileService {
 
     private static final int READ_BUFFER_SIZE = 128 * 1024;
 
@@ -146,21 +152,41 @@ public class FTPFiles implements org.xbib.files.Files {
         return performWithContext(ctx -> Files.getOwner(ctx.fileSystem.getPath(path)));
     }
 
-    @Override
-    public DirectoryStream<Path> stream(String path, String glob) throws IOException {
-        return performWithContext(ctx -> Files.newDirectoryStream(ctx.fileSystem.getPath(path), glob));
-    }
-
-    @Override
-    public DirectoryStream<Path> filter(String path, DirectoryStream.Filter<Path> filter) throws IOException {
-        return performWithContext(ctx -> Files.newDirectoryStream(ctx.fileSystem.getPath(path), filter));
-    }
-
     @Override
     public void upload(Path source, Path target, CopyOption... copyOptions) throws IOException {
         upload(source, target, DEFAULT_DIR_PERMISSIONS, DEFAULT_FILE_PERMISSIONS, copyOptions);
     }
 
+    @Override
+    public DirectoryStream<Path> stream(String path, String glob) throws IOException {
+        FTPContext ctx = new FTPContext(uri, env);
+        return new WrappedDirectoryStream<>(ctx, Files.newDirectoryStream(ctx.fileSystem.getPath(path), glob));
+    }
+
+    @Override
+    public DirectoryStream<Path> stream(String path, DirectoryStream.Filter<Path> filter) throws IOException {
+        FTPContext ctx = new FTPContext(uri, env);
+        return new WrappedDirectoryStream<>(ctx, Files.newDirectoryStream(ctx.fileSystem.getPath(path), filter));
+    }
+
+    @Override
+    public Stream<Path> list(String path) throws IOException {
+        FTPContext ctx = new FTPContext(uri, env);
+        return FileWalker.list(new WrappedDirectoryStream<>(ctx, Files.newDirectoryStream(ctx.fileSystem.getPath(path))));
+    }
+
+    @Override
+    public Stream<Path> walk(String path, FileVisitOption... options) throws IOException {
+        FTPContext ctx = new FTPContext(uri, env);
+        return FileWalker.walk(ctx, ctx.fileSystem.getPath(path), Integer.MAX_VALUE, options);
+    }
+
+    @Override
+    public Stream<Path> walk(String path, int maxdepth, FileVisitOption... options) throws IOException {
+        FTPContext ctx = new FTPContext(uri, env);
+        return FileWalker.walk(ctx, ctx.fileSystem.getPath(path), maxdepth, options);
+    }
+
     public void upload(Path source, Path target,
                        Set<PosixFilePermission> dirPerms,
                        Set<PosixFilePermission> filePerms,

files-ftp-fs/src/main/java/org/xbib/io/ftp/fs/spi/FTPFilesProvider.java

@@ -1,14 +1,14 @@
 package org.xbib.io.ftp.fs.spi;
 
-import org.xbib.files.Files;
+import org.xbib.files.FileService;
-import org.xbib.files.Provider;
+import org.xbib.files.FileServiceProvider;
 
 import java.net.URI;
 import java.util.Map;
 
-public class FTPFilesProvider implements Provider {
+public class FTPFilesProvider implements FileServiceProvider {
     @Override
-    public Files provide(URI uri, Map<String, ?> env) {
+    public FileService provide(URI uri, Map<String, ?> env) {
-        return !uri.isAbsolute() || uri.getScheme().equals("file") ?  new FTPFiles(uri, env) : null;
+        return uri.isAbsolute() && uri.getScheme().equals("ftp") ?  new FTPFiles(uri, env) : null;
     }
 }

files-sftp-fs/src/main/java/module-info.java

@@ -1,7 +1,7 @@
 import org.apache.sshd.fs.SftpFileSystemProvider;
 import java.nio.file.spi.FileSystemProvider;
-import org.xbib.files.Provider;
+import org.xbib.files.FileServiceProvider;
-import org.apache.sshd.fs.spi.SFTPFilesProvider;
+import org.apache.sshd.fs.spi.SFTPFileServiceProvider;
 
 module org.xbib.files.sftp.fs {
     exports org.apache.sshd.fs;
@@ -9,5 +9,6 @@ module org.xbib.files.sftp.fs {
     requires org.xbib.files;
     requires transitive org.xbib.files.sftp;
     provides FileSystemProvider with SftpFileSystemProvider;
-    provides Provider with SFTPFilesProvider;
+    provides FileServiceProvider with SFTPFileServiceProvider;
+    requires java.logging;
 }

files-sftp-fs/src/main/java/org/apache/sshd/fs/SftpFileSystem.java

@@ -71,7 +71,7 @@ public class SftpFileSystem
     private final ThreadLocal<Wrapper> wrappers = new ThreadLocal<>();
     private final int version;
     private final Set<String> supportedViews;
-    private SftpPath defaultDir;
+    private final SftpPath defaultDir;
     private int readBufferSize;
     private int writeBufferSize;
     private final List<FileStore> stores;

files-sftp-fs/src/main/java/org/apache/sshd/fs/spi/SFTPContext.java

@@ -5,11 +5,12 @@ import org.apache.sshd.client.SshClient;
 import org.apache.sshd.fs.SftpFileSystem;
 import org.apache.sshd.fs.SftpFileSystemProvider;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.net.URI;
 import java.util.Map;
 
-class SFTPContext {
+public class SFTPContext implements Closeable {
 
     private final SshClient sshClient;
 
@@ -17,7 +18,7 @@ class SFTPContext {
 
     final SftpFileSystem fileSystem;
 
-    SFTPContext(URI uri, Map<String, ?> env) throws IOException {
+    public SFTPContext(URI uri, Map<String, ?> env) throws IOException {
         this.sshClient = ClientBuilder.builder().build();
         Object object = env.get("workers");
         if (object instanceof Integer) {
@@ -33,7 +34,8 @@ class SFTPContext {
         this.fileSystem = provider.newFileSystem(uri, env);
     }
 
-    void close() throws IOException {
+    @Override
+    public void close() throws IOException {
         sshClient.stop();
         fileSystem.close();
     }

files-sftp-fs/src/main/java/org/apache/sshd/fs/spi/SFTPFileService.java

@@ -1,5 +1,9 @@
 package org.apache.sshd.fs.spi;
 
+import org.xbib.files.FileService;
+import org.xbib.files.FileWalker;
+import org.xbib.files.WrappedDirectoryStream;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -10,6 +14,7 @@ import java.nio.channels.ReadableByteChannel;
 import java.nio.channels.WritableByteChannel;
 import java.nio.file.CopyOption;
 import java.nio.file.DirectoryStream;
+import java.nio.file.FileVisitOption;
 import java.nio.file.Files;
 import java.nio.file.OpenOption;
 import java.nio.file.Path;
@@ -24,8 +29,9 @@ import java.nio.file.attribute.UserPrincipal;
 import java.util.EnumSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Stream;
 
-public class SFTPFiles implements org.xbib.files.Files {
+public class SFTPFileService implements FileService {
 
     private static final int READ_BUFFER_SIZE = 128 * 1024;
 
@@ -41,7 +47,7 @@ public class SFTPFiles implements org.xbib.files.Files {
 
     private final Map<String, ?> env;
 
-    public SFTPFiles(URI uri, Map<String, ?> env) {
+    public SFTPFileService(URI uri, Map<String, ?> env) {
         this.uri = uri;
         this.env = env;
     }
@@ -146,16 +152,6 @@ public class SFTPFiles implements org.xbib.files.Files {
         return performWithContext(ctx -> Files.getOwner(ctx.fileSystem.getPath(path)));
     }
 
-    @Override
-    public DirectoryStream<Path> stream(String path, String glob) throws IOException {
-        return performWithContext(ctx -> Files.newDirectoryStream(ctx.fileSystem.getPath(path), glob));
-    }
-
-    @Override
-    public DirectoryStream<Path> filter(String path, DirectoryStream.Filter<Path> filter) throws IOException {
-        return performWithContext(ctx -> Files.newDirectoryStream(ctx.fileSystem.getPath(path), filter));
-    }
-
     @Override
     public void upload(Path source, Path target, CopyOption... copyOptions) throws IOException {
         upload(source, target, DEFAULT_DIR_PERMISSIONS, DEFAULT_FILE_PERMISSIONS, copyOptions);
@@ -223,6 +219,36 @@ public class SFTPFiles implements org.xbib.files.Files {
         performWithContext(ctx -> Files.deleteIfExists(ctx.fileSystem.getPath(source)));
     }
 
+    @Override
+    public DirectoryStream<Path> stream(String path, String glob) throws IOException {
+        SFTPContext ctx = new SFTPContext(uri, env);
+        return new WrappedDirectoryStream<>(ctx, Files.newDirectoryStream(ctx.fileSystem.getPath(path), glob));
+    }
+
+    @Override
+    public DirectoryStream<Path> stream(String path, DirectoryStream.Filter<Path> filter) throws IOException {
+        SFTPContext ctx = new SFTPContext(uri, env);
+        return new WrappedDirectoryStream<>(ctx, Files.newDirectoryStream(ctx.fileSystem.getPath(path), filter));
+    }
+
+    @Override
+    public Stream<Path> list(String path) throws IOException {
+        SFTPContext ctx = new SFTPContext(uri, env);
+        return FileWalker.list(new WrappedDirectoryStream<>(ctx, Files.newDirectoryStream(ctx.fileSystem.getPath(path))));
+    }
+
+    @Override
+    public Stream<Path> walk(String path, FileVisitOption... options) throws IOException {
+        SFTPContext ctx = new SFTPContext(uri, env);
+        return FileWalker.walk(ctx, ctx.fileSystem.getPath(path), Integer.MAX_VALUE, options);
+    }
+
+    @Override
+    public Stream<Path> walk(String path, int maxdepth, FileVisitOption... options) throws IOException {
+        SFTPContext ctx = new SFTPContext(uri, env);
+        return FileWalker.walk(ctx, ctx.fileSystem.getPath(path), maxdepth, options);
+    }
+
     public void upload(Path source, Path target,
                        Set<PosixFilePermission> dirPermissions,
                        Set<PosixFilePermission> filePermissions,
@@ -389,4 +415,8 @@ public class SFTPFiles implements org.xbib.files.Files {
             }
         }
     }
+
+    private <T> T performWithoutClose(SFTPContext ctx, WithContext<T> action) throws IOException {
+        return action.perform(ctx);
+    }
 }

files-sftp-fs/src/main/java/org/apache/sshd/fs/spi/SFTPFileServiceProvider.java

@@ -0,0 +1,14 @@
+package org.apache.sshd.fs.spi;
+
+import org.xbib.files.FileService;
+import org.xbib.files.FileServiceProvider;
+
+import java.net.URI;
+import java.util.Map;
+
+public class SFTPFileServiceProvider implements FileServiceProvider {
+    @Override
+    public FileService provide(URI uri, Map<String, ?> env) {
+        return uri.isAbsolute() && uri.getScheme().equals("sftp") ?  new SFTPFileService(uri, env) : null;
+    }
+}

files-sftp-fs/src/main/java/org/apache/sshd/fs/spi/SFTPFilesProvider.java

@@ -1,14 +0,0 @@
-package org.apache.sshd.fs.spi;
-
-import org.xbib.files.Files;
-import org.xbib.files.Provider;
-
-import java.net.URI;
-import java.util.Map;
-
-public class SFTPFilesProvider implements Provider {
-    @Override
-    public Files provide(URI uri, Map<String, ?> env) {
-        return !uri.isAbsolute() || uri.getScheme().equals("file") ?  new SFTPFiles(uri, env) : null;
-    }
-}

files-sftp-fs/src/main/resources/META-INF/services/org.xbib.files.FileServiceProvider

@@ -0,0 +1 @@
+org.apache.sshd.fs.spi.SFTPFileServiceProvider

files-sftp-fs/src/main/resources/META-INF/services/org.xbib.files.Provider

@@ -1 +0,0 @@
-org.apache.sshd.fs.spi.SFTPFilesProvider

files-sftp-fs/src/test/java/org/apache/sshd/fs/test/FileServiceProviderTest.java

@@ -0,0 +1,30 @@
+package org.apache.sshd.fs.test;
+
+import org.junit.jupiter.api.Test;
+import org.xbib.files.FileService;
+
+import java.io.IOException;
+import java.security.Provider;
+import java.security.Security;
+import java.util.Map;
+import java.util.ServiceLoader;
+import java.util.logging.Logger;
+
+public class FileServiceProviderTest {
+
+    static {
+        // load bouncy castle provider (and other security providers)
+        for (Provider p : ServiceLoader.load(Provider.class)) {
+            if (Security.getProvider(p.getName()) == null) {
+                Security.addProvider(p);
+            }
+        }
+    }
+
+    @Test
+    public void testSFTP() throws IOException {
+        Map<String, ?> env = Map.of("username", "joerg");
+        FileService fs = FileService.newInstance("sftp://alkmene:22", env);
+        fs.list(".").forEach(p -> Logger.getAnonymousLogger().info(p.toString()));
+    }
+}

files-sftp-fs/src/test/java/org/apache/sshd/fs/test/SFTPFileSystemTest.java

@@ -13,9 +13,11 @@ import java.net.URI;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.security.KeyPair;
+import java.security.Provider;
 import java.security.Security;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.ServiceLoader;
 import java.util.logging.Logger;
 
 public class SFTPFileSystemTest {
@@ -23,8 +25,12 @@ public class SFTPFileSystemTest {
     private static final Logger logger = Logger.getLogger(SFTPFileSystemTest.class.getName());
 
     static {
-        Security.addProvider(new EdDSASecurityProvider());
+        // load bouncy castle provider (and other security providers)
-        Security.addProvider(new BouncyCastleProvider());
+        for (Provider p : ServiceLoader.load(Provider.class)) {
+            if (Security.getProvider(p.getName()) == null) {
+                Security.addProvider(p);
+            }
+        }
     }
 
     @Test
@@ -44,6 +50,7 @@ public class SFTPFileSystemTest {
         sshClient.setNioWorkers(1);
         sshClient.start();
         SftpFileSystem fileSystem = new SftpFileSystemProvider(sshClient).newFileSystem(uri, env);
+        // ...
         sshClient.stop();
     }
 }

files-sftp/src/main/java/org/apache/sshd/client/impl/AbstractSftpClient.java

@@ -675,8 +675,6 @@ public abstract class AbstractSftpClient extends AbstractSubsystemClient impleme
         if (!isOpen()) {
             throw new IOException("close(" + handle + ") client is closed");
         }
-
-
         byte[] id = Objects.requireNonNull(handle, "No handle").getIdentifier();
         Buffer buffer = new ByteArrayBuffer(id.length + Long.SIZE /* some extra fields */, false);
         buffer.putBytes(id);
@@ -688,8 +686,6 @@ public abstract class AbstractSftpClient extends AbstractSubsystemClient impleme
         if (!isOpen()) {
             throw new IOException("remove(" + path + ") client is closed");
         }
-
-
         Buffer buffer = new ByteArrayBuffer(path.length() + Long.SIZE /* some extra fields */, false);
         buffer = putReferencedName(SftpConstants.SSH_FXP_REMOVE, buffer, path, 0);
         checkCommandStatus(SftpConstants.SSH_FXP_REMOVE, buffer);