xrootd4j: Add kXR_dstat support

xrootd 3.0.0 adds the kXR_dstat option to return stat information inline with a kXR_dirlist response. This patch adds support for this option to xrootd4j. Target: master, 3.0 Acked-by: Tigran Mkrtchyan <[email protected]> Patch: https://rb.dcache.org/r/9003/
vingar · Feb 12, 2016 · c27606a · c27606a
1 parent 9fa1645
commit c27606a
Show file tree

Hide file tree

Showing 6 changed files with 248 additions and 16 deletions.
diff --git a/xrootd4j-standalone/src/main/java/org/dcache/xrootd/standalone/DataServerHandler.java b/xrootd4j-standalone/src/main/java/org/dcache/xrootd/standalone/DataServerHandler.java
@@ -20,11 +20,7 @@
 
 import com.google.common.hash.HashCode;
 import com.google.common.hash.Hashing;
-import com.google.common.io.Files;
-import io.netty.buffer.ByteBuf;
 import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.DefaultFileRegion;
-import io.netty.util.ReferenceCountUtil;
 import org.apache.commons.io.FilenameUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -36,8 +32,11 @@
 import java.net.InetSocketAddress;
 import java.nio.channels.ClosedChannelException;
 import java.nio.channels.FileChannel;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.NotDirectoryException;
+import java.nio.file.Path;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 
 import org.dcache.xrootd.core.XrootdException;
@@ -263,12 +262,23 @@ protected DirListResponse doOnDirList(ChannelHandlerContext context,
             throw new XrootdException(kXR_ArgMissing, "no source path specified");
         }
 
-        File dir = getFile(listPath);
-        String[] list = dir.list();
-        if (list == null) {
+        Path dir = getFile(listPath).toPath();
+        try (DirectoryStream<Path> paths = Files.newDirectoryStream(dir)) {
+            DirListResponse.Builder builder = DirListResponse.builder(request);
+            for (Path path : paths) {
+                builder.add(path.getFileName().toString(), request.isDirectoryStat() ? getFileStatusOf(path.toFile()) : null);
+                if (builder.count() >= 1000) {
+                    respond(context, builder.buildPartial());
+                }
+            }
+            return builder.buildFinal();
+        } catch (FileNotFoundException e) {
             throw new XrootdException(kXR_NotFound, "No such directory: " + dir);
+        } catch (NotDirectoryException e) {
+            throw new XrootdException(kXR_IOError, "Not a directory: " + dir);
+        } catch (IOException e) {
+            throw new XrootdException(kXR_IOError, "IO Error: " + dir);
         }
-        return new DirListResponse(request, kXR_ok, Arrays.asList(list));
     }
 
     @Override
@@ -499,7 +509,7 @@ protected QueryResponse doOnQuery(ChannelHandlerContext ctx, QueryRequest msg) t
 
         case kXR_Qcksum:
             try {
-                HashCode hash = Files.asByteSource(getFile(msg.getArgs())).hash(Hashing.adler32());
+                HashCode hash = com.google.common.io.Files.asByteSource(getFile(msg.getArgs())).hash(Hashing.adler32());
                 return new QueryResponse(msg, "ADLER32 " + hash);
             } catch (FileNotFoundException e) {
                 throw new XrootdException(kXR_NotFound, e.getMessage());

diff --git a/xrootd4j/src/main/java/org/dcache/xrootd/protocol/XrootdProtocol.java b/xrootd4j/src/main/java/org/dcache/xrootd/protocol/XrootdProtocol.java
@@ -187,6 +187,7 @@ public interface XrootdProtocol {
 
     // dirlist options
     public static final int kXR_online = 1;
+    public static final int kXR_dstat  = 2;
 
     // mkdir options
     public static final int kXR_mknone    = 0;

diff --git a/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/DirListRequest.java b/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/DirListRequest.java
@@ -24,14 +24,28 @@
 
 public class DirListRequest extends PathRequest
 {
+    private final short options;
+
     public DirListRequest(ByteBuf buffer)
     {
         super(buffer, kXR_dirlist);
+        options = buffer.getUnsignedByte(19);
+    }
+
+    public boolean isDirectoryStat()
+    {
+        return (options & kXR_dstat) == kXR_dstat;
+    }
+
+    private short getOptions()
+    {
+        return options;
     }
 
     @Override
     public String toString()
     {
-        return "dirlist[" + getPath() + "," + getOpaque() + "]";
+        return String.format("dirlist[%#x,%s,%s]",
+                             getOptions(), getPath(), getOpaque());
     }
 }
diff --git a/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/DirListResponse.java b/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/DirListResponse.java
@@ -20,15 +20,18 @@
 
 import io.netty.buffer.ByteBuf;
 
+import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
 
 import org.dcache.xrootd.protocol.XrootdProtocol;
+import org.dcache.xrootd.util.FileStatus;
 
 import static java.nio.charset.StandardCharsets.US_ASCII;
 
 public class DirListResponse extends AbstractXrootdResponse<DirListRequest>
 {
-    private final Iterable<String> names;
+    protected final Iterable<String> names;
 
     public DirListResponse(DirListRequest request, int statusCode, Iterable<String> names)
     {
@@ -66,14 +69,123 @@ protected void getBytes(ByteBuf buffer)
                 buffer.writeByte('\n');
                 buffer.writeBytes(i.next().getBytes(US_ASCII));
             }
-            /* Last entry in the list is terminated by a 0 rather than by
-             * a \n, if not more entries follow because the message is an
-             * intermediate message */
+            /* If no more entries follow, the last entry in the list is terminated
+             * by a 0 rather than by a \n.
+             */
             if (stat == XrootdProtocol.kXR_oksofar) {
                 buffer.writeByte('\n');
             } else {
                 buffer.writeByte(0);
             }
         }
     }
+
+    public static Builder builder(DirListRequest request)
+    {
+        return request.isDirectoryStat() ? new StatBuilder(request) : new SimpleBuilder(request);
+    }
+
+    public interface Builder
+    {
+        void add(String name);
+        void add(String name, FileStatus status);
+        DirListResponse buildPartial();
+        DirListResponse buildFinal();
+        int count();
+    }
+
+    private static class SimpleBuilder implements Builder
+    {
+        private final DirListRequest request;
+        private List<String> names = new ArrayList<>();
+
+        public SimpleBuilder(DirListRequest request)
+        {
+            this.request = request;
+        }
+
+        @Override
+        public void add(String name)
+        {
+            names.add(name);
+        }
+
+        @Override
+        public void add(String name, FileStatus status)
+        {
+            names.add(name);
+        }
+
+        @Override
+        public DirListResponse buildPartial()
+        {
+            DirListResponse response = new DirListResponse(request, XrootdProtocol.kXR_oksofar, names);
+            names = new ArrayList<>();
+            return response;
+        }
+
+        @Override
+        public DirListResponse buildFinal()
+        {
+            DirListResponse response = new DirListResponse(request, XrootdProtocol.kXR_ok, names);
+            names = null;
+            return response;
+        }
+
+        @Override
+        public int count()
+        {
+            return names.size();
+        }
+    }
+
+    private static class StatBuilder implements Builder
+    {
+        private final DirListRequest request;
+        private List<String> names = new ArrayList<>();
+        private List<FileStatus> fileStatus = new ArrayList<>();
+
+        public StatBuilder(DirListRequest request)
+        {
+            this.request = request;
+        }
+
+        @Override
+        public void add(String name)
+        {
+            names.add(name);
+            fileStatus.add(new FileStatus(0, 0, 0, 0));
+        }
+
+        @Override
+        public void add(String name, FileStatus status)
+        {
+            names.add(name);
+            fileStatus.add(status);
+        }
+
+        @Override
+        public DirListResponse buildPartial()
+        {
+            DirListResponse response = new DirListStatResponse(request, XrootdProtocol.kXR_oksofar, names, fileStatus);
+            names = new ArrayList<>();
+            fileStatus = new ArrayList<>();
+            return response;
+        }
+
+        @Override
+        public DirListResponse buildFinal()
+        {
+            DirListResponse response = new DirListStatResponse(request, XrootdProtocol.kXR_ok, names, fileStatus);
+            names = null;
+            fileStatus = null;
+            return response;
+        }
+
+        @Override
+        public int count()
+        {
+            return names.size();
+        }
+    }
 }
diff --git a/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/DirListStatResponse.java b/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/DirListStatResponse.java
@@ -0,0 +1,89 @@
+/* dCache Endit Nearline Storage Provider
+ *
+ * Copyright (C) 2016 Gerd Behrmann
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.dcache.xrootd.protocol.messages;
+
+import io.netty.buffer.ByteBuf;
+
+import java.util.Iterator;
+
+import org.dcache.xrootd.protocol.XrootdProtocol;
+import org.dcache.xrootd.util.FileStatus;
+
+import static java.nio.charset.StandardCharsets.US_ASCII;
+
+public class DirListStatResponse extends DirListResponse
+{
+    private final Iterable<FileStatus> status;
+
+    public DirListStatResponse(DirListRequest request, int statusCode, Iterable<String> names, Iterable<FileStatus> status)
+    {
+        super(request, statusCode, names);
+        this.status = status;
+    }
+
+    public DirListStatResponse(DirListRequest request, Iterable<String> names, Iterable<FileStatus> status)
+    {
+        super(request, names);
+        this.status = status;
+    }
+
+    public Iterable<FileStatus> getFileStatus()
+    {
+        return status;
+    }
+
+    @Override
+    public int getDataLength()
+    {
+        if (!names.iterator().hasNext()) {
+            return 0;
+        }
+        int length = 10;
+        Iterator<String> names = this.names.iterator();
+        Iterator<FileStatus> status = this.status.iterator();
+        while (names.hasNext() && status.hasNext()) {
+            length += names.next().length() + 1 + status.next().toString().length() + 1;
+        }
+        return length;
+    }
+
+    @Override
+    protected void getBytes(ByteBuf buffer)
+    {
+        Iterator<String> names = this.names.iterator();
+        Iterator<FileStatus> status = this.status.iterator();
+        if (names.hasNext() && status.hasNext()) {
+            buffer.writeBytes(".\n0 0 0 0".getBytes(US_ASCII));
+            do {
+                buffer.writeByte('\n');
+                buffer.writeBytes(names.next().getBytes(US_ASCII));
+                buffer.writeByte('\n');
+                buffer.writeBytes(status.next().toString().getBytes(US_ASCII));
+            } while (names.hasNext());
+
+            /* If no more entries follow, the last entry in the list is terminated
+             * by a 0 rather than by a \n.
+             */
+            if (stat == XrootdProtocol.kXR_oksofar) {
+                buffer.writeByte('\n');
+            } else {
+                buffer.writeByte(0);
+            }
+        }
+    }
+}
diff --git a/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/StatRequest.java b/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/StatRequest.java
@@ -36,9 +36,15 @@ public boolean isVfsSet()
         return (options & kXR_vfs) == kXR_vfs;
     }
 
+    private short getOptions()
+    {
+        return options;
+    }
+
     @Override
     public String toString()
     {
-        return "stat[" + getPath() + "," + getOpaque() + "," + options + "]";
+        return String.format("stat[%#x,%s,%s]",
+                             getOptions(), getPath(), getOpaque());
     }
 }