diff --git a/xrootd4j-standalone/src/main/java/org/dcache/xrootd/standalone/DataServerHandler.java b/xrootd4j-standalone/src/main/java/org/dcache/xrootd/standalone/DataServerHandler.java index 070798dc..b3cef4c3 100644 --- a/xrootd4j-standalone/src/main/java/org/dcache/xrootd/standalone/DataServerHandler.java +++ b/xrootd4j-standalone/src/main/java/org/dcache/xrootd/standalone/DataServerHandler.java @@ -20,11 +20,7 @@ import com.google.common.hash.HashCode; import com.google.common.hash.Hashing; -import com.google.common.io.Files; -import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.DefaultFileRegion; -import io.netty.util.ReferenceCountUtil; import org.apache.commons.io.FilenameUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,8 +32,11 @@ import java.net.InetSocketAddress; import java.nio.channels.ClosedChannelException; import java.nio.channels.FileChannel; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.NotDirectoryException; +import java.nio.file.Path; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import org.dcache.xrootd.core.XrootdException; @@ -263,12 +262,23 @@ protected DirListResponse doOnDirList(ChannelHandlerContext context, throw new XrootdException(kXR_ArgMissing, "no source path specified"); } - File dir = getFile(listPath); - String[] list = dir.list(); - if (list == null) { + Path dir = getFile(listPath).toPath(); + try (DirectoryStream paths = Files.newDirectoryStream(dir)) { + DirListResponse.Builder builder = DirListResponse.builder(request); + for (Path path : paths) { + builder.add(path.getFileName().toString(), request.isDirectoryStat() ? getFileStatusOf(path.toFile()) : null); + if (builder.count() >= 1000) { + respond(context, builder.buildPartial()); + } + } + return builder.buildFinal(); + } catch (FileNotFoundException e) { throw new XrootdException(kXR_NotFound, "No such directory: " + dir); + } catch (NotDirectoryException e) { + throw new XrootdException(kXR_IOError, "Not a directory: " + dir); + } catch (IOException e) { + throw new XrootdException(kXR_IOError, "IO Error: " + dir); } - return new DirListResponse(request, kXR_ok, Arrays.asList(list)); } @Override @@ -499,7 +509,7 @@ protected QueryResponse doOnQuery(ChannelHandlerContext ctx, QueryRequest msg) t case kXR_Qcksum: try { - HashCode hash = Files.asByteSource(getFile(msg.getArgs())).hash(Hashing.adler32()); + HashCode hash = com.google.common.io.Files.asByteSource(getFile(msg.getArgs())).hash(Hashing.adler32()); return new QueryResponse(msg, "ADLER32 " + hash); } catch (FileNotFoundException e) { throw new XrootdException(kXR_NotFound, e.getMessage()); diff --git a/xrootd4j/src/main/java/org/dcache/xrootd/protocol/XrootdProtocol.java b/xrootd4j/src/main/java/org/dcache/xrootd/protocol/XrootdProtocol.java index 2a3f8075..d9f136cf 100644 --- a/xrootd4j/src/main/java/org/dcache/xrootd/protocol/XrootdProtocol.java +++ b/xrootd4j/src/main/java/org/dcache/xrootd/protocol/XrootdProtocol.java @@ -187,6 +187,7 @@ public interface XrootdProtocol { // dirlist options public static final int kXR_online = 1; + public static final int kXR_dstat = 2; // mkdir options public static final int kXR_mknone = 0; diff --git a/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/DirListRequest.java b/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/DirListRequest.java index 9efecd99..504a5998 100644 --- a/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/DirListRequest.java +++ b/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/DirListRequest.java @@ -24,14 +24,28 @@ public class DirListRequest extends PathRequest { + private final short options; + public DirListRequest(ByteBuf buffer) { super(buffer, kXR_dirlist); + options = buffer.getUnsignedByte(19); + } + + public boolean isDirectoryStat() + { + return (options & kXR_dstat) == kXR_dstat; + } + + private short getOptions() + { + return options; } @Override public String toString() { - return "dirlist[" + getPath() + "," + getOpaque() + "]"; + return String.format("dirlist[%#x,%s,%s]", + getOptions(), getPath(), getOpaque()); } } diff --git a/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/DirListResponse.java b/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/DirListResponse.java index ab35e6a2..011937a9 100644 --- a/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/DirListResponse.java +++ b/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/DirListResponse.java @@ -20,15 +20,18 @@ import io.netty.buffer.ByteBuf; +import java.util.ArrayList; import java.util.Iterator; +import java.util.List; import org.dcache.xrootd.protocol.XrootdProtocol; +import org.dcache.xrootd.util.FileStatus; import static java.nio.charset.StandardCharsets.US_ASCII; public class DirListResponse extends AbstractXrootdResponse { - private final Iterable names; + protected final Iterable names; public DirListResponse(DirListRequest request, int statusCode, Iterable names) { @@ -66,9 +69,9 @@ protected void getBytes(ByteBuf buffer) buffer.writeByte('\n'); buffer.writeBytes(i.next().getBytes(US_ASCII)); } - /* Last entry in the list is terminated by a 0 rather than by - * a \n, if not more entries follow because the message is an - * intermediate message */ + /* If no more entries follow, the last entry in the list is terminated + * by a 0 rather than by a \n. + */ if (stat == XrootdProtocol.kXR_oksofar) { buffer.writeByte('\n'); } else { @@ -76,4 +79,113 @@ protected void getBytes(ByteBuf buffer) } } } + + public static Builder builder(DirListRequest request) + { + return request.isDirectoryStat() ? new StatBuilder(request) : new SimpleBuilder(request); + } + + public interface Builder + { + void add(String name); + void add(String name, FileStatus status); + DirListResponse buildPartial(); + DirListResponse buildFinal(); + int count(); + } + + private static class SimpleBuilder implements Builder + { + private final DirListRequest request; + private List names = new ArrayList<>(); + + public SimpleBuilder(DirListRequest request) + { + this.request = request; + } + + @Override + public void add(String name) + { + names.add(name); + } + + @Override + public void add(String name, FileStatus status) + { + names.add(name); + } + + @Override + public DirListResponse buildPartial() + { + DirListResponse response = new DirListResponse(request, XrootdProtocol.kXR_oksofar, names); + names = new ArrayList<>(); + return response; + } + + @Override + public DirListResponse buildFinal() + { + DirListResponse response = new DirListResponse(request, XrootdProtocol.kXR_ok, names); + names = null; + return response; + } + + @Override + public int count() + { + return names.size(); + } + } + + private static class StatBuilder implements Builder + { + private final DirListRequest request; + private List names = new ArrayList<>(); + private List fileStatus = new ArrayList<>(); + + public StatBuilder(DirListRequest request) + { + this.request = request; + } + + @Override + public void add(String name) + { + names.add(name); + fileStatus.add(new FileStatus(0, 0, 0, 0)); + } + + @Override + public void add(String name, FileStatus status) + { + names.add(name); + fileStatus.add(status); + } + + @Override + public DirListResponse buildPartial() + { + DirListResponse response = new DirListStatResponse(request, XrootdProtocol.kXR_oksofar, names, fileStatus); + names = new ArrayList<>(); + fileStatus = new ArrayList<>(); + return response; + } + + @Override + public DirListResponse buildFinal() + { + DirListResponse response = new DirListStatResponse(request, XrootdProtocol.kXR_ok, names, fileStatus); + names = null; + fileStatus = null; + return response; + } + + @Override + public int count() + { + return names.size(); + } + } } diff --git a/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/DirListStatResponse.java b/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/DirListStatResponse.java new file mode 100644 index 00000000..05cd59d3 --- /dev/null +++ b/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/DirListStatResponse.java @@ -0,0 +1,89 @@ +/* dCache Endit Nearline Storage Provider + * + * Copyright (C) 2016 Gerd Behrmann + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.dcache.xrootd.protocol.messages; + +import io.netty.buffer.ByteBuf; + +import java.util.Iterator; + +import org.dcache.xrootd.protocol.XrootdProtocol; +import org.dcache.xrootd.util.FileStatus; + +import static java.nio.charset.StandardCharsets.US_ASCII; + +public class DirListStatResponse extends DirListResponse +{ + private final Iterable status; + + public DirListStatResponse(DirListRequest request, int statusCode, Iterable names, Iterable status) + { + super(request, statusCode, names); + this.status = status; + } + + public DirListStatResponse(DirListRequest request, Iterable names, Iterable status) + { + super(request, names); + this.status = status; + } + + public Iterable getFileStatus() + { + return status; + } + + @Override + public int getDataLength() + { + if (!names.iterator().hasNext()) { + return 0; + } + int length = 10; + Iterator names = this.names.iterator(); + Iterator status = this.status.iterator(); + while (names.hasNext() && status.hasNext()) { + length += names.next().length() + 1 + status.next().toString().length() + 1; + } + return length; + } + + @Override + protected void getBytes(ByteBuf buffer) + { + Iterator names = this.names.iterator(); + Iterator status = this.status.iterator(); + if (names.hasNext() && status.hasNext()) { + buffer.writeBytes(".\n0 0 0 0".getBytes(US_ASCII)); + do { + buffer.writeByte('\n'); + buffer.writeBytes(names.next().getBytes(US_ASCII)); + buffer.writeByte('\n'); + buffer.writeBytes(status.next().toString().getBytes(US_ASCII)); + } while (names.hasNext()); + + /* If no more entries follow, the last entry in the list is terminated + * by a 0 rather than by a \n. + */ + if (stat == XrootdProtocol.kXR_oksofar) { + buffer.writeByte('\n'); + } else { + buffer.writeByte(0); + } + } + } +} diff --git a/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/StatRequest.java b/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/StatRequest.java index f6ef8b95..6ae062b2 100644 --- a/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/StatRequest.java +++ b/xrootd4j/src/main/java/org/dcache/xrootd/protocol/messages/StatRequest.java @@ -36,9 +36,15 @@ public boolean isVfsSet() return (options & kXR_vfs) == kXR_vfs; } + private short getOptions() + { + return options; + } + @Override public String toString() { - return "stat[" + getPath() + "," + getOpaque() + "," + options + "]"; + return String.format("stat[%#x,%s,%s]", + getOptions(), getPath(), getOpaque()); } }