iostream: Fix \r\n handling in reading buffers (#22621)

src/flisp/iostream.c

@@ -333,7 +333,7 @@ value_t fl_ioreaduntil(fl_context_t *fl_ctx, value_t *args, uint32_t nargs)
     ios_setbuf(&dest, data, 80, 0);
     char delim = get_delim_arg(fl_ctx, args[1], "io.readuntil");
     ios_t *src = toiostream(fl_ctx, args[0], "io.readuntil");
-    size_t n = ios_copyuntil(&dest, src, delim, 0);
+    size_t n = ios_copyuntil(&dest, src, delim);
     cv->len = n;
     if (dest.buf != data) {
         // outgrew initial space
@@ -352,7 +352,7 @@ value_t fl_iocopyuntil(fl_context_t *fl_ctx, value_t *args, uint32_t nargs)
     ios_t *dest = toiostream(fl_ctx, args[0], "io.copyuntil");
     ios_t *src = toiostream(fl_ctx, args[1], "io.copyuntil");
     char delim = get_delim_arg(fl_ctx, args[2], "io.copyuntil");
-    return size_wrap(fl_ctx, ios_copyuntil(dest, src, delim, 0));
+    return size_wrap(fl_ctx, ios_copyuntil(dest, src, delim));
 }
 
 value_t fl_iocopy(fl_context_t *fl_ctx, value_t *args, uint32_t nargs)

src/support/ios.c

@@ -803,7 +803,7 @@ size_t ios_copyall(ios_t *to, ios_t *from)
 
 #define LINE_CHUNK_SIZE 160
 
-size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, uint8_t chomp)
+size_t ios_copyuntil(ios_t *to, ios_t *from, char delim)
 {
     size_t total = 0, avail = (size_t)(from->size - from->bpos);
     while (!ios_eof(from)) {
@@ -822,11 +822,7 @@ size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, uint8_t chomp)
         }
         else {
             size_t ntowrite = pd - (from->buf+from->bpos) + 1;
-            size_t nchomp = 0;
-            if (chomp) {
-                nchomp = ios_nchomp(from, ntowrite);
-            }
-            written = ios_write(to, from->buf+from->bpos, ntowrite - nchomp);
+            written = ios_write(to, from->buf+from->bpos, ntowrite);
             from->bpos += ntowrite;
             total += written;
             return total;
@@ -1167,7 +1163,7 @@ char *ios_readline(ios_t *s)
 {
     ios_t dest;
     ios_mem(&dest, 0);
-    ios_copyuntil(&dest, s, '\n', 0);
+    ios_copyuntil(&dest, s, '\n');
     size_t n;
     return ios_take_buffer(&dest, &n);
 }

src/support/ios.h

@@ -94,7 +94,7 @@ JL_DLLEXPORT int ios_get_writable(ios_t *s);
 JL_DLLEXPORT void ios_set_readonly(ios_t *s);
 JL_DLLEXPORT size_t ios_copy(ios_t *to, ios_t *from, size_t nbytes);
 JL_DLLEXPORT size_t ios_copyall(ios_t *to, ios_t *from);
-JL_DLLEXPORT size_t ios_copyuntil(ios_t *to, ios_t *from, char delim, uint8_t chomp);
+JL_DLLEXPORT size_t ios_copyuntil(ios_t *to, ios_t *from, char delim);
 JL_DLLEXPORT size_t ios_nchomp(ios_t *from, size_t ntowrite);
 // ensure at least n bytes are buffered if possible. returns # available.
 JL_DLLEXPORT size_t ios_readprep(ios_t *from, size_t n);

src/sys.c

@@ -256,9 +256,9 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str, uint
 {
     jl_array_t *a;
     // manually inlined common case
-    char *pd = (char*)memchr(s->buf+s->bpos, delim, (size_t)(s->size - s->bpos));
+    char *pd = (char*)memchr(s->buf + s->bpos, delim, (size_t)(s->size - s->bpos));
     if (pd) {
-        size_t n = pd-(s->buf+s->bpos)+1;
+        size_t n = pd - (s->buf + s->bpos) + 1;
         if (str) {
             size_t nchomp = 0;
             if (chomp) {
@@ -277,7 +277,15 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str, uint
         ios_t dest;
         ios_mem(&dest, 0);
         ios_setbuf(&dest, (char*)a->data, 80, 0);
-        size_t n = ios_copyuntil(&dest, s, delim, chomp);
+        size_t n = ios_copyuntil(&dest, s, delim);
+        if (chomp && n > 0 && dest.buf[n - 1] == '\n') {
+            n--;
+            if (n > 0 && dest.buf[n - 1] == '\r') {
+                n--;
+            }
+            int truncret = ios_trunc(&dest, n); // it should always be possible to truncate dest
+            assert(truncret == 0);
+        }
         if (dest.buf != a->data) {
             a = jl_take_buffer(&dest);
         }

test/iobuffer.jl

@@ -90,6 +90,25 @@ write(io,"\n\r\n\n\r \n") > 0
 @test readlines(IOBuffer(""), chomp=true) == []
 @test readlines(IOBuffer("first\nsecond"), chomp=false) == String["first\n", "second"]
 @test readlines(IOBuffer("first\nsecond"), chomp=true) == String["first", "second"]
+
+let fname = tempname()
+    for dochomp in [true, false],
+        endline in ["\n", "\r\n"],
+        i in -5:5
+
+        ref = ("1"^(2^17 - i)) * endline
+        open(fname, "w") do io
+            write(io, ref)
+        end
+        x = readlines(fname, chomp = dochomp)
+        if dochomp
+            ref = chomp(ref)
+        end
+        @test ref == x[1]
+    end
+    rm(fname)
+end
+
 Base.compact(io)
 @test position(io) == 0
 @test ioslength(io) == 0