Process hangs on futex when indexing

[ehiggs]

$ tantivy new --index .

Creating new index 
Let's define it's schema! 



New field name  ? data
Text or unsigned 32-bit integer (T/I) ? t
Should the field be stored (Y/N) ? n
Should the field be indexed (Y/N) ? t
Error: Invalid input. Options are (Y/N)
Should the field be indexed (Y/N) ? y
Should the term be tokenized? (Y/N) ? y
Should the term frequencies (per doc) be in the index (Y/N) ? y
Should the term positions (per doc) be in the index (Y/N) ? y
Add another field (Y/N) ? n

[
  {
    "name": "data",
    "type": "text",
    "options": {
      "indexing": {
        "record": "position",
        "tokenizer": "en_stem"
      },
      "stored": false
    }
  }
]

$ tantivy index --index .

Then the process appears to hang (maybe it takes over half an hour to index 1033 files?). I dump the stack as follows:

$ gdb -batch -ex "thread apply all bt" -p 20259
[New LWP 20260]
[New LWP 20261]
[New LWP 20262]
[New LWP 20263]
[New LWP 20264]
[New LWP 20265]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
0x00007fb05935e072 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x7fb0586213e8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
88	../sysdeps/unix/sysv/linux/futex-internal.h: No such file or directory.

Thread 7 (Thread 0x7fb026bff700 (LWP 20265)):
#0  0x00007fb05935e072 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x7fb0586215f8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0x7fb058621600, cond=0x7fb0586215d0) at pthread_cond_wait.c:502
#2  __pthread_cond_wait (cond=0x7fb0586215d0, mutex=0x7fb058621600) at pthread_cond_wait.c:655
#3  0x0000560bb5a93156 in <chan::Iter<T> as core::iter::iterator::Iterator>::next ()
#4  0x0000560bb5a60eb9 in std::sys_common::backtrace::__rust_begin_short_backtrace ()
#5  0x0000560bb5a825a7 in <F as alloc::boxed::FnBox<A>>::call_box ()
#6  0x0000560bb5c4ec48 in _$LT$alloc..boxed..Box$LT$alloc..boxed..FnBox$LT$A$C$$u20$Output$u3d$R$GT$$u20$$u2b$$u20$$u27$a$GT$$u20$as$u20$core..ops..function..FnOnce$LT$A$GT$$GT$::call_once::h290a492c91ea9d98 () at /checkout/src/liballoc/boxed.rs:798
#7  std::sys_common::thread::start_thread () at libstd/sys_common/thread.rs:24
#8  std::sys::unix::thread::Thread::new::thread_start () at libstd/sys/unix/thread.rs:90
#9  0x00007fb0593577fc in start_thread (arg=0x7fb026bff700) at pthread_create.c:465
#10 0x00007fb058e6db5f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

Thread 6 (Thread 0x7fb0373ff700 (LWP 20264)):
#0  0x00007fb05935e072 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x7fb0586215f8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0x7fb058621600, cond=0x7fb0586215d0) at pthread_cond_wait.c:502
#2  __pthread_cond_wait (cond=0x7fb0586215d0, mutex=0x7fb058621600) at pthread_cond_wait.c:655
#3  0x0000560bb5a93156 in <chan::Iter<T> as core::iter::iterator::Iterator>::next ()
#4  0x0000560bb5a60eb9 in std::sys_common::backtrace::__rust_begin_short_backtrace ()
#5  0x0000560bb5a825a7 in <F as alloc::boxed::FnBox<A>>::call_box ()
#6  0x0000560bb5c4ec48 in _$LT$alloc..boxed..Box$LT$alloc..boxed..FnBox$LT$A$C$$u20$Output$u3d$R$GT$$u20$$u2b$$u20$$u27$a$GT$$u20$as$u20$core..ops..function..FnOnce$LT$A$GT$$GT$::call_once::h290a492c91ea9d98 () at /checkout/src/liballoc/boxed.rs:798
#7  std::sys_common::thread::start_thread () at libstd/sys_common/thread.rs:24
#8  std::sys::unix::thread::Thread::new::thread_start () at libstd/sys/unix/thread.rs:90
#9  0x00007fb0593577fc in start_thread (arg=0x7fb0373ff700) at pthread_create.c:465
#10 0x00007fb058e6db5f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

Thread 5 (Thread 0x7fb0477ff700 (LWP 20263)):
#0  0x00007fb05935e072 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x7fb0586215f8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0x7fb058621600, cond=0x7fb0586215d0) at pthread_cond_wait.c:502
#2  __pthread_cond_wait (cond=0x7fb0586215d0, mutex=0x7fb058621600) at pthread_cond_wait.c:655
#3  0x0000560bb5a93156 in <chan::Iter<T> as core::iter::iterator::Iterator>::next ()
#4  0x0000560bb5a60eb9 in std::sys_common::backtrace::__rust_begin_short_backtrace ()
#5  0x0000560bb5a825a7 in <F as alloc::boxed::FnBox<A>>::call_box ()
#6  0x0000560bb5c4ec48 in _$LT$alloc..boxed..Box$LT$alloc..boxed..FnBox$LT$A$C$$u20$Output$u3d$R$GT$$u20$$u2b$$u20$$u27$a$GT$$u20$as$u20$core..ops..function..FnOnce$LT$A$GT$$GT$::call_once::h290a492c91ea9d98 () at /checkout/src/liballoc/boxed.rs:798
#7  std::sys_common::thread::start_thread () at libstd/sys_common/thread.rs:24
#8  std::sys::unix::thread::Thread::new::thread_start () at libstd/sys/unix/thread.rs:90
#9  0x00007fb0593577fc in start_thread (arg=0x7fb0477ff700) at pthread_create.c:465
#10 0x00007fb058e6db5f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

Thread 4 (Thread 0x7fb057bff700 (LWP 20262)):
#0  0x00007fb05935e072 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x7fb058621748) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0x7fb0586216f0, cond=0x7fb058621720) at pthread_cond_wait.c:502
#2  __pthread_cond_wait (cond=0x7fb058621720, mutex=0x7fb0586216f0) at pthread_cond_wait.c:655
#3  0x0000560bb5c40b82 in std::sys::unix::condvar::Condvar::wait () at libstd/sys/unix/condvar.rs:78
#4  std::sys_common::condvar::Condvar::wait () at libstd/sys_common/condvar.rs:51
#5  std::sync::condvar::Condvar::wait () at libstd/sync/condvar.rs:212
#6  std::thread::park () at libstd/thread/mod.rs:800
#7  0x0000560bb5b7f3c4 in <std::sync::mpsc::Receiver<T>>::recv ()
#8  0x0000560bb5b82764 in std::sys_common::backtrace::__rust_begin_short_backtrace ()
#9  0x0000560bb5b7bbcf in <F as alloc::boxed::FnBox<A>>::call_box ()
#10 0x0000560bb5c4ec48 in _$LT$alloc..boxed..Box$LT$alloc..boxed..FnBox$LT$A$C$$u20$Output$u3d$R$GT$$u20$$u2b$$u20$$u27$a$GT$$u20$as$u20$core..ops..function..FnOnce$LT$A$GT$$GT$::call_once::h290a492c91ea9d98 () at /checkout/src/liballoc/boxed.rs:798
#11 std::sys_common::thread::start_thread () at libstd/sys_common/thread.rs:24
#12 std::sys::unix::thread::Thread::new::thread_start () at libstd/sys/unix/thread.rs:90
#13 0x00007fb0593577fc in start_thread (arg=0x7fb057bff700) at pthread_create.c:465
#14 0x00007fb058e6db5f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

Thread 3 (Thread 0x7fb0583fe700 (LWP 20261)):
#0  0x00007fb05935e072 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x7fb058621388) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0x7fb058621390, cond=0x7fb058621360) at pthread_cond_wait.c:502
#2  __pthread_cond_wait (cond=0x7fb058621360, mutex=0x7fb058621390) at pthread_cond_wait.c:655
#3  0x0000560bb5a03c59 in std::sys_common::backtrace::__rust_begin_short_backtrace ()
#4  0x0000560bb5c62976 in <F as alloc::boxed::FnBox<A>>::call_box ()
#5  0x0000560bb5c4ec48 in _$LT$alloc..boxed..Box$LT$alloc..boxed..FnBox$LT$A$C$$u20$Output$u3d$R$GT$$u20$$u2b$$u20$$u27$a$GT$$u20$as$u20$core..ops..function..FnOnce$LT$A$GT$$GT$::call_once::h290a492c91ea9d98 () at /checkout/src/liballoc/boxed.rs:798
#6  std::sys_common::thread::start_thread () at libstd/sys_common/thread.rs:24
#7  std::sys::unix::thread::Thread::new::thread_start () at libstd/sys/unix/thread.rs:90
#8  0x00007fb0593577fc in start_thread (arg=0x7fb0583fe700) at pthread_create.c:465
#9  0x00007fb058e6db5f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

Thread 2 (Thread 0x7fb0585ff700 (LWP 20260)):
#0  0x00007fb059361d5d in __libc_read (fd=0, buf=0x7fb057e12000, nbytes=8192) at ../sysdeps/unix/sysv/linux/read.c:26
#1  0x0000560bb5c44617 in std::sys::unix::fd::FileDesc::read () at libstd/sys/unix/fd.rs:58
#2  std::sys::unix::stdio::Stdin::read () at libstd/sys/unix/stdio.rs:24
#3  <std::io::stdio::StdinRaw as std::io::Read>::read () at libstd/io/stdio.rs:77
#4  <std::io::stdio::Maybe<R> as std::io::Read>::read () at libstd/io/stdio.rs:117
#5  <std::io::buffered::BufReader<R> as std::io::Read>::read () at libstd/io/buffered.rs:229
#6  <std::io::stdio::StdinLock<'a> as std::io::Read>::read () at libstd/io/stdio.rs:309
#7  0x0000560bb5c4448b in <std::io::stdio::Stdin as std::io::Read>::read () at libstd/io/stdio.rs:289
#8  0x0000560bb5a03166 in std::sys_common::backtrace::__rust_begin_short_backtrace ()
#9  0x0000560bb5c62d16 in <F as alloc::boxed::FnBox<A>>::call_box ()
#10 0x0000560bb5c4ec48 in _$LT$alloc..boxed..Box$LT$alloc..boxed..FnBox$LT$A$C$$u20$Output$u3d$R$GT$$u20$$u2b$$u20$$u27$a$GT$$u20$as$u20$core..ops..function..FnOnce$LT$A$GT$$GT$::call_once::h290a492c91ea9d98 () at /checkout/src/liballoc/boxed.rs:798
#11 std::sys_common::thread::start_thread () at libstd/sys_common/thread.rs:24
#12 std::sys::unix::thread::Thread::new::thread_start () at libstd/sys/unix/thread.rs:90
#13 0x00007fb0593577fc in start_thread (arg=0x7fb0585ff700) at pthread_create.c:465
#14 0x00007fb058e6db5f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

Thread 1 (Thread 0x7fb059b81900 (LWP 20259)):
#0  0x00007fb05935e072 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x7fb0586213e8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0x7fb0586213f0, cond=0x7fb0586213c0) at pthread_cond_wait.c:502
#2  __pthread_cond_wait (cond=0x7fb0586213c0, mutex=0x7fb0586213f0) at pthread_cond_wait.c:655
#3  0x0000560bb5c91ac0 in tantivy::commands::index::run_index_cli ()
#4  0x0000560bb5c8eb5c in tantivy::main ()
#5  0x0000560bb5c68de3 in std::rt::lang_start::{{closure}} ()
#6  0x0000560bb5c88ac8 in main ()

I've reproduced this on OS X 10.11 and Ubuntu 17.10.

[ehiggs]

It's reading from stdin. derp. 🤒

[fulmicoton]

You got me scared here :)

You can either pass a file to it, or uses stdin.
I'm sure you are not the first one to be confused. Let's open an issue. (Thanks for opening the other ticket too.)