Skip to content

Commit

Permalink
libfabric: Initial commit
Browse files Browse the repository at this point in the history 
libfabric overview

libfabric is an extensible framework for application access
to fabric services.  The framework supports multiple providers,
including providers built into the library.

The layout of the libfabric source tree is outlined below.
Note that this is under development, and that full functionality
is missing.

include/rdma
------------
Contains header files for the framework, including the base
framework APIs in fabric.h.  There are sample APIs available
for message queue operations, RDMA operations, and tagged
messages.

Proposed APIs and objects that support communication and data
transfer functionality are found in fi_domain.h.

fabric.h - Base framework APIs
fi_domain.h - General resource management objects
fi_socket.h - Base communication object

src
---
Contains the base implementation for the framework and kernel
supported APIs.

src/fabric - Base framework implementation
src/ucma - Interface to kernel rdma cm ABI
src/uverbs - Interface to kernel verbs ABI

examples
--------
Includes some simple examples that demonstrate how an application
can use the framework and various API sets.

examples/perf - Simple latency/bandwidth test.
examples/provinfo - List available provider information.

prov
----
Providers built into the libfabric library are under the prov
subdirectory.

prov/ibverbs -
This is a *sample* provider that sits over libibverbs.  It is NOT
meant as a real provider because of the overhead that results from
converting libfabric calls directly into libibverbs calls.  It is
intended to show how a hardware vendor can implement an optimized
version of their provider library for libfabric.

prov/mlx4 -
This is a sample provider that works in conjunction with the
ibverbs provider.  It is mostly unchanged from the existing libmlx4
verbs provider.

prov/psm -
This is a sample provider that sits over the Intel PSM library.

prov/rdmacm -
Incorporates the librdmacm functionality into libfabric.

Signed-off-by: Sean Hefty <[email protected]>
  • Loading branch information
@shefty
shefty committed Nov 7, 2013
0 parents commit 30ec628
Show file tree
Hide file tree
Showing 91 changed files with 31,963 additions and 0 deletions.
4 changes: 4 additions & 0 deletions AUTHORS
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Roland Dreier <[email protected]>
Dotan Barak <[email protected]>
Sean Hefty <[email protected]>
Michael S. Tsirkin <[email protected]>
378 changes: 378 additions & 0 deletions COPYING

Large diffs are not rendered by default.

141 changes: 141 additions & 0 deletions Makefile.am
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
AM_CPPFLAGS = -I$(srcdir)/include -I$(srcdir)/prov/ibverbs/include \
-I$(srcdir)/prov/rdmacm/include

lib_LTLIBRARIES = src/libfabric.la

ACLOCAL_AMFLAGS = -I config
AM_CFLAGS = -g -Wall -D_GNU_SOURCE

src_libfabric_la_CFLAGS = $(AM_CFLAGS) -DSYSCONFDIR=\"$(sysconfdir)\" -DRDMADIR=\"@rdmadir@\"

if HAVE_LD_VERSION_SCRIPT
libfabric_version_script = -Wl,--version-script=$(srcdir)/src/libfabric.map
else
libfabric_version_script =
endif

src_libfabric_la_SOURCES = src/fabric.c src/uverbs.c src/ucma.c \
prov/ibverbs/src/cmd.c \
prov/ibverbs/src/device.c \
prov/ibverbs/src/enum_strs.c \
prov/ibverbs/src/fi_verbs.c \
prov/ibverbs/src/init.c \
prov/ibverbs/src/marshall.c \
prov/ibverbs/src/memory.c \
prov/ibverbs/src/verbs.c \
prov/rdmacm/src/acm.c \
prov/rdmacm/src/addrinfo.c \
prov/rdmacm/src/cma.c \
prov/rdmacm/src/indexer.c \
prov/rdmacm/src/rsocket.c \
prov/mlx4/src/buf.c \
prov/mlx4/src/cq.c \
prov/mlx4/src/dbrec.c \
prov/mlx4/src/mlx4.c \
prov/mlx4/src/qp.c \
prov/mlx4/src/srq.c \
prov/mlx4/src/mlx4_verbs.c

if HAVE_PSM
src_libfabric_la_SOURCES += prov/psm/src/psmx_init.c \
prov/psm/src/psmx_domain.c \
prov/psm/src/psmx_ec.c \
prov/psm/src/psmx_av.c \
prov/psm/src/psmx_sock.c \
prov/psm/src/psmx_cm.c \
prov/psm/src/psmx_tagged.c \
prov/psm/src/psmx_util.c
endif

src_libfabric_la_LDFLAGS = -version-info 1 -export-dynamic \
$(libfabric_version_script)

src_libfabric_la_DEPENDENCIES = $(srcdir)/src/libfabric.map

bin_PROGRAMS = \
prov/rdmacm/examples/rstream \
prov/rdmacm/examples/rcopy \
prov/rdmacm/examples/riostream \
prov/rdmacm/examples/udpong \
examples/fi_provinfo \
examples/fi_perf

prov_rdmacm_examples_rstream_SOURCES = \
prov/rdmacm/examples/rstream.c \
prov/rdmacm/examples/common.c
prov_rdmacm_examples_rstream_LDADD = \
$(top_builddir)/src/libfabric.la
prov_rdmacm_examples_riostream_SOURCES = \
prov/rdmacm/examples/riostream.c \
prov/rdmacm/examples/common.c
prov_rdmacm_examples_riostream_LDADD = \
$(top_builddir)/src/libfabric.la
prov_rdmacm_examples_rcopy_SOURCES = \
prov/rdmacm/examples/rcopy.c
prov_rdmacm_examples_rcopy_LDADD = \
$(top_builddir)/src/libfabric.la
prov_rdmacm_examples_udpong_SOURCES = \
prov/rdmacm/examples/udpong.c \
prov/rdmacm/examples/common.c
prov_rdmacm_examples_udpong_LDADD = \
$(top_builddir)/src/libfabric.la
examples_fi_provinfo_SOURCES = \
examples/provinfo.c \
examples/shared.c
examples_fi_provinfo_LDADD = \
$(top_builddir)/src/libfabric.la
examples_fi_perf_SOURCES = \
examples/perf.c \
examples/shared.c
examples_fi_perf_LDADD = \
$(top_builddir)/src/libfabric.la

libfabricincludedir = $(includedir)/rdma
infinibandincludedir = $(includedir)/infiniband

libfabricinclude_HEADERS = $(top_srcdir)/include/rdma/fabric.h \
$(top_srcdir)/include/rdma/fi_arch.h \
$(top_srcdir)/include/rdma/fi_atomic.h \
$(top_srcdir)/include/rdma/fi_cm.h \
$(top_srcdir)/include/rdma/fi_domain.h \
$(top_srcdir)/include/rdma/fi_prov.h \
$(top_srcdir)/include/rdma/fi_rdma.h \
$(top_srcdir)/include/rdma/fi_socket.h \
$(top_srcdir)/include/rdma/fi_errno.h \
$(top_srcdir)/include/rdma/fi_tagged.h \
$(top_srcdir)/include/rdma/fi_ucma.h \
$(top_srcdir)/include/rdma/fi_umad.h \
$(top_srcdir)/include/rdma/fi_uverbs.h \
$(top_srcdir)/prov/rdmacm/include/rdma/rsocket.h

infinibandinclude_HEADERS = $(top_srcdir)/include/infiniband/ib.h

man_MANS = man/fi_getinfo.3 man/fi_socket.3 man/fi_open.3

EXTRA_DIST = include/fi.h src/libfabric.map libfabric.spec.in $(man_MANS) \
prov/ibverbs/include/infiniband/driver.h \
prov/ibverbs/include/infiniband/marshall.h \
prov/ibverbs/include/infiniband/opcode.h \
prov/ibverbs/include/infiniband/sa.h \
prov/ibverbs/include/infiniband/sa-kern-abi.h \
prov/ibverbs/include/infiniband/verbs.h \
prov/ibverbs/src/ibverbs.h \
prov/rdmacm/include/rdma/rdma_cma.h \
prov/rdmacm/include/rdma/rdma_verbs.h \
prov/rdmacm/src/cma.h \
prov/rdmacm/src/indexer.h \
prov/mlx4/src/doorbell.h \
prov/mlx4/src/mlx4.h \
prov/mlx4/src/mlx4-abi.h \
prov/mlx4/wqe.h \
examples/shared.h

dist-hook: libfabric.spec
cp libfabric.spec $(distdir)

install-data-hook:
cd $(DESTDIR)$(mandir)/man3 && \
$(RM) fi_freeinfo.3 && \
$(RM) fi_close.3 && \
$(LN_S) fi_getinfo.3 fi_freeinfo.3 && \
$(LN_S) fi_open.3 fi_close.3
10 changes: 10 additions & 0 deletions README
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
This README is for userspace RDMA fabric library.

Building
========
To make this directory, run:
./autogen.sh && ./configure && make && make install

Typically the autogen and configure steps only need be done the first
time unless configure.ac or Makefile.am changes.

9 changes: 9 additions & 0 deletions autogen.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#! /bin/sh

set -x
test -d ./config || mkdir ./config
aclocal -I config
libtoolize --force --copy
autoheader
automake --foreign --add-missing --copy
autoconf
139 changes: 139 additions & 0 deletions configure.ac
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
dnl Process this file with autoconf to produce a configure script.

AC_PREREQ(2.57)
AC_INIT(libfabric, 0.0.1, [email protected])
AC_CONFIG_SRCDIR([src/fabric.c])
AC_CONFIG_AUX_DIR(config)
AC_CONFIG_MACRO_DIR(config)
AC_CONFIG_HEADERS(config.h)
AM_INIT_AUTOMAKE(libfabric, 0.0.1)
m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])

AC_ARG_ENABLE([debug],
[AS_HELP_STRING([--enable-debug],
[Enable debugging @<:@default=no@:>@])
],
[CFLAGS="$CFLAGS -g -O0 -Wall"],
[enable_debug=no])

dnl Fix autoconf's habit of adding -g -O2 by default
AS_IF([test -z "$CFLAGS"],
[CFLAGS='-O2 -DNDEBUG -Wall'])

AM_PROG_LIBTOOL

AC_ARG_WITH([valgrind],
AC_HELP_STRING([--with-valgrind],
[Enable valgrind annotations - default NO]))

if test "$with_valgrind" != "" && test "$with_valgrind" != "no"; then
AC_DEFINE([INCLUDE_VALGRIND], 1,
[Define to 1 to enable valgrind annotations])
if test -d $with_valgrind; then
CPPFLAGS="$CPPLFAGS -I$with_valgrind/include"
fi
fi

AC_ARG_ENABLE(libcheck, [ --disable-libcheck do not test for presence of libraries],
[ if test "$enableval" = "no"; then
disable_libcheck=yes
fi
])

dnl Checks for programs
AC_PROG_CC

dnl Checks for typedefs, structures, and compiler characteristics.
AC_C_CONST
AC_CHECK_SIZEOF(long)

dnl Checks for libraries
AC_CHECK_LIB(dl, dlsym, [],
AC_MSG_ERROR([dlsym() not found. libfabric requires libdl.]))
AC_CHECK_LIB(pthread, pthread_mutex_init, [],
AC_MSG_ERROR([pthread_mutex_init() not found. libfabric requires libpthread.]))

dnl Check for gcc atomic intrinsics
AC_MSG_CHECKING(compiler support for atomics)
AC_TRY_LINK([int i = 0;],
[ return __sync_add_and_fetch(&i, 1) != __sync_sub_and_fetch(&i, 1); ],
[ AC_MSG_RESULT(yes) ],
[
AC_MSG_RESULT(no)
AC_DEFINE(DEFINE_ATOMICS, 1, [Set to 1 to implement atomics])
])

dnl Checks for header files.
AC_HEADER_STDC

if test "$disable_libcheck" != "yes"; then
if test "$with_valgrind" != "" && test "$with_valgrind" != "no"; then
AC_CHECK_HEADER(valgrind/memcheck.h, [],
AC_MSG_ERROR([valgrind requested but <valgrind/memcheck.h> not found.]))
fi
fi

AC_CACHE_CHECK(whether ld accepts --version-script, ac_cv_version_script,
if test -n "`$LD --help < /dev/null 2>/dev/null | grep version-script`"; then
ac_cv_version_script=yes
else
ac_cv_version_script=no
fi)

AM_CONDITIONAL(HAVE_LD_VERSION_SCRIPT, test "$ac_cv_version_script" = "yes")

AC_ARG_ENABLE([psm],
[AS_HELP_STRING([--enable-psm],
[Enable PSM provider @<:@default=no@:>@])
],
[AC_DEFINE([HAVE_PSM], [1], [Define if PSM is enabled])
LIBS="-lpsm_infinipath $LIBS"],
[enable_psm=no])

AC_ARG_WITH([psm],
[AS_HELP_STRING([--with-psm=@<:@PSM installation path@:>@],
[Provide path to PSM installation])
],
[AS_CASE([$with_psm],
[yes|no], [AC_DEFINE([HAVE_PSM], [1], [Define if PSM is enabled])],
[CPPFLAGS="-I$with_psm/include $CPPFLAGS"
LDFLAGS="-L$with_psm/lib64 -Wl,-rpath=$with_psm/lib64 $LDFLAGS"
LIBS="-lpsm_infinipath $LIBS"
AC_DEFINE([HAVE_PSM], [1], [Define if PSM is enabled])])
])

AC_ARG_WITH([psm-include],
[AS_HELP_STRING([--with-psm-include=@<:@PSM include path@:>@],
[Provide path to PSM include files])
],
[AS_CASE([$with_psm_include],
[yes|no], [AC_DEFINE([HAVE_PSM], [1], [Define if PSM is enabled])],
[CPPFLAGS="-I$with_psm_include $CPPFLAGS"
AC_DEFINE([HAVE_PSM], [1], [Define if PSM is enabled])
])
])

AC_ARG_WITH([psm-lib],
[AS_HELP_STRING([--with-psm-lib=@<:@PSM library path@:>@],
[Provide path to PSM library files])
],
[AS_CASE([$with_psm_lib],
[yes|no], [],
[LDFLAGS="-L$with_psm_lib -Wl,-rpath=$with_psm_lib $LDFLAGS"
LIBS="-lpsm_infinipath $LIBS"
AC_DEFINE([HAVE_PSM], [1], [Define if PSM is enabled])
])
])

AS_IF([test x"$enable_psm" = x"yes"],
[AC_CHECK_LIB(psm_infinipath, psm_init,
[AC_CHECK_HEADER([psm.h], [],
[AC_MSG_ERROR([psm.h not found. Provide the correct path to PSM with --with-psm-include (or --with-psm)])]
)],
AC_MSG_ERROR([psm_init() not found. Provide the correct path to PSM --with-psm-lib]))],
[AC_MSG_NOTICE(PSM not enabled)])

AM_CONDITIONAL([HAVE_PSM], [test x"$enable_psm" = x"yes"])

AC_CONFIG_FILES([Makefile libfabric.spec])
AC_OUTPUT
Loading

0 comments on commit 30ec628

Please sign in to comment.