From c86153ed2312c1a3e4d86b5df40f5be85437b05c Mon Sep 17 00:00:00 2001 From: Sean V Kelley Date: Thu, 30 Jan 2014 11:33:44 -0800 Subject: [PATCH] Initial commit Signed-off-by: Sean V Kelley --- AUTHORS | 0 COPYING | 28 + ChangeLog | 0 Makefile.am | 7 + NEWS | 0 README | 0 autogen.sh | 12 + configure.ac | 30 + libwsbm.pc.in | 10 + src/Android.mk | 39 ++ src/Makefile.am | 27 + src/config.h | 69 +++ src/wsbm_atomic.h | 76 +++ src/wsbm_driver.c | 243 ++++++++ src/wsbm_driver.h | 131 ++++ src/wsbm_fencemgr.c | 474 ++++++++++++++ src/wsbm_fencemgr.h | 162 +++++ src/wsbm_mallocpool.c | 187 ++++++ src/wsbm_manager.c | 1357 +++++++++++++++++++++++++++++++++++++++++ src/wsbm_manager.h | 187 ++++++ src/wsbm_mm.c | 289 +++++++++ src/wsbm_mm.h | 75 +++ src/wsbm_pool.h | 161 +++++ src/wsbm_priv.h | 45 ++ src/wsbm_slabpool.c | 1203 ++++++++++++++++++++++++++++++++++++ src/wsbm_ttmpool.c | 575 +++++++++++++++++ src/wsbm_userpool.c | 689 +++++++++++++++++++++ src/wsbm_util.h | 76 +++ 28 files changed, 6152 insertions(+) create mode 100644 AUTHORS create mode 100644 COPYING create mode 100644 ChangeLog create mode 100644 Makefile.am create mode 100644 NEWS create mode 100644 README create mode 100755 autogen.sh create mode 100644 configure.ac create mode 100644 libwsbm.pc.in create mode 100644 src/Android.mk create mode 100644 src/Makefile.am create mode 100644 src/config.h create mode 100644 src/wsbm_atomic.h create mode 100644 src/wsbm_driver.c create mode 100644 src/wsbm_driver.h create mode 100644 src/wsbm_fencemgr.c create mode 100644 src/wsbm_fencemgr.h create mode 100644 src/wsbm_mallocpool.c create mode 100644 src/wsbm_manager.c create mode 100644 src/wsbm_manager.h create mode 100644 src/wsbm_mm.c create mode 100644 src/wsbm_mm.h create mode 100644 src/wsbm_pool.h create mode 100644 src/wsbm_priv.h create mode 100644 src/wsbm_slabpool.c create mode 100644 src/wsbm_ttmpool.c create mode 100644 src/wsbm_userpool.c create mode 100644 src/wsbm_util.h diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..e69de29 diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..362bf8e --- /dev/null +++ b/COPYING @@ -0,0 +1,28 @@ +/************************************************************************** + * + * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 0000000..e69de29 diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..2940c78 --- /dev/null +++ b/Makefile.am @@ -0,0 +1,7 @@ +SUBDIRS = src +EXTRA_DIST = COPYING NEWS README libwsbm.pc.in + +pkgconfigdir = @pkgconfigdir@ +pkgconfig_DATA = libwsbm.pc + + diff --git a/NEWS b/NEWS new file mode 100644 index 0000000..e69de29 diff --git a/README b/README new file mode 100644 index 0000000..e69de29 diff --git a/autogen.sh b/autogen.sh new file mode 100755 index 0000000..354f254 --- /dev/null +++ b/autogen.sh @@ -0,0 +1,12 @@ +#! /bin/sh + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +autoreconf -v --install || exit 1 +cd $ORIGDIR || exit $? + +$srcdir/configure "$@" diff --git a/configure.ac b/configure.ac new file mode 100644 index 0000000..24191f2 --- /dev/null +++ b/configure.ac @@ -0,0 +1,30 @@ +AC_PREREQ(2.57) +AC_INIT([libwsbm], 1.1.0, [thomas@tungstengraphics.com], libwsbm) +AC_CONFIG_SRCDIR([Makefile.am]) +AM_INIT_AUTOMAKE([dist-bzip2]) + +AM_CONFIG_HEADER([config.h]) + +AC_DISABLE_STATIC +AC_PROG_LIBTOOL +AC_PROG_CC +PKG_PROG_PKG_CONFIG + +PKG_CHECK_MODULES(libdrm, libdrm) +AC_SUBST(libdrm_CFLAGS) +AC_SUBST(libdrm_LIBS) +AC_HEADER_STDC +AC_SYS_LARGEFILE + +AC_CHECK_HEADER(pthread.h, [ + AC_SEARCH_LIBS(pthread_cond_init, pthread, + [AC_DEFINE(HAVE_PTHREADS, 1, "os has pthreads")],,,) + ],,,) + +pkgconfigdir=${libdir}/pkgconfig +AC_SUBST(pkgconfigdir) + +AC_OUTPUT([ + Makefile + src/Makefile + libwsbm.pc]) diff --git a/libwsbm.pc.in b/libwsbm.pc.in new file mode 100644 index 0000000..dfff843 --- /dev/null +++ b/libwsbm.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: libwsbm +Description: Buffer manager abstraction library. +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lwsbm +Cflags: -I${includedir}/wsbm diff --git a/src/Android.mk b/src/Android.mk new file mode 100644 index 0000000..3232efe --- /dev/null +++ b/src/Android.mk @@ -0,0 +1,39 @@ +LOCAL_PATH:= $(call my-dir) + +include $(CLEAR_VARS) +LOCAL_SRC_FILES:= \ + wsbm_driver.c \ + wsbm_fencemgr.c \ + wsbm_mallocpool.c \ + wsbm_manager.c \ + wsbm_mm.c \ + wsbm_slabpool.c \ + wsbm_ttmpool.c \ + wsbm_userpool.c + +LOCAL_CFLAGS += -DHAVE_CONFIG_H +LOCAL_C_INCLUDES := \ + $(LOCAL_PATH)/../ \ + $(TARGET_OUT_HEADERS)/drm \ + $(TARGET_OUT_HEADERS)/ipp \ + $(TARGET_OUT_HEADERS)/libdrm \ + $(TARGET_OUT_HEADERS)/libdrm/shared-core \ + $(TARGET_OUT_HEADERS)/libttm + +LOCAL_MODULE_TAGS := optional +LOCAL_MODULE:= libwsbm +LOCAL_SHARED_LIBRARIES:= libdrm +include $(BUILD_SHARED_LIBRARY) + +include $(CLEAR_VARS) +LOCAL_COPY_HEADERS_TO := libwsbm/wsbm +LOCAL_COPY_HEADERS := \ + wsbm_atomic.h \ + wsbm_driver.h \ + wsbm_fencemgr.h \ + wsbm_manager.h \ + wsbm_mm.h \ + wsbm_pool.h \ + wsbm_priv.h \ + wsbm_util.h +include $(BUILD_COPY_HEADERS) diff --git a/src/Makefile.am b/src/Makefile.am new file mode 100644 index 0000000..4778f37 --- /dev/null +++ b/src/Makefile.am @@ -0,0 +1,27 @@ + +lib_LTLIBRARIES = libwsbm.la + +libwsbm_la_CFLAGS = @libdrm_CFLAGS@ -Wall +libwsbm_la_LDFLAGS = -version-number 1:1:0 -no-undefined @libdrm_LIBS@ +libwsbm_la_SOURCES = \ + wsbm_fencemgr.c \ + wsbm_fencemgr.h \ + wsbm_manager.c \ + wsbm_manager.h \ + wsbm_mm.c \ + wsbm_mm.h \ + wsbm_pool.h \ + wsbm_util.h \ + wsbm_mallocpool.c \ + wsbm_driver.h \ + wsbm_driver.c \ + wsbm_ttmpool.c \ + wsbm_slabpool.c \ + wsbm_userpool.c \ + wsbm_priv.h + + +libwsbmincludedir = ${includedir}/wsbm +libwsbminclude_HEADERS = wsbm_manager.h wsbm_pool.h wsbm_driver.h \ + wsbm_fencemgr.h wsbm_util.h wsbm_atomic.h + diff --git a/src/config.h b/src/config.h new file mode 100644 index 0000000..db26fd7 --- /dev/null +++ b/src/config.h @@ -0,0 +1,69 @@ +/* config.h. Generated from config.h.in by configure. */ +/* config.h.in. Generated from configure.ac by autoheader. */ + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* "os has pthreads" */ +#define HAVE_PTHREADS 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to the sub-directory in which libtool stores uninstalled libraries. + */ +#define LT_OBJDIR ".libs/" + +/* Name of package */ +#define PACKAGE "libwsbm" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "thomas@tungstengraphics.com" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "libwsbm" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "libwsbm 1.1.0" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "libwsbm" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "1.1.0" + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Version number of package */ +#define VERSION "1.1.0" + +/* Number of bits in a file offset, on hosts where this is settable. */ +#define _FILE_OFFSET_BITS 64 + +/* Define for large files, on AIX-style hosts. */ +/* #undef _LARGE_FILES */ diff --git a/src/wsbm_atomic.h b/src/wsbm_atomic.h new file mode 100644 index 0000000..7be2da9 --- /dev/null +++ b/src/wsbm_atomic.h @@ -0,0 +1,76 @@ +/** + * This file is under no copyright claims due to its + * simplicity. + */ + +#ifndef _WSBM_ATOMIC_H_ +#define _WSBM_ATOMIC_H_ + +#include + +struct _WsbmAtomic +{ + int32_t count; +}; + +#define wsbmAtomicInit(_i) {(i)} +#define wsbmAtomicSet(_v, _i) (((_v)->count) = (_i)) +#define wsbmAtomicRead(_v) ((_v)->count) + +static inline int +wsbmAtomicIncZero(struct _WsbmAtomic *v) +{ + unsigned char c; + __asm__ __volatile__("lock; incl %0; sete %1":"+m"(v->count), "=qm"(c) + ::"memory"); + + return c != 0; +} + +static inline int +wsbmAtomicDecNegative(struct _WsbmAtomic *v) +{ + unsigned char c; + int i = -1; + __asm__ __volatile__("lock; addl %2,%0; sets %1":"+m"(v->count), "=qm"(c) + :"ir"(i):"memory"); + + return c; +} + +static inline int +wsbmAtomicDecZero(struct _WsbmAtomic *v) +{ + unsigned char c; + + __asm__ __volatile__("lock; decl %0; sete %1":"+m"(v->count), "=qm"(c) + ::"memory"); + + return c != 0; +} + +static inline void +wsbmAtomicInc(struct _WsbmAtomic *v) +{ + __asm__ __volatile__("lock; incl %0":"+m"(v->count)); +} + +static inline void +wsbmAtomicDec(struct _WsbmAtomic *v) +{ + __asm__ __volatile__("lock; decl %0":"+m"(v->count)); +} + +static inline int32_t +wsbmAtomicCmpXchg(volatile struct _WsbmAtomic *v, int32_t old, int32_t new) +{ + int32_t previous; + + __asm__ __volatile__("lock; cmpxchgl %k1,%2":"=a"(previous) + :"r"(new), "m"(v->count), "0"(old) + :"memory"); + + return previous; +} + +#endif diff --git a/src/wsbm_driver.c b/src/wsbm_driver.c new file mode 100644 index 0000000..98f846d --- /dev/null +++ b/src/wsbm_driver.c @@ -0,0 +1,243 @@ +/************************************************************************** + * + * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: + * Thomas Hellstr�m + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include "wsbm_driver.h" + +struct _WsbmThreadFuncs *wsbmCurThreadFunc = NULL; +struct _WsbmVNodeFuncs *wsbmCurVNodeFunc = NULL; + +/* + * Single-threaded implementation. + */ + +static int +n_mutexInit(struct _WsbmMutex *mutex, struct _WsbmThreadFuncs *func) +{ + mutex->func = func; + return 0; +} + +static int +n_condInit(struct _WsbmCond *cond, struct _WsbmThreadFuncs *func) +{ + cond->func = func; + return 0; +} + +static void +n_mutexNone(struct _WsbmMutex *mutex) +{ + ; +} + +static void +n_condNone(struct _WsbmCond *cond) +{ + ; +} + +static void +n_condWait(struct _WsbmCond *cond, struct _WsbmMutex *mutex) +{ + ; +} + +static struct _WsbmThreadFuncs nullFunc = { + .mutexInit = n_mutexInit, + .mutexFree = n_mutexNone, + .mutexLock = n_mutexNone, + .mutexUnlock = n_mutexNone, + .condInit = n_condInit, + .condFree = n_condNone, + .condWait = n_condWait, + .condBroadcast = n_condNone +}; + +struct _WsbmThreadFuncs * +wsbmNullThreadFuncs(void) +{ + return &nullFunc; +} + +#if (HAVE_PTHREADS == 1) +#include "pthread.h" + +/* + * pthreads implementation: + */ + +struct _WsbmPMutex +{ + struct _WsbmThreadFuncs *func; + pthread_mutex_t mutex; +}; + +struct _WsbmPCond +{ + struct _WsbmThreadFuncs *func; + pthread_cond_t cond; +}; + +static inline struct _WsbmPMutex * +pMutexConvert(struct _WsbmMutex *m) +{ + union _PMutexConverter + { + struct _WsbmMutex wm; + struct _WsbmPMutex pm; + } *um = containerOf(m, union _PMutexConverter, wm); + + return &um->pm; +} + +static inline struct _WsbmPCond * +pCondConvert(struct _WsbmCond *c) +{ + union _PCondConverter + { + struct _WsbmCond wc; + struct _WsbmPCond pc; + } *uc = containerOf(c, union _PCondConverter, wc); + + return &uc->pc; +} + +static int +p_mutexInit(struct _WsbmMutex *mutex, struct _WsbmThreadFuncs *func) +{ + struct _WsbmPMutex *pMutex = pMutexConvert(mutex); + + if (sizeof(struct _WsbmMutex) < sizeof(struct _WsbmPMutex)) + return -EINVAL; + + pMutex->func = func; + pthread_mutex_init(&pMutex->mutex, NULL); + return 0; +} + +static void +p_mutexFree(struct _WsbmMutex *mutex) +{ + struct _WsbmPMutex *pMutex = pMutexConvert(mutex); + + pthread_mutex_destroy(&pMutex->mutex); +} + +static void +p_mutexLock(struct _WsbmMutex *mutex) +{ + struct _WsbmPMutex *pMutex = pMutexConvert(mutex); + + pthread_mutex_lock(&pMutex->mutex); +} + +static void +p_mutexUnlock(struct _WsbmMutex *mutex) +{ + struct _WsbmPMutex *pMutex = pMutexConvert(mutex); + + pthread_mutex_unlock(&pMutex->mutex); +} + +static int +p_condInit(struct _WsbmCond *cond, struct _WsbmThreadFuncs *func) +{ + struct _WsbmPCond *pCond = pCondConvert(cond); + + if (sizeof(struct _WsbmCond) < sizeof(struct _WsbmPCond)) + return -EINVAL; + + pCond->func = func; + pthread_cond_init(&pCond->cond, NULL); + return 0; +} + +static void +p_condFree(struct _WsbmCond *cond) +{ + struct _WsbmPCond *pCond = pCondConvert(cond); + + pthread_cond_destroy(&pCond->cond); +} + +static void +p_condBroadcast(struct _WsbmCond *cond) +{ + struct _WsbmPCond *pCond = pCondConvert(cond); + + pthread_cond_broadcast(&pCond->cond); +} + +static void +p_condWait(struct _WsbmCond *cond, struct _WsbmMutex *mutex) +{ + struct _WsbmPCond *pCond = pCondConvert(cond); + struct _WsbmPMutex *pMutex = pMutexConvert(mutex); + + pthread_cond_wait(&pCond->cond, &pMutex->mutex); +} + +static struct _WsbmThreadFuncs pthreadFunc = { + .mutexInit = p_mutexInit, + .mutexFree = p_mutexFree, + .mutexLock = p_mutexLock, + .mutexUnlock = p_mutexUnlock, + .condInit = p_condInit, + .condFree = p_condFree, + .condWait = p_condWait, + .condBroadcast = p_condBroadcast +}; + +struct _WsbmThreadFuncs * +wsbmPThreadFuncs(void) +{ + return &pthreadFunc; +} + +#else +#warning Pthreads is not present. Compiling without. + +struct _WsbmThreadFuncs * +wsbmPThreadFuncs(void) +{ + return &pthreadFunc; +} + +#endif diff --git a/src/wsbm_driver.h b/src/wsbm_driver.h new file mode 100644 index 0000000..d7da44e --- /dev/null +++ b/src/wsbm_driver.h @@ -0,0 +1,131 @@ +/************************************************************************** + * + * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: + * Thomas Hellstr�m + */ + +#ifndef _WSBM_DRIVER_H +#define _WSBM_DRIVER_H +#include +#include "wsbm_util.h" + +#define WSBM_MUTEX_SPACE 16 +#define WSBM_COND_SPACE 16 + +struct _WsbmMutex +{ + struct _WsbmThreadFuncs *func; + unsigned long storage[WSBM_MUTEX_SPACE]; +}; + +struct _WsbmCond +{ + struct _WsbmThreadFuncs *func; + unsigned long storage[WSBM_COND_SPACE]; +}; + +struct _WsbmThreadFuncs +{ + int (*mutexInit) (struct _WsbmMutex *, struct _WsbmThreadFuncs *); + void (*mutexFree) (struct _WsbmMutex *); + void (*mutexLock) (struct _WsbmMutex *); + void (*mutexUnlock) (struct _WsbmMutex *); + int (*condInit) (struct _WsbmCond *, struct _WsbmThreadFuncs *); + void (*condFree) (struct _WsbmCond *); + void (*condWait) (struct _WsbmCond *, struct _WsbmMutex *); + void (*condBroadcast) (struct _WsbmCond *); +}; + +extern struct _WsbmThreadFuncs *wsbmCurThreadFunc; + +#define WSBM_MUTEX_INIT(_mutex) \ + wsbmThreadFuncs()->mutexInit(_mutex,wsbmThreadFuncs()) +#define WSBM_MUTEX_FREE(_mutex) \ + { \ + (_mutex)->func->mutexFree(_mutex); \ + } +#define WSBM_MUTEX_LOCK(_mutex) \ + (_mutex)->func->mutexLock(_mutex); +#define WSBM_MUTEX_UNLOCK(_mutex) \ + (_mutex)->func->mutexUnlock(_mutex); + +#define WSBM_COND_INIT(_mutex) \ + wsbmThreadFuncs()->condInit(_mutex, wsbmThreadFuncs()) +#define WSBM_COND_FREE(_cond) \ + { \ + (_cond)->func->condFree(_cond); \ + } +#define WSBM_COND_WAIT(_cond, _mutex) \ + (_cond)->func->condWait(_cond, _mutex); +#define WSBM_COND_BROADCAST(_cond) \ + (_cond)->func->condBroadcast(_cond); + +struct _WsbmVNodeFuncs +{ + struct _ValidateNode *(*alloc) (struct _WsbmVNodeFuncs *, int); + void (*free) (struct _ValidateNode *); + void (*clear) (struct _ValidateNode *); +}; + +extern struct _WsbmVNodeFuncs *wsbmCurVNodeFunc; + +struct _WsbmBufStorage; +struct _WsbmKernelBuf; + +struct _ValidateNode +{ + uint32_t hash; + int type_id; + struct _WsbmListHead head; + struct _WsbmListHead hashHead; + int listItem; + uint64_t set_flags; + uint64_t clr_flags; + void *buf; + struct _WsbmVNodeFuncs *func; +}; + +static inline struct _WsbmVNodeFuncs * +wsbmVNodeFuncs(void) +{ + return wsbmCurVNodeFunc; +} + +static inline struct _WsbmThreadFuncs * +wsbmThreadFuncs(void) +{ + return wsbmCurThreadFunc; +} + +extern struct _WsbmThreadFuncs *wsbmNullThreadFuncs(void); + +extern struct _WsbmThreadFuncs *wsbmPThreadFuncs(void); + +#endif diff --git a/src/wsbm_fencemgr.c b/src/wsbm_fencemgr.c new file mode 100644 index 0000000..4010ac9 --- /dev/null +++ b/src/wsbm_fencemgr.c @@ -0,0 +1,474 @@ +/************************************************************************** + * + * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, Tx., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "wsbm_fencemgr.h" +#include "wsbm_pool.h" +#include "wsbm_manager.h" +#include +#include +#include +#include + +struct _WsbmFenceClass +{ + struct _WsbmListHead head; + struct _WsbmMutex mutex; + struct _WsbmMutex cmd_mutex; +}; + +/* + * Note: The struct _WsbmFenceMgr::Mutex should never be held + * during sleeps, since that may block fast concurrent access to + * fence data. + */ + +struct _WsbmFenceMgr +{ + /* + * Constant members. Need no mutex protection. + */ + struct _WsbmFenceMgrCreateInfo info; + void *private; + + /* + * Atomic members. No mutex protection. + */ + + struct _WsbmAtomic count; + + /* + * These members are protected by this->mutex + */ + + struct _WsbmFenceClass *classes; + uint32_t num_classes; +}; + +struct _WsbmFenceObject +{ + + /* + * These members are constant and need no mutex protection. + * Note that @private may point to a structure with its own + * mutex protection, that we don't care about. + */ + + struct _WsbmFenceMgr *mgr; + uint32_t fence_class; + uint32_t fence_type; + void *private; + + /* + * Atomic members. No mutex protection. note that + * @signaled types is updated using a compare-and-swap + * scheme to guarantee atomicity. + */ + + struct _WsbmAtomic refCount; + struct _WsbmAtomic signaled_types; + + /* + * These members are protected by mgr->mutex. + */ + struct _WsbmListHead head; +}; + +uint32_t +wsbmFenceType(struct _WsbmFenceObject *fence) +{ + return fence->fence_type; +} + +struct _WsbmFenceMgr * +wsbmFenceMgrCreate(const struct _WsbmFenceMgrCreateInfo *info) +{ + struct _WsbmFenceMgr *tmp; + uint32_t i, j; + int ret; + + tmp = calloc(1, sizeof(*tmp)); + if (!tmp) + return NULL; + + tmp->info = *info; + tmp->classes = calloc(tmp->info.num_classes, sizeof(*tmp->classes)); + if (!tmp->classes) + goto out_err; + + for (i = 0; i < tmp->info.num_classes; ++i) { + struct _WsbmFenceClass *fc = &tmp->classes[i]; + + WSBMINITLISTHEAD(&fc->head); + ret = WSBM_MUTEX_INIT(&fc->mutex); + if (ret) + goto out_err1; + ret = WSBM_MUTEX_INIT(&fc->cmd_mutex); + if (ret) { + WSBM_MUTEX_FREE(&fc->mutex); + goto out_err1; + } + } + wsbmAtomicSet(&tmp->count, 0); + + return tmp; + + out_err1: + for (j = 0; j < i; ++j) { + WSBM_MUTEX_FREE(&tmp->classes[j].mutex); + WSBM_MUTEX_FREE(&tmp->classes[j].cmd_mutex); + } + free(tmp->classes); + out_err: + if (tmp) + free(tmp); + return NULL; +} + +void +wsbmFenceUnreference(struct _WsbmFenceObject **pFence) +{ + struct _WsbmFenceObject *fence = *pFence; + struct _WsbmFenceMgr *mgr; + + *pFence = NULL; + if (fence == NULL) + return; + + mgr = fence->mgr; + if (wsbmAtomicDecZero(&fence->refCount)) { + struct _WsbmFenceClass *fc = &mgr->classes[fence->fence_class]; + + WSBM_MUTEX_LOCK(&fc->mutex); + WSBMLISTDELINIT(&fence->head); + WSBM_MUTEX_UNLOCK(&fc->mutex); + if (fence->private) + mgr->info.unreference(mgr, &fence->private); + fence->mgr = NULL; + wsbmAtomicDecZero(&mgr->count); + free(fence); + } +} + +static void +wsbmSignalPreviousFences(struct _WsbmFenceMgr *mgr, + struct _WsbmListHead *list, + uint32_t fence_class, uint32_t signaled_types) +{ + struct _WsbmFenceClass *fc = &mgr->classes[fence_class]; + struct _WsbmFenceObject *entry; + struct _WsbmListHead *prev; + uint32_t old_signaled_types; + uint32_t ret_st; + + WSBM_MUTEX_LOCK(&fc->mutex); + while (list != &fc->head && list->next != list) { + entry = WSBMLISTENTRY(list, struct _WsbmFenceObject, head); + + prev = list->prev; + + do { + old_signaled_types = wsbmAtomicRead(&entry->signaled_types); + signaled_types = + old_signaled_types | (signaled_types & entry->fence_type); + if (signaled_types == old_signaled_types) + break; + + ret_st = + wsbmAtomicCmpXchg(&entry->signaled_types, old_signaled_types, + signaled_types); + } while (ret_st != old_signaled_types); + + if (signaled_types == entry->fence_type) + WSBMLISTDELINIT(list); + + list = prev; + } + WSBM_MUTEX_UNLOCK(&fc->mutex); +} + +int +wsbmFenceFinish(struct _WsbmFenceObject *fence, uint32_t fence_type, + int lazy_hint) +{ + struct _WsbmFenceMgr *mgr = fence->mgr; + int ret = 0; + + if ((wsbmAtomicRead(&fence->signaled_types) & fence_type) == fence_type) + goto out; + + ret = mgr->info.finish(mgr, fence->private, fence_type, lazy_hint); + if (ret) + goto out; + + wsbmSignalPreviousFences(mgr, &fence->head, fence->fence_class, + fence_type); + out: + return ret; +} + +uint32_t +wsbmFenceSignaledTypeCached(struct _WsbmFenceObject * fence) +{ + return wsbmAtomicRead(&fence->signaled_types); +} + +int +wsbmFenceSignaledType(struct _WsbmFenceObject *fence, uint32_t flush_type, + uint32_t * signaled) +{ + int ret = 0; + struct _WsbmFenceMgr *mgr; + uint32_t signaled_types; + uint32_t old_signaled_types; + uint32_t ret_st; + + mgr = fence->mgr; + *signaled = wsbmAtomicRead(&fence->signaled_types); + if ((*signaled & flush_type) == flush_type) + goto out0; + + ret = mgr->info.signaled(mgr, fence->private, flush_type, signaled); + if (ret) { + *signaled = wsbmAtomicRead(&fence->signaled_types); + goto out0; + } + + do { + old_signaled_types = wsbmAtomicRead(&fence->signaled_types); + signaled_types = old_signaled_types | *signaled; + if (signaled_types == old_signaled_types) + break; + + ret_st = wsbmAtomicCmpXchg(&fence->signaled_types, old_signaled_types, + signaled_types); + if (old_signaled_types == ret_st) + wsbmSignalPreviousFences(mgr, &fence->head, fence->fence_class, + *signaled); + } while (old_signaled_types != ret_st); + + return 0; + out0: + return ret; +} + +struct _WsbmFenceObject * +wsbmFenceReference(struct _WsbmFenceObject *fence) +{ + if (fence == NULL) + return NULL; + wsbmAtomicInc(&fence->refCount); + return fence; +} + +struct _WsbmFenceObject * +wsbmFenceCreateSig(struct _WsbmFenceMgr *mgr, uint32_t fence_class, + uint32_t fence_type, uint32_t signaled_types, + void *private, size_t private_size) +{ + struct _WsbmFenceClass *fc = &mgr->classes[fence_class]; + struct _WsbmFenceObject *fence; + size_t fence_size = sizeof(*fence); + + if (private_size) + fence_size = ((fence_size + 15) & ~15); + + fence = calloc(1, fence_size + private_size); + + if (!fence) + goto out_err; + + wsbmAtomicSet(&fence->refCount, 1); + fence->mgr = mgr; + fence->fence_class = fence_class; + fence->fence_type = fence_type; + wsbmAtomicSet(&fence->signaled_types, signaled_types); + fence->private = private; + if (private_size) { + fence->private = (void *)(((uint8_t *) fence) + fence_size); + memcpy(fence->private, private, private_size); + } + + WSBM_MUTEX_LOCK(&fc->mutex); + WSBMLISTADDTAIL(&fence->head, &fc->head); + WSBM_MUTEX_UNLOCK(&fc->mutex); + wsbmAtomicInc(&mgr->count); + return fence; + + out_err: + { + int ret = mgr->info.finish(mgr, private, fence_type, 0); + + if (ret) + usleep(10000000); + } + if (fence) + free(fence); + + mgr->info.unreference(mgr, &private); + return NULL; +} + +struct _WsbmFenceObject * +wsbmFenceCreate(struct _WsbmFenceMgr *mgr, uint32_t fence_class, + uint32_t fence_type, void *private, size_t private_size) +{ + return wsbmFenceCreateSig(mgr, fence_class, fence_type, 0, private, + private_size); +} + +struct _WsbmTTMFenceMgrPriv +{ + int fd; + unsigned int devOffset; +}; + +static int +tSignaled(struct _WsbmFenceMgr *mgr, void *private, uint32_t flush_type, + uint32_t * signaled_type) +{ + struct _WsbmTTMFenceMgrPriv *priv = + (struct _WsbmTTMFenceMgrPriv *)mgr->private; + union ttm_fence_signaled_arg arg; + int ret; + + arg.req.handle = (unsigned long)private; + arg.req.fence_type = flush_type; + arg.req.flush = 1; + *signaled_type = 0; + + ret = drmCommandWriteRead(priv->fd, priv->devOffset + TTM_FENCE_SIGNALED, + &arg, sizeof(arg)); + if (ret) + return ret; + + *signaled_type = arg.rep.signaled_types; + return 0; +} + +static int +tFinish(struct _WsbmFenceMgr *mgr, void *private, uint32_t fence_type, + int lazy_hint) +{ + struct _WsbmTTMFenceMgrPriv *priv = + (struct _WsbmTTMFenceMgrPriv *)mgr->private; + union ttm_fence_finish_arg arg = + {.req = {.handle = (unsigned long)private, + .fence_type = fence_type, + .mode = (lazy_hint) ? TTM_FENCE_FINISH_MODE_LAZY : 0} + }; + int ret; + + do { + ret = drmCommandWriteRead(priv->fd, priv->devOffset + TTM_FENCE_FINISH, + &arg, sizeof(arg)); + } while (ret == -EAGAIN || ret == -ERESTART); + + return ret; +} + +static int +tUnref(struct _WsbmFenceMgr *mgr, void **private) +{ + struct _WsbmTTMFenceMgrPriv *priv = + (struct _WsbmTTMFenceMgrPriv *)mgr->private; + struct ttm_fence_unref_arg arg = {.handle = (unsigned long)*private }; + + *private = NULL; + + return drmCommandWrite(priv->fd, priv->devOffset + TTM_FENCE_UNREF, + &arg, sizeof(arg)); +} + +struct _WsbmFenceMgr * +wsbmFenceMgrTTMInit(int fd, unsigned int numClass, unsigned int devOffset) +{ + struct _WsbmFenceMgrCreateInfo info; + struct _WsbmFenceMgr *mgr; + struct _WsbmTTMFenceMgrPriv *priv = malloc(sizeof(*priv)); + + if (!priv) + return NULL; + + priv->fd = fd; + priv->devOffset = devOffset; + + info.flags = WSBM_FENCE_CLASS_ORDERED; + info.num_classes = numClass; + info.signaled = tSignaled; + info.finish = tFinish; + info.unreference = tUnref; + + mgr = wsbmFenceMgrCreate(&info); + if (mgr == NULL) { + free(priv); + return NULL; + } + + mgr->private = (void *)priv; + return mgr; +} + +void +wsbmFenceCmdLock(struct _WsbmFenceMgr *mgr, uint32_t fence_class) +{ + WSBM_MUTEX_LOCK(&mgr->classes[fence_class].cmd_mutex); +} + +void +wsbmFenceCmdUnlock(struct _WsbmFenceMgr *mgr, uint32_t fence_class) +{ + WSBM_MUTEX_UNLOCK(&mgr->classes[fence_class].cmd_mutex); +} + +void +wsbmFenceMgrTTMTakedown(struct _WsbmFenceMgr *mgr) +{ + int i; + + if (!mgr) + return; + + if (mgr->private) + free(mgr->private); + + for (i = 0; i < mgr->info.num_classes; ++i) { + WSBM_MUTEX_FREE(&mgr->classes[i].mutex); + WSBM_MUTEX_FREE(&mgr->classes[i].cmd_mutex); + } + free(mgr); + + return; +} diff --git a/src/wsbm_fencemgr.h b/src/wsbm_fencemgr.h new file mode 100644 index 0000000..cb10ae2 --- /dev/null +++ b/src/wsbm_fencemgr.h @@ -0,0 +1,162 @@ +/************************************************************************** + * + * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, Tx., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#ifndef WSBM_FENCEMGR_H +#define WSBM_FENCEMGR_H + +#include +#include + +struct _WsbmFenceObject; +struct _WsbmFenceMgr; + +/* + * Do a quick check to see if the fence manager has registered the fence + * object as signaled. Note that this function may return a false negative + * answer. + */ +extern uint32_t wsbmFenceSignaledTypeCached(struct _WsbmFenceObject *fence); + +/* + * Check if the fence object is signaled. This function can be substantially + * more expensive to call than the above function, but will not return a false + * negative answer. The argument "flush_type" sets the types that the + * underlying mechanism must make sure will eventually signal. + */ +extern int wsbmFenceSignaledType(struct _WsbmFenceObject *fence, + uint32_t flush_type, uint32_t * signaled); + +/* + * Convenience functions. + */ + +static inline int +wsbmFenceSignaled(struct _WsbmFenceObject *fence, uint32_t flush_type) +{ + uint32_t signaled_types; + int ret = wsbmFenceSignaledType(fence, flush_type, &signaled_types); + + if (ret) + return 0; + return ((signaled_types & flush_type) == flush_type); +} + +static inline int +wsbmFenceSignaledCached(struct _WsbmFenceObject *fence, uint32_t flush_type) +{ + uint32_t signaled_types = wsbmFenceSignaledTypeCached(fence); + + return ((signaled_types & flush_type) == flush_type); +} + +/* + * Reference a fence object. + */ +extern struct _WsbmFenceObject *wsbmFenceReference(struct _WsbmFenceObject + *fence); + +/* + * Unreference a fence object. The fence object pointer will be reset to NULL. + */ + +extern void wsbmFenceUnreference(struct _WsbmFenceObject **pFence); + +/* + * Wait for a fence to signal the indicated fence_type. + * If "lazy_hint" is true, it indicates that the wait may sleep to avoid + * busy-wait polling. + */ +extern int wsbmFenceFinish(struct _WsbmFenceObject *fence, + uint32_t fence_type, int lazy_hint); + +/* + * Create a WsbmFenceObject for manager "mgr". + * + * "private" is a pointer that should be used for the callbacks in + * struct _WsbmFenceMgrCreateInfo. + * + * if private_size is nonzero, then the info stored at *private, with size + * private size will be copied and the fence manager will instead use a + * pointer to the copied data for the callbacks in + * struct _WsbmFenceMgrCreateInfo. In that case, the object pointed to by + * "private" may be destroyed after the call to wsbmFenceCreate. + */ +extern struct _WsbmFenceObject *wsbmFenceCreate(struct _WsbmFenceMgr *mgr, + uint32_t fence_class, + uint32_t fence_type, + void *private, + size_t private_size); + + +extern struct _WsbmFenceObject *wsbmFenceCreateSig(struct _WsbmFenceMgr *mgr, + uint32_t fence_class, + uint32_t fence_type, + uint32_t signaled_types, + void *private, + size_t private_size); + +extern uint32_t wsbmFenceType(struct _WsbmFenceObject *fence); + +/* + * Fence creations are ordered. If a fence signals a fence_type, + * it is safe to assume that all fences of the same class that was + * created before that fence has signaled the same type. + */ + +#define WSBM_FENCE_CLASS_ORDERED (1 << 0) + +struct _WsbmFenceMgrCreateInfo +{ + uint32_t flags; + uint32_t num_classes; + int (*signaled) (struct _WsbmFenceMgr * mgr, void *private, + uint32_t flush_type, uint32_t * signaled_type); + int (*finish) (struct _WsbmFenceMgr * mgr, void *private, + uint32_t fence_type, int lazy_hint); + int (*unreference) (struct _WsbmFenceMgr * mgr, void **private); +}; + +extern struct _WsbmFenceMgr *wsbmFenceMgrCreate(const struct + _WsbmFenceMgrCreateInfo + *info); +extern void wsbmFenceCmdLock(struct _WsbmFenceMgr *mgr, uint32_t fence_class); +extern void wsbmFenceCmdUnlock(struct _WsbmFenceMgr *mgr, + uint32_t fence_class); +/* + * Builtin drivers. + */ + +extern struct _WsbmFenceMgr *wsbmFenceMgrTTMInit(int fd, + unsigned int numClass, + unsigned int devOffset); +extern void wsbmFenceMgrTTMTakedown(struct _WsbmFenceMgr *mgr); +#endif diff --git a/src/wsbm_mallocpool.c b/src/wsbm_mallocpool.c new file mode 100644 index 0000000..b5252d6 --- /dev/null +++ b/src/wsbm_mallocpool.c @@ -0,0 +1,187 @@ +/************************************************************************** + * + * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstr�m + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include "wsbm_pool.h" +#include "wsbm_manager.h" + +struct _WsbmMallocBuffer +{ + struct _WsbmBufStorage buf; + size_t size; + void *mem; +}; + +static inline struct _WsbmMallocBuffer * +mallocBuf(struct _WsbmBufStorage *buf) +{ + return containerOf(buf, struct _WsbmMallocBuffer, buf); +} + +static struct _WsbmBufStorage * +pool_create(struct _WsbmBufferPool *pool, + unsigned long size, uint32_t placement, unsigned alignment) +{ + struct _WsbmMallocBuffer *mBuf = malloc(size + sizeof(*mBuf) + 16); + + if (!mBuf) + return NULL; + + wsbmBufStorageInit(&mBuf->buf, pool); + mBuf->size = size; + mBuf->mem = (void *)((unsigned long)mBuf + sizeof(*mBuf)); + if ((placement & WSBM_PL_MASK_MEM) != WSBM_PL_FLAG_SYSTEM) + abort(); + + return &mBuf->buf; +} + +static void +pool_destroy(struct _WsbmBufStorage **buf) +{ + free(mallocBuf(*buf)); + *buf = NULL; +} + +static int +pool_waitIdle(struct _WsbmBufStorage *buf, int lazy) +{ + return 0; +} + +static int +pool_map(struct _WsbmBufStorage *buf, unsigned mode, void **virtual) +{ + *virtual = mallocBuf(buf)->mem; + return 0; +} + +static void +pool_unmap(struct _WsbmBufStorage *buf) +{ + ; +} + +static int +pool_syncforcpu(struct _WsbmBufStorage *buf, unsigned mode) +{ + return 0; +} + +static void +pool_releasefromcpu(struct _WsbmBufStorage *buf, unsigned mode) +{ + ; +} + +static unsigned long +pool_offset(struct _WsbmBufStorage *buf) +{ + /* + * BUG + */ + abort(); + return 0UL; +} + +static unsigned long +pool_poolOffset(struct _WsbmBufStorage *buf) +{ + /* + * BUG + */ + abort(); +} + +static uint32_t +pool_placement(struct _WsbmBufStorage *buf) +{ + return WSBM_PL_FLAG_SYSTEM | WSBM_PL_FLAG_CACHED; +} + +static unsigned long +pool_size(struct _WsbmBufStorage *buf) +{ + return mallocBuf(buf)->size; +} + +static void +pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence) +{ + abort(); +} + +static struct _WsbmKernelBuf * +pool_kernel(struct _WsbmBufStorage *buf) +{ + abort(); + return NULL; +} + +static void +pool_takedown(struct _WsbmBufferPool *pool) +{ + free(pool); +} + +struct _WsbmBufferPool * +wsbmMallocPoolInit(void) +{ + struct _WsbmBufferPool *pool; + + pool = (struct _WsbmBufferPool *)calloc(1, sizeof(*pool)); + if (!pool) + return NULL; + + pool->fd = -1; + pool->map = &pool_map; + pool->unmap = &pool_unmap; + pool->syncforcpu = &pool_syncforcpu; + pool->releasefromcpu = &pool_releasefromcpu; + pool->destroy = &pool_destroy; + pool->offset = &pool_offset; + pool->poolOffset = &pool_poolOffset; + pool->placement = &pool_placement; + pool->size = &pool_size; + pool->create = &pool_create; + pool->fence = &pool_fence; + pool->kernel = &pool_kernel; + pool->validate = NULL; + pool->waitIdle = &pool_waitIdle; + pool->takeDown = &pool_takedown; + return pool; +} diff --git a/src/wsbm_manager.c b/src/wsbm_manager.c new file mode 100644 index 0000000..e95309d --- /dev/null +++ b/src/wsbm_manager.c @@ -0,0 +1,1357 @@ +/************************************************************************** + * + * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright 2009 Vmware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstr�m + * Keith Whitwell + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include "errno.h" +#include "string.h" +#include "wsbm_pool.h" +#include "wsbm_manager.h" +#include "wsbm_fencemgr.h" +#include "wsbm_driver.h" +#include "wsbm_priv.h" +#include "wsbm_util.h" +#include "wsbm_atomic.h" +#include "assert.h" + +#define WSBM_BODATA_SIZE_ACCEPT 4096 + +#define WSBM_BUFFER_COMPLEX 0 +#define WSBM_BUFFER_SIMPLE 1 +#define WSBM_BUFFER_REF 2 + +struct _ValidateList +{ + unsigned numTarget; + unsigned numCurrent; + unsigned numOnList; + unsigned hashSize; + uint32_t hashMask; + int driverData; + struct _WsbmListHead list; + struct _WsbmListHead free; + struct _WsbmListHead *hashTable; +}; + +struct _WsbmBufferObject +{ + /* Left to the client to protect this data for now. */ + + struct _WsbmAtomic refCount; + struct _WsbmBufStorage *storage; + + uint32_t placement; + unsigned alignment; + unsigned bufferType; + struct _WsbmBufferPool *pool; +}; + +struct _WsbmBufferList +{ + int hasKernelBuffers; + + struct _ValidateList kernelBuffers; /* List of kernel buffers needing validation */ + struct _ValidateList userBuffers; /* List of user-space buffers needing validation */ +}; + +static struct _WsbmMutex bmMutex; +static struct _WsbmCond bmCond; +static int initialized = 0; +static void *commonData = NULL; + +static int kernelReaders = 0; +static int kernelLocked = 0; + +int +wsbmInit(struct _WsbmThreadFuncs *tf, struct _WsbmVNodeFuncs *vf) +{ + int ret; + + wsbmCurThreadFunc = tf; + wsbmCurVNodeFunc = vf; + + ret = WSBM_MUTEX_INIT(&bmMutex); + if (ret) + return -ENOMEM; + ret = WSBM_COND_INIT(&bmCond); + if (ret) { + WSBM_MUTEX_FREE(&bmMutex); + return -ENOMEM; + } + + initialized = 1; + return 0; +} + +void +wsbmCommonDataSet(void *d) +{ + commonData = d; +} + +void * +wsbmCommonDataGet(void) +{ + return commonData; +} + +int +wsbmIsInitialized(void) +{ + return initialized; +} + +void +wsbmTakedown(void) +{ + initialized = 0; + commonData = NULL; + WSBM_COND_FREE(&bmCond); + WSBM_MUTEX_FREE(&bmMutex); +} + +static struct _ValidateNode * +validateListAddNode(struct _ValidateList *list, void *item, + uint32_t hash, uint64_t flags, uint64_t mask) +{ + struct _ValidateNode *node; + struct _WsbmListHead *l; + struct _WsbmListHead *hashHead; + + l = list->free.next; + if (l == &list->free) { + node = wsbmVNodeFuncs()->alloc(wsbmVNodeFuncs(), 0); + if (!node) { + return NULL; + } + list->numCurrent++; + } else { + WSBMLISTDEL(l); + node = WSBMLISTENTRY(l, struct _ValidateNode, head); + } + node->buf = item; + node->set_flags = flags & mask; + node->clr_flags = (~flags) & mask; + node->listItem = list->numOnList; + WSBMLISTADDTAIL(&node->head, &list->list); + list->numOnList++; + hashHead = list->hashTable + hash; + WSBMLISTADDTAIL(&node->hashHead, hashHead); + + return node; +} + +static uint32_t +wsbmHashFunc(uint8_t * key, uint32_t len, uint32_t mask) +{ + uint32_t hash, i; + + for (hash = 0, i = 0; i < len; ++i) { + hash += *key++; + hash += (hash << 10); + hash ^= (hash >> 6); + } + + hash += (hash << 3); + hash ^= (hash >> 11); + hash += (hash << 15); + + return hash & mask; +} + +static void +validateFreeList(struct _ValidateList *list) +{ + struct _ValidateNode *node; + struct _WsbmListHead *l; + + l = list->list.next; + while (l != &list->list) { + WSBMLISTDEL(l); + node = WSBMLISTENTRY(l, struct _ValidateNode, head); + + WSBMLISTDEL(&node->hashHead); + node->func->free(node); + l = list->list.next; + list->numCurrent--; + list->numOnList--; + } + + l = list->free.next; + while (l != &list->free) { + WSBMLISTDEL(l); + node = WSBMLISTENTRY(l, struct _ValidateNode, head); + + node->func->free(node); + l = list->free.next; + list->numCurrent--; + } + free(list->hashTable); +} + +static int +validateListAdjustNodes(struct _ValidateList *list) +{ + struct _ValidateNode *node; + struct _WsbmListHead *l; + int ret = 0; + + while (list->numCurrent < list->numTarget) { + node = wsbmVNodeFuncs()->alloc(wsbmVNodeFuncs(), list->driverData); + if (!node) { + ret = -ENOMEM; + break; + } + list->numCurrent++; + WSBMLISTADD(&node->head, &list->free); + } + + while (list->numCurrent > list->numTarget) { + l = list->free.next; + if (l == &list->free) + break; + WSBMLISTDEL(l); + node = WSBMLISTENTRY(l, struct _ValidateNode, head); + + node->func->free(node); + list->numCurrent--; + } + return ret; +} + +static inline int +wsbmPot(unsigned int val) +{ + unsigned int shift = 0; + while(val > (1 << shift)) + shift++; + + return shift; +} + + + +static int +validateCreateList(int numTarget, struct _ValidateList *list, int driverData) +{ + int i; + unsigned int shift = wsbmPot(numTarget); + int ret; + + list->hashSize = (1 << shift); + list->hashMask = list->hashSize - 1; + + list->hashTable = malloc(list->hashSize * sizeof(*list->hashTable)); + if (!list->hashTable) + return -ENOMEM; + + for (i = 0; i < list->hashSize; ++i) + WSBMINITLISTHEAD(&list->hashTable[i]); + + WSBMINITLISTHEAD(&list->list); + WSBMINITLISTHEAD(&list->free); + list->numTarget = numTarget; + list->numCurrent = 0; + list->numOnList = 0; + list->driverData = driverData; + ret = validateListAdjustNodes(list); + if (ret != 0) + free(list->hashTable); + + return ret; +} + +static int +validateResetList(struct _ValidateList *list) +{ + struct _WsbmListHead *l; + struct _ValidateNode *node; + int ret; + + ret = validateListAdjustNodes(list); + if (ret) + return ret; + + l = list->list.next; + while (l != &list->list) { + WSBMLISTDEL(l); + node = WSBMLISTENTRY(l, struct _ValidateNode, head); + + WSBMLISTDEL(&node->hashHead); + WSBMLISTADD(l, &list->free); + list->numOnList--; + l = list->list.next; + } + return validateListAdjustNodes(list); +} + +void +wsbmWriteLockKernelBO(void) +{ + WSBM_MUTEX_LOCK(&bmMutex); + while (kernelReaders != 0) + WSBM_COND_WAIT(&bmCond, &bmMutex); + kernelLocked = 1; +} + +void +wsbmWriteUnlockKernelBO(void) +{ + kernelLocked = 0; + WSBM_MUTEX_UNLOCK(&bmMutex); +} + +void +wsbmReadLockKernelBO(void) +{ + WSBM_MUTEX_LOCK(&bmMutex); + if (kernelReaders++ == 0) + kernelLocked = 1; + WSBM_MUTEX_UNLOCK(&bmMutex); +} + +void +wsbmReadUnlockKernelBO(void) +{ + WSBM_MUTEX_LOCK(&bmMutex); + if (--kernelReaders == 0) { + kernelLocked = 0; + WSBM_COND_BROADCAST(&bmCond); + } + WSBM_MUTEX_UNLOCK(&bmMutex); +} + +void +wsbmBOWaitIdle(struct _WsbmBufferObject *buf, int lazy) +{ + struct _WsbmBufStorage *storage; + + storage = buf->storage; + if (!storage) + return; + + (void)storage->pool->waitIdle(storage, lazy); +} + +void * +wsbmBOMap(struct _WsbmBufferObject *buf, unsigned mode) +{ + struct _WsbmBufStorage *storage = buf->storage; + void *virtual; + int retval; + + retval = storage->pool->map(storage, mode, &virtual); + + return (retval == 0) ? virtual : NULL; +} + +void +wsbmBOUnmap(struct _WsbmBufferObject *buf) +{ + struct _WsbmBufStorage *storage = buf->storage; + + if (!storage) + return; + + storage->pool->unmap(storage); +} + +int +wsbmBOSyncForCpu(struct _WsbmBufferObject *buf, unsigned mode) +{ + struct _WsbmBufStorage *storage = buf->storage; + + return storage->pool->syncforcpu(storage, mode); +} + +void +wsbmBOReleaseFromCpu(struct _WsbmBufferObject *buf, unsigned mode) +{ + struct _WsbmBufStorage *storage = buf->storage; + + storage->pool->releasefromcpu(storage, mode); +} + +unsigned long +wsbmBOOffsetHint(struct _WsbmBufferObject *buf) +{ + struct _WsbmBufStorage *storage = buf->storage; + + return storage->pool->offset(storage); +} + +unsigned long +wsbmBOPoolOffset(struct _WsbmBufferObject *buf) +{ + struct _WsbmBufStorage *storage = buf->storage; + + return storage->pool->poolOffset(storage); +} + +uint32_t +wsbmBOPlacementHint(struct _WsbmBufferObject * buf) +{ + struct _WsbmBufStorage *storage = buf->storage; + + assert(buf->storage != NULL); + + return storage->pool->placement(storage); +} + +struct _WsbmBufferObject * +wsbmBOReference(struct _WsbmBufferObject *buf) +{ + if (buf->bufferType == WSBM_BUFFER_SIMPLE) { + wsbmAtomicInc(&buf->storage->refCount); + } else { + wsbmAtomicInc(&buf->refCount); + } + return buf; +} + +int +wsbmBOSetStatus(struct _WsbmBufferObject *buf, + uint32_t setFlags, uint32_t clrFlags) +{ + struct _WsbmBufStorage *storage = buf->storage; + + if (!storage) + return 0; + + if (storage->pool->setStatus == NULL) + return -EINVAL; + + return storage->pool->setStatus(storage, setFlags, clrFlags); +} + +void +wsbmBOUnreference(struct _WsbmBufferObject **p_buf) +{ + struct _WsbmBufferObject *buf = *p_buf; + + *p_buf = NULL; + + if (!buf) + return; + + if (buf->bufferType == WSBM_BUFFER_SIMPLE) { + struct _WsbmBufStorage *dummy = buf->storage; + + wsbmBufStorageUnref(&dummy); + return; + } + + if (wsbmAtomicDecZero(&buf->refCount)) { + wsbmBufStorageUnref(&buf->storage); + free(buf); + } +} + +int +wsbmBOData(struct _WsbmBufferObject *buf, + unsigned size, const void *data, + struct _WsbmBufferPool *newPool, uint32_t placement) +{ + void *virtual = NULL; + int newBuffer; + int retval = 0; + struct _WsbmBufStorage *storage; + int synced = 0; + uint32_t placement_diff; + struct _WsbmBufferPool *curPool; + + if (buf->bufferType == WSBM_BUFFER_SIMPLE) + return -EINVAL; + + storage = buf->storage; + + if (newPool == NULL) + newPool = buf->pool; + + if (newPool == NULL) + return -EINVAL; + + newBuffer = (!storage || storage->pool != newPool || + storage->pool->size(storage) < size || + storage->pool->size(storage) > + size + WSBM_BODATA_SIZE_ACCEPT); + + if (!placement) + placement = buf->placement; + + if (newBuffer) { + if (buf->bufferType == WSBM_BUFFER_REF) + return -EINVAL; + + wsbmBufStorageUnref(&buf->storage); + + if (size == 0) { + buf->pool = newPool; + buf->placement = placement; + retval = 0; + goto out; + } + + buf->storage = + newPool->create(newPool, size, placement, buf->alignment); + if (!buf->storage) { + retval = -ENOMEM; + goto out; + } + + buf->placement = placement; + buf->pool = newPool; + } else if (wsbmAtomicRead(&storage->onList) || + 0 != storage->pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE | + WSBM_SYNCCPU_DONT_BLOCK)) { + /* + * Buffer is busy. need to create a new one. + */ + + struct _WsbmBufStorage *tmp_storage; + + curPool = storage->pool; + + tmp_storage = + curPool->create(curPool, size, placement, buf->alignment); + + if (tmp_storage) { + wsbmBufStorageUnref(&buf->storage); + buf->storage = tmp_storage; + buf->placement = placement; + } else { + retval = curPool->syncforcpu(storage, WSBM_SYNCCPU_WRITE); + if (retval) + goto out; + synced = 1; + } + } else + synced = 1; + + placement_diff = placement ^ buf->placement; + + /* + * We might need to change buffer placement. + */ + + storage = buf->storage; + curPool = storage->pool; + + if (placement_diff) { + assert(curPool->setStatus != NULL); + curPool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); + retval = curPool->setStatus(storage, + placement_diff & placement, + placement_diff & ~placement); + if (retval) + goto out; + + buf->placement = placement; + + } + + if (!synced) { + retval = curPool->syncforcpu(buf->storage, WSBM_SYNCCPU_WRITE); + + if (retval) + goto out; + synced = 1; + } + + storage = buf->storage; + curPool = storage->pool; + + if (data) { + retval = curPool->map(storage, WSBM_ACCESS_WRITE, &virtual); + if (retval) + goto out; + memcpy(virtual, data, size); + curPool->unmap(storage); + } + + out: + + if (synced) + curPool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); + + return retval; +} + +int +wsbmBODataUB(struct _WsbmBufferObject *buf, + unsigned size, const void *data, struct _WsbmBufferPool *newPool, + uint32_t placement, const unsigned long *user_ptr) +{ + int newBuffer; + int retval = 0; + struct _WsbmBufStorage *storage; + int synced = 0; + uint32_t placement_diff; + struct _WsbmBufferPool *curPool; + extern struct _WsbmBufStorage * + ttm_pool_ub_create(struct _WsbmBufferPool *pool, + unsigned long size, uint32_t placement, unsigned alignment, + const unsigned long *user_ptr); + + if (buf->bufferType == WSBM_BUFFER_SIMPLE) + return -EINVAL; + + storage = buf->storage; + + if (newPool == NULL) + newPool = buf->pool; + + if (newPool == NULL) + return -EINVAL; + + newBuffer = (!storage || storage->pool != newPool || + storage->pool->size(storage) < size || + storage->pool->size(storage) > + size + WSBM_BODATA_SIZE_ACCEPT); + + if (!placement) + placement = buf->placement; + + if (newBuffer) { + if (buf->bufferType == WSBM_BUFFER_REF) + return -EINVAL; + + wsbmBufStorageUnref(&buf->storage); + + if (size == 0) { + buf->pool = newPool; + buf->placement = placement; + retval = 0; + goto out; + } + + buf->storage = + ttm_pool_ub_create(newPool, size, placement, buf->alignment, user_ptr); + if (!buf->storage) { + retval = -ENOMEM; + goto out; + } + + buf->placement = placement; + buf->pool = newPool; + } else if (wsbmAtomicRead(&storage->onList) || + 0 != storage->pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE | + WSBM_SYNCCPU_DONT_BLOCK)) { + /* + * Buffer is busy. need to create a new one. + * Actually such case will not be encountered for current ICS implementation + * TODO: maybe need refine the following code when such usage case is required + */ + + struct _WsbmBufStorage *tmp_storage; + + curPool = storage->pool; + + tmp_storage = + ttm_pool_ub_create(curPool, size, placement, buf->alignment, user_ptr); + + if (tmp_storage) { + wsbmBufStorageUnref(&buf->storage); + buf->storage = tmp_storage; + buf->placement = placement; + } else { + retval = curPool->syncforcpu(storage, WSBM_SYNCCPU_WRITE); + if (retval) + goto out; + synced = 1; + } + } else { + synced = 1; + } + + placement_diff = placement ^ buf->placement; + + /* + * We might need to change buffer placement. + */ + + storage = buf->storage; + curPool = storage->pool; + + if (placement_diff) { + assert(curPool->setStatus != NULL); + curPool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); + retval = curPool->setStatus(storage, + placement_diff & placement, + placement_diff & ~placement); + if (retval) + goto out; + + buf->placement = placement; + } + + if (!synced) { + retval = curPool->syncforcpu(buf->storage, WSBM_SYNCCPU_WRITE); + if (retval) + goto out; + synced = 1; + } + + storage = buf->storage; + curPool = storage->pool; + + if (data) { + memcpy(user_ptr, data, size); + } + + out: + + if (synced) + curPool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); + + return retval; +} + +static struct _WsbmBufStorage * +wsbmStorageClone(struct _WsbmBufferObject *buf) +{ + struct _WsbmBufStorage *storage = buf->storage; + struct _WsbmBufferPool *pool = storage->pool; + + return pool->create(pool, pool->size(storage), buf->placement, + buf->alignment); +} + +struct _WsbmBufferObject * +wsbmBOClone(struct _WsbmBufferObject *buf, + int (*accelCopy) (struct _WsbmBufferObject *, + struct _WsbmBufferObject *)) +{ + struct _WsbmBufferObject *newBuf; + int ret; + + newBuf = malloc(sizeof(*newBuf)); + if (!newBuf) + return NULL; + + *newBuf = *buf; + newBuf->storage = wsbmStorageClone(buf); + if (!newBuf->storage) + goto out_err0; + + wsbmAtomicSet(&newBuf->refCount, 1); + if (!accelCopy || accelCopy(newBuf, buf) != 0) { + + struct _WsbmBufferPool *pool = buf->storage->pool; + struct _WsbmBufStorage *storage = buf->storage; + struct _WsbmBufStorage *newStorage = newBuf->storage; + void *virtual; + void *nVirtual; + + ret = pool->syncforcpu(storage, WSBM_SYNCCPU_READ); + if (ret) + goto out_err1; + ret = pool->map(storage, WSBM_ACCESS_READ, &virtual); + if (ret) + goto out_err2; + ret = pool->map(newStorage, WSBM_ACCESS_WRITE, &nVirtual); + if (ret) + goto out_err3; + + memcpy(nVirtual, virtual, pool->size(storage)); + pool->unmap(newBuf->storage); + pool->unmap(buf->storage); + pool->releasefromcpu(storage, WSBM_SYNCCPU_READ); + } + + return newBuf; + out_err3: + buf->pool->unmap(buf->storage); + out_err2: + buf->pool->releasefromcpu(buf->storage, WSBM_SYNCCPU_READ); + out_err1: + wsbmBufStorageUnref(&newBuf->storage); + out_err0: + free(newBuf); + return 0; +} + +int +wsbmBOSubData(struct _WsbmBufferObject *buf, + unsigned long offset, unsigned long size, const void *data, + int (*accelCopy) (struct _WsbmBufferObject *, + struct _WsbmBufferObject *)) +{ + int ret = 0; + + if (buf->bufferType == WSBM_BUFFER_SIMPLE) + return -EINVAL; + + if (size && data) { + void *virtual; + struct _WsbmBufStorage *storage = buf->storage; + struct _WsbmBufferPool *pool = storage->pool; + + ret = pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE); + if (ret) + goto out; + + if (wsbmAtomicRead(&storage->onList)) { + + struct _WsbmBufferObject *newBuf; + + /* + * Another context has this buffer on its validate list. + * This should be a very rare situation, but it can be valid, + * and therefore we must deal with it by cloning the storage. + */ + + pool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); + newBuf = wsbmBOClone(buf, accelCopy); + + /* + * If clone fails we have the choice of either bailing. + * (The other context will be happy), or go on and update + * the old buffer anyway. (We will be happy). We choose the + * latter. + */ + + if (newBuf) { + storage = newBuf->storage; + wsbmAtomicInc(&storage->refCount); + wsbmBufStorageUnref(&buf->storage); + buf->storage = storage; + wsbmBOUnreference(&newBuf); + pool = storage->pool; + } + + ret = pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE); + if (ret) + goto out; + } + + ret = pool->map(storage, WSBM_ACCESS_WRITE, &virtual); + if (ret) { + pool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); + goto out; + } + + memcpy((unsigned char *)virtual + offset, data, size); + pool->unmap(storage); + pool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); + } + out: + return ret; +} + +int +wsbmBOGetSubData(struct _WsbmBufferObject *buf, + unsigned long offset, unsigned long size, void *data) +{ + int ret = 0; + + if (size && data) { + void *virtual; + struct _WsbmBufStorage *storage = buf->storage; + struct _WsbmBufferPool *pool = storage->pool; + + ret = pool->syncforcpu(storage, WSBM_SYNCCPU_READ); + if (ret) + goto out; + ret = pool->map(storage, WSBM_ACCESS_READ, &virtual); + if (ret) { + pool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); + goto out; + } + memcpy(data, (unsigned char *)virtual + offset, size); + pool->unmap(storage); + pool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); + } + out: + return ret; +} + +int +wsbmBOSetReferenced(struct _WsbmBufferObject *buf, unsigned long handle) +{ + int ret = 0; + + wsbmBufStorageUnref(&buf->storage); + if (buf->pool->createByReference == NULL) { + ret = -EINVAL; + goto out; + } + buf->storage = buf->pool->createByReference(buf->pool, handle); + if (!buf->storage) { + ret = -EINVAL; + goto out; + } + buf->bufferType = WSBM_BUFFER_REF; + out: + return ret; +} + +void +wsbmBOFreeSimple(void *ptr) +{ + free(ptr); +} + +struct _WsbmBufferObject * +wsbmBOCreateSimple(struct _WsbmBufferPool *pool, + unsigned long size, + uint32_t placement, + unsigned alignment, size_t extra_size, size_t * offset) +{ + struct _WsbmBufferObject *buf; + struct _WsbmBufStorage *storage; + + *offset = (sizeof(*buf) + 15) & ~15; + + if (extra_size) { + extra_size += *offset - sizeof(*buf); + } + + buf = (struct _WsbmBufferObject *)calloc(1, sizeof(*buf) + extra_size); + if (!buf) + return NULL; + + storage = pool->create(pool, size, placement, alignment); + if (!storage) + goto out_err0; + + storage->destroyContainer = &wsbmBOFreeSimple; + storage->destroyArg = buf; + + buf->storage = storage; + buf->alignment = alignment; + buf->pool = pool; + buf->placement = placement; + buf->bufferType = WSBM_BUFFER_SIMPLE; + + return buf; + + out_err0: + free(buf); + return NULL; +} + +int +wsbmGenBuffers(struct _WsbmBufferPool *pool, + unsigned n, + struct _WsbmBufferObject *buffers[], + unsigned alignment, uint32_t placement) +{ + struct _WsbmBufferObject *buf; + int i; + + placement = (placement) ? placement : + WSBM_PL_FLAG_SYSTEM | WSBM_PL_FLAG_CACHED; + + for (i = 0; i < n; ++i) { + buf = (struct _WsbmBufferObject *)calloc(1, sizeof(*buf)); + if (!buf) + return -ENOMEM; + + wsbmAtomicSet(&buf->refCount, 1); + buf->placement = placement; + buf->alignment = alignment; + buf->pool = pool; + buf->bufferType = WSBM_BUFFER_COMPLEX; + buffers[i] = buf; + } + return 0; +} + +void +wsbmDeleteBuffers(unsigned n, struct _WsbmBufferObject *buffers[]) +{ + int i; + + for (i = 0; i < n; ++i) { + wsbmBOUnreference(&buffers[i]); + } +} + +/* + * Note that lists are per-context and don't need mutex protection. + */ + +struct _WsbmBufferList * +wsbmBOCreateList(int target, int hasKernelBuffers) +{ + struct _WsbmBufferList *list = calloc(sizeof(*list), 1); + int ret; + + if (!list) + return NULL; + list->hasKernelBuffers = hasKernelBuffers; + if (hasKernelBuffers) { + ret = validateCreateList(target, &list->kernelBuffers, 0); + if (ret) + return NULL; + } + + ret = validateCreateList(target, &list->userBuffers, 1); + if (ret) { + validateFreeList(&list->kernelBuffers); + return NULL; + } + + return list; +} + +int +wsbmBOResetList(struct _WsbmBufferList *list) +{ + int ret; + + if (list->hasKernelBuffers) { + ret = validateResetList(&list->kernelBuffers); + if (ret) + return ret; + } + ret = validateResetList(&list->userBuffers); + return ret; +} + +void +wsbmBOFreeList(struct _WsbmBufferList *list) +{ + if (list->hasKernelBuffers) + validateFreeList(&list->kernelBuffers); + validateFreeList(&list->userBuffers); + free(list); +} + +static int +wsbmAddValidateItem(struct _ValidateList *list, void *buf, uint64_t flags, + uint64_t mask, int *itemLoc, + struct _ValidateNode **pnode, int *newItem) +{ + struct _ValidateNode *node, *cur; + struct _WsbmListHead *l; + struct _WsbmListHead *hashHead; + uint32_t hash; + uint32_t count = 0; + uint32_t key = (unsigned long) buf; + + cur = NULL; + hash = wsbmHashFunc((uint8_t *) &key, 4, list->hashMask); + hashHead = list->hashTable + hash; + *newItem = 0; + + for (l = hashHead->next; l != hashHead; l = l->next) { + count++; + node = WSBMLISTENTRY(l, struct _ValidateNode, hashHead); + + if (node->buf == buf) { + cur = node; + break; + } + } + + if (!cur) { + cur = validateListAddNode(list, buf, hash, flags, mask); + if (!cur) + return -ENOMEM; + *newItem = 1; + cur->func->clear(cur); + } else { + uint64_t set_flags = flags & mask; + uint64_t clr_flags = (~flags) & mask; + + if (((cur->clr_flags | clr_flags) & WSBM_PL_MASK_MEM) == + WSBM_PL_MASK_MEM) { + /* + * No available memory type left. Bail. + */ + return -EINVAL; + } + + if ((cur->set_flags | set_flags) & + (cur->clr_flags | clr_flags) & ~WSBM_PL_MASK_MEM) { + /* + * Conflicting flags. Bail. + */ + return -EINVAL; + } + + cur->set_flags &= ~(clr_flags & WSBM_PL_MASK_MEM); + cur->set_flags |= (set_flags & ~WSBM_PL_MASK_MEM); + cur->clr_flags |= clr_flags; + } + *itemLoc = cur->listItem; + if (pnode) + *pnode = cur; + return 0; +} + +int +wsbmBOAddListItem(struct _WsbmBufferList *list, + struct _WsbmBufferObject *buf, + uint64_t flags, uint64_t mask, int *itemLoc, + struct _ValidateNode **node) +{ + int newItem; + struct _WsbmBufStorage *storage = buf->storage; + int ret; + int dummy; + struct _ValidateNode *dummyNode; + + if (list->hasKernelBuffers) { + ret = wsbmAddValidateItem(&list->kernelBuffers, + storage->pool->kernel(storage), + flags, mask, itemLoc, node, &dummy); + if (ret) + goto out_unlock; + } else { + *node = NULL; + *itemLoc = -1000; + } + + ret = wsbmAddValidateItem(&list->userBuffers, storage, + flags, mask, &dummy, &dummyNode, &newItem); + if (ret) + goto out_unlock; + + if (newItem) { + wsbmAtomicInc(&storage->refCount); + wsbmAtomicInc(&storage->onList); + } + + out_unlock: + return ret; +} + +void +wsbmBOFence(struct _WsbmBufferObject *buf, struct _WsbmFenceObject *fence) +{ + struct _WsbmBufStorage *storage; + + storage = buf->storage; + if (storage->pool->fence) + storage->pool->fence(storage, fence); + +} + +int +wsbmBOOnList(const struct _WsbmBufferObject *buf) +{ + if (buf->storage == NULL) + return 0; + return wsbmAtomicRead(&buf->storage->onList); +} + +int +wsbmBOUnrefUserList(struct _WsbmBufferList *list) +{ + struct _WsbmBufStorage *storage; + void *curBuf; + + curBuf = validateListIterator(&list->userBuffers); + + while (curBuf) { + storage = (struct _WsbmBufStorage *)(validateListNode(curBuf)->buf); + wsbmAtomicDec(&storage->onList); + wsbmBufStorageUnref(&storage); + curBuf = validateListNext(&list->userBuffers, curBuf); + } + + return wsbmBOResetList(list); +} + + +int +wsbmBOFenceUserList(struct _WsbmBufferList *list, + struct _WsbmFenceObject *fence) +{ + struct _WsbmBufStorage *storage; + void *curBuf; + + curBuf = validateListIterator(&list->userBuffers); + + /* + * User-space fencing callbacks. + */ + + while (curBuf) { + storage = (struct _WsbmBufStorage *)(validateListNode(curBuf)->buf); + + storage->pool->fence(storage, fence); + wsbmAtomicDec(&storage->onList); + wsbmBufStorageUnref(&storage); + curBuf = validateListNext(&list->userBuffers, curBuf); + } + + return wsbmBOResetList(list); +} + +int +wsbmBOValidateUserList(struct _WsbmBufferList *list) +{ + void *curBuf; + struct _WsbmBufStorage *storage; + struct _ValidateNode *node; + int ret; + + curBuf = validateListIterator(&list->userBuffers); + + /* + * User-space validation callbacks. + */ + + while (curBuf) { + node = validateListNode(curBuf); + storage = (struct _WsbmBufStorage *)node->buf; + if (storage->pool->validate) { + ret = storage->pool->validate(storage, node->set_flags, + node->clr_flags); + if (ret) + return ret; + } + curBuf = validateListNext(&list->userBuffers, curBuf); + } + return 0; +} + +int +wsbmBOUnvalidateUserList(struct _WsbmBufferList *list) +{ + void *curBuf; + struct _WsbmBufStorage *storage; + struct _ValidateNode *node; + + curBuf = validateListIterator(&list->userBuffers); + + /* + * User-space validation callbacks. + */ + + while (curBuf) { + node = validateListNode(curBuf); + storage = (struct _WsbmBufStorage *)node->buf; + if (storage->pool->unvalidate) { + storage->pool->unvalidate(storage); + } + wsbmAtomicDec(&storage->onList); + wsbmBufStorageUnref(&storage); + curBuf = validateListNext(&list->userBuffers, curBuf); + } + return wsbmBOResetList(list); +} + +void +wsbmPoolTakeDown(struct _WsbmBufferPool *pool) +{ + pool->takeDown(pool); + +} + +unsigned long +wsbmBOSize(struct _WsbmBufferObject *buf) +{ + unsigned long size; + struct _WsbmBufStorage *storage; + + storage = buf->storage; + size = storage->pool->size(storage); + + return size; + +} + +struct _ValidateList * +wsbmGetKernelValidateList(struct _WsbmBufferList *list) +{ + return (list->hasKernelBuffers) ? &list->kernelBuffers : NULL; +} + +struct _ValidateList * +wsbmGetUserValidateList(struct _WsbmBufferList *list) +{ + return &list->userBuffers; +} + +struct _ValidateNode * +validateListNode(void *iterator) +{ + struct _WsbmListHead *l = (struct _WsbmListHead *)iterator; + + return WSBMLISTENTRY(l, struct _ValidateNode, head); +} + +void * +validateListIterator(struct _ValidateList *list) +{ + void *ret = list->list.next; + + if (ret == &list->list) + return NULL; + return ret; +} + +void * +validateListNext(struct _ValidateList *list, void *iterator) +{ + void *ret; + + struct _WsbmListHead *l = (struct _WsbmListHead *)iterator; + + ret = l->next; + if (ret == &list->list) + return NULL; + return ret; +} + +uint32_t +wsbmKBufHandle(const struct _WsbmKernelBuf * kBuf) +{ + return kBuf->handle; +} + +extern void +wsbmUpdateKBuf(struct _WsbmKernelBuf *kBuf, + uint64_t gpuOffset, uint32_t placement, + uint32_t fence_type_mask) +{ + kBuf->gpuOffset = gpuOffset; + kBuf->placement = placement; + kBuf->fence_type_mask = fence_type_mask; +} + +extern struct _WsbmKernelBuf * +wsbmKBuf(const struct _WsbmBufferObject *buf) +{ + struct _WsbmBufStorage *storage = buf->storage; + + return storage->pool->kernel(storage); +} diff --git a/src/wsbm_manager.h b/src/wsbm_manager.h new file mode 100644 index 0000000..aa2eb5f --- /dev/null +++ b/src/wsbm_manager.h @@ -0,0 +1,187 @@ +/************************************************************************** + * + * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright 2009 Vmware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstr�m + * Keith Whitwell + */ + +#ifndef _WSBM_MANAGER_H_ +#define _WSBM_MANAGER_H_ +#include "wsbm_fencemgr.h" +#include "wsbm_util.h" +#include "wsbm_driver.h" + +#define WSBM_VERSION_MAJOR 1 +#define WSBM_VERSION_MINOR 1 +#define WSBM_VERSION_PL 0 + +struct _WsbmFenceObject; +struct _WsbmBufferObject; +struct _WsbmBufferPool; +struct _WsbmBufferList; + +/* + * These flags mimics the TTM closely, but since + * this library is not dependant on TTM, we need to + * replicate them here, and if there is a discrepancy, + * that needs to be resolved in the buffer pool using + * the TTM flags. + */ + +#define WSBM_PL_MASK_MEM 0x0000FFFF + +#define WSBM_PL_FLAG_SYSTEM (1 << 0) +#define WSBM_PL_FLAG_TT (1 << 1) +#define WSBM_PL_FLAG_VRAM (1 << 2) +#define WSBM_PL_FLAG_PRIV0 (1 << 3) +#define WSBM_PL_FLAG_SWAPPED (1 << 15) +#define WSBM_PL_FLAG_CACHED (1 << 16) +#define WSBM_PL_FLAG_UNCACHED (1 << 17) +#define WSBM_PL_FLAG_WC (1 << 18) +#define WSBM_PL_FLAG_SHARED (1 << 20) +#define WSBM_PL_FLAG_NO_EVICT (1 << 21) + +#define WSBM_ACCESS_READ (1 << 0) +#define WSBM_ACCESS_WRITE (1 << 1) + +#define WSBM_SYNCCPU_READ WSBM_ACCESS_READ +#define WSBM_SYNCCPU_WRITE WSBM_ACCESS_WRITE +#define WSBM_SYNCCPU_DONT_BLOCK (1 << 2) +#define WSBM_SYNCCPU_TRY_CACHED (1 << 3) + +extern void *wsbmBOMap(struct _WsbmBufferObject *buf, unsigned mode); +extern void wsbmBOUnmap(struct _WsbmBufferObject *buf); +extern int wsbmBOSyncForCpu(struct _WsbmBufferObject *buf, unsigned mode); +extern void wsbmBOReleaseFromCpu(struct _WsbmBufferObject *buf, + unsigned mode); + +extern unsigned long wsbmBOOffsetHint(struct _WsbmBufferObject *buf); +extern unsigned long wsbmBOPoolOffset(struct _WsbmBufferObject *buf); + +extern uint32_t wsbmBOPlacementHint(struct _WsbmBufferObject *buf); +extern struct _WsbmBufferObject *wsbmBOReference(struct _WsbmBufferObject + *buf); +extern void wsbmBOUnreference(struct _WsbmBufferObject **p_buf); + +extern int wsbmBOData(struct _WsbmBufferObject *r_buf, + unsigned size, const void *data, + struct _WsbmBufferPool *pool, uint32_t placement); + +extern int wsbmBODataUB(struct _WsbmBufferObject *buf, + unsigned size, const void *data, struct _WsbmBufferPool *newPool, + uint32_t placement, const unsigned long *user_ptr); + +extern int wsbmBOSetStatus(struct _WsbmBufferObject *buf, + uint32_t setPlacement, uint32_t clrPlacement); +extern int wsbmBOSubData(struct _WsbmBufferObject *buf, + unsigned long offset, unsigned long size, + const void *data, + int (*accelCopy) (struct _WsbmBufferObject *, + struct _WsbmBufferObject *)); +extern struct _WsbmBufferObject *wsbmBOClone(struct _WsbmBufferObject *buf, + int (*accelCopy) (struct + _WsbmBufferObject + *, + struct + _WsbmBufferObject + *)); + +extern int wsbmBOGetSubData(struct _WsbmBufferObject *buf, + unsigned long offset, unsigned long size, + void *data); +extern int wsbmGenBuffers(struct _WsbmBufferPool *pool, + unsigned n, + struct _WsbmBufferObject *buffers[], + unsigned alignment, uint32_t placement); + +struct _WsbmBufferObject *wsbmBOCreateSimple(struct _WsbmBufferPool *pool, + unsigned long size, + uint32_t placement, + unsigned alignment, + size_t extra_size, + size_t * offset); + +extern void wsbmDeleteBuffers(unsigned n, + struct _WsbmBufferObject *buffers[]); +extern struct _WsbmBufferList *wsbmBOCreateList(int target, + int hasKernelBuffers); +extern int wsbmBOResetList(struct _WsbmBufferList *list); +extern int wsbmBOAddListItem(struct _WsbmBufferList *list, + struct _WsbmBufferObject *buf, + uint64_t flags, uint64_t mask, int *itemLoc, + struct _ValidateNode **node); + +extern void wsbmBOFreeList(struct _WsbmBufferList *list); +extern int wsbmBOFenceUserList(struct _WsbmBufferList *list, + struct _WsbmFenceObject *fence); + +extern int wsbmBOUnrefUserList(struct _WsbmBufferList *list); +extern int wsbmBOValidateUserList(struct _WsbmBufferList *list); +extern int wsbmBOUnvalidateUserList(struct _WsbmBufferList *list); + +extern void wsbmBOFence(struct _WsbmBufferObject *buf, + struct _WsbmFenceObject *fence); + +extern void wsbmPoolTakeDown(struct _WsbmBufferPool *pool); +extern int wsbmBOSetReferenced(struct _WsbmBufferObject *buf, + unsigned long handle); +unsigned long wsbmBOSize(struct _WsbmBufferObject *buf); +extern void wsbmBOWaitIdle(struct _WsbmBufferObject *buf, int lazy); +extern int wsbmBOOnList(const struct _WsbmBufferObject *buf); + +extern void wsbmPoolTakeDown(struct _WsbmBufferPool *pool); + +extern void wsbmReadLockKernelBO(void); +extern void wsbmReadUnlockKernelBO(void); +extern void wsbmWriteLockKernelBO(void); +extern void wsbmWriteUnlockKernelBO(void); + +extern int wsbmInit(struct _WsbmThreadFuncs *tf, struct _WsbmVNodeFuncs *vf); +extern void wsbmTakedown(void); +extern int wsbmIsInitialized(void); +extern void wsbmCommonDataSet(void *d); +extern void *wsbmCommonDataGet(void); + +extern struct _ValidateList *wsbmGetKernelValidateList(struct _WsbmBufferList + *list); +extern struct _ValidateList *wsbmGetUserValidateList(struct _WsbmBufferList + *list); + +extern struct _ValidateNode *validateListNode(void *iterator); +extern void *validateListIterator(struct _ValidateList *list); +extern void *validateListNext(struct _ValidateList *list, void *iterator); + +extern uint32_t wsbmKBufHandle(const struct _WsbmKernelBuf *); +extern void wsbmUpdateKBuf(struct _WsbmKernelBuf *, + uint64_t gpuOffset, + uint32_t placement, uint32_t fence_flags); + +extern struct _WsbmKernelBuf *wsbmKBuf(const struct _WsbmBufferObject *buf); + +#endif diff --git a/src/wsbm_mm.c b/src/wsbm_mm.c new file mode 100644 index 0000000..88871ba --- /dev/null +++ b/src/wsbm_mm.c @@ -0,0 +1,289 @@ +/************************************************************************** + * + * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA. + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ + +/* + * Generic simple memory manager implementation. Intended to be used as a base + * class implementation for more advanced memory managers. + * + * Note that the algorithm used is quite simple and there might be substantial + * performance gains if a smarter free list is implemented. Currently it is just an + * unordered stack of free regions. This could easily be improved if an RB-tree + * is used instead. At least if we expect heavy fragmentation. + * Note that this implementation is more or less identical to the drm core manager + * in the linux kernel. + * + * Authors: + * Thomas Hellstr�m + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "wsbm_mm.h" +#include +#include + +unsigned long +wsbmMMTailSpace(struct _WsbmMM *mm) +{ + struct _WsbmListHead *tail_node; + struct _WsbmMMNode *entry; + + tail_node = mm->ml_entry.prev; + entry = WSBMLISTENTRY(tail_node, struct _WsbmMMNode, ml_entry); + + if (!entry->free) + return 0; + + return entry->size; +} + +int +wsbmMMRemoveSpaceFromTail(struct _WsbmMM *mm, unsigned long size) +{ + struct _WsbmListHead *tail_node; + struct _WsbmMMNode *entry; + + tail_node = mm->ml_entry.prev; + entry = WSBMLISTENTRY(tail_node, struct _WsbmMMNode, ml_entry); + + if (!entry->free) + return -ENOMEM; + + if (entry->size <= size) + return -ENOMEM; + + entry->size -= size; + return 0; +} + +static int +wsbmMMCreateTailNode(struct _WsbmMM *mm, + unsigned long start, unsigned long size) +{ + struct _WsbmMMNode *child; + + child = (struct _WsbmMMNode *)malloc(sizeof(*child)); + if (!child) + return -ENOMEM; + + child->free = 1; + child->size = size; + child->start = start; + child->mm = mm; + + WSBMLISTADDTAIL(&child->ml_entry, &mm->ml_entry); + WSBMLISTADDTAIL(&child->fl_entry, &mm->fl_entry); + + return 0; +} + +static struct _WsbmMMNode * +wsbmMMSplitAtStart(struct _WsbmMMNode *parent, unsigned long size) +{ + struct _WsbmMMNode *child; + + child = (struct _WsbmMMNode *)malloc(sizeof(*child)); + if (!child) + return NULL; + + WSBMINITLISTHEAD(&child->fl_entry); + + child->free = 0; + child->size = size; + child->start = parent->start; + child->mm = parent->mm; + + WSBMLISTADDTAIL(&child->ml_entry, &parent->ml_entry); + WSBMINITLISTHEAD(&child->fl_entry); + + parent->size -= size; + parent->start += size; + return child; +} + +struct _WsbmMMNode * +wsbmMMGetBlock(struct _WsbmMMNode *parent, + unsigned long size, unsigned alignment) +{ + + struct _WsbmMMNode *align_splitoff = NULL; + struct _WsbmMMNode *child; + unsigned tmp = 0; + + if (alignment) + tmp = parent->start % alignment; + + if (tmp) { + align_splitoff = wsbmMMSplitAtStart(parent, alignment - tmp); + if (!align_splitoff) + return NULL; + } + + if (parent->size == size) { + WSBMLISTDELINIT(&parent->fl_entry); + parent->free = 0; + return parent; + } else { + child = wsbmMMSplitAtStart(parent, size); + } + + if (align_splitoff) + wsbmMMPutBlock(align_splitoff); + + return child; +} + +/* + * Put a block. Merge with the previous and / or next block if they are free. + * Otherwise add to the free stack. + */ + +void +wsbmMMPutBlock(struct _WsbmMMNode *cur) +{ + + struct _WsbmMM *mm = cur->mm; + struct _WsbmListHead *cur_head = &cur->ml_entry; + struct _WsbmListHead *root_head = &mm->ml_entry; + struct _WsbmMMNode *prev_node = NULL; + struct _WsbmMMNode *next_node; + + int merged = 0; + + if (cur_head->prev != root_head) { + prev_node = + WSBMLISTENTRY(cur_head->prev, struct _WsbmMMNode, ml_entry); + if (prev_node->free) { + prev_node->size += cur->size; + merged = 1; + } + } + if (cur_head->next != root_head) { + next_node = + WSBMLISTENTRY(cur_head->next, struct _WsbmMMNode, ml_entry); + if (next_node->free) { + if (merged) { + prev_node->size += next_node->size; + WSBMLISTDEL(&next_node->ml_entry); + WSBMLISTDEL(&next_node->fl_entry); + free(next_node); + } else { + next_node->size += cur->size; + next_node->start = cur->start; + merged = 1; + } + } + } + if (!merged) { + cur->free = 1; + WSBMLISTADD(&cur->fl_entry, &mm->fl_entry); + } else { + WSBMLISTDEL(&cur->ml_entry); + free(cur); + } +} + +struct _WsbmMMNode * +wsbmMMSearchFree(const struct _WsbmMM *mm, + unsigned long size, unsigned alignment, int best_match) +{ + struct _WsbmListHead *list; + const struct _WsbmListHead *free_stack = &mm->fl_entry; + struct _WsbmMMNode *entry; + struct _WsbmMMNode *best; + unsigned long best_size; + unsigned wasted; + + best = NULL; + best_size = ~0UL; + + WSBMLISTFOREACH(list, free_stack) { + entry = WSBMLISTENTRY(list, struct _WsbmMMNode, fl_entry); + + wasted = 0; + + if (entry->size < size) + continue; + + if (alignment) { + register unsigned tmp = entry->start % alignment; + + if (tmp) + wasted += alignment - tmp; + } + + if (entry->size >= size + wasted) { + if (!best_match) + return entry; + if (size < best_size) { + best = entry; + best_size = entry->size; + } + } + } + + return best; +} + +int +wsbmMMclean(struct _WsbmMM *mm) +{ + struct _WsbmListHead *head = &mm->ml_entry; + + return (head->next->next == head); +} + +int +wsbmMMinit(struct _WsbmMM *mm, unsigned long start, unsigned long size) +{ + WSBMINITLISTHEAD(&mm->ml_entry); + WSBMINITLISTHEAD(&mm->fl_entry); + + return wsbmMMCreateTailNode(mm, start, size); +} + +void +wsbmMMtakedown(struct _WsbmMM *mm) +{ + struct _WsbmListHead *bnode = mm->fl_entry.next; + struct _WsbmMMNode *entry; + + entry = WSBMLISTENTRY(bnode, struct _WsbmMMNode, fl_entry); + + if (entry->ml_entry.next != &mm->ml_entry || + entry->fl_entry.next != &mm->fl_entry) { + return; + } + + WSBMLISTDEL(&entry->fl_entry); + WSBMLISTDEL(&entry->ml_entry); + free(entry); +} diff --git a/src/wsbm_mm.h b/src/wsbm_mm.h new file mode 100644 index 0000000..d3bbe94 --- /dev/null +++ b/src/wsbm_mm.h @@ -0,0 +1,75 @@ +/************************************************************************** + * + * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA. + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ + +/* + * Generic simple memory manager implementation. Intended to be used as a base + * class implementation for more advanced memory managers. + * + * Note that the algorithm used is quite simple and there might be substantial + * performance gains if a smarter free list is implemented. Currently it is just an + * unordered stack of free regions. This could easily be improved if an RB-tree + * is used instead. At least if we expect heavy fragmentation. + * + * Authors: + * Thomas Hellstrom + */ + +#ifndef _WSBM_MM_H_ +#define _WSBM_MM_H_ + +#include "wsbm_util.h" +struct _WsbmMM +{ + struct _WsbmListHead fl_entry; + struct _WsbmListHead ml_entry; +}; + +struct _WsbmMMNode +{ + struct _WsbmListHead fl_entry; + struct _WsbmListHead ml_entry; + int free; + unsigned long start; + unsigned long size; + struct _WsbmMM *mm; +}; + +extern struct _WsbmMMNode *wsbmMMSearchFree(const struct _WsbmMM *mm, + unsigned long size, + unsigned alignment, + int best_match); +extern struct _WsbmMMNode *wsbmMMGetBlock(struct _WsbmMMNode *parent, + unsigned long size, + unsigned alignment); +extern void wsbmMMPutBlock(struct _WsbmMMNode *cur); +extern void wsbmMMtakedown(struct _WsbmMM *mm); +extern int wsbmMMinit(struct _WsbmMM *mm, unsigned long start, + unsigned long size); +extern int wsbmMMclean(struct _WsbmMM *mm); +#endif diff --git a/src/wsbm_pool.h b/src/wsbm_pool.h new file mode 100644 index 0000000..1d07523 --- /dev/null +++ b/src/wsbm_pool.h @@ -0,0 +1,161 @@ +/************************************************************************** + * + * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstr�m + */ + +#ifndef _WSBM_BUFPOOL_H_ +#define _WSBM_BUFPOOL_H_ + +#include +#include "wsbm_util.h" +#include "wsbm_driver.h" +#include "wsbm_atomic.h" + +struct _WsbmFenceObject; + +struct _WsbmBufStorage +{ + struct _WsbmBufferPool *pool; + struct _WsbmMutex mutex; + struct _WsbmAtomic refCount; + struct _WsbmAtomic onList; + void *destroyArg; + void (*destroyContainer) (void *); +}; + +struct _WsbmKernelBuf; + +struct _WsbmBufferPool +{ + int fd; + int (*map) (struct _WsbmBufStorage * buf, unsigned mode, void **virtual); + void (*unmap) (struct _WsbmBufStorage * buf); + int (*syncforcpu) (struct _WsbmBufStorage * buf, unsigned mode); + void (*releasefromcpu) (struct _WsbmBufStorage * buf, unsigned mode); + void (*destroy) (struct _WsbmBufStorage ** buf); + unsigned long (*offset) (struct _WsbmBufStorage * buf); + unsigned long (*poolOffset) (struct _WsbmBufStorage * buf); + uint32_t(*placement) (struct _WsbmBufStorage * buf); + unsigned long (*size) (struct _WsbmBufStorage * buf); + struct _WsbmKernelBuf *(*kernel) (struct _WsbmBufStorage * buf); + struct _WsbmBufStorage *(*create) (struct _WsbmBufferPool * pool, + unsigned long size, + uint32_t placement, + unsigned alignment); + struct _WsbmBufStorage *(*createByReference) (struct _WsbmBufferPool * + pool, uint32_t handle); + void (*fence) (struct _WsbmBufStorage * buf, + struct _WsbmFenceObject * fence); + void (*unvalidate) (struct _WsbmBufStorage * buf); + int (*validate) (struct _WsbmBufStorage * buf, uint64_t set_flags, + uint64_t clr_flags); + int (*waitIdle) (struct _WsbmBufStorage * buf, int lazy); + int (*setStatus) (struct _WsbmBufStorage * buf, + uint32_t set_placement, uint32_t clr_placement); + void (*takeDown) (struct _WsbmBufferPool * pool); +}; + +static inline int +wsbmBufStorageInit(struct _WsbmBufStorage *storage, + struct _WsbmBufferPool *pool) +{ + int ret = WSBM_MUTEX_INIT(&storage->mutex); + + if (ret) + return -ENOMEM; + storage->pool = pool; + wsbmAtomicSet(&storage->refCount, 1); + wsbmAtomicSet(&storage->onList, 0); + storage->destroyContainer = NULL; + return 0; +} + +static inline void +wsbmBufStorageTakedown(struct _WsbmBufStorage *storage) +{ + WSBM_MUTEX_FREE(&storage->mutex); +} + +static inline void +wsbmBufStorageUnref(struct _WsbmBufStorage **pStorage) +{ + struct _WsbmBufStorage *storage = *pStorage; + + *pStorage = NULL; + if (storage == NULL) + return; + + if (wsbmAtomicDecZero(&storage->refCount)) { + if (storage->destroyContainer) + storage->destroyContainer(storage->destroyArg); + storage->pool->destroy(&storage); + return; + } +} + +/* + * Builtin pools. + */ + +/* + * Kernel buffer objects. Size in multiples of page size. Page size aligned. + */ + +extern struct _WsbmBufferPool *wsbmTTMPoolInit(int fd, + unsigned int devOffset); +extern struct _WsbmBufferPool *wsbmMallocPoolInit(void); + +struct _WsbmSlabCache; +extern struct _WsbmBufferPool *wsbmSlabPoolInit(int fd, uint32_t devOffset, + uint32_t placement, + uint32_t validMask, + uint32_t smallestSize, + uint32_t numSizes, + uint32_t desiredNumBuffers, + uint32_t maxSlabSize, + uint32_t pageAlignment, + struct _WsbmSlabCache *cache); +extern struct _WsbmSlabCache *wsbmSlabCacheInit(uint32_t checkIntervalMsec, + uint32_t slabTimeoutMsec); +extern void wsbmSlabCacheFinish(struct _WsbmSlabCache *cache); + +extern struct _WsbmBufferPool *wsbmUserPoolInit(void *vramAddr, + unsigned long vramStart, + unsigned long vramSize, + void *agpAddr, + unsigned long agpStart, + unsigned long agpSize, + uint32_t(*fenceTypes) + (uint64_t set_flags)); + +extern void wsbmUserPoolClean(struct _WsbmBufferPool *pool, + int cleanVram, int cleanAgp); + +#endif diff --git a/src/wsbm_priv.h b/src/wsbm_priv.h new file mode 100644 index 0000000..89e4313 --- /dev/null +++ b/src/wsbm_priv.h @@ -0,0 +1,45 @@ +/************************************************************************** + * + * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: + * Thomas Hellstr�m + */ + +#ifndef _WSBM_PRIV_H_ +#define _WSBM_PRIV_H_ + +struct _WsbmKernelBuf +{ + uint64_t gpuOffset; + uint32_t handle; + uint32_t placement; + uint32_t fence_type_mask; +}; + +#endif diff --git a/src/wsbm_slabpool.c b/src/wsbm_slabpool.c new file mode 100644 index 0000000..56f7d65 --- /dev/null +++ b/src/wsbm_slabpool.c @@ -0,0 +1,1203 @@ +/************************************************************************** + * + * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "wsbm_pool.h" +#include "wsbm_fencemgr.h" +#include "wsbm_priv.h" +#include "wsbm_manager.h" + +#define WSBM_SLABPOOL_ALLOC_RETRIES 100 +#define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret) \ + do { \ + (_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \ + } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ + +#define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret) \ + do { \ + (_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \ + } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ + + +#ifdef DEBUG_FENCESIGNALED +static int createbuffer = 0; +static int fencesignaled = 0; +#endif + +struct _WsbmSlab; + +struct _WsbmSlabBuffer +{ + struct _WsbmKernelBuf kBuf; + struct _WsbmBufStorage storage; + struct _WsbmCond event; + + /* + * Remains constant after creation. + */ + + int isSlabBuffer; + struct _WsbmSlab *parent; + uint32_t start; + void *virtual; + unsigned long requestedSize; + uint64_t mapHandle; + + /* + * Protected by struct _WsbmSlabSizeHeader::mutex + */ + + struct _WsbmListHead head; + + /* + * Protected by this::mutex + */ + + struct _WsbmFenceObject *fence; + uint32_t fenceType; + struct _WsbmAtomic writers; /* (Only upping) */ + int unFenced; +}; + +struct _WsbmSlabPool; +struct _WsbmSlabKernelBO +{ + + /* + * Constant at creation + */ + + struct _WsbmKernelBuf kBuf; + uint32_t pageAlignment; + void *virtual; + unsigned long actualSize; + uint64_t mapHandle; + + /* + * Protected by struct _WsbmSlabCache::mutex + */ + + struct _WsbmSlabPool *slabPool; + uint32_t proposedPlacement; + struct _WsbmListHead timeoutHead; + struct _WsbmListHead head; + struct timeval timeFreed; +}; + +struct _WsbmSlab +{ + struct _WsbmListHead head; + struct _WsbmListHead freeBuffers; + uint32_t numBuffers; + uint32_t numFree; + struct _WsbmSlabBuffer *buffers; + struct _WsbmSlabSizeHeader *header; + struct _WsbmSlabKernelBO *kbo; +}; + +struct _WsbmSlabSizeHeader +{ + /* + * Constant at creation. + */ + struct _WsbmSlabPool *slabPool; + uint32_t bufSize; + + /* + * Protected by this::mutex + */ + + struct _WsbmListHead slabs; + struct _WsbmListHead freeSlabs; + struct _WsbmListHead delayedBuffers; + uint32_t numDelayed; + struct _WsbmMutex mutex; +}; + +struct _WsbmSlabCache +{ + struct timeval slabTimeout; + struct timeval checkInterval; + struct timeval nextCheck; + struct _WsbmListHead timeoutList; + struct _WsbmListHead unCached; + struct _WsbmListHead cached; + struct _WsbmMutex mutex; +}; + +struct _WsbmSlabPool +{ + struct _WsbmBufferPool pool; + + /* + * The data of this structure remains constant after + * initialization and thus needs no mutex protection. + */ + + unsigned int devOffset; + struct _WsbmSlabCache *cache; + uint32_t proposedPlacement; + uint32_t validMask; + uint32_t *bucketSizes; + uint32_t numBuckets; + uint32_t pageSize; + int pageAlignment; + int maxSlabSize; + int desiredNumBuffers; + struct _WsbmSlabSizeHeader *headers; +}; + +static inline struct _WsbmSlabPool * +slabPoolFromPool(struct _WsbmBufferPool *pool) +{ + return containerOf(pool, struct _WsbmSlabPool, pool); +} + +static inline struct _WsbmSlabPool * +slabPoolFromBuf(struct _WsbmSlabBuffer *sBuf) +{ + return slabPoolFromPool(sBuf->storage.pool); +} + +static inline struct _WsbmSlabBuffer * +slabBuffer(struct _WsbmBufStorage *buf) +{ + return containerOf(buf, struct _WsbmSlabBuffer, storage); +} + +/* + * FIXME: Perhaps arrange timeout slabs in size buckets for fast + * retreival?? + */ + +static inline int +wsbmTimeAfterEq(struct timeval *arg1, struct timeval *arg2) +{ + return ((arg1->tv_sec > arg2->tv_sec) || + ((arg1->tv_sec == arg2->tv_sec) && + (arg1->tv_usec > arg2->tv_usec))); +} + +static inline void +wsbmTimeAdd(struct timeval *arg, struct timeval *add) +{ + unsigned int sec; + + arg->tv_sec += add->tv_sec; + arg->tv_usec += add->tv_usec; + sec = arg->tv_usec / 1000000; + arg->tv_sec += sec; + arg->tv_usec -= sec * 1000000; +} + +static void +wsbmFreeKernelBO(struct _WsbmSlabKernelBO *kbo) +{ + struct ttm_pl_reference_req arg; + struct _WsbmSlabPool *slabPool; + + if (!kbo) + return; + + slabPool = kbo->slabPool; + arg.handle = kbo->kBuf.handle; + (void)munmap(kbo->virtual, kbo->actualSize); + (void)drmCommandWrite(slabPool->pool.fd, + slabPool->devOffset + TTM_PL_UNREF, &arg, + sizeof(arg)); + free(kbo); +} + +static void +wsbmFreeTimeoutKBOsLocked(struct _WsbmSlabCache *cache, struct timeval *time) +{ + struct _WsbmListHead *list, *next; + struct _WsbmSlabKernelBO *kbo; + + if (!wsbmTimeAfterEq(time, &cache->nextCheck)) + return; + + WSBMLISTFOREACHSAFE(list, next, &cache->timeoutList) { + kbo = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, timeoutHead); + + if (!wsbmTimeAfterEq(time, &kbo->timeFreed)) + break; + + WSBMLISTDELINIT(&kbo->timeoutHead); + WSBMLISTDELINIT(&kbo->head); + wsbmFreeKernelBO(kbo); + } + + cache->nextCheck = *time; + wsbmTimeAdd(&cache->nextCheck, &cache->checkInterval); +} + +/* + * Add a _SlabKernelBO to the free slab manager. + * This means that it is available for reuse, but if it's not + * reused in a while, it will be freed. + */ + +static void +wsbmSetKernelBOFree(struct _WsbmSlabCache *cache, + struct _WsbmSlabKernelBO *kbo) +{ + struct timeval time; + struct timeval timeFreed; + + gettimeofday(&time, NULL); + timeFreed = time; + WSBM_MUTEX_LOCK(&cache->mutex); + wsbmTimeAdd(&timeFreed, &cache->slabTimeout); + kbo->timeFreed = timeFreed; + + if (kbo->kBuf.placement & TTM_PL_FLAG_CACHED) + WSBMLISTADD(&kbo->head, &cache->cached); + else + WSBMLISTADD(&kbo->head, &cache->unCached); + + WSBMLISTADDTAIL(&kbo->timeoutHead, &cache->timeoutList); + wsbmFreeTimeoutKBOsLocked(cache, &time); + + WSBM_MUTEX_UNLOCK(&cache->mutex); +} + +/* + * Get a _SlabKernelBO for us to use as storage for a slab. + */ + +static struct _WsbmSlabKernelBO * +wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header) +{ + struct _WsbmSlabPool *slabPool = header->slabPool; + struct _WsbmSlabCache *cache = slabPool->cache; + struct _WsbmListHead *list, *head; + uint32_t size = header->bufSize * slabPool->desiredNumBuffers; + struct _WsbmSlabKernelBO *kbo; + struct _WsbmSlabKernelBO *kboTmp; + int ret; + + /* + * FIXME: We should perhaps allow some variation in slabsize in order + * to efficiently reuse slabs. + */ + + size = (size <= slabPool->maxSlabSize) ? size : slabPool->maxSlabSize; + if (size < header->bufSize) + size = header->bufSize; + size = (size + slabPool->pageSize - 1) & ~(slabPool->pageSize - 1); + WSBM_MUTEX_LOCK(&cache->mutex); + + kbo = NULL; + + retry: + head = (slabPool->proposedPlacement & TTM_PL_FLAG_CACHED) ? + &cache->cached : &cache->unCached; + + WSBMLISTFOREACH(list, head) { + kboTmp = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, head); + + if ((kboTmp->actualSize == size) && + (slabPool->pageAlignment == 0 || + (kboTmp->pageAlignment % slabPool->pageAlignment) == 0)) { + + if (!kbo) + kbo = kboTmp; + + if ((kbo->proposedPlacement ^ slabPool->proposedPlacement) == 0) + break; + + } + } + + if (kbo) { + WSBMLISTDELINIT(&kbo->head); + WSBMLISTDELINIT(&kbo->timeoutHead); + } + + WSBM_MUTEX_UNLOCK(&cache->mutex); + + if (kbo) { + uint32_t new_mask = + kbo->proposedPlacement ^ slabPool->proposedPlacement; + + ret = 0; + if (new_mask) { + union ttm_pl_setstatus_arg arg; + struct ttm_pl_setstatus_req *req = &arg.req; + struct ttm_pl_rep *rep = &arg.rep; + + req->handle = kbo->kBuf.handle; + req->set_placement = slabPool->proposedPlacement & new_mask; + req->clr_placement = ~slabPool->proposedPlacement & new_mask; + DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd, + slabPool->devOffset + TTM_PL_SETSTATUS, + arg, ret); + if (ret == 0) { + kbo->kBuf.gpuOffset = rep->gpu_offset; + kbo->kBuf.placement = rep->placement; + } + kbo->proposedPlacement = slabPool->proposedPlacement; + } + + if (ret == 0) + return kbo; + + wsbmFreeKernelBO(kbo); + kbo = NULL; + goto retry; + } + + kbo = calloc(1, sizeof(*kbo)); + if (!kbo) + return NULL; + + { + union ttm_pl_create_arg arg; + + kbo->slabPool = slabPool; + WSBMINITLISTHEAD(&kbo->head); + WSBMINITLISTHEAD(&kbo->timeoutHead); + + arg.req.size = size; + arg.req.placement = slabPool->proposedPlacement; + arg.req.page_alignment = slabPool->pageAlignment; + + DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd, + slabPool->devOffset + TTM_PL_CREATE, + arg, ret); + if (ret) + goto out_err0; + + kbo->kBuf.gpuOffset = arg.rep.gpu_offset; + kbo->kBuf.placement = arg.rep.placement; + kbo->kBuf.handle = arg.rep.handle; + + kbo->actualSize = arg.rep.bo_size; + kbo->mapHandle = arg.rep.map_handle; + kbo->proposedPlacement = slabPool->proposedPlacement; + } + + kbo->virtual = mmap(0, kbo->actualSize, + PROT_READ | PROT_WRITE, MAP_SHARED, + slabPool->pool.fd, kbo->mapHandle); + + if (kbo->virtual == MAP_FAILED) { + ret = -errno; + goto out_err1; + } + + return kbo; + + out_err1: + { + struct ttm_pl_reference_req arg = {.handle = kbo->kBuf.handle }; + + (void)drmCommandWrite(slabPool->pool.fd, + slabPool->devOffset + TTM_PL_UNREF, + &arg, sizeof(arg)); + } + out_err0: + free(kbo); + return NULL; +} + +static int +wsbmAllocSlab(struct _WsbmSlabSizeHeader *header) +{ + struct _WsbmSlab *slab; + struct _WsbmSlabBuffer *sBuf; + uint32_t numBuffers; + int ret; + int i; + + slab = calloc(1, sizeof(*slab)); + if (!slab) + return -ENOMEM; + + slab->kbo = wsbmAllocKernelBO(header); + if (!slab->kbo) { + ret = -ENOMEM; + goto out_err0; + } + + numBuffers = slab->kbo->actualSize / header->bufSize; + + slab->buffers = calloc(numBuffers, sizeof(*slab->buffers)); + if (!slab->buffers) { + ret = -ENOMEM; + goto out_err1; + } + + WSBMINITLISTHEAD(&slab->head); + WSBMINITLISTHEAD(&slab->freeBuffers); + slab->numBuffers = numBuffers; + slab->numFree = 0; + slab->header = header; + + sBuf = slab->buffers; + for (i = 0; i < numBuffers; ++i) { + ret = wsbmBufStorageInit(&sBuf->storage, &header->slabPool->pool); + if (ret) + goto out_err2; + sBuf->parent = slab; + sBuf->start = i * header->bufSize; + sBuf->virtual = (void *)((uint8_t *) slab->kbo->virtual + + sBuf->start); + wsbmAtomicSet(&sBuf->writers, 0); + sBuf->isSlabBuffer = 1; + WSBM_COND_INIT(&sBuf->event); + WSBMLISTADDTAIL(&sBuf->head, &slab->freeBuffers); + slab->numFree++; + sBuf++; + } + + WSBMLISTADDTAIL(&slab->head, &header->slabs); + + return 0; + + out_err2: + sBuf = slab->buffers; + for (i = 0; i < numBuffers; ++i) { + if (sBuf->parent == slab) { + WSBM_COND_FREE(&sBuf->event); + wsbmBufStorageTakedown(&sBuf->storage); + } + sBuf++; + } + free(slab->buffers); + out_err1: + wsbmSetKernelBOFree(header->slabPool->cache, slab->kbo); + out_err0: + free(slab); + return ret; +} + +/* + * Delete a buffer from the slab header delayed list and put + * it on the slab free list. + */ + +static void +wsbmSlabFreeBufferLocked(struct _WsbmSlabBuffer *buf) +{ + struct _WsbmSlab *slab = buf->parent; + struct _WsbmSlabSizeHeader *header = slab->header; + struct _WsbmListHead *list = &buf->head; + + WSBMLISTDEL(list); + WSBMLISTADDTAIL(list, &slab->freeBuffers); + slab->numFree++; + + if (slab->head.next == &slab->head) + WSBMLISTADDTAIL(&slab->head, &header->slabs); + + if (slab->numFree == slab->numBuffers) { + list = &slab->head; + WSBMLISTDEL(list); + WSBMLISTADDTAIL(list, &header->freeSlabs); + } + + if (header->slabs.next == &header->slabs || + slab->numFree != slab->numBuffers) { + + struct _WsbmListHead *next; + struct _WsbmSlabCache *cache = header->slabPool->cache; + + WSBMLISTFOREACHSAFE(list, next, &header->freeSlabs) { + int i; + struct _WsbmSlabBuffer *sBuf; + + slab = WSBMLISTENTRY(list, struct _WsbmSlab, head); + + WSBMLISTDELINIT(list); + + sBuf = slab->buffers; + for (i = 0; i < slab->numBuffers; ++i) { + if (sBuf->parent == slab) { + WSBM_COND_FREE(&sBuf->event); + wsbmBufStorageTakedown(&sBuf->storage); + } + sBuf++; + } + wsbmSetKernelBOFree(cache, slab->kbo); + free(slab->buffers); + free(slab); + } + } +} + +static void +wsbmSlabCheckFreeLocked(struct _WsbmSlabSizeHeader *header, int wait) +{ + struct _WsbmListHead *list, *prev, *first, *head; + struct _WsbmSlabBuffer *sBuf; + struct _WsbmSlab *slab; + int firstWasSignaled = 1; + int signaled; + int i; + int ret; + + /* + * Rerun the freeing test if the youngest tested buffer + * was signaled, since there might be more idle buffers + * in the delay list. + */ + + while (firstWasSignaled) { + firstWasSignaled = 0; + signaled = 0; + first = header->delayedBuffers.next; + + /* Only examine the oldest 1/3 of delayed buffers: + */ + if (header->numDelayed > 3) { + for (i = 0; i < header->numDelayed; i += 3) { + first = first->next; + } + } + + /* + * No need to take the buffer mutex for each buffer we loop + * through since we're currently the only user. + */ + + head = first->next; + WSBMLISTFOREACHPREVSAFE(list, prev, head) { + + if (list == &header->delayedBuffers) + break; + + sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head); + + slab = sBuf->parent; + + if (!signaled) { + if (wait) { + ret = wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0); + if (ret) + break; + signaled = 1; + wait = 0; + } else { + signaled = + wsbmFenceSignaled(sBuf->fence, sBuf->fenceType); +#ifdef DEBUG_FENCESIGNALED + fencesignaled++; +#endif + } + if (signaled) { + if (list == first) + firstWasSignaled = 1; + wsbmFenceUnreference(&sBuf->fence); + header->numDelayed--; + wsbmSlabFreeBufferLocked(sBuf); + } else + break; + } else if (wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) { + wsbmFenceUnreference(&sBuf->fence); + header->numDelayed--; + wsbmSlabFreeBufferLocked(sBuf); + } + } + } +} + +static struct _WsbmSlabBuffer * +wsbmSlabAllocBuffer(struct _WsbmSlabSizeHeader *header) +{ + static struct _WsbmSlabBuffer *buf; + struct _WsbmSlab *slab; + struct _WsbmListHead *list; + int count = WSBM_SLABPOOL_ALLOC_RETRIES; + + WSBM_MUTEX_LOCK(&header->mutex); + while (header->slabs.next == &header->slabs && count > 0) { + wsbmSlabCheckFreeLocked(header, 0); + if (header->slabs.next != &header->slabs) + break; + + WSBM_MUTEX_UNLOCK(&header->mutex); + if (count != WSBM_SLABPOOL_ALLOC_RETRIES) + usleep(1000); + WSBM_MUTEX_LOCK(&header->mutex); + (void)wsbmAllocSlab(header); + count--; + } + + list = header->slabs.next; + if (list == &header->slabs) { + WSBM_MUTEX_UNLOCK(&header->mutex); + return NULL; + } + slab = WSBMLISTENTRY(list, struct _WsbmSlab, head); + if (--slab->numFree == 0) + WSBMLISTDELINIT(list); + + list = slab->freeBuffers.next; + WSBMLISTDELINIT(list); + + WSBM_MUTEX_UNLOCK(&header->mutex); + buf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head); + + buf->storage.destroyContainer = NULL; + +#ifdef DEBUG_FENCESIGNALED + createbuffer++; +#endif + return buf; +} + +static struct _WsbmBufStorage * +pool_create(struct _WsbmBufferPool *pool, unsigned long size, + uint32_t placement, unsigned alignment) +{ + struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool); + struct _WsbmSlabSizeHeader *header; + struct _WsbmSlabBuffer *sBuf; + int i; + int ret; + + /* + * FIXME: Check for compatibility. + */ + + header = slabPool->headers; + for (i = 0; i < slabPool->numBuckets; ++i) { + if (header->bufSize >= size) + break; + header++; + } + + if (i < slabPool->numBuckets) { + sBuf = wsbmSlabAllocBuffer(header); + return ((sBuf) ? &sBuf->storage : NULL); + } + + /* + * Fall back to allocate a buffer object directly from DRM. + * and wrap it in a wsbmBO structure. + */ + + sBuf = calloc(1, sizeof(*sBuf)); + + if (!sBuf) + return NULL; + + if (alignment) { + if ((alignment < slabPool->pageSize) + && (slabPool->pageSize % alignment)) + goto out_err0; + if ((alignment > slabPool->pageSize) + && (alignment % slabPool->pageSize)) + goto out_err0; + } + + ret = wsbmBufStorageInit(&sBuf->storage, pool); + if (ret) + goto out_err0; + + ret = WSBM_COND_INIT(&sBuf->event); + if (ret) + goto out_err1; + + { + union ttm_pl_create_arg arg; + + arg.req.size = size; + arg.req.placement = placement; + arg.req.page_alignment = alignment / slabPool->pageSize; + + DRMRESTARTCOMMANDWRITEREAD(pool->fd, + slabPool->devOffset + TTM_PL_CREATE, + arg, ret); + + if (ret) + goto out_err2; + + sBuf->kBuf.gpuOffset = arg.rep.gpu_offset; + sBuf->kBuf.placement = arg.rep.placement; + sBuf->kBuf.handle = arg.rep.handle; + sBuf->mapHandle = arg.rep.map_handle; + sBuf->requestedSize = size; + + sBuf->virtual = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, + pool->fd, sBuf->mapHandle); + + if (sBuf->virtual == MAP_FAILED) + goto out_err3; + } + + wsbmAtomicSet(&sBuf->writers, 0); + return &sBuf->storage; + out_err3: + { + struct ttm_pl_reference_req arg; + + arg.handle = sBuf->kBuf.handle; + (void)drmCommandWriteRead(pool->fd, + slabPool->devOffset + TTM_PL_UNREF, + &arg, sizeof(arg)); + } + out_err2: + WSBM_COND_FREE(&sBuf->event); + out_err1: + wsbmBufStorageTakedown(&sBuf->storage); + out_err0: + free(sBuf); + return NULL; +} + +static void +pool_destroy(struct _WsbmBufStorage **p_buf) +{ + struct _WsbmBufStorage *buf = *p_buf; + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + struct _WsbmSlab *slab; + struct _WsbmSlabSizeHeader *header; + + *p_buf = NULL; + + if (!sBuf->isSlabBuffer) { + struct _WsbmSlabPool *slabPool = slabPoolFromBuf(sBuf); + struct ttm_pl_reference_req arg; + + if (sBuf->virtual != NULL) { + (void)munmap(sBuf->virtual, sBuf->requestedSize); + sBuf->virtual = NULL; + } + + arg.handle = sBuf->kBuf.handle; + (void)drmCommandWrite(slabPool->pool.fd, + slabPool->devOffset + TTM_PL_UNREF, + &arg, sizeof(arg)); + + WSBM_COND_FREE(&sBuf->event); + wsbmBufStorageTakedown(&sBuf->storage); + free(sBuf); + return; + } + + slab = sBuf->parent; + header = slab->header; + + /* + * No need to take the buffer mutex below since we're the only user. + */ + + WSBM_MUTEX_LOCK(&header->mutex); + sBuf->unFenced = 0; + wsbmAtomicSet(&sBuf->writers, 0); + wsbmAtomicSet(&sBuf->storage.refCount, 1); + + if (sBuf->fence && !wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) { + WSBMLISTADDTAIL(&sBuf->head, &header->delayedBuffers); + header->numDelayed++; + } else { + if (sBuf->fence) + wsbmFenceUnreference(&sBuf->fence); + wsbmSlabFreeBufferLocked(sBuf); + } + + WSBM_MUTEX_UNLOCK(&header->mutex); +} + +static void +waitIdleLocked(struct _WsbmSlabBuffer *sBuf, int lazy) +{ + struct _WsbmBufStorage *storage = &sBuf->storage; + + while (sBuf->unFenced || sBuf->fence != NULL) { + + if (sBuf->unFenced) + WSBM_COND_WAIT(&sBuf->event, &storage->mutex); + + if (sBuf->fence != NULL) { + if (!wsbmFenceSignaled(sBuf->fence, sBuf->fenceType)) { + struct _WsbmFenceObject *fence = + wsbmFenceReference(sBuf->fence); + + WSBM_MUTEX_UNLOCK(&storage->mutex); + (void)wsbmFenceFinish(fence, sBuf->fenceType, lazy); + WSBM_MUTEX_LOCK(&storage->mutex); + if (sBuf->fence == fence) + wsbmFenceUnreference(&sBuf->fence); + + wsbmFenceUnreference(&fence); + } else { + wsbmFenceUnreference(&sBuf->fence); + } + } + } +} + +static int +pool_waitIdle(struct _WsbmBufStorage *buf, int lazy) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + WSBM_MUTEX_LOCK(&buf->mutex); + waitIdleLocked(sBuf, lazy); + WSBM_MUTEX_UNLOCK(&buf->mutex); + + return 0; +} + +static int +pool_map(struct _WsbmBufStorage *buf, unsigned mode, void **virtual) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + *virtual = sBuf->virtual; + + return 0; +} + +static void +pool_releaseFromCpu(struct _WsbmBufStorage *buf, unsigned mode) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + if (wsbmAtomicDecZero(&sBuf->writers)) + WSBM_COND_BROADCAST(&sBuf->event); +} + +static int +pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + int ret = 0; + + WSBM_MUTEX_LOCK(&buf->mutex); + if ((mode & WSBM_SYNCCPU_DONT_BLOCK)) { + int signaled; + + if (sBuf->unFenced) { + ret = -EBUSY; + goto out_unlock; + } + + if (sBuf->isSlabBuffer) + signaled = (sBuf->fence == NULL) || + wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType); + else + signaled = (sBuf->fence == NULL) || + wsbmFenceSignaled(sBuf->fence, sBuf->fenceType); + + ret = 0; + if (signaled) { + wsbmFenceUnreference(&sBuf->fence); + wsbmAtomicInc(&sBuf->writers); + } else + ret = -EBUSY; + goto out_unlock; + } + waitIdleLocked(sBuf, 0); + wsbmAtomicInc(&sBuf->writers); + out_unlock: + WSBM_MUTEX_UNLOCK(&buf->mutex); + return ret; +} + +static void +pool_unmap(struct _WsbmBufStorage *buf) +{ + ; +} + +static unsigned long +pool_poolOffset(struct _WsbmBufStorage *buf) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + return sBuf->start; +} + +static unsigned long +pool_size(struct _WsbmBufStorage *buf) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + if (!sBuf->isSlabBuffer) + return sBuf->requestedSize; + + return sBuf->parent->header->bufSize; +} + +static struct _WsbmKernelBuf * +pool_kernel(struct _WsbmBufStorage *buf) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + return (sBuf->isSlabBuffer) ? &sBuf->parent->kbo->kBuf : &sBuf->kBuf; +} + +static unsigned long +pool_offset(struct _WsbmBufStorage *buf) +{ + return pool_kernel(buf)->gpuOffset + pool_poolOffset(buf); +} + +static void +pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + struct _WsbmKernelBuf *kBuf; + + WSBM_MUTEX_LOCK(&buf->mutex); + if (sBuf->fence) + wsbmFenceUnreference(&sBuf->fence); + + kBuf = pool_kernel(buf); + sBuf->fenceType = kBuf->fence_type_mask; + if (!wsbmFenceSignaledCached(fence, sBuf->fenceType)) + sBuf->fence = wsbmFenceReference(fence); + + sBuf->unFenced = 0; + WSBM_COND_BROADCAST(&sBuf->event); + WSBM_MUTEX_UNLOCK(&buf->mutex); +} + +static int +pool_validate(struct _WsbmBufStorage *buf, + uint64_t set_flags, uint64_t clr_flags) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + WSBM_MUTEX_LOCK(&buf->mutex); + while (wsbmAtomicRead(&sBuf->writers) != 0) { + WSBM_COND_WAIT(&sBuf->event, &buf->mutex); + } + + sBuf->unFenced = 1; + WSBM_MUTEX_UNLOCK(&buf->mutex); + return 0; +} + +static void +pool_unvalidate(struct _WsbmBufStorage *buf) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + WSBM_MUTEX_LOCK(&buf->mutex); + if (sBuf->unFenced) { + sBuf->unFenced = 0; + WSBM_COND_BROADCAST(&sBuf->event); + } + WSBM_MUTEX_UNLOCK(&buf->mutex); +} + +struct _WsbmSlabCache * +wsbmSlabCacheInit(uint32_t checkIntervalMsec, uint32_t slabTimeoutMsec) +{ + struct _WsbmSlabCache *tmp; + + tmp = calloc(1, sizeof(*tmp)); + if (!tmp) + return NULL; + + WSBM_MUTEX_INIT(&tmp->mutex); + WSBM_MUTEX_LOCK(&tmp->mutex); + tmp->slabTimeout.tv_usec = slabTimeoutMsec * 1000; + tmp->slabTimeout.tv_sec = tmp->slabTimeout.tv_usec / 1000000; + tmp->slabTimeout.tv_usec -= tmp->slabTimeout.tv_sec * 1000000; + + tmp->checkInterval.tv_usec = checkIntervalMsec * 1000; + tmp->checkInterval.tv_sec = tmp->checkInterval.tv_usec / 1000000; + tmp->checkInterval.tv_usec -= tmp->checkInterval.tv_sec * 1000000; + + gettimeofday(&tmp->nextCheck, NULL); + wsbmTimeAdd(&tmp->nextCheck, &tmp->checkInterval); + WSBMINITLISTHEAD(&tmp->timeoutList); + WSBMINITLISTHEAD(&tmp->unCached); + WSBMINITLISTHEAD(&tmp->cached); + WSBM_MUTEX_UNLOCK(&tmp->mutex); + + return tmp; +} + +void +wsbmSlabCacheFinish(struct _WsbmSlabCache *cache) +{ + struct timeval time; + + time = cache->nextCheck; + WSBM_MUTEX_LOCK(&cache->mutex); + wsbmTimeAdd(&time, &cache->checkInterval); + wsbmFreeTimeoutKBOsLocked(cache, &time); + WSBM_MUTEX_UNLOCK(&cache->mutex); + + assert(cache->timeoutList.next == &cache->timeoutList); + assert(cache->unCached.next == &cache->unCached); + assert(cache->cached.next == &cache->cached); + + WSBM_MUTEX_FREE(&cache->mutex); + free(cache); +} + +static void +wsbmInitSizeHeader(struct _WsbmSlabPool *slabPool, uint32_t size, + struct _WsbmSlabSizeHeader *header) +{ + WSBM_MUTEX_INIT(&header->mutex); + WSBM_MUTEX_LOCK(&header->mutex); + + WSBMINITLISTHEAD(&header->slabs); + WSBMINITLISTHEAD(&header->freeSlabs); + WSBMINITLISTHEAD(&header->delayedBuffers); + + header->numDelayed = 0; + header->slabPool = slabPool; + header->bufSize = size; + + WSBM_MUTEX_UNLOCK(&header->mutex); +} + +static void +wsbmFinishSizeHeader(struct _WsbmSlabSizeHeader *header) +{ + struct _WsbmListHead *list, *next; + struct _WsbmSlabBuffer *sBuf; + + WSBM_MUTEX_LOCK(&header->mutex); + WSBMLISTFOREACHSAFE(list, next, &header->delayedBuffers) { + sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head); + + if (sBuf->fence) { + (void)wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0); + wsbmFenceUnreference(&sBuf->fence); + } + header->numDelayed--; + wsbmSlabFreeBufferLocked(sBuf); + } + WSBM_MUTEX_UNLOCK(&header->mutex); + WSBM_MUTEX_FREE(&header->mutex); +} + +static void +pool_takedown(struct _WsbmBufferPool *pool) +{ + struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool); + int i; + + for (i = 0; i < slabPool->numBuckets; ++i) { + wsbmFinishSizeHeader(&slabPool->headers[i]); + } + + free(slabPool->headers); + free(slabPool->bucketSizes); + free(slabPool); +} + +struct _WsbmBufferPool * +wsbmSlabPoolInit(int fd, + uint32_t devOffset, + uint32_t placement, + uint32_t validMask, + uint32_t smallestSize, + uint32_t numSizes, + uint32_t desiredNumBuffers, + uint32_t maxSlabSize, + uint32_t pageAlignment, struct _WsbmSlabCache *cache) +{ + struct _WsbmBufferPool *pool; + struct _WsbmSlabPool *slabPool; + uint32_t i; + + slabPool = calloc(1, sizeof(*slabPool)); + if (!slabPool) + return NULL; + + pool = &slabPool->pool; + + slabPool->bucketSizes = calloc(numSizes, sizeof(*slabPool->bucketSizes)); + if (!slabPool->bucketSizes) + goto out_err0; + + slabPool->headers = calloc(numSizes, sizeof(*slabPool->headers)); + if (!slabPool->headers) + goto out_err1; + + slabPool->devOffset = devOffset; + slabPool->cache = cache; + slabPool->proposedPlacement = placement; + slabPool->validMask = validMask; + slabPool->numBuckets = numSizes; + slabPool->pageSize = getpagesize(); + slabPool->pageAlignment = pageAlignment; + slabPool->maxSlabSize = maxSlabSize; + slabPool->desiredNumBuffers = desiredNumBuffers; + + for (i = 0; i < slabPool->numBuckets; ++i) { + slabPool->bucketSizes[i] = (smallestSize << i); + wsbmInitSizeHeader(slabPool, slabPool->bucketSizes[i], + &slabPool->headers[i]); + } + + pool->fd = fd; + pool->map = &pool_map; + pool->unmap = &pool_unmap; + pool->destroy = &pool_destroy; + pool->offset = &pool_offset; + pool->poolOffset = &pool_poolOffset; + pool->size = &pool_size; + pool->create = &pool_create; + pool->fence = &pool_fence; + pool->kernel = &pool_kernel; + pool->validate = &pool_validate; + pool->unvalidate = &pool_unvalidate; + pool->waitIdle = &pool_waitIdle; + pool->takeDown = &pool_takedown; + pool->releasefromcpu = &pool_releaseFromCpu; + pool->syncforcpu = &pool_syncForCpu; + + return pool; + + out_err1: + free(slabPool->bucketSizes); + out_err0: + free(slabPool); + + return NULL; +} diff --git a/src/wsbm_ttmpool.c b/src/wsbm_ttmpool.c new file mode 100644 index 0000000..b6b13f3 --- /dev/null +++ b/src/wsbm_ttmpool.c @@ -0,0 +1,575 @@ +/************************************************************************** + * + * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, Tx., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstr�m + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include "wsbm_pool.h" +#include "assert.h" +#include "wsbm_priv.h" +#include "wsbm_manager.h" +#include "psb_ttm_placement_user.h" + +#define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret) \ + do { \ + (_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \ + } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ + +#define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret) \ + do { \ + (_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \ + } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ + +/* + * Buffer pool implementation using DRM buffer objects as wsbm buffer objects. + */ + +struct _TTMBuffer +{ + struct _WsbmBufStorage buf; + struct _WsbmCond event; + + /* + * Remains constant after creation. + */ + + uint64_t requestedSize; + uint64_t mapHandle; + uint64_t realSize; + + /* + * Protected by the kernel lock. + */ + + struct _WsbmKernelBuf kBuf; + + /* + * Protected by the mutex. + */ + + void *virtual; + int syncInProgress; + unsigned readers; + unsigned writers; +}; + +struct _TTMPool +{ + struct _WsbmBufferPool pool; + unsigned int pageSize; + unsigned int devOffset; +}; + +static inline struct _TTMPool * +ttmGetPool(struct _TTMBuffer *dBuf) +{ + return containerOf(dBuf->buf.pool, struct _TTMPool, pool); +} + +static inline struct _TTMBuffer * +ttmBuffer(struct _WsbmBufStorage *buf) +{ + return containerOf(buf, struct _TTMBuffer, buf); +} + +static struct _WsbmBufStorage * +pool_create(struct _WsbmBufferPool *pool, + unsigned long size, uint32_t placement, unsigned alignment) +{ + struct _TTMBuffer *dBuf = (struct _TTMBuffer *) + calloc(1, sizeof(*dBuf)); + struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool); + int ret; + unsigned pageSize = ttmPool->pageSize; + union ttm_pl_create_arg arg; + + if (!dBuf) + return NULL; + + if ((alignment > pageSize) && (alignment % pageSize)) + goto out_err0; + + ret = wsbmBufStorageInit(&dBuf->buf, pool); + if (ret) + goto out_err0; + + ret = WSBM_COND_INIT(&dBuf->event); + if (ret) + goto out_err1; + + arg.req.size = size; + arg.req.placement = placement; + arg.req.page_alignment = alignment / pageSize; + + DRMRESTARTCOMMANDWRITEREAD(pool->fd, ttmPool->devOffset + TTM_PL_CREATE, + arg, ret); + + if (ret) + goto out_err2; + + dBuf->requestedSize = size; + dBuf->kBuf.gpuOffset = arg.rep.gpu_offset; + dBuf->mapHandle = arg.rep.map_handle; + dBuf->realSize = arg.rep.bo_size; + dBuf->kBuf.placement = arg.rep.placement; + dBuf->kBuf.handle = arg.rep.handle; + + return &dBuf->buf; + + out_err2: + WSBM_COND_FREE(&dBuf->event); + out_err1: + wsbmBufStorageTakedown(&dBuf->buf); + out_err0: + free(dBuf); + return NULL; +} + +static struct _WsbmBufStorage * +pool_reference(struct _WsbmBufferPool *pool, unsigned handle) +{ + struct _TTMBuffer *dBuf = (struct _TTMBuffer *)calloc(1, sizeof(*dBuf)); + struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool); + union ttm_pl_reference_arg arg; + int ret; + + if (!dBuf) + return NULL; + + ret = wsbmBufStorageInit(&dBuf->buf, pool); + if (ret) + goto out_err0; + + ret = WSBM_COND_INIT(&dBuf->event); + if (ret) + goto out_err1; + + arg.req.handle = handle; + ret = drmCommandWriteRead(pool->fd, ttmPool->devOffset + TTM_PL_REFERENCE, + &arg, sizeof(arg)); + + if (ret) + goto out_err2; + + dBuf->requestedSize = arg.rep.bo_size; + dBuf->kBuf.gpuOffset = arg.rep.gpu_offset; + dBuf->mapHandle = arg.rep.map_handle; + dBuf->realSize = arg.rep.bo_size; + dBuf->kBuf.placement = arg.rep.placement; + dBuf->kBuf.handle = arg.rep.handle; + dBuf->kBuf.fence_type_mask = arg.rep.sync_object_arg; + + return &dBuf->buf; + + out_err2: + WSBM_COND_FREE(&dBuf->event); + out_err1: + wsbmBufStorageTakedown(&dBuf->buf); + out_err0: + free(dBuf); + return NULL; +} + +static void +pool_destroy(struct _WsbmBufStorage **buf) +{ + struct _TTMBuffer *dBuf = ttmBuffer(*buf); + struct _TTMPool *ttmPool = ttmGetPool(dBuf); + struct ttm_pl_reference_req arg; + + *buf = NULL; + if (dBuf->virtual != NULL) { + (void)munmap(dBuf->virtual, dBuf->requestedSize); + dBuf->virtual = NULL; + } + arg.handle = dBuf->kBuf.handle; + (void)drmCommandWrite(dBuf->buf.pool->fd, + ttmPool->devOffset + TTM_PL_UNREF, + &arg, sizeof(arg)); + + WSBM_COND_FREE(&dBuf->event); + wsbmBufStorageTakedown(&dBuf->buf); + free(dBuf); +} + +static int +syncforcpu_locked(struct _WsbmBufStorage *buf, unsigned mode) +{ + uint32_t kmode = 0; + struct _TTMBuffer *dBuf = ttmBuffer(buf); + struct _TTMPool *ttmPool = ttmGetPool(dBuf); + unsigned int readers; + unsigned int writers; + int ret = 0; + + while (dBuf->syncInProgress) + WSBM_COND_WAIT(&dBuf->event, &buf->mutex); + + readers = dBuf->readers; + writers = dBuf->writers; + + if ((mode & WSBM_SYNCCPU_READ) && (++dBuf->readers == 1)) + kmode |= TTM_PL_SYNCCPU_MODE_READ; + + if ((mode & WSBM_SYNCCPU_WRITE) && (++dBuf->writers == 1)) + kmode |= TTM_PL_SYNCCPU_MODE_WRITE; + + if (kmode) { + struct ttm_pl_synccpu_arg arg; + + if (mode & WSBM_SYNCCPU_DONT_BLOCK) + kmode |= TTM_PL_SYNCCPU_MODE_NO_BLOCK; + + dBuf->syncInProgress = 1; + + /* + * This might be a lengthy wait, so + * release the mutex. + */ + + WSBM_MUTEX_UNLOCK(&buf->mutex); + + arg.handle = dBuf->kBuf.handle; + arg.access_mode = kmode; + arg.op = TTM_PL_SYNCCPU_OP_GRAB; + + DRMRESTARTCOMMANDWRITE(dBuf->buf.pool->fd, + ttmPool->devOffset + TTM_PL_SYNCCPU, arg, ret); + + WSBM_MUTEX_LOCK(&buf->mutex); + dBuf->syncInProgress = 0; + WSBM_COND_BROADCAST(&dBuf->event); + + if (ret) { + dBuf->readers = readers; + dBuf->writers = writers; + } + } + + return ret; +} + +static int +releasefromcpu_locked(struct _WsbmBufStorage *buf, unsigned mode) +{ + uint32_t kmode = 0; + struct _TTMBuffer *dBuf = ttmBuffer(buf); + struct _TTMPool *ttmPool = ttmGetPool(dBuf); + int ret = 0; + + while (dBuf->syncInProgress) + WSBM_COND_WAIT(&dBuf->event, &buf->mutex); + + if ((mode & WSBM_SYNCCPU_READ) && (--dBuf->readers == 0)) + kmode |= TTM_PL_SYNCCPU_MODE_READ; + + if ((mode & WSBM_SYNCCPU_WRITE) && (--dBuf->writers == 0)) + kmode |= TTM_PL_SYNCCPU_MODE_WRITE; + + if (kmode) { + struct ttm_pl_synccpu_arg arg; + + arg.handle = dBuf->kBuf.handle; + arg.access_mode = kmode; + arg.op = TTM_PL_SYNCCPU_OP_RELEASE; + + DRMRESTARTCOMMANDWRITE(dBuf->buf.pool->fd, + ttmPool->devOffset + TTM_PL_SYNCCPU, arg, ret); + + } + + return ret; +} + +static int +pool_syncforcpu(struct _WsbmBufStorage *buf, unsigned mode) +{ + int ret; + + WSBM_MUTEX_LOCK(&buf->mutex); + ret = syncforcpu_locked(buf, mode); + WSBM_MUTEX_UNLOCK(&buf->mutex); + return ret; +} + +static void +pool_releasefromcpu(struct _WsbmBufStorage *buf, unsigned mode) +{ + WSBM_MUTEX_LOCK(&buf->mutex); + (void)releasefromcpu_locked(buf, mode); + WSBM_MUTEX_UNLOCK(&buf->mutex); +} + +#ifdef ANDROID +static void* _temp_mmap(void *addr, size_t size, int prot, int flags, int fd, long long offset) +{ + return __mmap2(addr, size, prot, flags, fd, (unsigned long)(offset >> PAGE_SHIFT)); +} +#endif + +static int +pool_map(struct _WsbmBufStorage *buf, unsigned mode, void **virtual) +{ + struct _TTMBuffer *dBuf = ttmBuffer(buf); + void *virt; + int ret = 0; + + WSBM_MUTEX_LOCK(&buf->mutex); + + /* + * mmaps are expensive, so we only really unmap if + * we destroy the buffer. + */ + + if (dBuf->virtual == NULL) { +#ifndef ANDROID + virt = mmap(0, dBuf->requestedSize, + PROT_READ | PROT_WRITE, MAP_SHARED, + buf->pool->fd, dBuf->mapHandle); +#else + virt = _temp_mmap(0, dBuf->requestedSize, + PROT_READ | PROT_WRITE, MAP_SHARED, + buf->pool->fd, dBuf->mapHandle); +#endif + if (virt == MAP_FAILED) { + ret = -errno; + goto out_unlock; + } + dBuf->virtual = virt; + } + + *virtual = dBuf->virtual; + out_unlock: + + WSBM_MUTEX_UNLOCK(&buf->mutex); + + return ret; +} + +static void +pool_unmap(struct _WsbmBufStorage *buf) +{ + ; +} + +static unsigned long +pool_offset(struct _WsbmBufStorage *buf) +{ + struct _TTMBuffer *dBuf = ttmBuffer(buf); + + return dBuf->kBuf.gpuOffset; +} + +static unsigned long +pool_poolOffset(struct _WsbmBufStorage *buf) +{ + return 0; +} + +static uint32_t +pool_placement(struct _WsbmBufStorage *buf) +{ + struct _TTMBuffer *dBuf = ttmBuffer(buf); + + return dBuf->kBuf.placement; +} + +static unsigned long +pool_size(struct _WsbmBufStorage *buf) +{ + struct _TTMBuffer *dBuf = ttmBuffer(buf); + + return dBuf->realSize; +} + +static void +pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence) +{ + /* + * Noop. The kernel handles all fencing. + */ +} + +static int +pool_waitIdle(struct _WsbmBufStorage *buf, int lazy) +{ + struct _TTMBuffer *dBuf = ttmBuffer(buf); + struct _TTMPool *ttmPool = ttmGetPool(dBuf); + struct ttm_pl_waitidle_arg req; + struct _WsbmBufferPool *pool = buf->pool; + int ret; + + req.handle = dBuf->kBuf.handle; + req.mode = (lazy) ? TTM_PL_WAITIDLE_MODE_LAZY : 0; + + DRMRESTARTCOMMANDWRITE(pool->fd, ttmPool->devOffset + TTM_PL_WAITIDLE, + req, ret); + + return ret; +} + +static void +pool_takedown(struct _WsbmBufferPool *pool) +{ + struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool); + + free(ttmPool); +} + +static int +pool_setStatus(struct _WsbmBufStorage *buf, uint32_t set_placement, + uint32_t clr_placement) +{ + struct _TTMBuffer *dBuf = ttmBuffer(buf); + struct _TTMPool *ttmPool = ttmGetPool(dBuf); + union ttm_pl_setstatus_arg arg; + struct ttm_pl_setstatus_req *req = &arg.req; + struct ttm_pl_rep *rep = &arg.rep; + struct _WsbmBufferPool *pool = buf->pool; + int ret; + + req->handle = dBuf->kBuf.handle; + req->set_placement = set_placement; + req->clr_placement = clr_placement; + + DRMRESTARTCOMMANDWRITEREAD(pool->fd, + ttmPool->devOffset + TTM_PL_SETSTATUS, + arg, ret); + + if (!ret) { + dBuf->kBuf.gpuOffset = rep->gpu_offset; + dBuf->kBuf.placement = rep->placement; + } + + return ret; +} + +static struct _WsbmKernelBuf * +pool_kernel(struct _WsbmBufStorage *buf) +{ + return (void *)&ttmBuffer(buf)->kBuf; +} + +struct _WsbmBufferPool * +wsbmTTMPoolInit(int fd, unsigned int devOffset) +{ + struct _TTMPool *ttmPool; + struct _WsbmBufferPool *pool; + + ttmPool = (struct _TTMPool *)calloc(1, sizeof(*ttmPool)); + + if (!ttmPool) + return NULL; + + ttmPool->pageSize = getpagesize(); + ttmPool->devOffset = devOffset; + pool = &ttmPool->pool; + + pool->fd = fd; + pool->map = &pool_map; + pool->unmap = &pool_unmap; + pool->syncforcpu = &pool_syncforcpu; + pool->releasefromcpu = &pool_releasefromcpu; + pool->destroy = &pool_destroy; + pool->offset = &pool_offset; + pool->poolOffset = &pool_poolOffset; + pool->placement = &pool_placement; + pool->size = &pool_size; + pool->create = &pool_create; + pool->fence = &pool_fence; + pool->kernel = &pool_kernel; + pool->validate = NULL; + pool->unvalidate = NULL; + pool->waitIdle = &pool_waitIdle; + pool->takeDown = &pool_takedown; + pool->createByReference = &pool_reference; + pool->setStatus = &pool_setStatus; + return pool; +} + +struct _WsbmBufStorage * +ttm_pool_ub_create(struct _WsbmBufferPool *pool, unsigned long size, uint32_t placement, unsigned alignment, const unsigned long *user_ptr) +{ + struct _TTMBuffer *dBuf = (struct _TTMBuffer *) + calloc(1, sizeof(*dBuf)); + struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool); + int ret; + unsigned pageSize = ttmPool->pageSize; + union ttm_pl_create_ub_arg arg; + + if (!dBuf) + return NULL; + + if ((alignment > pageSize) && (alignment % pageSize)) + goto out_err0; + + ret = wsbmBufStorageInit(&dBuf->buf, pool); + if (ret) + goto out_err0; + + ret = WSBM_COND_INIT(&dBuf->event); + if (ret) + goto out_err1; + + arg.req.size = size; + arg.req.placement = placement; + arg.req.page_alignment = alignment / pageSize; + arg.req.user_address = (unsigned long)user_ptr; + + DRMRESTARTCOMMANDWRITEREAD(pool->fd, ttmPool->devOffset + TTM_PL_CREATE_UB, + arg, ret); + if (ret) + goto out_err2; + + dBuf->requestedSize = size; + dBuf->kBuf.gpuOffset = arg.rep.gpu_offset; + dBuf->mapHandle = arg.rep.map_handle; + dBuf->realSize = arg.rep.bo_size; + dBuf->kBuf.placement = arg.rep.placement; + dBuf->kBuf.handle = arg.rep.handle; + + return &dBuf->buf; + + out_err2: + WSBM_COND_FREE(&dBuf->event); + out_err1: + wsbmBufStorageTakedown(&dBuf->buf); + out_err0: + free(dBuf); + return NULL; +} + diff --git a/src/wsbm_userpool.c b/src/wsbm_userpool.c new file mode 100644 index 0000000..9c7b79a --- /dev/null +++ b/src/wsbm_userpool.c @@ -0,0 +1,689 @@ +/************************************************************************** + * + * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include "wsbm_pool.h" +#include "wsbm_fencemgr.h" +#include "wsbm_manager.h" +#include "wsbm_mm.h" +#include "wsbm_priv.h" + +/* + * Malloced memory must be aligned to 16 bytes, since that's what + * the DMA bitblt requires. + */ + +#define WSBM_USER_ALIGN_ADD 16 +#define WSBM_USER_ALIGN_SYSMEM(_val) \ + ((void *)(((unsigned long) (_val) + 15) & ~15)) + +struct _WsbmUserBuffer +{ + struct _WsbmBufStorage buf; + struct _WsbmKernelBuf kBuf; + + /* Protected by the pool mutex */ + + struct _WsbmListHead lru; + struct _WsbmListHead delayed; + + /* Protected by the buffer mutex */ + + unsigned long size; + unsigned long alignment; + + struct _WsbmCond event; + uint32_t proposedPlacement; + uint32_t newFenceType; + + void *map; + void *sysmem; + int unFenced; + struct _WsbmFenceObject *fence; + struct _WsbmMMNode *node; + + struct _WsbmAtomic writers; +}; + +struct _WsbmUserPool +{ + /* + * Constant after initialization. + */ + + struct _WsbmBufferPool pool; + unsigned long agpOffset; + unsigned long agpMap; + unsigned long agpSize; + unsigned long vramOffset; + unsigned long vramMap; + unsigned long vramSize; + struct _WsbmMutex mutex; + struct _WsbmListHead delayed; + struct _WsbmListHead vramLRU; + struct _WsbmListHead agpLRU; + struct _WsbmMM vramMM; + struct _WsbmMM agpMM; + uint32_t(*fenceTypes) (uint64_t); +}; + +static inline struct _WsbmUserPool * +userPool(struct _WsbmUserBuffer *buf) +{ + return containerOf(buf->buf.pool, struct _WsbmUserPool, pool); +} + +static inline struct _WsbmUserBuffer * +userBuf(struct _WsbmBufStorage *buf) +{ + return containerOf(buf, struct _WsbmUserBuffer, buf); +} + +static void +waitIdleLocked(struct _WsbmBufStorage *buf, int lazy) +{ + struct _WsbmUserBuffer *vBuf = userBuf(buf); + + while (vBuf->unFenced || vBuf->fence != NULL) { + if (vBuf->unFenced) + WSBM_COND_WAIT(&vBuf->event, &buf->mutex); + + if (vBuf->fence != NULL) { + if (!wsbmFenceSignaled(vBuf->fence, vBuf->kBuf.fence_type_mask)) { + struct _WsbmFenceObject *fence = + wsbmFenceReference(vBuf->fence); + + WSBM_MUTEX_UNLOCK(&buf->mutex); + (void)wsbmFenceFinish(fence, vBuf->kBuf.fence_type_mask, + lazy); + WSBM_MUTEX_LOCK(&buf->mutex); + + if (vBuf->fence == fence) + wsbmFenceUnreference(&vBuf->fence); + + wsbmFenceUnreference(&fence); + } else { + wsbmFenceUnreference(&vBuf->fence); + } + } + } +} + +static int +pool_waitIdle(struct _WsbmBufStorage *buf, int lazy) +{ + WSBM_MUTEX_UNLOCK(&buf->mutex); + waitIdleLocked(buf, lazy); + WSBM_MUTEX_UNLOCK(&buf->mutex); + + return 0; +} + +static int +evict_lru(struct _WsbmListHead *lru) +{ + struct _WsbmUserBuffer *vBuf; + struct _WsbmUserPool *p; + struct _WsbmListHead *list = lru->next; + int err; + + if (list == lru) { + return -ENOMEM; + } + + vBuf = WSBMLISTENTRY(list, struct _WsbmUserBuffer, lru); + p = userPool(vBuf); + WSBM_MUTEX_UNLOCK(&p->mutex); + WSBM_MUTEX_LOCK(&vBuf->buf.mutex); + WSBM_MUTEX_LOCK(&p->mutex); + + vBuf->sysmem = malloc(vBuf->size + WSBM_USER_ALIGN_ADD); + + if (!vBuf->sysmem) { + err = -ENOMEM; + goto out_unlock; + } + + (void)wsbmFenceFinish(vBuf->fence, vBuf->kBuf.fence_type_mask, 0); + wsbmFenceUnreference(&vBuf->fence); + + memcpy(WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem), vBuf->map, vBuf->size); + WSBMLISTDELINIT(&vBuf->lru); + vBuf->kBuf.placement = WSBM_PL_FLAG_SYSTEM; + vBuf->map = WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem); + + /* + * FIXME: Free memory. + */ + + err = 0; + out_unlock: + WSBM_MUTEX_UNLOCK(&vBuf->buf.mutex); + return err; +} + +static struct _WsbmBufStorage * +pool_create(struct _WsbmBufferPool *pool, + unsigned long size, uint32_t placement, unsigned alignment) +{ + struct _WsbmUserPool *p = containerOf(pool, struct _WsbmUserPool, pool); + struct _WsbmUserBuffer *vBuf = calloc(1, sizeof(*vBuf)); + + if (!vBuf) + return NULL; + + wsbmBufStorageInit(&vBuf->buf, pool); + vBuf->sysmem = NULL; + vBuf->proposedPlacement = placement; + vBuf->size = size; + vBuf->alignment = alignment; + + WSBMINITLISTHEAD(&vBuf->lru); + WSBMINITLISTHEAD(&vBuf->delayed); + WSBM_MUTEX_LOCK(&p->mutex); + + if (placement & WSBM_PL_FLAG_TT) { + vBuf->node = wsbmMMSearchFree(&p->agpMM, size, alignment, 1); + if (vBuf->node) + vBuf->node = wsbmMMGetBlock(vBuf->node, size, alignment); + + if (vBuf->node) { + vBuf->kBuf.placement = WSBM_PL_FLAG_TT; + vBuf->kBuf.gpuOffset = p->agpOffset + vBuf->node->start; + vBuf->map = (void *)(p->agpMap + vBuf->node->start); + WSBMLISTADDTAIL(&vBuf->lru, &p->agpLRU); + goto have_mem; + } + } + + if (placement & WSBM_PL_FLAG_VRAM) { + vBuf->node = wsbmMMSearchFree(&p->vramMM, size, alignment, 1); + if (vBuf->node) + vBuf->node = wsbmMMGetBlock(vBuf->node, size, alignment); + + if (vBuf->node) { + vBuf->kBuf.placement = WSBM_PL_FLAG_VRAM; + vBuf->kBuf.gpuOffset = p->vramOffset + vBuf->node->start; + vBuf->map = (void *)(p->vramMap + vBuf->node->start); + WSBMLISTADDTAIL(&vBuf->lru, &p->vramLRU); + goto have_mem; + } + } + + if ((placement & WSBM_PL_FLAG_NO_EVICT) + && !(placement & WSBM_PL_FLAG_SYSTEM)) { + WSBM_MUTEX_UNLOCK(&p->mutex); + goto out_err; + } + + vBuf->sysmem = malloc(size + WSBM_USER_ALIGN_ADD); + vBuf->kBuf.placement = WSBM_PL_FLAG_SYSTEM; + vBuf->map = WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem); + + have_mem: + + WSBM_MUTEX_UNLOCK(&p->mutex); + if (vBuf->sysmem != NULL + || (!(vBuf->kBuf.placement & WSBM_PL_FLAG_SYSTEM))) + return &vBuf->buf; + out_err: + free(vBuf); + return NULL; +} + +static int +pool_validate(struct _WsbmBufStorage *buf, uint64_t set_flags, + uint64_t clr_flags) +{ + struct _WsbmUserBuffer *vBuf = userBuf(buf); + struct _WsbmUserPool *p = userPool(vBuf); + int err = -ENOMEM; + + WSBM_MUTEX_LOCK(&buf->mutex); + + while (wsbmAtomicRead(&vBuf->writers) != 0) + WSBM_COND_WAIT(&vBuf->event, &buf->mutex); + + vBuf->unFenced = 1; + + WSBM_MUTEX_LOCK(&p->mutex); + WSBMLISTDELINIT(&vBuf->lru); + + vBuf->proposedPlacement = + (vBuf->proposedPlacement | set_flags) & ~clr_flags; + + if ((vBuf->proposedPlacement & vBuf->kBuf.placement & WSBM_PL_MASK_MEM) == + vBuf->kBuf.placement) { + err = 0; + goto have_mem; + } + + /* + * We're moving to another memory region, so evict first and we'll + * do a sw copy to the other region. + */ + + if (!(vBuf->kBuf.placement & WSBM_PL_FLAG_SYSTEM)) { + struct _WsbmListHead tmpLRU; + + WSBMINITLISTHEAD(&tmpLRU); + WSBMLISTADDTAIL(&tmpLRU, &vBuf->lru); + err = evict_lru(&tmpLRU); + if (err) + goto have_mem; + } + + if (vBuf->proposedPlacement & WSBM_PL_FLAG_TT) { + do { + vBuf->node = + wsbmMMSearchFree(&p->agpMM, vBuf->size, vBuf->alignment, 1); + if (vBuf->node) + vBuf->node = + wsbmMMGetBlock(vBuf->node, vBuf->size, vBuf->alignment); + + if (vBuf->node) { + vBuf->kBuf.placement = WSBM_PL_FLAG_TT; + vBuf->kBuf.gpuOffset = p->agpOffset + vBuf->node->start; + vBuf->map = (void *)(p->agpMap + vBuf->node->start); + memcpy(vBuf->map, WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem), + vBuf->size); + free(vBuf->sysmem); + goto have_mem; + } + } while (evict_lru(&p->agpLRU) == 0); + } + + if (vBuf->proposedPlacement & WSBM_PL_FLAG_VRAM) { + do { + vBuf->node = + wsbmMMSearchFree(&p->vramMM, vBuf->size, vBuf->alignment, 1); + if (vBuf->node) + vBuf->node = + wsbmMMGetBlock(vBuf->node, vBuf->size, vBuf->alignment); + + if (!err && vBuf->node) { + vBuf->kBuf.placement = WSBM_PL_FLAG_VRAM; + vBuf->kBuf.gpuOffset = p->vramOffset + vBuf->node->start; + vBuf->map = (void *)(p->vramMap + vBuf->node->start); + memcpy(vBuf->map, WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem), + vBuf->size); + free(vBuf->sysmem); + goto have_mem; + } + } while (evict_lru(&p->vramLRU) == 0); + } + + if (vBuf->proposedPlacement & WSBM_PL_FLAG_SYSTEM) + goto have_mem; + + err = -ENOMEM; + + have_mem: + vBuf->newFenceType = p->fenceTypes(set_flags); + WSBM_MUTEX_UNLOCK(&p->mutex); + WSBM_MUTEX_UNLOCK(&buf->mutex); + return err; +} + +static int +pool_setStatus(struct _WsbmBufStorage *buf, + uint32_t set_placement, uint32_t clr_placement) +{ + struct _WsbmUserBuffer *vBuf = userBuf(buf); + int ret; + + ret = pool_validate(buf, set_placement, clr_placement); + vBuf->unFenced = 0; + return ret; +} + +void +release_delayed_buffers(struct _WsbmUserPool *p) +{ + struct _WsbmUserBuffer *vBuf; + struct _WsbmListHead *list, *next; + + WSBM_MUTEX_LOCK(&p->mutex); + + /* + * We don't need to take the buffer mutexes in this loop, since + * the only other user is the evict_lru function, which has the + * pool mutex held when accessing the buffer fence member. + */ + + WSBMLISTFOREACHSAFE(list, next, &p->delayed) { + vBuf = WSBMLISTENTRY(list, struct _WsbmUserBuffer, delayed); + + if (!vBuf->fence + || wsbmFenceSignaled(vBuf->fence, vBuf->kBuf.fence_type_mask)) { + if (vBuf->fence) + wsbmFenceUnreference(&vBuf->fence); + + WSBMLISTDEL(&vBuf->delayed); + WSBMLISTDEL(&vBuf->lru); + + if ((vBuf->kBuf.placement & WSBM_PL_FLAG_SYSTEM) == 0) + wsbmMMPutBlock(vBuf->node); + else + free(vBuf->sysmem); + + free(vBuf); + } else + break; + + } + WSBM_MUTEX_UNLOCK(&p->mutex); +} + +static void +pool_destroy(struct _WsbmBufStorage **buf) +{ + struct _WsbmUserBuffer *vBuf = userBuf(*buf); + struct _WsbmUserPool *p = userPool(vBuf); + + *buf = NULL; + + WSBM_MUTEX_LOCK(&vBuf->buf.mutex); + if ((vBuf->fence + && !wsbmFenceSignaled(vBuf->fence, vBuf->kBuf.fence_type_mask))) { + WSBM_MUTEX_LOCK(&p->mutex); + WSBMLISTADDTAIL(&vBuf->delayed, &p->delayed); + WSBM_MUTEX_UNLOCK(&p->mutex); + WSBM_MUTEX_UNLOCK(&vBuf->buf.mutex); + return; + } + + if (vBuf->fence) + wsbmFenceUnreference(&vBuf->fence); + + WSBM_MUTEX_LOCK(&p->mutex); + WSBMLISTDEL(&vBuf->lru); + WSBM_MUTEX_UNLOCK(&p->mutex); + + if (!(vBuf->kBuf.placement & WSBM_PL_FLAG_SYSTEM)) + wsbmMMPutBlock(vBuf->node); + else + free(vBuf->sysmem); + + free(vBuf); + return; +} + +static int +pool_map(struct _WsbmBufStorage *buf, unsigned mode, void **virtual) +{ + struct _WsbmUserBuffer *vBuf = userBuf(buf); + + *virtual = vBuf->map; + return 0; +} + +static void +pool_unmap(struct _WsbmBufStorage *buf) +{ + ; +} + +static void +pool_releaseFromCpu(struct _WsbmBufStorage *buf, unsigned mode) +{ + struct _WsbmUserBuffer *vBuf = userBuf(buf); + + if (wsbmAtomicDecZero(&vBuf->writers)) + WSBM_COND_BROADCAST(&vBuf->event); + +} + +static int +pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode) +{ + struct _WsbmUserBuffer *vBuf = userBuf(buf); + int ret = 0; + + WSBM_MUTEX_LOCK(&buf->mutex); + if ((mode & WSBM_SYNCCPU_DONT_BLOCK)) { + + if (vBuf->unFenced) { + ret = -EBUSY; + goto out_unlock; + } + + ret = 0; + if ((vBuf->fence == NULL) || + wsbmFenceSignaled(vBuf->fence, vBuf->kBuf.fence_type_mask)) { + wsbmFenceUnreference(&vBuf->fence); + wsbmAtomicInc(&vBuf->writers); + } else + ret = -EBUSY; + + goto out_unlock; + } + waitIdleLocked(buf, 0); + wsbmAtomicInc(&vBuf->writers); + out_unlock: + WSBM_MUTEX_UNLOCK(&buf->mutex); + return ret; +} + +static unsigned long +pool_offset(struct _WsbmBufStorage *buf) +{ + return userBuf(buf)->kBuf.gpuOffset; +} + +static unsigned long +pool_poolOffset(struct _WsbmBufStorage *buf) +{ + return 0UL; +} + +static unsigned long +pool_size(struct _WsbmBufStorage *buf) +{ + return userBuf(buf)->size; +} + +static void +pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence) +{ + struct _WsbmUserBuffer *vBuf = userBuf(buf); + struct _WsbmUserPool *p = userPool(vBuf); + + WSBM_MUTEX_LOCK(&buf->mutex); + + if (vBuf->fence) + wsbmFenceUnreference(&vBuf->fence); + + vBuf->fence = wsbmFenceReference(fence); + vBuf->unFenced = 0; + vBuf->kBuf.fence_type_mask = vBuf->newFenceType; + + WSBM_COND_BROADCAST(&vBuf->event); + WSBM_MUTEX_LOCK(&p->mutex); + if (vBuf->kBuf.placement & WSBM_PL_FLAG_VRAM) + WSBMLISTADDTAIL(&vBuf->lru, &p->vramLRU); + else if (vBuf->kBuf.placement & WSBM_PL_FLAG_TT) + WSBMLISTADDTAIL(&vBuf->lru, &p->agpLRU); + WSBM_MUTEX_UNLOCK(&p->mutex); + WSBM_MUTEX_UNLOCK(&buf->mutex); +} + +static void +pool_unvalidate(struct _WsbmBufStorage *buf) +{ + struct _WsbmUserBuffer *vBuf = userBuf(buf); + struct _WsbmUserPool *p = userPool(vBuf); + + WSBM_MUTEX_LOCK(&buf->mutex); + + if (!vBuf->unFenced) + goto out_unlock; + + vBuf->unFenced = 0; + WSBM_COND_BROADCAST(&vBuf->event); + WSBM_MUTEX_LOCK(&p->mutex); + if (vBuf->kBuf.placement & WSBM_PL_FLAG_VRAM) + WSBMLISTADDTAIL(&vBuf->lru, &p->vramLRU); + else if (vBuf->kBuf.placement & WSBM_PL_FLAG_TT) + WSBMLISTADDTAIL(&vBuf->lru, &p->agpLRU); + WSBM_MUTEX_UNLOCK(&p->mutex); + + out_unlock: + + WSBM_MUTEX_UNLOCK(&buf->mutex); +} + +static struct _WsbmKernelBuf * +pool_kernel(struct _WsbmBufStorage *buf) +{ + struct _WsbmUserBuffer *vBuf = userBuf(buf); + + return &vBuf->kBuf; +} + +static void +pool_takedown(struct _WsbmBufferPool *pool) +{ + struct _WsbmUserPool *p = containerOf(pool, struct _WsbmUserPool, pool); + int empty; + + do { + release_delayed_buffers(p); + WSBM_MUTEX_LOCK(&p->mutex); + empty = (p->delayed.next == &p->delayed); + WSBM_MUTEX_UNLOCK(&p->mutex); + + if (!empty) + usleep(1000); + + } while (!empty); + WSBM_MUTEX_LOCK(&p->mutex); + + while (evict_lru(&p->vramLRU) == 0) ; + while (evict_lru(&p->agpLRU) == 0) ; + + WSBM_MUTEX_UNLOCK(&p->mutex); + + wsbmMMtakedown(&p->agpMM); + wsbmMMtakedown(&p->vramMM); + + free(p); +} + +void +wsbmUserPoolClean(struct _WsbmBufferPool *pool, int cleanVram, int cleanAgp) +{ + struct _WsbmUserPool *p = containerOf(pool, struct _WsbmUserPool, pool); + + WSBM_MUTEX_LOCK(&p->mutex); + if (cleanVram) + while (evict_lru(&p->vramLRU) == 0) ; + if (cleanAgp) + while (evict_lru(&p->agpLRU) == 0) ; + WSBM_MUTEX_UNLOCK(&p->mutex); +} + +struct _WsbmBufferPool * +wsbmUserPoolInit(void *vramAddr, + unsigned long vramStart, unsigned long vramSize, + void *agpAddr, unsigned long agpStart, + unsigned long agpSize, + uint32_t(*fenceTypes) (uint64_t set_flags)) +{ + struct _WsbmBufferPool *pool; + struct _WsbmUserPool *uPool; + int ret; + + uPool = calloc(1, sizeof(*uPool)); + if (!uPool) + goto out_err0; + + ret = WSBM_MUTEX_INIT(&uPool->mutex); + if (ret) + goto out_err0; + + ret = wsbmMMinit(&uPool->vramMM, 0, vramSize); + if (ret) + goto out_err1; + + ret = wsbmMMinit(&uPool->agpMM, 0, agpSize); + if (ret) + goto out_err2; + + WSBMINITLISTHEAD(&uPool->delayed); + WSBMINITLISTHEAD(&uPool->vramLRU); + WSBMINITLISTHEAD(&uPool->agpLRU); + + uPool->agpOffset = agpStart; + uPool->agpMap = (unsigned long)agpAddr; + uPool->vramOffset = vramStart; + uPool->vramMap = (unsigned long)vramAddr; + uPool->fenceTypes = fenceTypes; + + pool = &uPool->pool; + pool->map = &pool_map; + pool->unmap = &pool_unmap; + pool->destroy = &pool_destroy; + pool->offset = &pool_offset; + pool->poolOffset = &pool_poolOffset; + pool->size = &pool_size; + pool->create = &pool_create; + pool->fence = &pool_fence; + pool->unvalidate = &pool_unvalidate; + pool->kernel = &pool_kernel; + pool->validate = &pool_validate; + pool->waitIdle = &pool_waitIdle; + pool->takeDown = &pool_takedown; + pool->setStatus = &pool_setStatus; + pool->syncforcpu = &pool_syncForCpu; + pool->releasefromcpu = &pool_releaseFromCpu; + + return pool; + + out_err2: + wsbmMMtakedown(&uPool->vramMM); + out_err1: + WSBM_MUTEX_FREE(&uPool->mutex); + out_err0: + free(uPool); + + return NULL; +} diff --git a/src/wsbm_util.h b/src/wsbm_util.h new file mode 100644 index 0000000..13212c1 --- /dev/null +++ b/src/wsbm_util.h @@ -0,0 +1,76 @@ +/* + * This file is not copyrighted. + */ + +#ifndef _WSBM_UTIL_H_ +#define _WSBM_UTIL_H_ + +#include + +#ifndef containerOf +#define containerOf(__item, __type, __field) \ + ((__type *)(((char *) (__item)) - offsetof(__type, __field))) +#endif + +struct _WsbmListHead +{ + struct _WsbmListHead *prev; + struct _WsbmListHead *next; +}; + +#define WSBMINITLISTHEAD(__item) \ + do{ \ + (__item)->prev = (__item); \ + (__item)->next = (__item); \ + } while (0) + +#define WSBMLISTADD(__item, __list) \ + do { \ + (__item)->prev = (__list); \ + (__item)->next = (__list)->next; \ + (__list)->next->prev = (__item); \ + (__list)->next = (__item); \ + } while (0) + +#define WSBMLISTADDTAIL(__item, __list) \ + do { \ + (__item)->next = (__list); \ + (__item)->prev = (__list)->prev; \ + (__list)->prev->next = (__item); \ + (__list)->prev = (__item); \ + } while(0) + +#define WSBMLISTDEL(__item) \ + do { \ + (__item)->prev->next = (__item)->next; \ + (__item)->next->prev = (__item)->prev; \ + } while(0) + +#define WSBMLISTDELINIT(__item) \ + do { \ + (__item)->prev->next = (__item)->next; \ + (__item)->next->prev = (__item)->prev; \ + (__item)->next = (__item); \ + (__item)->prev = (__item); \ + } while(0) + +#define WSBMLISTFOREACH(__item, __list) \ + for((__item) = (__list)->next; (__item) != (__list); (__item) = (__item)->next) + +#define WSBMLISTFOREACHPREV(__item, __list) \ + for((__item) = (__list)->prev; (__item) != (__list); (__item) = (__item)->prev) + +#define WSBMLISTFOREACHSAFE(__item, __next, __list) \ + for((__item) = (__list)->next, (__next) = (__item)->next; \ + (__item) != (__list); \ + (__item) = (__next), (__next) = (__item)->next) + +#define WSBMLISTFOREACHPREVSAFE(__item, __prev, __list) \ + for((__item) = (__list)->prev, (__prev) = (__item->prev); \ + (__item) != (__list); \ + (__item) = (__prev), (__prev) = (__item)->prev) + +#define WSBMLISTENTRY(__item, __type, __field) \ + containerOf(__item, __type, __field) + +#endif