diff --git a/vendored/chainweb-storage/.github/workflows/build.yml b/vendored/chainweb-storage/.github/workflows/build.yml index 53f98b1a9..1bced9982 100644 --- a/vendored/chainweb-storage/.github/workflows/build.yml +++ b/vendored/chainweb-storage/.github/workflows/build.yml @@ -1,6 +1,7 @@ name: Build on: + workflow_dispatch: push: schedule: - cron: '0 8 * * *' @@ -13,11 +14,11 @@ jobs: strategy: fail-fast: false matrix: - ghc: ['8.8.4', '8.10.5', '9.0.1'] - cabal: ['3.4'] + ghc: ['8.10.7', '9.0.2', '9.2'] + cabal: ['3.6'] os: - - 'ubuntu-18.04' - 'ubuntu-20.04' + - 'ubuntu-22.04' - 'macOS-latest' # - 'windows-latest' # windows builds generally work but are flaky on the github runners @@ -25,9 +26,9 @@ jobs: # Setup - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install GHC and Cabal - uses: haskell/actions/setup@v1.2.3 + uses: haskell/actions/setup@v2 with: ghc-version: ${{ matrix.ghc }} cabal-version: ${{ matrix.cabal }} @@ -36,7 +37,7 @@ jobs: run: | case "${{ runner.os }}" in Linux) - sudo apt-get install -y librocksdb-dev + sudo apt-get install -y libgflags-dev liblz4-dev libzstd-dev libsnappy-dev libbz2-dev ;; mac*) brew install rocksdb @@ -54,20 +55,19 @@ jobs: documentation: True benchmarks: True tests: True - write-ghc-environment-files: always EOF # Restore Packages from Caches - - uses: actions/cache@v2.1.6 + - uses: actions/cache@v3 name: Cache ~/.cabal/packages with: path: | ~/.cabal/packages ~/.cabal/store dist-newstyle - key: ${{ matrix.os }}-${{ matrix.ghc }}-cabal-cache-${{ github.sha }} + key: ${{ matrix.os }}-${{ matrix.ghc }}-cabal-${{ github.sha }} restore-keys: | - ${{ matrix.os }}-${{ matrix.ghc }}-cabal-cache- + ${{ matrix.os }}-${{ matrix.ghc }}-cabal- # Build - name: Update package database diff --git a/vendored/chainweb-storage/cabal.project b/vendored/chainweb-storage/cabal.project index b4fb0a867..b4524e733 100644 --- a/vendored/chainweb-storage/cabal.project +++ b/vendored/chainweb-storage/cabal.project @@ -3,4 +3,4 @@ packages: chainweb-storage.cabal source-repository-package type: git location: https://github.com/kadena-io/rocksdb-haskell.git - tag: 2f0bae730d137b7edee090c4014d891675933ba5 + tag: 42cfb81426fe1cadfe411a723942c55bdb54285e diff --git a/vendored/chainweb-storage/chainweb-storage.cabal b/vendored/chainweb-storage/chainweb-storage.cabal index 65f79241b..abce3922d 100644 --- a/vendored/chainweb-storage/chainweb-storage.cabal +++ b/vendored/chainweb-storage/chainweb-storage.cabal @@ -31,13 +31,13 @@ library ghc-options: -Wall exposed-modules: - Data.CAS - Data.CAS.Forgetful - Data.CAS.HashMap - Data.CAS.RocksDB - Data.DedupStore + Chainweb.Storage.Table + Chainweb.Storage.Table.Forgetful + Chainweb.Storage.Table.HashMap + Chainweb.Storage.Table.RocksDB + Chainweb.Storage.DedupStore build-depends: - , base >=4.10 && <4.16 + , base >=4.10 && <5 , bytestring >=0.10 , containers >=0.5 , cryptonite >= 0.25 @@ -48,6 +48,7 @@ library , hashable >=1.2 , lens >=4.16 , memory >=0.14 + , mtl >= 2.2 , nothunks >= 0.1.0.0 , rocksdb-haskell-kadena >=1.1.0 , stm >=2.4 @@ -74,7 +75,7 @@ test-suite rocksdb-tests -- external , async >=2.2 - , base >=4.10 && <4.16 + , base >=4.10 && <5 , bytestring >=0.10 , exceptions >=0.10 , lens >=4.16 diff --git a/vendored/chainweb-storage/src/Data/DedupStore.hs b/vendored/chainweb-storage/src/Chainweb/Storage/DedupStore.hs similarity index 93% rename from vendored/chainweb-storage/src/Data/DedupStore.hs rename to vendored/chainweb-storage/src/Chainweb/Storage/DedupStore.hs index 974f00be9..57991d399 100644 --- a/vendored/chainweb-storage/src/Data/DedupStore.hs +++ b/vendored/chainweb-storage/src/Chainweb/Storage/DedupStore.hs @@ -8,9 +8,10 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} {-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE FlexibleContexts #-} -- | --- Module: Data.DedupStore +-- Module: Chainweb.Storage.DedupStore -- Copyright: Copyright © 2019 Kadena LLC. -- License: MIT -- Maintainer: Lars Kuhtz @@ -18,7 +19,7 @@ -- -- A Deduplicated Key-Value Store -- -module Data.DedupStore +module Chainweb.Storage.DedupStore ( -- * Deduplicated Key-Value Store DedupStore(..) @@ -46,7 +47,6 @@ import qualified Data.ByteString as B import qualified Data.ByteString.Builder as BB import qualified Data.ByteString.Lazy as BL import qualified Data.ByteString.Unsafe as BU -import Data.CAS import Data.Hashable import Data.Int import qualified Data.List as L @@ -64,7 +64,8 @@ import System.IO.Unsafe -- internal modules -import Data.CAS.RocksDB +import Chainweb.Storage.Table +import Chainweb.Storage.Table.RocksDB -- -------------------------------------------------------------------------- -- -- Utils @@ -111,7 +112,7 @@ instance Exception DedupStoreException -- data DedupStore k v = DedupStore { _dedupRoots :: !(RocksDbTable k DedupHash) - , _dedupChunks :: !(RocksDbCas Chunk) + , _dedupChunks :: !(Casify RocksDbTable Chunk) , _dedupValueCodec :: !(Codec v) } @@ -126,7 +127,7 @@ newDedupStore -> DedupStore k v newDedupStore rdb vc kc n = DedupStore { _dedupRoots = newTable rdb dedupHashCodec kc (n <> ["roots"]) - , _dedupChunks = newCas rdb dedupChunkCodec dedupHashCodec (n <> ["chunks"]) + , _dedupChunks = Casify $ newTable rdb dedupChunkCodec dedupHashCodec (n <> ["chunks"]) , _dedupValueCodec = vc } where @@ -155,7 +156,7 @@ dedupInsert :: DedupStore k v -> k -> v -> IO () dedupInsert store k v = do dedupKey <- dedupStore (_dedupChunks store) $ BL.fromStrict $ _codecEncode (_dedupValueCodec store) v - tableInsert (_dedupRoots store) k dedupKey + tableInsert (_dedupRoots store) k dedupKey {-# INLINE dedupInsert #-} -- | @dedupLookup db k@ returns 'Just' the value at key @k@ in the @@ -185,7 +186,7 @@ instance IsCasValue Chunk where type CasKeyType Chunk = DedupHash casKey (Chunk _ h _) = h -type DedupCas cas = (IsCas cas, CasValueType cas ~ Chunk) +type DedupCas cas = Cas cas Chunk -- | Store a sequence of bytes in a deduplicated content addressed store. -- Returns the hash of the bytes, that can be used to query the data from the @@ -217,7 +218,7 @@ dedupStore' store = go0 where go0 :: HasCallStack => BL.ByteString -> IO (Int, Int, DedupHash) go0 bytes = mapM (hashAndStore 0x0) (toChunks $ roll bytes) >>= \case - [] -> error "Data.DedupStore.dedupStore.go0: Data.ByteString.Lazy.toChunks must not return an empty list" + [] -> error "Chainweb.Storage.DedupStore.dedupStore.go0: Data.ByteString.Lazy.toChunks must not return an empty list" [x] -> return x l -> do (h, m, r) <- go1 (third3 <$> l) @@ -225,7 +226,7 @@ dedupStore' store = go0 go1 :: HasCallStack => [DedupHash] -> IO (Int, Int, DedupHash) go1 hashes = mapM (hashAndStore 0x1) (rollHashes hashes) >>= \case - [] -> error "Data.DedupStore.dedupStore.go1: Data.ByteString.Lazy.toChunks must not return an empty list" + [] -> error "Chainweb.Storage.DedupStore.dedupStore.go1: Data.ByteString.Lazy.toChunks must not return an empty list" [x] -> return x l -> do (h, m, r) <- go1 (third3 <$> l) @@ -245,10 +246,10 @@ dedupStore' store = go0 hashAndStore :: Word8 -> B.ByteString -> IO (Int, Int, DedupHash) hashAndStore tag c = do let h = DedupHash $ BA.convert $ C.hash @_ @DedupHashAlg (B.cons tag c) - (hit, miss) <- casMember store h >>= \x -> if x + (hit, miss) <- tableMember store h >>= \x -> if x then return (1, 0) else do - casInsert store (Chunk tag h c) + casInsert store (Chunk tag h c) return (0, 1) return (hit, miss, h) {-# INLINE hashAndStore #-} @@ -275,7 +276,7 @@ dedupRestore -> DedupHash -> IO (Maybe BL.ByteString) dedupRestore store key = fmap BB.toLazyByteString <$> do - casLookup store key >>= \case + tableLookup store key >>= \case Nothing -> return Nothing Just (Chunk 0x0 _ b) -> return $ Just $ BB.byteString b Just (Chunk 0x1 _ b) -> Just <$> splitHashes b @@ -283,7 +284,7 @@ dedupRestore store key = fmap BB.toLazyByteString <$> do where go h = do - casLookup store h >>= \case + tableLookup store h >>= \case Nothing -> throwM $ DedupStoreCorruptedData "Missing chunk from store" Just (Chunk 0x0 _ b) -> return (BB.byteString b) Just (Chunk 0x1 _ b) -> splitHashes b @@ -310,7 +311,7 @@ roll :: BL.ByteString -> BL.ByteString roll z = BL.fromChunks $ go seed 0 z za ze where z' = BL.unpack z - za = (BL.unpack $ BL.replicate window 0) <> z' + za = BL.unpack (BL.replicate window 0) <> z' {-# SCC za #-} ze = z' {-# SCC ze #-} diff --git a/vendored/chainweb-storage/src/Chainweb/Storage/Table.hs b/vendored/chainweb-storage/src/Chainweb/Storage/Table.hs new file mode 100644 index 000000000..484a1719d --- /dev/null +++ b/vendored/chainweb-storage/src/Chainweb/Storage/Table.hs @@ -0,0 +1,184 @@ +{-# LANGUAGE ConstraintKinds #-} +{-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE DerivingStrategies #-} +{-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE FunctionalDependencies #-} +{-# LANGUAGE GADTs #-} +{-# LANGUAGE InstanceSigs #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE QuantifiedConstraints #-} +{-# LANGUAGE TypeApplications #-} +{-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE UndecidableInstances #-} + +-- | +-- Module: Chainweb.Storage.Table +-- Copyright: Copyright © 2019 Kadena LLC. +-- License: MIT +-- Maintainer: Lars Kuhtz , Edmund Noble +-- Stability: experimental +-- Description: Key Value Store +-- +-- API for Key-Value Stores +module Chainweb.Storage.Table + ( IsCasValue (..) + , ReadableTable (..) + , tableLookupBatch + , ReadableTable1 + , ReadableCas + , Table (..) + , Table1 + , Casify(..) + , Cas + , casInsert + , casInsertBatch + , casDelete + , casDeleteBatch + , IterableTable (..) + , IterableTable1 + , IterableCas + , Entry (..) + , Iterator (..) + , Iterator1 + , CasIterator + , tableLookupM + , casLookupM + , TableException (..) + ) +where + +import Control.Exception (Exception, SomeException) +import Control.Lens +import Control.Monad.Catch (throwM) + +import Data.Coerce +import Data.Foldable +import Data.Maybe +import Data.Text (Text) + +import GHC.Generics +import GHC.Stack + +-- | The class of content-addressable values. +-- +-- The casKey function must be morally injective: +-- +-- prop> casKey a /= casKey b || a == b +-- +-- Usually, 'casKey' is a cryptographic, i.e. collision resistant, hash +-- function. +class Eq (CasKeyType v) => IsCasValue v where + type CasKeyType v + casKey :: v -> CasKeyType v + +-- | Read-Only View of a Key-Value Store +-- +class ReadableTable t k v | t -> k v where + tableLookup :: t -> k -> IO (Maybe v) + tableLookupBatch' :: t -> Traversal s r k (Maybe v) -> s -> IO r + tableLookupBatch' t l = l (tableLookup t) + tableMember :: t -> k -> IO Bool + tableMember t k = isJust <$> tableLookup t k + +tableLookupBatch :: (ReadableTable t k v, Each s t' k (Maybe v)) => t -> s -> IO t' +tableLookupBatch t = tableLookupBatch' t each + +type ReadableCas t v = ReadableTable t (CasKeyType v) v +type ReadableTable1 t = forall k v. ReadableTable (t k v) k v + +class ReadableTable t k v => Table t k v | t -> k v where + tableInsert :: t -> k -> v -> IO () + tableInsertBatch :: t -> [(k, v)] -> IO () + tableInsertBatch t kvs = traverse_ (uncurry (tableInsert t)) kvs + tableDelete :: t -> k -> IO () + tableDeleteBatch :: t -> [k] -> IO () + tableDeleteBatch t ks = traverse_ (tableDelete t) ks +type Table1 t = forall k v. Table (t k v) k v + +type Cas t v = Table t (CasKeyType v) v + +casInsert :: (IsCasValue v, Cas t v) => t -> v -> IO () +casInsert t v = tableInsert t (casKey v) v + +casInsertBatch :: (IsCasValue v, Cas t v) => t -> [v] -> IO () +casInsertBatch t vs = tableInsertBatch t [(casKey v, v) | v <- vs] + +casDelete :: (IsCasValue v, Cas t v) => t -> v -> IO () +casDelete t = tableDelete t . casKey + +casDeleteBatch :: (IsCasValue v, Cas t v) => t -> [v] -> IO () +casDeleteBatch t = tableDeleteBatch t . fmap casKey + +class (Table t k v, Iterator i k v) => IterableTable t i k v | t -> i k v where + -- the created iterator must be positioned at the start of the table. + withTableIterator :: t -> (i -> IO a) -> IO a +type IterableTable1 t i = forall k v. IterableTable (t k v) (i k v) k v + +type IterableCas t v = IterableTable t (CasKeyType v) v + +data Entry k v = Entry !k !v + deriving (Eq, Show, Ord) + +class Iterator i k v | i -> k v where + iterSeek :: i -> k -> IO () + iterLast :: i -> IO () + iterFirst :: i -> IO () + iterNext :: i -> IO () + iterPrev :: i -> IO () + iterEntry :: i -> IO (Maybe (Entry k v)) + iterKey :: i -> IO (Maybe k) + iterKey i = (fmap . fmap) (\(Entry k _) -> k) $ iterEntry i + iterValue :: i -> IO (Maybe v) + iterValue i = (fmap . fmap) (\(Entry _ v) -> v) $ iterEntry i + iterValid :: i -> IO Bool + iterValid i = isJust <$> iterKey i +type Iterator1 i = forall k v. Iterator (i k v) k v + +type CasIterator i v = Iterator i (CasKeyType v) v + +-- | A newtype wrapper that takes only a single type constructor. This useful in +-- situations where a Higher Order type constructor for a CAS is required. A +-- type synonym doesn't work in this situation because type synonyms must be +-- fully applied. +-- +newtype Casify t v = Casify { unCasify :: t (CasKeyType v) v } +instance forall t k v. (CasKeyType v ~ k, ReadableTable (t k v) k v) => ReadableTable (Casify t v) k v where + tableLookup = coerce @(t k v -> k -> IO (Maybe v)) tableLookup + -- can't seem to write `coerce` without the resulting instantiation being impredicative + tableLookupBatch' (Casify t) b = tableLookupBatch' t b + tableMember = coerce @(t k v -> k -> IO Bool) tableMember +instance forall t k v. (CasKeyType v ~ k, Table (t k v) k v) => Table (Casify t v) k v where + tableInsert = coerce @(t k v -> k -> v -> IO ()) tableInsert + tableInsertBatch = coerce @(t k v -> [(k, v)] -> IO ()) tableInsertBatch + tableDelete = coerce @(t k v -> k -> IO ()) tableDelete + tableDeleteBatch = coerce @(t k v -> [k] -> IO ()) tableDeleteBatch +-- TODO: why is this Iterator superclass needed? +instance forall t i k v. (CasKeyType v ~ k, IterableTable (t k v) i k v, Iterator i k v) => IterableTable (Casify t v) i k v where + withTableIterator :: forall a. Casify t v -> (i -> IO a) -> IO a + withTableIterator = coerce @(t k v -> (i -> IO a) -> IO a) withTableIterator + +-- | Lookup a value by its key in a key-value store and throw an +-- 'TableException' if the value doesn't exist in the store +tableLookupM :: (HasCallStack, ReadableTable t k v) => t -> k -> IO v +tableLookupM cas k = + tableLookup cas k >>= \case + Nothing -> + throwM . TableException $ + "tableLookupM: lookup failed for table key" + Just v -> return $! v + +casLookupM :: (HasCallStack, ReadableCas t v) => t -> CasKeyType v -> IO v +casLookupM = tableLookupM + +-- | Exceptions that are thrown by instances of 'IsCas'. +data TableException + = TableException !Text + | TableImplementationException !SomeException + deriving (Show, Generic) + +instance Exception TableException + diff --git a/vendored/chainweb-storage/src/Chainweb/Storage/Table/Forgetful.hs b/vendored/chainweb-storage/src/Chainweb/Storage/Table/Forgetful.hs new file mode 100644 index 000000000..a1dfff307 --- /dev/null +++ b/vendored/chainweb-storage/src/Chainweb/Storage/Table/Forgetful.hs @@ -0,0 +1,34 @@ +{-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE MultiParamTypeClasses #-} + +-- | +-- Module: Chainweb.Storage.Table.Forgetful +-- Copyright: Copyright © 2019 Kadena LLC. +-- License: MIT +-- Maintainer: Lars Kuhtz +-- Stability: experimental +-- +-- A key-value store that forgets everything that is stored in it. +-- +module Chainweb.Storage.Table.Forgetful +( ForgetfulTable(..) +) where + +import Chainweb.Storage.Table + +-- | A key-value store that forgets everything that is added to it. +-- +data ForgetfulTable k v = ForgetfulTable + +instance ReadableTable (ForgetfulTable k v) k v where + tableLookup _ _ = return Nothing + tableMember _ _ = return False + {-# INLINE tableLookup #-} + +instance Table (ForgetfulTable k v) k v where + tableInsert _ _ _ = return () + tableDelete _ _ = return () + {-# INLINE tableInsert #-} + {-# INLINE tableDelete #-} + diff --git a/vendored/chainweb-storage/src/Chainweb/Storage/Table/HashMap.hs b/vendored/chainweb-storage/src/Chainweb/Storage/Table/HashMap.hs new file mode 100644 index 000000000..62565979c --- /dev/null +++ b/vendored/chainweb-storage/src/Chainweb/Storage/Table/HashMap.hs @@ -0,0 +1,65 @@ +{-# LANGUAGE ExistentialQuantification #-} +{-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE FunctionalDependencies #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE UndecidableInstances #-} + +-- | +-- Module: Chainweb.Storage.Table.HashMap +-- Copyright: Copyright © 2019 Kadena LLC. +-- License: MIT +-- Maintainer: Lars Kuhtz +-- Stability: experimental +-- Description: +-- +-- A thread-safe in-memory 'Table' implementation based on 'HM.HashMap'. +-- +module Chainweb.Storage.Table.HashMap +( HashMapTable +, emptyTable +, toList +, size +) where + +import Control.Concurrent.STM.TVar +import Control.Monad ((<$!>)) +import Control.Monad.STM + +import Data.Hashable +import qualified Data.HashMap.Strict as HM + +-- internal modules + +import Chainweb.Storage.Table + +-- | An 'IsTable' implementation that is base on 'HM.HashMap'. +-- +newtype HashMapTable k v = HashMapTable (TVar (HM.HashMap k v)) + +instance (Hashable k, Eq k) => ReadableTable (HashMapTable k v) k v where + tableLookup (HashMapTable var) k = HM.lookup k <$!> readTVarIO var + tableMember (HashMapTable var) k = + HM.member k <$> readTVarIO var + +instance (Hashable k, Eq k) => Table (HashMapTable k v) k v where + tableInsert (HashMapTable var) k v = + atomically $ modifyTVar' var (HM.insert k v) + tableDelete (HashMapTable var) k = + atomically $ modifyTVar' var (HM.delete k) + +-- | Create new empty CAS +-- +emptyTable :: (Hashable k, Eq k) => IO (HashMapTable k v) +emptyTable = HashMapTable <$> newTVarIO mempty + +-- | Return all entries of CAS as List +-- +toList :: HashMapTable k v -> IO [v] +toList (HashMapTable var) = HM.elems <$!> readTVarIO var + +-- | The number of items in the CAS +-- +size :: HashMapTable k v -> IO Int +size (HashMapTable var) = HM.size <$!> readTVarIO var + diff --git a/vendored/chainweb-storage/src/Data/CAS/RocksDB.hs b/vendored/chainweb-storage/src/Chainweb/Storage/Table/RocksDB.hs similarity index 62% rename from vendored/chainweb-storage/src/Data/CAS/RocksDB.hs rename to vendored/chainweb-storage/src/Chainweb/Storage/Table/RocksDB.hs index ffd08a3e3..d46e6b91e 100644 --- a/vendored/chainweb-storage/src/Data/CAS/RocksDB.hs +++ b/vendored/chainweb-storage/src/Chainweb/Storage/Table/RocksDB.hs @@ -3,31 +3,29 @@ {-# LANGUAGE DeriveAnyClass #-} {-# LANGUAGE DeriveFunctor #-} {-# LANGUAGE DeriveGeneric #-} -{-# LANGUAGE DerivingStrategies #-} {-# LANGUAGE DerivingVia #-} {-# LANGUAGE ExistentialQuantification #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE GeneralizedNewtypeDeriving #-} {-# LANGUAGE LambdaCase #-} +{-# LANGUAGE MultiParamTypeClasses #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} -{-# LANGUAGE StandaloneDeriving #-} {-# LANGUAGE TypeApplications #-} {-# LANGUAGE TypeFamilies #-} -{-# LANGUAGE ViewPatterns #-} {-# LANGUAGE UndecidableInstances #-} -{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE ViewPatterns #-} -- | --- Module: Data.CAS.RocksDB +-- Module: Chainweb.Storage.Table.RocksDB -- Copyright: Copyright © 2019 Kadena LLC. -- License: MIT -- Maintainer: Lars Kuhtz -- Stability: experimental -- --- Persisted iey-value and and content-addressable-key-value stores with a +-- Persisted key-value and and content-addressable-key-value stores with a -- RocksDB backend. -- -- A 'RocksDbTable' provides a typed key-value store with an additional iterator @@ -39,75 +37,56 @@ -- TODO: Abstract the 'RocksDbTable' API into a typeclass so that one can -- provide alternative implementations for it. -- -module Data.CAS.RocksDB -( RocksDb(..) --- , rocksDbHandle --- , rocksDbNamespace -, openRocksDb -, closeRocksDb -, withRocksDb -, withTempRocksDb -, destroyRocksDb -, resetOpenRocksDb -, modernDefaultOptions - --- * Rocks DB Table -, Codec(..) -, RocksDbTable -, newTable -, tableLookup -, tableLookupBatch -, tableInsert -, tableDelete - --- * Batch Updates -, RocksDbUpdate(..) -, updateBatch - --- * Rocks DB Table Iterator -, RocksDbTableIter -, withTableIter -, tableIterValid - --- ** Seeking -, tableIterSeek -, tableIterFirst -, tableIterLast - --- ** Advance Iterator -, tableIterNext -, tableIterPrev - --- ** Query Iterator -, tableIterEntry -, tableIterValue -, tableIterKey - --- ** Streams -, iterToEntryStream -, iterToValueStream -, iterToReverseValueStream -, iterToKeyStream - --- ** Extremal Table Entries -, tableMaxKey -, tableMaxValue -, tableMaxEntry -, tableMinKey -, tableMinValue -, tableMinEntry - --- * RocksDbCas -, RocksDbCas(..) -, newCas - --- * RocksDB-specific tools -, checkpointRocksDb -, deleteRangeRocksDb -, compactRangeRocksDb -) where +module Chainweb.Storage.Table.RocksDB + ( RocksDb(..) + -- , rocksDbHandle + -- , rocksDbNamespace + , openRocksDb + , closeRocksDb + , withRocksDb + , withTempRocksDb + , destroyRocksDb + , resetOpenRocksDb + , modernDefaultOptions + + -- * Rocks DB Table + , Codec(..) + , RocksDbTable + , newTable + , tableLookup + , tableLookupBatch + , tableInsert + , tableDelete + + -- * Batch Updates + , RocksDbUpdate(..) + , updateBatch + + -- * Rocks DB Table Iterator + , RocksDbTableIter + + -- ** Streams + , iterToEntryStream + , iterToValueStream + , iterToReverseValueStream + , iterToKeyStream + + -- ** Extremal Table Entries + , tableMaxKey + , tableMaxValue + , tableMaxEntry + , tableMinKey + , tableMinValue + , tableMinEntry + + -- * RocksDB-specific tools + , checkpointRocksDb + , deleteRangeRocksDb + , compactRangeRocksDb + ) where import Control.Exception(evaluate) +import Control.Lens import Control.Monad import Control.Monad.Catch import Control.Monad.IO.Class @@ -115,14 +94,15 @@ import Control.Monad.IO.Class import Data.ByteString(ByteString) import qualified Data.ByteString as B import qualified Data.ByteString.Char8 as B8 +import qualified Data.ByteString.Internal as BI import qualified Data.ByteString.Unsafe as BU import Data.Coerce import Data.Foldable +import Data.Maybe -import Data.CAS +import Chainweb.Storage.Table import Data.String import qualified Data.Text as T -import qualified Data.Vector as V import Foreign import Foreign.C @@ -174,7 +154,6 @@ unexpectedMsg msg expected actual = msg -- sshow :: Show a => IsString b => a -> b sshow = fromString . show -{-# INLINE sshow #-} -- -------------------------------------------------------------------------- -- -- RocksDb @@ -193,9 +172,7 @@ instance NoThunks RocksDb where [ noThunks ctx (InspectHeapNamed @"Data.RocksDB.Base.DB" a) , noThunks ctx b ] - showTypeOf _ = "Data.CAS.RocksDB.RocksDb" - {-# INLINE wNoThunks #-} - {-# INLINE showTypeOf #-} + showTypeOf _ = "Chainweb.Storage.Table.RocksDB.RocksDb" -- makeLenses ''RocksDb @@ -215,7 +192,7 @@ openRocksDb path opts_ptr = do GHC.setFileSystemEncoding GHC.utf8 createDirectoryIfMissing True path db <- withFilePath path $ \path_ptr -> - liftM R.DB + fmap R.DB $ R.throwIfErr "open" $ C.c_rocksdb_open opts_ptr path_ptr let rdb = RocksDb db mempty @@ -302,9 +279,7 @@ data Codec a = Codec instance NoThunks (Codec a) where -- NoThunks does not look inside of closures for captured thunks wNoThunks _ _ = return Nothing - showTypeOf _ = "Data.CAS.RocksDB.Codec" - {-# INLINE wNoThunks #-} - {-# INLINE showTypeOf #-} + showTypeOf _ = "Chainweb.Storage.Table.RocksDB.Codec" -- | A logical table in a 'RocksDb'. Tables in a rocks db are have isolated key -- namespaces. @@ -323,9 +298,7 @@ instance NoThunks (RocksDbTable k v) where , noThunks ctx c , noThunks ctx (InspectHeapNamed @"Data.RocksDB.Base.DB" d) ] - showTypeOf _ = "Data.CAS.RocksDB.RocksDbTable" - {-# INLINE wNoThunks #-} - {-# INLINE showTypeOf #-} + showTypeOf _ = "Chainweb.Storage.Table.RocksDB.RocksDbTable" -- | Create a new 'RocksDbTable' in the given 'RocksDb'. -- @@ -341,59 +314,12 @@ newTable -> RocksDbTable k v newTable db valCodec keyCodec namespace | any (B8.any (\x -> x `elem` ['$', '%', '/'])) namespace - = error $ "Data.CAS.RocksDb.newTable: invalid character in table namespace: " <> sshow namespace + = error $ "Chainweb.Storage.Table.RocksDb.newTable: invalid character in table namespace: " <> sshow namespace | otherwise = RocksDbTable valCodec keyCodec ns (_rocksDbHandle db) where ns = _rocksDbNamespace db <> "-" <> B.intercalate "/" namespace -{-# INLINE newTable #-} - --- | @tableInsert db k v@ inserts the value @v@ at key @k@ in the rocks db table --- @db@. --- -tableInsert :: RocksDbTable k v -> k -> v -> IO () -tableInsert db k v = R.put - (_rocksDbTableDb db) - R.defaultWriteOptions - (encKey db k) - (encVal db v) -{-# INLINE tableInsert #-} --- | @tableLookup db k@ returns 'Just' the value at key @k@ in the --- 'RocksDbTable' @db@ if it exists, or 'Nothing' if the @k@ doesn't exist in --- the table. --- -tableLookup :: RocksDbTable k v -> k -> IO (Maybe v) -tableLookup db k = do - maybeBytes <- get (_rocksDbTableDb db) mempty (encKey db k) - traverse (decVal db) maybeBytes -{-# INLINE tableLookup #-} - --- | @tableLookupBatch db ks@ returns for each @k@ in @ks@ 'Just' the value at --- key @k@ in the 'RocksDbTable' @db@ if it exists, or 'Nothing' if the @k@ --- doesn't exist in the table. --- -tableLookupBatch - :: HasCallStack - => RocksDbTable k v - -> V.Vector k - -> IO (V.Vector (Maybe v)) -tableLookupBatch db ks = do - results <- multiGet (_rocksDbTableDb db) mempty (V.map (encKey db) ks) - V.forM results $ \case - Left e -> error $ "Data.CAS.RocksDB.tableLookupBatch: " <> e - Right x -> traverse (decVal db) x -{-# INLINE tableLookupBatch #-} - --- | @tableDelete db k@ deletes the value at the key @k@ from the 'RocksDbTable' --- db. If the @k@ doesn't exist in @db@ this function does nothing. --- -tableDelete :: RocksDbTable k v -> k -> IO () -tableDelete db k = R.delete - (_rocksDbTableDb db) - R.defaultWriteOptions - (encKey db k) -{-# INLINE tableDelete #-} -- -------------------------------------------------------------------------- -- -- Batches @@ -414,7 +340,6 @@ data RocksDbUpdate rocksDbUpdateDb :: RocksDbUpdate -> R.DB rocksDbUpdateDb (RocksDbDelete t _) = _rocksDbTableDb t rocksDbUpdateDb (RocksDbInsert t _ _) = _rocksDbTableDb t -{-# INLINE rocksDbUpdateDb #-} -- | Atomically execute a batch of rocks db updates. -- @@ -429,7 +354,7 @@ updateBatch batch = R.write rdb R.defaultWriteOptions $ checkMkOp <$> batch checkMkOp o | rdb == rocksDbUpdateDb o = mkOp o - | otherwise = error "Data.CAS.RocksDB.updateBatch: all operations in a batch must be for the same RocksDB instance." + | otherwise = error "Chainweb.Storage.Table.RocksDB.updateBatch: all operations in a batch must be for the same RocksDB instance." mkOp (RocksDbDelete t k) = R.Del (encKey t k) mkOp (RocksDbInsert t k v) = R.Put (encKey t k) (encVal t v) @@ -446,7 +371,7 @@ updateBatch batch = R.write rdb R.defaultWriteOptions $ checkMkOp <$> batch -- periods of time. After usage it should be released in a timely manner. -- -- The recommended way to created a 'RocksDbTableIter' is to use the function --- `withTableIter`. +-- `withTableIterator`. -- data RocksDbTableIter k v = RocksDbTableIter { _rocksDbTableIterValueCodec :: !(Codec v) @@ -462,118 +387,59 @@ instance NoThunks (RocksDbTableIter k v) where , noThunks ctx c , noThunks ctx (InspectHeapNamed @"Data.RocksDB.Iterator.Iterator" d) ] - showTypeOf _ = "Data.CAS.RocksDB.RocksDbTableIterator" - {-# INLINE wNoThunks #-} - {-# INLINE showTypeOf #-} - --- | Provide an computation with a 'RocksDbTableIterator' and release the iterator --- after after the computation has finished either by returning a result or --- throwing an exception. If the 'RocksDbTable' input is not empty, the iterator --- will point to the first key in the 'RocksdbTable' when created. --- --- This is function provides the preferred way of creating and using a --- 'RocksDbTableIter'. --- -withTableIter :: RocksDbTable k v -> (RocksDbTableIter k v -> IO a) -> IO a -withTableIter db k = R.withReadOptions readOptions $ \opts_ptr -> - I.withIter (_rocksDbTableDb db) opts_ptr $ \iter -> do - let tableIter = makeTableIter iter - tableIterFirst tableIter - k tableIter - where - readOptions = fold - [ R.setLowerBound (namespaceFirst $ _rocksDbTableNamespace db) - , R.setUpperBound (namespaceLast $ _rocksDbTableNamespace db) - -- TODO: this setting tells rocksdb to use prefix seek *when it can*. - -- the question remains: is it actually being used? - , R.setAutoPrefixMode True - ] - makeTableIter = - RocksDbTableIter - (_rocksDbTableValueCodec db) - (_rocksDbTableKeyCodec db) - (_rocksDbTableNamespace db) -{-# INLINE withTableIter #-} + showTypeOf _ = "Chainweb.Storage.Table.RocksDB.RocksDbTableIterator" --- | Checks if an 'RocksDbTableIterator' is valid. --- --- A valid iterator returns a value when 'tableIterEntry', 'tableIterValue', or --- 'tableIterKey' is called on it. --- -tableIterValid :: RocksDbTableIter k v -> IO Bool -tableIterValid it = - I.iterValid (_rocksDbTableIter it) -{-# INLINE tableIterValid #-} +instance Iterator (RocksDbTableIter k v) k v where + iterValid it = + I.iterValid (_rocksDbTableIter it) --- | Efficiently seek to a key in a 'RocksDbTableIterator' iteration. --- -tableIterSeek :: RocksDbTableIter k v -> k -> IO () -tableIterSeek it = I.iterSeek (_rocksDbTableIter it) . encIterKey it -{-# INLINE tableIterSeek #-} + iterSeek it = I.iterSeek (_rocksDbTableIter it) . encIterKey it --- | Seek to the first key in a 'RocksDbTable'. --- -tableIterFirst :: RocksDbTableIter k v -> IO () -tableIterFirst it = - I.iterFirst (_rocksDbTableIter it) -{-# INLINE tableIterFirst #-} + iterFirst it = + I.iterFirst (_rocksDbTableIter it) --- | Seek to the last value in a 'RocksDbTable' --- -tableIterLast :: RocksDbTableIter k v -> IO () -tableIterLast it = - I.iterLast (_rocksDbTableIter it) -{-# INLINE tableIterLast #-} + iterLast it = + I.iterLast (_rocksDbTableIter it) --- | Move a 'RocksDbTableIter' to the next key in a 'RocksDbTable'. --- -tableIterNext :: RocksDbTableIter k v -> IO () -tableIterNext = I.iterNext . _rocksDbTableIter -{-# INLINE tableIterNext #-} + iterNext = I.iterNext . _rocksDbTableIter --- | Move a 'RocksDbTableIter' to the previous key in a 'RocksDbTable'. --- -tableIterPrev :: RocksDbTableIter k v -> IO () -tableIterPrev = I.iterPrev . _rocksDbTableIter -{-# INLINE tableIterPrev #-} + iterPrev = I.iterPrev . _rocksDbTableIter --- | Returns the key and the value at the current position of a --- 'RocksDbTableIter'. Returns 'Nothing' if the iterator is invalid. --- -tableIterEntry - :: RocksDbTableIter k v - -> IO (Maybe (k, v)) -tableIterEntry it = - I.iterEntry (_rocksDbTableIter it) >>= \case - Nothing -> return Nothing - Just (k, v) -> do - !k' <- decIterKey it k - !v' <- decIterVal it v - return $ Just (k', v') + iterEntry it = + I.iterEntry (_rocksDbTableIter it) >>= \case + Nothing -> return Nothing + Just (k, v) -> do + k' <- decIterKey it k + v' <- decIterVal it v + return $! Just $! Entry k' v' -{-# INLINE tableIterEntry #-} + iterValue it = I.iterValue (_rocksDbTableIter it) >>= \case + Nothing -> return Nothing + Just v -> Just <$> (evaluate =<< decIterVal it v) --- | Returns the value at the current position of a 'RocksDbTableIter'. Returns --- 'Nothing' if the iterator is invalid. --- -tableIterValue - :: RocksDbTableIter k v - -> IO (Maybe v) -tableIterValue it = I.iterValue (_rocksDbTableIter it) >>= \case - Nothing -> return Nothing - Just v -> pure . Just =<< evaluate =<< decIterVal it v -{-# INLINE tableIterValue #-} + iterKey it = I.iterKey (_rocksDbTableIter it) >>= \case + Nothing -> return Nothing + Just k -> Just <$> (evaluate =<< decIterKey it k) --- | Returns the key at the current position of a 'RocksDbTableIter'. Returns --- 'Nothing' if the iterator is invalid. --- -tableIterKey - :: RocksDbTableIter k v - -> IO (Maybe k) -tableIterKey it = I.iterKey (_rocksDbTableIter it) >>= \case - Nothing -> return Nothing - Just k -> pure . Just =<< evaluate =<< decIterKey it k -{-# INLINE tableIterKey #-} +instance IterableTable (RocksDbTable k v) (RocksDbTableIter k v) k v where + withTableIterator db k = R.withReadOptions readOptions $ \opts_ptr -> + I.withIter (_rocksDbTableDb db) opts_ptr $ \iter -> do + let tableIter = makeTableIter iter + iterFirst tableIter + k tableIter + where + readOptions = fold + [ R.setLowerBound (namespaceFirst $ _rocksDbTableNamespace db) + , R.setUpperBound (namespaceLast $ _rocksDbTableNamespace db) + -- TODO: this setting tells rocksdb to use prefix seek *when it can*. + -- the question remains: is it actually being used? + , R.setAutoPrefixMode True + ] + makeTableIter = + RocksDbTableIter + (_rocksDbTableValueCodec db) + (_rocksDbTableKeyCodec db) + (_rocksDbTableNamespace db) -- | Returns the stream of key-value pairs of an 'RocksDbTableIter'. -- @@ -582,12 +448,11 @@ tableIterKey it = I.iterKey (_rocksDbTableIter it) >>= \case -- error. Not releasing the iterator after the processing of the stream has -- finished results in a memory leak. -- -iterToEntryStream :: RocksDbTableIter k v -> S.Stream (S.Of (k,v)) IO () +iterToEntryStream :: RocksDbTableIter k v -> S.Stream (S.Of (Entry k v)) IO () iterToEntryStream it = - liftIO (tableIterEntry it) >>= \case + liftIO (iterEntry it) >>= \case Nothing -> return () - Just x -> S.yield x >> liftIO (tableIterNext it) >> iterToEntryStream it -{-# INLINE iterToEntryStream #-} + Just x -> S.yield x >> liftIO (iterNext it) >> iterToEntryStream it -- | Returns the stream of values of an 'RocksDbTableIter'. -- @@ -598,10 +463,9 @@ iterToEntryStream it = -- iterToValueStream :: Show k => RocksDbTableIter k v -> S.Stream (S.Of v) IO () iterToValueStream it = do - liftIO (tableIterValue it) >>= \case + liftIO (iterValue it) >>= \case Nothing -> return () - Just x -> S.yield x >> liftIO (tableIterNext it) >> iterToValueStream it -{-# INLINE iterToValueStream #-} + Just x -> S.yield x >> liftIO (iterNext it) >> iterToValueStream it -- Returns the stream of key-value pairs of an 'RocksDbTableIter' in reverse -- order. @@ -613,10 +477,9 @@ iterToValueStream it = do -- iterToReverseValueStream :: RocksDbTableIter k v -> S.Stream (S.Of v) IO () iterToReverseValueStream it = - liftIO (tableIterValue it) >>= \case + liftIO (iterValue it) >>= \case Nothing -> return () - Just x -> S.yield x >> liftIO (tableIterPrev it) >> iterToReverseValueStream it -{-# INLINE iterToReverseValueStream #-} + Just x -> S.yield x >> liftIO (iterPrev it) >> iterToReverseValueStream it -- | Returns the stream of keys of an 'RocksDbTableIter'. -- @@ -627,48 +490,41 @@ iterToReverseValueStream it = -- iterToKeyStream :: RocksDbTableIter k v -> S.Stream (S.Of k) IO () iterToKeyStream it = - liftIO (tableIterKey it) >>= \case + liftIO (iterKey it) >>= \case Nothing -> return () - Just x -> S.yield x >> liftIO (tableIterNext it) >> iterToKeyStream it -{-# INLINE iterToKeyStream #-} + Just x -> S.yield x >> liftIO (iterNext it) >> iterToKeyStream it -- Extremal Table Entries -- | The maximum key in a 'RocksDbTable'. -- tableMaxKey :: RocksDbTable k v -> IO (Maybe k) -tableMaxKey = flip withTableIter $ \i -> tableIterLast i *> tableIterKey i -{-# INLINE tableMaxKey #-} +tableMaxKey = flip withTableIterator $ \i -> iterLast i *> iterKey i -- | The maximum value in a 'RocksDbTable'. -- tableMaxValue :: RocksDbTable k v -> IO (Maybe v) -tableMaxValue = flip withTableIter $ \i -> tableIterLast i *> tableIterValue i -{-# INLINE tableMaxValue #-} +tableMaxValue = flip withTableIterator $ \i -> iterLast i *> iterValue i -- | The maximum key-value in a 'RocksDbTable'. -- -tableMaxEntry :: RocksDbTable k v -> IO (Maybe (k, v)) -tableMaxEntry = flip withTableIter $ \i -> tableIterLast i *> tableIterEntry i -{-# INLINE tableMaxEntry #-} +tableMaxEntry :: RocksDbTable k v -> IO (Maybe (Entry k v)) +tableMaxEntry = flip withTableIterator $ \i -> iterLast i *> iterEntry i -- | The minimum key in a 'RocksDbTable'. -- tableMinKey :: RocksDbTable k v -> IO (Maybe k) -tableMinKey = flip withTableIter $ \i -> tableIterFirst i *> tableIterKey i -{-# INLINE tableMinKey #-} +tableMinKey = flip withTableIterator $ \i -> iterFirst i *> iterKey i -- | The minimum value in a 'RocksDbTable'. -- tableMinValue :: RocksDbTable k v -> IO (Maybe v) -tableMinValue = flip withTableIter $ \i -> tableIterFirst i *> tableIterValue i -{-# INLINE tableMinValue #-} +tableMinValue = flip withTableIterator $ \i -> iterFirst i *> iterValue i -- | The minimum key-value in a 'RocksDbTable'. -- -tableMinEntry :: RocksDbTable k v -> IO (Maybe (k, v)) -tableMinEntry = flip withTableIter $ \i -> tableIterFirst i *> tableIterEntry i -{-# INLINE tableMinEntry #-} +tableMinEntry :: RocksDbTable k v -> IO (Maybe (Entry k v)) +tableMinEntry = flip withTableIterator $ \i -> iterFirst i *> iterEntry i -- -------------------------------------------------------------------------- -- -- CAS @@ -676,70 +532,44 @@ tableMinEntry = flip withTableIter $ \i -> tableIterFirst i *> tableIterEntry i -- | For a 'IsCasValue' @v@ with 'CasKeyType v ~ k@, a 'RocksDbTable k v' is an -- instance of 'HasCasLookup'. -- -instance (IsCasValue v, CasKeyType v ~ k) => HasCasLookup (RocksDbTable k v) where - type CasValueType (RocksDbTable k v) = v - casLookup = tableLookup - casLookupBatch = tableLookupBatch +instance forall k v. ReadableTable (RocksDbTable k v) k v where + tableLookup db k = do + maybeBytes <- getRocksDb (_rocksDbTableDb db) mempty (encKey db k) + traverse (decVal db) maybeBytes + + tableMember db k = + isJust <$> getRocksDb (_rocksDbTableDb db) mempty (encKey db k) - {-# INLINE casLookup #-} - {-# INLINE casLookupBatch #-} + -- | @tableLookupBatch db ks@ returns for each @k@ in @ks@ 'Just' the value at + -- key @k@ in the 'RocksDbTable' @db@ if it exists, or 'Nothing' if the @k@ + -- doesn't exist in the table. + -- + tableLookupBatch' db t = unsafePartsOf t $ \ks -> do + results <- multiGet (_rocksDbTableDb db) mempty (map (encKey db) ks) + forM results $ \case + Left e -> error $ "Chainweb.Storage.Table.RocksDB.tableLookupBatch: " <> e + Right x -> traverse (decVal db) x -- | For a 'IsCasValue' @v@ with 'CasKeyType v ~ k@, a 'RocksDbTable k v' is an -- instance of 'IsCas'. -- -instance (IsCasValue v, CasKeyType v ~ k) => IsCas (RocksDbTable k v) where - casInsert db a = tableInsert db (casKey a) a - casDelete = tableDelete +instance Table (RocksDbTable k v) k v where + tableInsert db k v = R.put + (_rocksDbTableDb db) + R.defaultWriteOptions + (encKey db k) + (encVal db v) - casInsertBatch db vs = updateBatch (mkOp <$> V.toList vs) - where - mkOp v = RocksDbInsert db (casKey v) v - - casDeleteBatch db vs = updateBatch (RocksDbDelete db <$> V.toList vs) - - {-# INLINE casInsert #-} - {-# INLINE casDelete #-} - {-# INLINE casInsertBatch #-} - {-# INLINE casDeleteBatch #-} + tableDelete db k = R.delete + (_rocksDbTableDb db) + R.defaultWriteOptions + (encKey db k) --- | A newtype wrapper that takes only a single type constructor. This useful in --- situations where a Higher Order type constructor for a CAS is required. A --- type synonym doesn't work in this situation because type synonyms must be --- fully applied. --- -newtype RocksDbCas v = RocksDbCas { _getRocksDbCas :: RocksDbTable (CasKeyType v) v } - deriving newtype (NoThunks) + tableInsertBatch db kvs = + updateBatch (uncurry (RocksDbInsert db) <$> kvs) -instance IsCasValue v => HasCasLookup (RocksDbCas v) where - type CasValueType (RocksDbCas v) = v - casLookup (RocksDbCas x) = casLookup x - casLookupBatch (RocksDbCas x) = casLookupBatch x - - {-# INLINE casLookup #-} - {-# INLINE casLookupBatch #-} - -instance IsCasValue v => IsCas (RocksDbCas v) where - casInsert (RocksDbCas x) = casInsert x - casDelete (RocksDbCas x) = casDelete x - casInsertBatch (RocksDbCas x) = casInsertBatch x - casDeleteBatch (RocksDbCas x) = casDeleteBatch x - - {-# INLINE casInsert #-} - {-# INLINE casDelete #-} - {-# INLINE casInsertBatch #-} - {-# INLINE casDeleteBatch #-} - --- | Create a new 'RocksDbCas'. --- -newCas - :: CasKeyType v ~ k - => RocksDb - -> Codec v - -> Codec k - -> [B.ByteString] - -> RocksDbCas v -newCas db vc kc n = RocksDbCas $ newTable db vc kc n -{-# INLINE newCas #-} + tableDeleteBatch db ks = + updateBatch (RocksDbDelete db <$> ks) -- -------------------------------------------------------------------------- -- -- Exceptions @@ -747,62 +577,52 @@ newCas db vc kc n = RocksDbCas $ newTable db vc kc n -- | Excpeptions that can be thrown by functions in this module. -- data RocksDbException - = RocksDbTableIterInvalidKeyNamespace (Expected B.ByteString) (Actual B.ByteString) + = RocksDbTableIterInvalidKeyNamespace !(Expected B.ByteString) !(Actual B.ByteString) deriving (Eq, Ord, Generic, NoThunks) instance Exception RocksDbException where displayException (RocksDbTableIterInvalidKeyNamespace e a) - = T.unpack $ unexpectedMsg "Data.CAS.RocksDB: invalid table key" e a - {-# INLINE displayException #-} + = T.unpack $ unexpectedMsg "Chainweb.Storage.Table.RocksDB: invalid table key" e a instance Show RocksDbException where show = displayException - {-# INLINE show #-} -- -------------------------------------------------------------------------- -- -- Table Utils encVal :: RocksDbTable k v -> v -> B.ByteString encVal = _codecEncode . _rocksDbTableValueCodec -{-# INLINE encVal #-} encKey :: RocksDbTable k v -> k -> B.ByteString encKey it k = namespaceFirst ns <> _codecEncode (_rocksDbTableKeyCodec it) k where ns = _rocksDbTableNamespace it -{-# INLINE encKey #-} decVal :: MonadThrow m => RocksDbTable k v -> B.ByteString -> m v decVal tbl = _codecDecode $ _rocksDbTableValueCodec tbl -{-# INLINE decVal #-} -- -------------------------------------------------------------------------- -- -- Iter Utils namespaceFirst :: B.ByteString -> B.ByteString namespaceFirst ns = ns <> "$" -{-# INLINE namespaceFirst #-} namespaceLast :: B.ByteString -> B.ByteString namespaceLast ns = ns <> "%" -{-# INLINE namespaceLast #-} encIterKey :: RocksDbTableIter k v -> k -> B.ByteString encIterKey it k = namespaceFirst ns <> _codecEncode (_rocksDbTableIterKeyCodec it) k where ns = _rocksDbTableIterNamespace it -{-# INLINE encIterKey #-} decIterVal :: MonadThrow m => RocksDbTableIter k v -> B.ByteString -> m v -decIterVal i bs = _codecDecode (_rocksDbTableIterValueCodec i) bs -{-# INLINE decIterVal #-} +decIterVal i = _codecDecode (_rocksDbTableIterValueCodec i) decIterKey :: MonadThrow m => RocksDbTableIter k v -> B.ByteString -> m k decIterKey it k = _codecDecode (_rocksDbTableIterKeyCodec it) (B.drop (B.length prefix) k) where prefix = namespaceFirst $ _rocksDbTableIterNamespace it -{-# INLINE decIterKey #-} checked :: HasCallStack => String -> (Ptr CString -> IO a) -> IO a checked whatWasIDoing act = alloca $ \errPtr -> do @@ -811,7 +631,7 @@ checked whatWasIDoing act = alloca $ \errPtr -> do err <- peek errPtr unless (err == nullPtr) $ do errStr <- B.packCString err - let msg = unwords ["Data.CAS.RocksDB.checked: error while", whatWasIDoing <> ":", B8.unpack errStr] + let msg = unwords ["Chainweb.Storage.Table.RocksDB.checked: error while", whatWasIDoing <> ":", B8.unpack errStr] C.c_rocksdb_free err error msg return r @@ -833,7 +653,7 @@ checkpointRocksDb RocksDb { _rocksDbHandle = R.DB dbPtr } logSizeFlushThreshold validateRangeOrdered :: HasCallStack => RocksDbTable k v -> (Maybe k, Maybe k) -> (B.ByteString, B.ByteString) validateRangeOrdered table (Just (encKey table -> l), Just (encKey table -> u)) | l >= u = - error "Data.CAS.RocksDB.validateRangeOrdered: range bounds not ordered according to codec" + error "Chainweb.Storage.Table.RocksDB.validateRangeOrdered: range bounds not ordered according to codec" | otherwise = (l, u) validateRangeOrdered table (l, u) = ( maybe (namespaceFirst (_rocksDbTableNamespace table)) (encKey table) l @@ -849,7 +669,7 @@ deleteRangeRocksDb table range = do R.withCWriteOpts R.defaultWriteOptions $ \optsPtr -> BU.unsafeUseAsCStringLen (fst range') $ \(minKeyPtr, minKeyLen) -> BU.unsafeUseAsCStringLen (snd range') $ \(maxKeyPtr, maxKeyLen) -> - checked "Data.CAS.RocksDB.deleteRangeRocksDb" $ + checked "Chainweb.Storage.Table.RocksDB.deleteRangeRocksDb" $ C.rocksdb_delete_range dbPtr optsPtr minKeyPtr (fromIntegral minKeyLen :: CSize) maxKeyPtr (fromIntegral maxKeyLen :: CSize) @@ -867,11 +687,11 @@ compactRangeRocksDb table range = -- | Read a value by key. -- One less copy than the version in rocksdb-haskell by using unsafePackCStringFinalizer. -get :: MonadIO m => R.DB -> R.ReadOptions -> ByteString -> m (Maybe ByteString) -get (R.DB db_ptr) opts key = liftIO $ R.withReadOptions opts $ \opts_ptr -> +getRocksDb :: MonadIO m => R.DB -> R.ReadOptions -> ByteString -> m (Maybe ByteString) +getRocksDb (R.DB db_ptr) opts key = liftIO $ R.withReadOptions opts $ \opts_ptr -> BU.unsafeUseAsCStringLen key $ \(key_ptr, klen) -> alloca $ \vlen_ptr -> do - val_ptr <- checked "Data.CAS.RocksDB.get" $ + val_ptr <- checked "Chainweb.Storage.Table.RocksDB.get" $ C.c_rocksdb_get db_ptr opts_ptr key_ptr (R.intToCSize klen) vlen_ptr vlen <- peek vlen_ptr if val_ptr == nullPtr @@ -898,8 +718,8 @@ multiGet :: MonadIO m => R.DB -> R.ReadOptions - -> V.Vector ByteString - -> m (V.Vector (Either String (Maybe ByteString))) + -> [ByteString] + -> m [Either String (Maybe ByteString)] multiGet (R.DB db_ptr) opts keys = liftIO $ R.withReadOptions opts $ \opts_ptr -> allocaArray len $ \keysArray -> allocaArray len $ \keySizesArray -> @@ -917,21 +737,22 @@ multiGet (R.DB db_ptr) opts keys = liftIO $ R.withReadOptions opts $ \opts_ptr - keysArray keySizesArray valuesArray valueSizesArray errsArray - V.generateM len $ \i -> do + forM [0..len-1] $ \i -> do valuePtr <- peekElemOff valuesArray i if valuePtr /= nullPtr then do valueLen <- R.cSizeToInt <$> peekElemOff valueSizesArray i - r <- BU.unsafePackMallocCStringLen (valuePtr, valueLen) + r <- BU.unsafePackCStringFinalizer (castPtr valuePtr) valueLen (C.c_rocksdb_free valuePtr) return $ Right $ Just r else do errPtr <- peekElemOff errsArray i if errPtr /= nullPtr then do - err <- B8.unpack <$> BU.unsafePackMallocCString errPtr + errLen <- BI.c_strlen errPtr + err <- B8.unpack <$> BU.unsafePackCStringFinalizer (castPtr errPtr) (fromIntegral errLen :: Int) (C.c_rocksdb_free errPtr) return $ Left err else return $ Right Nothing - in go 0 $ V.toList keys + in go 0 keys where - len = V.length keys + len = length keys diff --git a/vendored/chainweb-storage/src/Data/CAS.hs b/vendored/chainweb-storage/src/Data/CAS.hs deleted file mode 100644 index 9ebf4fdbd..000000000 --- a/vendored/chainweb-storage/src/Data/CAS.hs +++ /dev/null @@ -1,137 +0,0 @@ -{-# LANGUAGE BangPatterns #-} -{-# LANGUAGE ConstraintKinds #-} -{-# LANGUAGE DeriveAnyClass #-} -{-# LANGUAGE DeriveGeneric #-} -{-# LANGUAGE DerivingStrategies #-} -{-# LANGUAGE FlexibleContexts #-} -{-# LANGUAGE GADTs #-} -{-# LANGUAGE GeneralizedNewtypeDeriving #-} -{-# LANGUAGE LambdaCase #-} -{-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE TypeFamilies #-} --- | --- Module: Data.CAS --- Copyright: Copyright © 2019 Kadena LLC. --- License: MIT --- Maintainer: Lars Kuhtz --- Stability: experimental --- Description: Content Addressed Key Value Store (CAS) --- --- API for Content-Addressable Stores (CAS) --- -module Data.CAS -( IsCasValue(..) -, HasCasLookup(..) -, IsCas(..) -, casLookupM -, HasCasLookupConstraint -, CasConstraint -) where - -import Control.Exception (Exception, SomeException) -import Control.Monad.Catch (throwM) - -import Data.Foldable -import Data.Kind -import Data.Maybe -import Data.Text (Text) -import qualified Data.Vector as V - -import GHC.Generics - --- | The class of content-addressable values. --- --- The casKey function must be morally injective: --- --- prop> casKey a /= casKey b || a == b --- --- Usually, 'casKey' is a cryptographic, i.e. collision resistant, hash --- function. --- -class Eq (CasKeyType v) => IsCasValue v where - type CasKeyType v - casKey :: v -> CasKeyType v - --- | Read-Only View of a Content Addressed Key-Value Store --- --- Since the key uniquely determines the content of the store a value for a key --- is either available or not available. There is no dispute about the value --- itself. --- -class IsCasValue (CasValueType a) => HasCasLookup a where - type CasValueType a :: Type - - -- | Lookup a value in a content-addressable store - -- - casLookup :: a -> CasKeyType (CasValueType a) -> IO (Maybe (CasValueType a)) - - -- | Lookup a batch of values in a content-addressable store - -- - casLookupBatch :: a -> V.Vector (CasKeyType (CasValueType a)) -> IO (V.Vector (Maybe (CasValueType a))) - casLookupBatch = traverse . casLookup - {-# INLINE casLookupBatch #-} - - -- | Check for the existence of a value in a content addressable store - -- - casMember :: a -> CasKeyType (CasValueType a) -> IO Bool - casMember db = fmap isJust . casLookup db - {-# INLINE casMember #-} - - --- | Content Addressed Key-Value Stores --- --- Since the key uniquely determines the content of the store a value for a key --- is either available or not available. There is no dispute about the value --- itself. Thus there are only 'casInsert' and 'casDelete' functions but there --- is no @casUpdate@ function. --- -class HasCasLookup a => IsCas a where - - -- | Insert a value into a content-addressasble store - -- - casInsert :: a -> CasValueType a -> IO () - - -- | Delete a value from a content-addressable store - -- - casDelete :: a -> CasKeyType (CasValueType a) -> IO () - - -- | Insert a batch of values into a content-addressasble store - -- - casInsertBatch :: a -> V.Vector (CasValueType a) -> IO () - casInsertBatch = traverse_ . casInsert - {-# INLINE casInsertBatch #-} - - -- | Delete a batch of values from a content-addressable store - -- - casDeleteBatch :: a -> V.Vector (CasKeyType (CasValueType a)) -> IO () - casDeleteBatch = traverse_ . casDelete - {-# INLINE casDeleteBatch #-} - --- | Lookup a value by its key in a content-addressable store and throw an --- 'CasException' if the value doesn't exist in the store --- -casLookupM - :: HasCasLookup a - => a -> CasKeyType (CasValueType a) -> IO (CasValueType a) -casLookupM cas k = casLookup cas k >>= \case - Nothing -> throwM . CasException $ - "casLookupM: lookup failed for cas key" - (Just !x) -> return x - --- | Exceptions that are thrown by instances of 'IsCas'. --- -data CasException = CasException Text | CasImplementationException SomeException - deriving (Show, Generic) - -instance Exception CasException - --- | @HasCasLookupConstraint cas x@ asserts that @cas x@ is an instance if --- 'HasCasLookup' with value type 'x'. --- -type HasCasLookupConstraint cas x = (HasCasLookup (cas x), CasValueType (cas x) ~ x) - --- | @CasConstraint cas x@ asserts that @cas x@ is an instance if 'IsCas' with --- value type 'x'. --- -type CasConstraint cas x = (HasCasLookupConstraint cas x, IsCas (cas x)) - diff --git a/vendored/chainweb-storage/src/Data/CAS/Forgetful.hs b/vendored/chainweb-storage/src/Data/CAS/Forgetful.hs deleted file mode 100644 index 09432bfd4..000000000 --- a/vendored/chainweb-storage/src/Data/CAS/Forgetful.hs +++ /dev/null @@ -1,32 +0,0 @@ -{-# LANGUAGE TypeFamilies #-} - --- | --- Module: Data.CAS.Forgetful --- Copyright: Copyright © 2019 Kadena LLC. --- License: MIT --- Maintainer: Lars Kuhtz --- Stability: experimental --- --- A content addressable store that forgets everything that is stored in it. --- -module Data.CAS.Forgetful -( ForgetfulCas(..) -) where - -import Data.CAS - --- | A content addressable store that forgets everything that is added to it. --- -data ForgetfulCas a = ForgetfulCas - -instance IsCasValue a => HasCasLookup (ForgetfulCas a) where - type CasValueType (ForgetfulCas a) = a - casLookup _ _ = return Nothing - {-# INLINE casLookup #-} - -instance IsCasValue a => IsCas (ForgetfulCas a) where - casInsert _ _ = return () - casDelete _ _ = return () - {-# INLINE casInsert #-} - {-# INLINE casDelete #-} - diff --git a/vendored/chainweb-storage/src/Data/CAS/HashMap.hs b/vendored/chainweb-storage/src/Data/CAS/HashMap.hs deleted file mode 100644 index 2c7b4eabf..000000000 --- a/vendored/chainweb-storage/src/Data/CAS/HashMap.hs +++ /dev/null @@ -1,68 +0,0 @@ -{-# LANGUAGE ExistentialQuantification #-} -{-# LANGUAGE FlexibleContexts #-} -{-# LANGUAGE LambdaCase #-} -{-# LANGUAGE TypeFamilies #-} -{-# LANGUAGE UndecidableInstances #-} - --- | --- Module: Data.CAS.HashMap --- Copyright: Copyright © 2019 Kadena LLC. --- License: MIT --- Maintainer: Lars Kuhtz --- Stability: experimental --- Description: --- --- A thread-safe in-memory 'IsCas' implementation based on 'HM.HashMap'. --- -module Data.CAS.HashMap -( HashMapCas -, emptyCas -, toList -, size -) where - -import Control.Concurrent.STM.TVar -import Control.Monad ((<$!>)) -import Control.Monad.STM - -import Data.Hashable -import qualified Data.HashMap.Strict as HM - --- internal modules - -import Data.CAS - --- | An 'IsCas' implementation that is base on 'HM.HashMap'. --- -data HashMapCas v = IsCasValue v => HashMapCas !(TVar (HM.HashMap (CasKeyType v) v)) - -instance (Show (CasKeyType v), Hashable (CasKeyType v), IsCasValue v) => HasCasLookup (HashMapCas v) where - type CasValueType (HashMapCas v) = v - casLookup (HashMapCas var) k = HM.lookup k <$!> readTVarIO var - {-# INLINE casLookup #-} - -instance (Show (CasKeyType v), Hashable (CasKeyType v), IsCasValue v) => IsCas (HashMapCas v) where - casInsert cas@(HashMapCas var) a = casLookup cas (casKey a) >>= \case - Just _ -> return () - Nothing -> atomically $ modifyTVar' var $ HM.insert (casKey a) a - casDelete cas@(HashMapCas var) k = casLookup cas k >>= \case - Nothing -> return () - Just _ -> atomically $ modifyTVar' var $ HM.delete k - {-# INLINE casInsert #-} - {-# INLINE casDelete #-} - --- | Create new empty CAS --- -emptyCas :: Hashable (CasKeyType v) => IsCasValue v => IO (HashMapCas v) -emptyCas = HashMapCas <$> newTVarIO mempty - --- | Return all entries of CAS as List --- -toList :: HashMapCas v -> IO [v] -toList (HashMapCas var) = HM.elems <$!> readTVarIO var - --- | The number of items in the CAS --- -size :: HashMapCas v -> IO Int -size (HashMapCas var) = HM.size <$!> readTVarIO var - diff --git a/vendored/chainweb-storage/test/Main.hs b/vendored/chainweb-storage/test/Main.hs index 5331cec28..eeb536b2d 100644 --- a/vendored/chainweb-storage/test/Main.hs +++ b/vendored/chainweb-storage/test/Main.hs @@ -1,9 +1,7 @@ {-# LANGUAGE BangPatterns #-} -{-# LANGUAGE OverloadedLists #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE TypeApplications #-} {-# LANGUAGE TypeFamilies #-} - {-# OPTIONS_GHC -fno-warn-orphans #-} -- | @@ -13,57 +11,71 @@ -- Maintainer: Lars Kuhtz -- Stability: experimental -- --- Tests for "Data.CAS.RocksDB" --- +-- Tests for "Chainweb.Storage.RocksDB" module Main -( main -) where + ( main, + ) +where + +-- internal modules +import Chainweb.Storage.Table +import Chainweb.Storage.Table.RocksDB import Control.Concurrent.Async import Control.Exception import Control.Lens import Control.Monad import Control.Monad.Catch - import qualified Data.ByteString.Char8 as B8 import Data.Foldable import Data.List - import GHC.Stack - import NoThunks.Class - import Text.Read -import qualified Data.Vector as V - --- internal modules - -import Data.CAS -import Data.CAS.RocksDB - -- -------------------------------------------------------------------------- -- -- Utils data RocksDbTableTestException - = CodecException String - | RocksDbTableTestFailure String - deriving (Show) + = CodecException !String + | RocksDbTableTestFailure !String + +instance Show RocksDbTableTestException where + show (CodecException str) = + unlines + [ "Codec exception" + , str + ] + show (RocksDbTableTestFailure str) = + unlines + [ "RocksDb table test failure" + , str + ] instance Exception RocksDbTableTestException assertIO :: HasCallStack => Eq a => Show a => IO a -> a -> IO () -assertIO f r = f >>= \a -> unless (a == r) $ throwM - $ RocksDbTableTestFailure - $ "test failed:\n expected: " <> show r <> "\n actual: " <> show a - <> " " <> prettyCallStack callStack +assertIO f r = + f >>= \a -> + unless (a == r) $ + throwM $ + RocksDbTableTestFailure $ unlines + [ "test failed:" + , unwords ["expected:", show r] + , unwords ["actual:", show a] + , prettyCallStack callStack + ] assertNoThunks :: HasCallStack => NoThunks a => a -> IO () -assertNoThunks !a = case unsafeNoThunks $! a of - Nothing -> return () - Just e -> throwM $ RocksDbTableTestFailure - $ "test failed:\n unexpected thunk: " <> show e - <> " " <> prettyCallStack callStack +assertNoThunks a = case unsafeNoThunks $! a of + Nothing -> return () + Just e -> + throwM $ + RocksDbTableTestFailure $ unlines + [ "test failed:" + , unwords ["unexpected thunk:", show e] + , prettyCallStack callStack + ] -- -------------------------------------------------------------------------- -- -- Test Table @@ -88,35 +100,34 @@ assertEmptyTable t = do assertEntries :: HasCallStack => RocksDbTable Int Int -> [(Int, Int)] -> IO () assertEntries t l_ = do assertNoThunks t - forM_ l $ \(k,v) -> assertIO (tableLookup t k) (Just v) - assertIO (tableLookupBatch t (V.fromList ks)) (Just <$> V.fromList vs) + forM_ l $ \(k, v) -> assertIO (tableLookup t k) (Just v) + assertIO (tableLookupBatch t ks) (Just <$> vs) - assertIO (tableMinKey t) (firstOf (folded._1) l) - assertIO (tableMinValue t) (firstOf (folded._2) l) + assertIO (tableMinKey t) (firstOf (folded . _1) l) + assertIO (tableMinValue t) (firstOf (folded . _2) l) - assertIO (tableMaxKey t) (lastOf (folded._1) l) - assertIO (tableMaxValue t) (lastOf (folded._2) l) + assertIO (tableMaxKey t) (lastOf (folded . _1) l) + assertIO (tableMaxValue t) (lastOf (folded . _2) l) -- check forward iteration and first and last - withTableIter t $ \i -> do - assertIO (tableIterFirst i >> tableIterKey i) (firstOf (folded._1) l) - assertIO (tableIterLast i >> tableIterKey i) (lastOf (folded._1) l) - - tableIterFirst i - assertIO (tableIterValid i) (not $ null l) - forM_ l $ \(k,v) -> do - assertIO (tableIterEntry i) (Just (k,v)) - tableIterNext i - assertIO (tableIterValid i) False - - -- check backward iteration - withTableIter t $ \i -> do - tableIterLast i - assertIO (tableIterValid i) (not $ null l) - forM_ (reverse l) $ \(k,v) -> do - assertIO (tableIterEntry i) (Just (k,v)) - tableIterPrev i - assertIO (tableIterValid i) False + withTableIterator t $ \i -> do + assertIO (iterFirst i >> iterKey i) (firstOf (folded . _1) l) + assertIO (iterLast i >> iterKey i) (lastOf (folded . _1) l) + iterFirst i + assertIO (iterValid i) (not $ null l) + forM_ l $ \(k, v) -> do + assertIO (iterEntry i) (Just (Entry k v)) + iterNext i + assertIO (iterValid i) False + + -- check backward iteration + withTableIterator t $ \i -> do + iterLast i + assertIO (iterValid i) (not $ null l) + forM_ (reverse l) $ \(k, v) -> do + assertIO (iterEntry i) (Just (Entry k v)) + iterPrev i + assertIO (iterValid i) False where l = sort l_ (ks, vs) = unzip l @@ -126,19 +137,19 @@ tableTests db tableName = do assertNoThunks t assertEmptyTable t - tableInsert t 1 8 - assertEntries t [(1,8)] + tableInsert t 1 8 + assertEntries t [(1, 8)] - tableInsert t 2 9 - assertEntries t [(1,8), (2,9)] + tableInsert t 2 9 + assertEntries t [(1, 8), (2, 9)] - tableDelete t 1 - assertEntries t [(2,9)] + tableDelete t 1 + assertEntries t [(2, 9)] - tableInsert t 2 8 - assertEntries t [(2,8)] + tableInsert t 2 8 + assertEntries t [(2, 8)] - tableDelete t 2 + tableDelete t 2 assertEmptyTable t where !t = intTable db tableName @@ -151,81 +162,79 @@ tableBatchTests db tableName = do updateBatch [] assertEmptyTable t - updateBatch [ RocksDbInsert t 1 8] - assertEntries t [(1,8)] + updateBatch [RocksDbInsert t 1 8] + assertEntries t [(1, 8)] updateBatch [RocksDbInsert t 2 9] - assertEntries t [(1,8), (2,9)] + assertEntries t [(1, 8), (2, 9)] updateBatch [RocksDbDelete t 2] - assertEntries t [(1,8)] + assertEntries t [(1, 8)] updateBatch [RocksDbInsert t 2 9, RocksDbDelete t 2] - assertEntries t [(1,8)] + assertEntries t [(1, 8)] updateBatch [RocksDbInsert t 2 9, RocksDbDelete t 2, RocksDbInsert t 2 9] - assertEntries t [(1,8), (2,9)] + assertEntries t [(1, 8), (2, 9)] updateBatch [RocksDbInsert t 1 8, RocksDbDelete t 1] - assertEntries t [(2,9)] + assertEntries t [(2, 9)] updateBatch [RocksDbInsert t 1 7, RocksDbInsert t 1 8, RocksDbInsert t 1 8] - assertEntries t [(1,8), (2,9)] + assertEntries t [(1, 8), (2, 9)] updateBatch [RocksDbDelete t 1, RocksDbInsert t 3 7] - assertEntries t [(2,9), (3,7)] + assertEntries t [(2, 9), (3, 7)] updateBatch [RocksDbInsert t 4 6, RocksDbInsert t 5 5] - assertEntries t [(2,9), (3,7), (4,6), (5,5)] + assertEntries t [(2, 9), (3, 7), (4, 6), (5, 5)] updateBatch [RocksDbDelete t 2, RocksDbDelete t 3, RocksDbDelete t 4, RocksDbDelete t 5] assertEmptyTable t where t = intTable db tableName - -- Orphan instance -- instance IsCasValue Int where type CasKeyType Int = Int - casKey = (+10) - {-# INLINE casKey #-} + casKey = (+ 10) casBatchTests :: HasCallStack => RocksDb -> B8.ByteString -> IO () casBatchTests db tableName = do assertEmptyTable t - casInsertBatch t mempty + tableInsertBatch t mempty assertEmptyTable t - casInsertBatch t [1] + casInsertBatch t [1] assertCasEntries t [1] - casInsertBatch t [2] + casInsertBatch t [2] assertCasEntries t [1, 2] - casDeleteBatch t [casKey @Int 2] + casDeleteBatch t [2] assertCasEntries t [1] - casInsertBatch t [1] + casInsertBatch t [1] assertCasEntries t [1] - casInsertBatch t [2, 2, 2] + casInsertBatch t [2, 2, 2] assertCasEntries t [1, 2] - casInsertBatch t [1,2,3,4] + casInsertBatch t [1, 2, 3, 4] assertCasEntries t [1, 2, 3, 4] - casDeleteBatch t [casKey @Int 5] + casDeleteBatch t [5] assertCasEntries t [1, 2, 3, 4] - casDeleteBatch t $ casKey @Int <$> [1, 3, 1] + casDeleteBatch t [1, 3, 1] assertCasEntries t [2, 4] - casDeleteBatch t [] + casDeleteBatch t [] assertCasEntries t [2, 4] - casDeleteBatch t $ casKey @Int <$> [2, 4] + casDeleteBatch t [2, 4] assertEmptyTable t where t = intTable db tableName @@ -233,32 +242,32 @@ casBatchTests db tableName = do casTests :: HasCallStack => RocksDb -> B8.ByteString -> IO () casTests db tableName = do assertEmptyTable t - assertIO (casMember t 1) False - assertIO (casLookup t 1) Nothing + assertIO (tableMember t 1) False + assertIO (tableLookup t 1) Nothing - casInsertBatch t mempty + casInsertBatch t mempty assertEmptyTable t - casInsert t 1 + casInsert t 1 assertCasEntries t [1] - assertIO (casMember t $ casKey @Int 1) True - assertIO (casLookup t $ casKey @Int 1) (Just 1) + assertIO (tableMember t (casKey @Int 1)) True + assertIO (tableLookup t (casKey @Int 1)) (Just 1) - casInsert t 2 + casInsert t 2 assertCasEntries t [1, 2] - assertIO (casMember t $ casKey @Int 1) True - assertIO (casMember t $ casKey @Int 2) True - assertIO (casLookup t $ casKey @Int 1) (Just 1) - assertIO (casLookup t $ casKey @Int 2) (Just 2) - assertIO (casLookupBatch t [casKey @Int 1, casKey @Int 2]) [Just 1, Just 2] + assertIO (tableMember t (casKey @Int 1)) True + assertIO (tableMember t (casKey @Int 2)) True + assertIO (tableLookup t (casKey @Int 1)) (Just 1) + assertIO (tableLookup t (casKey @Int 2)) (Just 2) + assertIO (tableLookupBatch t [casKey @Int 1, casKey @Int 2]) [Just 1, Just 2] - casDelete t $ casKey @Int 2 + casDelete t 2 assertCasEntries t [1] - assertIO (casMember t $ casKey @Int 1) True - assertIO (casMember t $ casKey @Int 2) False - assertIO (casLookup t $ casKey @Int 1) (Just 1) - assertIO (casLookup t $ casKey @Int 2) Nothing - assertIO (casLookupBatch t [casKey @Int 1, casKey @Int 2]) [Just 1, Nothing] + assertIO (tableMember t (casKey @Int 1)) True + assertIO (tableMember t (casKey @Int 2)) False + assertIO (tableLookup t (casKey @Int 1)) (Just 1) + assertIO (tableLookup t (casKey @Int 2)) Nothing + assertIO (tableLookupBatch t [casKey @Int 1, casKey @Int 2]) [Just 1, Nothing] casInsert t 1 assertCasEntries t [1] @@ -266,16 +275,16 @@ casTests db tableName = do traverse_ @[] (casInsert t) [2, 2, 2] assertCasEntries t [1, 2] - traverse_ @[] (casInsert t) [1,2,3,4] + traverse_ @[] (casInsert t) [1, 2, 3, 4] assertCasEntries t [1, 2, 3, 4] - casDelete t $ casKey @Int 5 + casDelete t (casKey @Int 5) assertCasEntries t [1, 2, 3, 4] - traverse_ @[] (casDelete t) $ casKey @Int <$> [1, 3, 1] + traverse_ @[] (casDelete t) [1, 3, 1] assertCasEntries t [2, 4] - traverse_ @[] (casDelete t) $ casKey @Int <$> [2, 4] + traverse_ @[] (casDelete t) [2, 4] assertEmptyTable t where t = intTable db tableName @@ -283,7 +292,7 @@ casTests db tableName = do assertCasEntries :: HasCallStack => RocksDbTable Int Int -> [Int] -> IO () assertCasEntries t l = do assertNoThunks t - assertEntries t $ (\x -> (casKey x, x)) <$> l + assertEntries t [(casKey v, v) | v <- l] -- -------------------------------------------------------------------------- -- -- Main @@ -292,8 +301,7 @@ main :: IO () main = withTempRocksDb "testDb" $ \db -> do mapConcurrently_ (\i -> tableTests db $ "testTable" <> B8.pack (show i)) - ([0..100] :: [Int]) + ([0 .. 100] :: [Int]) tableBatchTests db "testTable" casTests db "testTable" casBatchTests db "testTable" -