address comments

Makefile

@@ -58,14 +58,14 @@ test-e2e: export OASIS_INDEXER_E2E = true
 test-e2e:
 	@$(GO) test -race -coverpkg=./... -coverprofile=coverage.txt -covermode=atomic -v ./tests/e2e
 
-dump-state: oasis-indexer
-	sed -E -i='' 's/query_on_cache_miss: false/query_on_cache_miss: true/g' "tests/e2e_regression/e2e_config.yml"
-	./oasis-indexer --config tests/e2e_regression/e2e_config.yml analyze
+fill-cache-for-e2e-regression: oasis-indexer
+	cp tests/e2e_regression/e2e_config.yml /tmp/indexer_fill_e2e_regression_cache.yml
+	sed -E -i='' 's/query_on_cache_miss: false/query_on_cache_miss: true/g' /tmp/indexer_fill_e2e_regression_cache.yml
+	./oasis-indexer --config /tmp/indexer_fill_e2e_regression_cache.yml analyze
 
 # Run the api tests locally, assuming the environment is set up with an oasis-node that is 
 # accessible as specified in the config file.
-test-api: oasis-indexer
-	sed -E -i='' 's/query_on_cache_miss: true/query_on_cache_miss: false/g' "tests/e2e_regression/e2e_config.yml"
+test-e2e-regression: oasis-indexer
 	./oasis-indexer --config tests/e2e_regression/e2e_config.yml analyze
 	@$(ECHO) "$(CYAN)*** Indexer finished; starting api tests...$(OFF)"
 	./tests/e2e_regression/run.sh

config/config.go

@@ -118,7 +118,7 @@ type AnalyzersList struct {
 // SourceConfig has some controls about what chain we're analyzing and how to connect.
 type SourceConfig struct {
 	// Cache holds the configuration for a file-based caching backend.
-	Cache *CacheList `koanf:"cache"`
+	Cache *CacheConfig `koanf:"cache"`
 
 	// ChainName is the name of the chain (e.g. mainnet/testnet). Set
 	// this to use one of the default chains.
@@ -171,14 +171,6 @@ func SingleNetworkLookup(rpc string) map[string]*NodeConfig {
 	}
 }
 
-type CacheList struct {
-	// Consensus holds the configuration for the consensus cache
-	Consensus *CacheConfig `koanf:"consensus"`
-
-	// Runtime holds the configuration for the runtime cache
-	Runtime *CacheConfig `koanf:"runtime"`
-}
-
 type CacheConfig struct {
 	// CacheDir is the directory where the cache data is stored
 	CacheDir string `koanf:"cache_dir"`

storage/oasis/client.go

@@ -5,6 +5,7 @@ package oasis
 import (
 	"context"
 	"fmt"
+	"os"
 
 	"github.com/oasisprotocol/oasis-indexer/analyzer"
 	"github.com/oasisprotocol/oasis-indexer/config"
@@ -21,8 +22,12 @@ const (
 // NewConsensusClient creates a new ConsensusClient.
 func NewConsensusClient(ctx context.Context, sourceConfig *config.SourceConfig) (*ConsensusClient, error) {
 	// If we are using purely file-backed indexer, do not connect to the node.
-	if sourceConfig.Cache != nil && sourceConfig.Cache.Consensus != nil && !sourceConfig.Cache.Consensus.QueryOnCacheMiss {
-		nodeApi, err := file.NewFileConsensusApiLite(sourceConfig.Cache.Consensus.CacheDir, nil)
+	if sourceConfig.Cache != nil && !sourceConfig.Cache.QueryOnCacheMiss {
+		cachePath := sourceConfig.Cache.CacheDir + "/consensus"
+		if err := os.MkdirAll(cachePath, os.ModeDir); err != nil {
+			return nil, fmt.Errorf("error creating cache dir for consensusApi: %w", err)
+		}
+		nodeApi, err := file.NewFileConsensusApiLite(cachePath, nil)
 		if err != nil {
 			return nil, fmt.Errorf("error instantiating cache-based consensusApi: %w", err)
 		}
@@ -31,13 +36,19 @@ func NewConsensusClient(ctx context.Context, sourceConfig *config.SourceConfig)
 			network: sourceConfig.SDKNetwork(),
 		}, nil
 	}
+
+	// Create an API that connects to the real node, then wrap it in a caching layer.
 	var nodeApi nodeapi.ConsensusApiLite
 	nodeApi, err := history.NewHistoryConsensusApiLite(ctx, sourceConfig.History(), sourceConfig.Nodes, sourceConfig.FastStartup)
 	if err != nil {
 		return nil, fmt.Errorf("instantiating history consensus API lite: %w", err)
 	}
-	if sourceConfig.Cache != nil && sourceConfig.Cache.Consensus != nil {
-		nodeApi, err = file.NewFileConsensusApiLite(sourceConfig.Cache.Consensus.CacheDir, nodeApi)
+	if sourceConfig.Cache != nil {
+		cachePath := sourceConfig.Cache.CacheDir + "/consensus"
+		if err := os.MkdirAll(cachePath, os.ModeDir); err != nil {
+			return nil, fmt.Errorf("error creating cache dir for consensusApi: %w", err)
+		}
+		nodeApi, err = file.NewFileConsensusApiLite(cachePath, nodeApi)
 		if err != nil {
 			return nil, fmt.Errorf("error instantiating cache-based consensusApi: %w", err)
 		}
@@ -72,12 +83,15 @@ func NewRuntimeClient(ctx context.Context, sourceConfig *config.SourceConfig, ru
 	// to the node at all. this requires storing runtime info offline.
 	var nodeApi nodeapi.RuntimeApiLite
 	nodeApi = nodeapi.NewUniversalRuntimeApiLite(info.ID, rawConn, &sdkClient)
-	if sourceConfig.Cache != nil && sourceConfig.Cache.Runtime != nil {
-		cacheConfig := sourceConfig.Cache.Runtime
-		if cacheConfig.QueryOnCacheMiss {
-			nodeApi, err = file.NewFileRuntimeApiLite(runtime.String(), cacheConfig.CacheDir, nodeApi)
+	if sourceConfig.Cache != nil {
+		cachePath := sourceConfig.Cache.CacheDir + runtime.String()
+		if err := os.MkdirAll(cachePath, os.ModeDir); err != nil {
+			return nil, fmt.Errorf("error creating cache dir for runtimeApi: %w", err)
+		}
+		if sourceConfig.Cache.QueryOnCacheMiss {
+			nodeApi, err = file.NewFileRuntimeApiLite(runtime.String(), cachePath, nodeApi)
 		} else {
-			nodeApi, err = file.NewFileRuntimeApiLite(runtime.String(), cacheConfig.CacheDir, nil)
+			nodeApi, err = file.NewFileRuntimeApiLite(runtime.String(), cachePath, nil)
 		}
 		if err != nil {
 			return nil, fmt.Errorf("error instantiating cache-based runtimeApi: %w", err)

tests/e2e_regression/e2e_config.yml

@@ -1,12 +1,8 @@
 analysis:
   source:
     cache:
-      consensus:
-        cache_dir: tests/e2e_regression/data/consensus
-        query_on_cache_miss: true
-      runtime:
-        cache_dir: tests/e2e_regression/data/runtime
-        query_on_cache_miss: true
+      cache_dir: tests/e2e_regression/data
+      query_on_cache_miss: true
     chain_name: mainnet
     nodes:
       damask: