Add hints to port conflict and lock file panics (#1535)

* add hint for port error
* add issue filter for port panic
* add lock file hint
* add metrics endpoint port conflict hint
* add hint for tracing endpoint port conflict
* add acceptance test for resource conflics
* Split out common conflict test code into a function
* Add state, metrics, and tracing conflict tests

* Add a full set of stderr acceptance test functions

This change makes the stdout and stderr acceptance test interfaces
identical.

* move Zcash listener opening
* add todo about hint for disk full
* add constant for lock file
* match path in state cache
* don't match windows cache path

* Use Display for state path logs

Avoids weird escaping on Windows when using Debug

* Add Windows conflict error messages

* Turn PORT_IN_USE_ERROR into a regex

And add another alternative Windows-specific port error

Co-authored-by: teor <[email protected]>
Co-authored-by: Jane Lusby <[email protected]>

zebra-network/Cargo.toml

@@ -16,8 +16,10 @@ hex = "0.4"
 # indexmap has rayon support for parallel iteration,
 # which we don't use, so disable it to drop the dependencies.
 indexmap = { version = "1.6", default-features = false }
+lazy_static = "1.4.0"
 pin-project = "0.4"
 rand = "0.7"
+regex = "1"
 serde = { version = "1", features = ["serde_derive"] }
 thiserror = "1"
 

zebra-network/src/constants.rs

@@ -2,6 +2,9 @@
 
 use std::time::Duration;
 
+use lazy_static::lazy_static;
+use regex::Regex;
+
 // XXX should these constants be split into protocol also?
 use crate::protocol::external::types::*;
 
@@ -95,6 +98,16 @@ pub const EWMA_DEFAULT_RTT: Duration = Duration::from_secs(20 + 1);
 /// better peers when we restart the sync.
 pub const EWMA_DECAY_TIME: Duration = Duration::from_secs(200);
 
+lazy_static! {
+    /// OS-specific error when the port attempting to be opened is already in use.
+    pub static ref PORT_IN_USE_ERROR: Regex = if cfg!(unix) {
+        #[allow(clippy::trivial_regex)]
+        Regex::new("already in use")
+    } else {
+        Regex::new("(access a socket in a way forbidden by its access permissions)|(Only one usage of each socket address)")
+    }.expect("regex is valid");
+}
+
 /// Magic numbers used to identify different Zcash networks.
 pub mod magics {
     use super::*;

zebra-network/src/lib.rs

@@ -66,7 +66,7 @@ pub type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
 
 mod address_book;
 mod config;
-mod constants;
+pub mod constants;
 mod isolated;
 mod meta_addr;
 mod peer;

zebra-network/src/peer_set/initialize.rs

@@ -115,16 +115,7 @@ where
     );
     let peer_set = Buffer::new(BoxService::new(peer_set), constants::PEERSET_BUFFER_SIZE);
 
-    // Connect the tx end to the 3 peer sources:
-
-    // 1. Initial peers, specified in the config.
-    let add_guard = tokio::spawn(add_initial_peers(
-        config.initial_peers(),
-        connector.clone(),
-        peerset_tx.clone(),
-    ));
-
-    // 2. Incoming peer connections, via a listener.
+    // 1. Incoming peer connections, via a listener.
 
     // Warn if we're configured using the wrong network port.
     // TODO: use the right port if the port is unspecified
@@ -144,6 +135,18 @@ where
 
     let listen_guard = tokio::spawn(listen(config.listen_addr, listener, peerset_tx.clone()));
 
+    let initial_peers_fut = {
+        let initial_peers = config.initial_peers();
+        let connector = connector.clone();
+        let tx = peerset_tx.clone();
+
+        // Connect the tx end to the 3 peer sources:
+        add_initial_peers(initial_peers, connector, tx)
+    };
+
+    // 2. Initial peers, specified in the config.
+    let add_guard = tokio::spawn(initial_peers_fut);
+
     // 3. Outgoing peers we connect to in response to load.
     let mut candidates = CandidateSet::new(address_book.clone(), peer_set.clone());
 
@@ -211,7 +214,18 @@ where
     S: Service<(TcpStream, SocketAddr), Response = peer::Client, Error = BoxError> + Clone,
     S::Future: Send + 'static,
 {
-    let listener = TcpListener::bind(addr).await?;
+    let listener_result = TcpListener::bind(addr).await;
+
+    let listener = match listener_result {
+        Ok(l) => l,
+        Err(e) => panic!(
+            "Opening Zcash network protocol listener {:?} failed: {:?}. \
+             Hint: Check if another zebrad or zcashd process is running. \
+             Try changing the network listen_addr in the Zebra config.",
+            addr, e,
+        ),
+    };
+
     let local_addr = listener.local_addr()?;
     info!("Opened Zcash protocol endpoint at {}", local_addr);
     loop {

zebra-state/Cargo.toml

@@ -13,6 +13,7 @@ zebra-chain = { path = "../zebra-chain" }
 dirs = "3.0.1"
 hex = "0.4.2"
 lazy_static = "1.4.0"
+regex = "1"
 serde = { version = "1", features = ["serde_derive"] }
 
 futures = "0.3.12"

zebra-state/src/constants.rs

@@ -1,3 +1,5 @@
+//! Definitions of constants.
+
 /// The maturity threshold for transparent coinbase outputs.
 ///
 /// A transaction MUST NOT spend a transparent output of a coinbase transaction
@@ -13,3 +15,11 @@ pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRANSPARENT_COINBASE_MATURITY - 1;
 
 /// The database format version, incremented each time the database format changes.
 pub const DATABASE_FORMAT_VERSION: u32 = 4;
+
+use lazy_static::lazy_static;
+use regex::Regex;
+
+lazy_static! {
+    /// Regex that matches the RocksDB error when its lock file is already open.
+    pub static ref LOCK_FILE_ERROR: Regex = Regex::new("(lock file).*(temporarily unavailable)|(in use)|(being used by another process)").expect("regex is valid");
+}

zebra-state/src/lib.rs

@@ -16,7 +16,7 @@
 #![allow(clippy::unnecessary_wraps)]
 
 mod config;
-mod constants;
+pub mod constants;
 mod error;
 mod request;
 mod response;

zebra-state/src/service/finalized_state.rs

@@ -45,8 +45,21 @@ impl FinalizedState {
             rocksdb::ColumnFamilyDescriptor::new("sprout_nullifiers", db_options.clone()),
             rocksdb::ColumnFamilyDescriptor::new("sapling_nullifiers", db_options.clone()),
         ];
-        let db = rocksdb::DB::open_cf_descriptors(&db_options, path, column_families)
-            .expect("database path and options are valid");
+        let db_result = rocksdb::DB::open_cf_descriptors(&db_options, &path, column_families);
+
+        let db = match db_result {
+            Ok(d) => {
+                tracing::info!("Opened Zebra state cache at {}", path.display());
+                d
+            }
+            // TODO: provide a different hint if the disk is full, see #1623
+            Err(e) => panic!(
+                "Opening database {:?} failed: {:?}. \
+                 Hint: Check if another zebrad process is running. \
+                 Try changing the state cache_dir in the Zebra config.",
+                path, e,
+            ),
+        };
 
         let new_state = Self {
             queued_by_prev_hash: HashMap::new(),