Merge pull request #1785 from subspace/farm-resizing

Farm resizing

crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs

@@ -2,7 +2,7 @@ mod dsn;
 
 use crate::commands::farm::dsn::configure_dsn;
 use crate::commands::shared::print_disk_farm_info;
-use crate::utils::{get_required_plot_space_with_overhead, shutdown_signal};
+use crate::utils::shutdown_signal;
 use crate::{DiskFarm, FarmingArgs};
 use anyhow::{anyhow, Result};
 use futures::stream::FuturesUnordered;
@@ -187,17 +187,17 @@ where
         let single_disk_plot = match single_disk_plot_fut.await {
             Ok(single_disk_plot) => single_disk_plot,
             Err(SingleDiskPlotError::InsufficientAllocatedSpace {
-                min_size,
+                min_space,
                 allocated_space,
             }) => {
-                let minimum_plot_size = get_required_plot_space_with_overhead(min_size as u64);
-                let allocated_plotting_space_with_overhead =
-                    get_required_plot_space_with_overhead(allocated_space);
-
                 return Err(anyhow::anyhow!(
-                    "Plot size is too low ({} bytes). Minimum is {}",
-                    allocated_plotting_space_with_overhead,
-                    minimum_plot_size
+                    "Allocated space {} ({}) is not enough, minimum is ~{} (~{}, {} bytes to be \
+                    exact)",
+                    bytesize::to_string(allocated_space, true),
+                    bytesize::to_string(allocated_space, false),
+                    bytesize::to_string(min_space, true),
+                    bytesize::to_string(min_space, false),
+                    min_space
                 ));
             }
             Err(error) => {

crates/subspace-farmer/src/bin/subspace-farmer/main.rs

@@ -212,10 +212,9 @@ struct Command {
     ///
     ///   path=/path/to/directory,size=5T
     ///
-    /// `size` is max plot size in human readable format (e.g. 10GB, 2TiB) or just bytes.
-    /// TODO: Update overhead number here or account for it automatically
-    /// Note that `size` is how much data will be plotted, you also need to account for metadata,
-    /// which right now occupies up to 8% of the disk space.
+    /// `size` is max allocated size in human readable format (e.g. 10GB, 2TiB) or just bytes that
+    /// farmer will make sure not not exceed (and will pre-allocated all the space on startup to
+    /// ensure it will not run out of space in runtime).
     #[arg(long)]
     farm: Vec<DiskFarm>,
     /// Run temporary farmer with specified plot size in human readable format (e.g. 10GB, 2TiB) or

crates/subspace-farmer/src/bin/subspace-farmer/utils.rs

@@ -18,15 +18,6 @@ pub(crate) fn raise_fd_limit() {
         }
     }
 }
-
-pub(crate) const DB_OVERHEAD_PERCENT: u64 = 92;
-
-pub(crate) fn get_required_plot_space_with_overhead(allocated_space: u64) -> u64 {
-    // TODO: Should account for database overhead of various additional databases.
-    //  For now assume 92% will go for plot itself
-    allocated_space * 100 / DB_OVERHEAD_PERCENT
-}
-
 #[cfg(unix)]
 pub(crate) async fn shutdown_signal() {
     use futures::FutureExt;

crates/subspace-farmer/src/identity.rs

@@ -45,6 +45,14 @@ impl Deref for Identity {
 }
 
 impl Identity {
+    /// Size of the identity file on disk
+    pub fn file_size() -> usize {
+        IdentityFileContents {
+            entropy: vec![0; ENTROPY_LENGTH],
+        }
+        .encoded_size()
+    }
+
     /// Opens the existing identity, or creates a new one.
     pub fn open_or_create<B: AsRef<Path>>(base_directory: B) -> Result<Self, Error> {
         if let Some(identity) = Self::open(base_directory.as_ref())? {

crates/subspace-farmer/src/lib.rs

@@ -4,6 +4,7 @@
     hash_extract_if,
     impl_trait_in_assoc_type,
     io_error_other,
+    int_roundings,
     iter_collect_into,
     let_chains,
     trait_alias,

crates/subspace-farmer/src/single_disk_plot.rs

@@ -13,7 +13,6 @@ use crate::single_disk_plot::piece_reader::PieceReader;
 pub use crate::single_disk_plot::plotting::PlottingError;
 use crate::single_disk_plot::plotting::{plotting, plotting_scheduler};
 use crate::utils::JoinOnDrop;
-use bytesize::ByteSize;
 use derive_more::{Display, From};
 use event_listener_primitives::{Bag, HandlerId};
 use futures::channel::{mpsc, oneshot};
@@ -41,6 +40,7 @@ use subspace_farmer_components::file_ext::FileExt;
 use subspace_farmer_components::plotting::{PieceGetter, PlottedSector};
 use subspace_farmer_components::sector::{sector_size, SectorMetadataChecksummed};
 use subspace_farmer_components::FarmerProtocolInfo;
+use subspace_networking::NetworkingParametersManager;
 use subspace_proof_of_space::Table;
 use subspace_rpc_primitives::{FarmerAppInfo, SolutionResponse};
 use thiserror::Error;
@@ -55,6 +55,8 @@ const_assert!(std::mem::size_of::<usize>() >= std::mem::size_of::<u64>());
 
 /// Reserve 1M of space for plot metadata (for potential future expansion)
 const RESERVED_PLOT_METADATA: u64 = 1024 * 1024;
+/// Reserve 1M of space for plot info (for potential future expansion)
+const RESERVED_PLOT_INFO: u64 = 1024 * 1024;
 
 /// Semaphore that limits disk access concurrency in strategic places to the number specified during
 /// initialization
@@ -280,22 +282,12 @@ pub enum SingleDiskPlotError {
     /// Piece cache error
     #[error("Piece cache error: {0}")]
     PieceCacheError(#[from] DiskPieceCacheError),
+    /// Can't preallocate metadata file, probably not enough space on disk
+    #[error("Can't preallocate metadata file, probably not enough space on disk: {0}")]
+    CantPreallocateMetadataFile(io::Error),
     /// Can't preallocate plot file, probably not enough space on disk
     #[error("Can't preallocate plot file, probably not enough space on disk: {0}")]
     CantPreallocatePlotFile(io::Error),
-    /// Can't resize plot after creation
-    #[error(
-        "Usable plotting space of plot {id} {new_space} is different from {old_space} when plot \
-        was created, resizing isn't supported yet"
-    )]
-    CantResize {
-        /// Plot ID
-        id: SingleDiskPlotId,
-        /// Space allocated during plot creation
-        old_space: ByteSize,
-        /// New desired plot size
-        new_space: ByteSize,
-    },
     /// Wrong chain (genesis hash)
     #[error(
         "Genesis hash of plot {id} {wrong_chain} is different from {correct_chain} when plot was \
@@ -348,12 +340,12 @@ pub enum SingleDiskPlotError {
     /// Allocated space is not enough for one sector
     #[error(
         "Allocated space is not enough for one sector. \
-        The lowest acceptable value for allocated space is: {min_size}, \
-        you provided: {allocated_space}."
+        The lowest acceptable value for allocated space is {min_space} bytes, \
+        provided {allocated_space} bytes."
     )]
     InsufficientAllocatedSpace {
         /// Minimal allocated space
-        min_size: usize,
+        min_space: u64,
         /// Current allocated space
         allocated_space: u64,
     },
@@ -471,15 +463,7 @@ impl SingleDiskPlot {
         let public_key = identity.public_key().to_bytes().into();
 
         let single_disk_plot_info = match SingleDiskPlotInfo::load_from(&directory)? {
-            Some(single_disk_plot_info) => {
-                if allocated_space != single_disk_plot_info.allocated_space() {
-                    return Err(SingleDiskPlotError::CantResize {
-                        id: *single_disk_plot_info.id(),
-                        old_space: ByteSize::b(single_disk_plot_info.allocated_space()),
-                        new_space: ByteSize::b(allocated_space),
-                    });
-                }
-
+            Some(mut single_disk_plot_info) => {
                 if &farmer_app_info.genesis_hash != single_disk_plot_info.genesis_hash() {
                     return Err(SingleDiskPlotError::WrongChain {
                         id: *single_disk_plot_info.id(),
@@ -515,20 +499,27 @@ impl SingleDiskPlot {
                     );
                 }
 
+                if allocated_space != single_disk_plot_info.allocated_space() {
+                    info!(
+                        old_space = %bytesize::to_string(single_disk_plot_info.allocated_space(), true),
+                        new_space = %bytesize::to_string(allocated_space, true),
+                        "Farm size has changed"
+                    );
+
+                    {
+                        let new_allocated_space = allocated_space;
+                        let SingleDiskPlotInfo::V0 {
+                            allocated_space, ..
+                        } = &mut single_disk_plot_info;
+                        *allocated_space = new_allocated_space;
+                    }
+
+                    single_disk_plot_info.store_to(&directory)?;
+                }
+
                 single_disk_plot_info
             }
             None => {
-                let sector_size = sector_size(max_pieces_in_sector);
-
-                // TODO: Account for plot overhead
-                let target_sector_count = (allocated_space / sector_size as u64) as usize;
-                if target_sector_count == 0 {
-                    return Err(SingleDiskPlotError::InsufficientAllocatedSpace {
-                        min_size: sector_size,
-                        allocated_space,
-                    });
-                }
-
                 let single_disk_plot_info = SingleDiskPlotInfo::new(
                     SingleDiskPlotId::new(),
                     farmer_app_info.genesis_hash,
@@ -546,21 +537,47 @@ impl SingleDiskPlot {
         let pieces_in_sector = single_disk_plot_info.pieces_in_sector();
         let sector_size = sector_size(max_pieces_in_sector);
         let sector_metadata_size = SectorMetadataChecksummed::encoded_size();
-        // TODO: Split allocated space into more components once all components are deterministic
-        //  to correctly size everything:
-        //  * plot info
-        //  * identity
-        //  * metadata
-        //  * plot
-        //  * known_addresses
-        let cache_size = allocated_space / 100 * u64::from(cache_percentage.get());
-        // We have a hardcoded value decreasing allocated space to account for things like cache,
-        // don't change plot size (yet) if cache is under 5% that is assumed to be accounted for,
-        // this will be removed once we have proper static allocation as described in TODO above
-        let cache_size_exceeding_5_percents =
-            allocated_space / 100 * u64::from(cache_percentage.get()).saturating_sub(5);
-        let plot_space = (allocated_space - cache_size_exceeding_5_percents) / sector_size as u64;
-        let target_sector_count = plot_space;
+        let single_sector_overhead = (sector_size + sector_metadata_size) as u64;
+        // Fixed space usage regardless of plot size
+        let fixed_space_usage = RESERVED_PLOT_METADATA
+            + RESERVED_PLOT_INFO
+            + Identity::file_size() as u64
+            + NetworkingParametersManager::file_size() as u64;
+        // Calculate how many sectors can fit
+        let target_sector_count = {
+            let potentially_plottable_space = allocated_space.saturating_sub(fixed_space_usage)
+                / 100
+                * (100 - u64::from(cache_percentage.get()));
+            // Do the rounding to make sure we have exactly as much space as fits whole number of
+            // sectors
+            potentially_plottable_space / single_sector_overhead
+        };
+
+        if target_sector_count == 0 {
+            let mut single_plot_with_cache_space =
+                single_sector_overhead.div_ceil(100 - u64::from(cache_percentage.get())) * 100;
+            // Cache must not be empty, ensure it contains at least one element even if
+            // percentage-wise it will use more space
+            if single_plot_with_cache_space - single_sector_overhead
+                < DiskPieceCache::element_size() as u64
+            {
+                single_plot_with_cache_space =
+                    single_sector_overhead + DiskPieceCache::element_size() as u64;
+            }
+
+            return Err(SingleDiskPlotError::InsufficientAllocatedSpace {
+                min_space: fixed_space_usage + single_plot_with_cache_space,
+                allocated_space,
+            });
+        }
+
+        // Remaining space will be used for caching purposes
+        let cache_capacity = {
+            let cache_space = allocated_space
+                - fixed_space_usage
+                - (target_sector_count * single_sector_overhead);
+            cache_space as usize / DiskPieceCache::element_size()
+        };
         let target_sector_count = match SectorIndex::try_from(target_sector_count) {
             Ok(target_sector_count) if target_sector_count < SectorIndex::MAX => {
                 target_sector_count
@@ -578,24 +595,25 @@ impl SingleDiskPlot {
             }
         };
 
-        // TODO: Consider file locking to prevent other apps from modifying itS
+        // TODO: Consider file locking to prevent other apps from modifying it
         let mut metadata_file = OpenOptions::new()
             .read(true)
             .write(true)
             .create(true)
             .open(directory.join(Self::METADATA_FILE))?;
 
-        let (metadata_header, metadata_header_mmap) = if metadata_file.seek(SeekFrom::End(0))? == 0
-        {
+        let metadata_size = metadata_file.seek(SeekFrom::End(0))?;
+        let expected_metadata_size =
+            RESERVED_PLOT_METADATA + sector_metadata_size as u64 * u64::from(target_sector_count);
+        let (metadata_header, metadata_header_mmap) = if metadata_size == 0 {
             let metadata_header = PlotMetadataHeader {
                 version: 0,
                 sector_count: 0,
             };
 
-            metadata_file.preallocate(
-                RESERVED_PLOT_METADATA
-                    + sector_metadata_size as u64 * u64::from(target_sector_count),
-            )?;
+            metadata_file
+                .preallocate(expected_metadata_size)
+                .map_err(SingleDiskPlotError::CantPreallocateMetadataFile)?;
             metadata_file.write_all_at(metadata_header.encode().as_slice(), 0)?;
 
             let metadata_header_mmap = unsafe {
@@ -606,21 +624,36 @@ impl SingleDiskPlot {
 
             (metadata_header, metadata_header_mmap)
         } else {
-            let metadata_header_mmap = unsafe {
+            if metadata_size != expected_metadata_size {
+                // Allocating the whole file (`set_len` below can create a sparse file, which will
+                // cause writes to fail later)
+                metadata_file
+                    .preallocate(expected_metadata_size)
+                    .map_err(SingleDiskPlotError::CantPreallocateMetadataFile)?;
+                // Truncating file (if necessary)
+                metadata_file.set_len(expected_metadata_size)?;
+            }
+            let mut metadata_header_mmap = unsafe {
                 MmapOptions::new()
                     .len(PlotMetadataHeader::encoded_size())
                     .map_mut(&metadata_file)?
             };
 
-            let metadata_header = PlotMetadataHeader::decode(&mut metadata_header_mmap.as_ref())
-                .map_err(SingleDiskPlotError::FailedToDecodeMetadataHeader)?;
+            let mut metadata_header =
+                PlotMetadataHeader::decode(&mut metadata_header_mmap.as_ref())
+                    .map_err(SingleDiskPlotError::FailedToDecodeMetadataHeader)?;
 
             if metadata_header.version != Self::SUPPORTED_PLOT_VERSION {
                 return Err(SingleDiskPlotError::UnexpectedMetadataVersion(
                     metadata_header.version,
                 ));
             }
 
+            if metadata_header.sector_count > target_sector_count {
+                metadata_header.sector_count = target_sector_count;
+                metadata_header.encode_to(&mut metadata_header_mmap.as_mut());
+            }
+
             (metadata_header, metadata_header_mmap)
         };
 
@@ -656,14 +689,15 @@ impl SingleDiskPlot {
                 .open(directory.join(Self::PLOT_FILE))?,
         );
 
+        // Allocating the whole file (`set_len` below can create a sparse file, which will cause
+        // writes to fail later)
         plot_file
             .preallocate(sector_size as u64 * u64::from(target_sector_count))
             .map_err(SingleDiskPlotError::CantPreallocatePlotFile)?;
+        // Truncating file (if necessary)
+        plot_file.set_len(sector_size as u64 * u64::from(target_sector_count))?;
 
-        let piece_cache = DiskPieceCache::open(
-            &directory,
-            cache_size as usize / DiskPieceCache::element_size(),
-        )?;
+        let piece_cache = DiskPieceCache::open(&directory, cache_capacity)?;
 
         let (error_sender, error_receiver) = oneshot::channel();
         let error_sender = Arc::new(Mutex::new(Some(error_sender)));