Data Not Saving After 500 Cycles in WLTC Simulation Script

[lappemic]

Hello,

I'm currently working on a WLTC simulation using PyBaMM, and I've encountered an issue where the simulation completes, but the data does not save when running more than 500 cycles. Here is a snippet of my script:

# cell_simulation_WLTC.py
import pybamm
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import os
from pathlib import Path
import pickle
import datetime
from src.paths import PYBAMM_DATA_DIR, PYBAMM_FIG_DIR

os.chdir(pybamm.__path__[0] + "/..")

pybamm.set_logging_level("NOTICE")

# Define simulation parameters
cycles = 750
OUTFOLDERNAME = "test"
parameter_reference = "Chen2020"

starttime = datetime.datetime.now().replace(microsecond=0)
print("*"*80)
print(f"Starting simulation at {starttime}")
print("Starting PyBaMM simulation")
print(f"Output directory: {PYBAMM_DATA_DIR}")
print(f"Output folder name: {OUTFOLDERNAME}")
print(f"Number of cycles: {cycles}")
print(f"Parameter values: {parameter_reference}")

parameter_values = pybamm.ParameterValues(parameter_reference)
parameter_values.update({"SEI kinetic rate constant [m.s-1]": 1e-12})

model = pybamm.lithium_ion.SPM(
  {
    "thermal": "lumped", 
    "SEI": "reaction limited",
    "SEI film resistance": "distributed", 
    "SEI porosity change": "false"
    }
  )

drive_cycle = pd.read_csv("pybamm/input/drive_cycles/WLTC.csv", comment='#', header=None).to_numpy()

# scale the current of the drive cycle
scale_factor = 0.3
drive_cycle[:, 1] *= scale_factor

# set up the experiment
experiment = pybamm.Experiment(
  [
      (
        "Hold at 4.2 V until C/100 (5 minute period)",
        "Rest for 4 hours (5 minute period)",
        "Discharge at 0.1C until 2.5 V (5 minute period)",  # initial capacity check
        "Charge at 0.3C until 4.2 V (5 minute period)",
        "Hold at 4.2 V until C/100 (5 minute period)",
      )
  ]
  + [
      (
        pybamm.step.current(drive_cycle),
        "Rest for 1 hour (5 minute period)",
        pybamm.step.current(drive_cycle),
        "Rest for 1 hour (5 minute period)",
        pybamm.step.current(drive_cycle),
        "Rest for 1 hour (5 minute period)",
        "Charge at 1C until 4.2V (5 minute period)",
        "Hold at 4.2V until C/50 (5 minute period)"
      ),
  ] * cycles
  + ["Discharge at 0.1C until 2.5 V (5 minute period)"]  # final capacity check
)

sim = pybamm.Simulation(
    model, parameter_values=parameter_values, experiment=experiment
)

solution = sim.solve()

outdir = PYBAMM_DATA_DIR.joinpath(OUTFOLDERNAME)
outdir.mkdir(parents=True, exist_ok=True)
modelname = "_".join(model.name.split(" ")).lower()
scale_factor_str = f"{scale_factor:.2f}".replace('.', 'p')

# saving to pickle
outfilename_pickle = f"{modelname}_{parameter_reference.lower()}_WLTC_{cycles:05}_{scale_factor_str}.pkl"
outfilepath_pickle = outdir.joinpath(outdir, outfilename_pickle)
solution.save_data(outfilepath_pickle, ["Time [s]", "Current [A]", "Voltage [V]"], to_format="pickle")

# saving to csv
outfilename_csv = f"{modelname}_{parameter_reference.lower()}_WLTC_{cycles:05}_{scale_factor_str}.csv"
outfilepath_csv = outdir.joinpath(outdir, outfilename_csv)
print(f"Saving to {outfilepath_csv}")
solution.save_data(outfilepath_csv, ["Time [s]", "Current [A]", "Voltage [V]"], to_format="csv")

endtime = datetime.datetime.now().replace(microsecond=0)
print(f"Simulation finished at {endtime}")
print(f"Total time taken: {endtime-starttime}")

I'm running this script in the terminal using:

nohup python3 cell_simulation_WLTC.py > cell_simulation_WLTC.log &

The last entries in the log file are:

2024-06-25 14:38:48.470 - [NOTICE] logger.func(15): Cycle 751/752, step 6/8: Rest for 1 hour (5 minute period)
2024-06-25 14:38:48.488 - [NOTICE] logger.func(15): Cycle 751/752, step 7/8: Charge at 1C until 4.2V (5 minute period)
2024-06-25 14:38:48.506 - [NOTICE] logger.func(15): Cycle 751/752, step 8/8: Hold at 4.2V until C/50 (5 minute period)
2024-06-25 14:38:51.824 - [NOTICE] logger.func(15): Cycle 752/752 (54 minutes, 27 seconds elapsed) --------------------
2024-06-25 14:38:51.825 - [NOTICE] logger.func(15): Cycle 752/752, step 1/1: Discharge at 0.1C until 2.5 V (5 minute period)
2024-06-25 14:38:52.382 - [NOTICE] logger.func(15): Finish experiment simulation, took 54 minutes, 27 seconds

The simulation works for up to 500 cycles, but anything above that causes the script to hang indefinitely after the logs mentioned above. The data is not being saved as expected.

Has anyone experienced similar issues or have any suggestions on how to resolve this? Any insights would be greatly appreciated!

Thanks in advance for your help.