From 12971b91f3f184cf6b8b30f449147916c904e95e Mon Sep 17 00:00:00 2001 From: zhenyu Date: Mon, 21 Feb 2022 22:11:23 +0800 Subject: [PATCH 01/46] feature: merge run.py --- dpgen/generator/lib/calypso.py | 352 ++++++++ dpgen/generator/run.py | 1409 +++++++++++++++++++++++++++----- 2 files changed, 1548 insertions(+), 213 deletions(-) create mode 100644 dpgen/generator/lib/calypso.py diff --git a/dpgen/generator/lib/calypso.py b/dpgen/generator/lib/calypso.py new file mode 100644 index 000000000..d3d7bd83a --- /dev/null +++ b/dpgen/generator/lib/calypso.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python3 + +def make_run_opt_script(fmax ) : + + ret ="import os,sys,glob,time \n" + ret+="import numpy as np \n" + ret+="from ase.io import read \n" + ret+="from ase.optimize import BFGS,QuasiNewton,LBFGS \n" + ret+="from ase.constraints import UnitCellFilter, ExpCellFilter\n" + ret+="from deepmd.calculator import DP \n" + ret+=" \n" + ret+="def Get_Element_Num(elements): \n" + ret+=" '''Using the Atoms.symples to Know Element&Num''' \n" + ret+=" element = [] \n" + ret+=" ele = {} \n" + ret+=" element.append(elements[0]) \n" + ret+=" for x in elements: \n" + ret+=" if x not in element : \n" + ret+=" element.append(x) \n" + ret+=" for x in element: \n" + ret+=" ele[x] = elements.count(x) \n" + ret+=" return element, ele \n" + ret+=" \n" + ret+="def Write_Contcar(element, ele, lat, pos): \n" + ret+=" '''Write CONTCAR''' \n" + ret+=" f = open('CONTCAR','w') \n" + ret+=" f.write('ASE-DPKit-Optimization\\n') \n" + ret+=" f.write('1.0\\n') \n" + ret+=" for i in range(3): \n" + ret+=" f.write('%15.10f %15.10f %15.10f\\n' % tuple(lat[i])) \n" + ret+=" for x in element: \n" + ret+=" f.write(x + ' ') \n" + ret+=" f.write('\\n') \n" + ret+=" for x in element: \n" + ret+=" f.write(str(ele[x]) + ' ') \n" + ret+=" f.write('\\n') \n" + ret+=" f.write('Direct\\n') \n" + ret+=" na = sum(ele.values()) \n" + ret+=" dpos = np.dot(pos,np.linalg.inv(lat)) \n" + ret+=" for i in range(na): \n" + ret+=" f.write('%15.10f %15.10f %15.10f\\n' % tuple(dpos[i])) \n" + ret+=" \n" + ret+="def Write_Outcar(element, ele, volume, lat, pos, ene, force, stress,pstress): \n" + ret+=" '''Write OUTCAR''' \n" + ret+=" f = open('OUTCAR','w') \n" + ret+=" for x in element: \n" + ret+=" f.write('VRHFIN =' + str(x) + '\\n') \n" + ret+=" f.write('ions per type =') \n" + ret+=" for x in element: \n" + ret+=" f.write('%5d' % ele[x]) \n" + ret+=" #f.write('\\nvolume of cell :\\n') \n" + ret+=" f.write('\\nDirection XX YY ZZ XY YZ ZX\\n') \n" + ret+=" f.write('in kB') \n" + ret+=" f.write('%15.6f' % stress[0]) \n" + ret+=" f.write('%15.6f' % stress[1]) \n" + ret+=" f.write('%15.6f' % stress[2]) \n" + ret+=" f.write('%15.6f' % stress[3]) \n" + ret+=" f.write('%15.6f' % stress[4]) \n" + ret+=" f.write('%15.6f' % stress[5]) \n" + ret+=" f.write('\\n') \n" + ret+=" ext_pressure = np.sum(stress[0] + stress[1] + stress[2])/3.0 - pstress \n" + ret+=" f.write('external pressure = %20.6f kB Pullay stress = %20.6f kB\\n'% (ext_pressure, pstress))\n" + ret+=" f.write('volume of cell : %20.6f\\n' % volume) \n" + ret+=" f.write('direct lattice vectors\\n') \n" + ret+=" for i in range(3): \n" + ret+=" f.write('%10.6f %10.6f %10.6f\\n' % tuple(lat[i])) \n" + ret+=" f.write('POSITION TOTAL-FORCE(eV/Angst)\\n') \n" + ret+=" f.write('-------------------------------------------------------------------\\n') \n" + ret+=" na = sum(ele.values()) \n" + ret+=" for i in range(na): \n" + ret+=" f.write('%15.6f %15.6f %15.6f' % tuple(pos[i])) \n" + ret+=" f.write('%15.6f %15.6f %15.6f\\n' % tuple(force[i])) \n" + ret+=" f.write('-------------------------------------------------------------------\\n') \n" + ret+=" f.write('energy without entropy= %20.6f %20.6f\\n' % (ene, ene/na)) \n" + ret+=" enthalpy = ene + pstress * volume / 1602.17733 \n" + ret+=" f.write('enthalpy is TOTEN = %20.6f %20.6f\\n' % (enthalpy, enthalpy/na)) \n" + ret+=" \n" + ret+="def Write_calylog(templog): \n" + ret+=" '''For Write Evolve Structures Log into caly.log''' \n" + ret+=" f = open('opt.log','a+') \n" + ret+=" f.write(templog+'\\n') \n" + ret+=" f.close() \n" + ret+="def read_stress():\n" + ret+=" pstress = 0\n" + ret+=" #assert os.path.exists('./input.dat'), 'input.dat does not exist!'\n" + ret+=" try:\n" + ret+=" f = open('input.dat','r')\n" + ret+=" except:\n" + ret+=" assert os.path.exists('../input.dat'),' now we are in %s, do not find ../input.dat'%(os.getcwd())\n" + ret+=" f = open('../input.dat','r')\n" + ret+=" lines = f.readlines()\n" + ret+=" f.close()\n" + ret+=" for line in lines:\n" + ret+=" if line[0] == '#':\n" + ret+=" continue\n" + ret+=" if 'PSTRESS' in line:\n" + ret+=" pstress = float(line.split('=')[1])\n" + ret+=" return pstress\n" + ret+=" \n" + ret+=" \n" + ret+="def run_opt(stress): \n" + ret+=" '''Using the ASE&DP to Optimize Configures''' \n" + ret+=" ''' > 600 Steps Called Failure Optimization''' \n" + ret+=" \n" + ret+=" os.system('mv OUTCAR OUTCAR-last')\n" + ret+=" is_dpgen = True\n" + ret+=" nn = 1\n" + ret+=" try:\n" + ret+=" model_path = sys.argv[1] \n" + ret+=" Model_List = glob.glob('%s/graph*pb'%model_path) \n" + ret+=" calc = DP(model=Model_List[1]) # init the model before iteration \n" + ret+=" except:\n" + ret+=" assert os.path.exists('graph.pb'), 'did not found graph.pb in this directory %s, or did you forget to add args?'%(os.getcwd())\n" + ret+=" calc = DP(model='graph.pb') # init the model before iteration \n" + ret+=" is_dpgen = False\n" + ret+=" nn = 3\n" + ret+=" \n" + ret+=" print('Start to Optimize Structures by DP----------') \n" + ret+=" \n" + ret+=" Opt_Step = 200 \n" + ret+=" start = time.time() \n" + ret+=" # pstress kbar\n" + ret+=" pstress = stress\n" + ret+=" # kBar to eV/A^3\n" + ret+=" # 1 eV/A^3 = 160.21766028 GPa\n" + ret+=" # 1 / 160.21766028 ~ 0.006242\n" + ret+=" aim_stress = 1.0 * pstress* 0.01 * 0.6242 / 10.0 \n" + ret+=" to_be_opti = read('POSCAR') \n" + ret+=" to_be_opti.calc = calc \n" + ret+=" ucf = UnitCellFilter(to_be_opti, scalar_pressure=aim_stress)\n" + ret+=" atoms_vol_2 = to_be_opti.get_volume() \n" + ret+=" for i in range(nn): \n" + ret+=" if is_dpgen:\n" + ret+=" opt = LBFGS(ucf,trajectory='traj.traj') \n" + ret+=" else:\n" + ret+=" opt = LBFGS(ucf) \n" + ret+=" #opt = QuasiNewton(to_be_opti) \n" + ret+=" #opt = BFGS(to_be_opti) \n" + ret+=" #opt = BFGS(to_be_opti,trajectory='traj.traj',logfile='opt.log') \n" + ret+=" opt.run(fmax=%s,steps=200) \n"%str(fmax) + ret+=" \n" + ret+=" atoms_lat = to_be_opti.cell \n" + ret+=" atoms_pos = to_be_opti.positions \n" + ret+=" atoms_force = to_be_opti.get_forces() \n" + ret+=" atoms_stress = to_be_opti.get_stress() \n" + ret+=" # eV/A^3 to GPa\n" + ret+=" atoms_stress = atoms_stress/(0.01*0.6242)\n" + ret+=" #atoms_num = to_be_opti.get_atomic_numbers() \n" + ret+=" atoms_symbols = to_be_opti.get_chemical_symbols() \n" + ret+=" #atoms_formula = to_be_opti.get_chemical_formula() \n" + ret+=" atoms_ene = to_be_opti.get_potential_energy() \n" + ret+=" atoms_vol = to_be_opti.get_volume() \n" + ret+=" \n" + ret+=" element, ele = Get_Element_Num(atoms_symbols) \n" + ret+=" \n" + ret+=" Write_Contcar(element, ele, atoms_lat, atoms_pos) \n" + ret+=" Write_Outcar(element, ele, atoms_vol, atoms_lat, atoms_pos,atoms_ene, atoms_force, atoms_stress * -10.0, pstress) \n" + ret+=" \n" + ret+=" \n" + ret+=" stop = time.time() \n" + ret+=" _cwd = os.getcwd() \n" + ret+=" _cwd = os.path.basename(_cwd) \n" + ret+=" print('%s is done, time: %s' % (_cwd,stop-start)) \n" + ret+="stress = read_stress()\n" + ret+="run_opt(stress) \n" + return ret + + +def make_check_outcar_script( ) : + + ret = "import numpy as np \n" + ret+= "import os,sys,glob,time \n" + ret+= "from deepmd.calculator import DP \n" + ret+= "from ase.io import read \n" + ret+= " \n" + ret+= "def Get_Element_Num(elements): \n" + ret+= " '''Using the Atoms.symples to Know Element&Num''' \n" + ret+= " element = [] \n" + ret+= " ele = {} \n" + ret+= " element.append(elements[0]) \n" + ret+= " for x in elements: \n" + ret+= " if x not in element : \n" + ret+= " element.append(x) \n" + ret+= " for x in element: \n" + ret+= " ele[x] = elements.count(x) \n" + ret+= " return element, ele \n" + ret+= " \n" + ret+= "def Write_Contcar(element, ele, lat, pos): \n" + ret+= " '''Write CONTCAR''' \n" + ret+= " f = open('CONTCAR','w') \n" + ret+= " f.write('ASE-DPKit-FAILED-nan\\n') \n" + ret+= " f.write('1.0\\n') \n" + ret+= " for i in range(3): \n" + ret+= " f.write('%15.10f %15.10f %15.10f\\n' % tuple(lat[i])) \n" + ret+= " for x in element: \n" + ret+= " f.write(x + ' ') \n" + ret+= " f.write('\\n') \n" + ret+= " for x in element: \n" + ret+= " f.write(str(ele[x]) + ' ') \n" + ret+= " f.write('\\n') \n" + ret+= " f.write('Direct\\n') \n" + ret+= " na = sum(ele.values()) \n" + ret+= " dpos = np.dot(pos,np.linalg.inv(lat)) \n" + ret+= " for i in range(na): \n" + ret+= " f.write('%15.10f %15.10f %15.10f\\n' % tuple(dpos[i])) \n" + ret+= " \n" + ret+= "def Write_Outcar(element, ele, volume, lat, pos, ene, force, stress,pstress): \n" + ret+= " '''Write OUTCAR''' \n" + ret+= " f = open('OUTCAR','w') \n" + ret+= " for x in element: \n" + ret+= " f.write('VRHFIN =' + str(x) + '\\n') \n" + ret+= " f.write('ions per type =') \n" + ret+= " for x in element: \n" + ret+= " f.write('%5d' % ele[x]) \n" + ret+= " #f.write('\\nvolume of cell :\\n') \n" + ret+= " f.write('\\nDirection XX YY ZZ XY YZ ZX\\n') \n" + ret+= " f.write('in kB') \n" + ret+= " f.write('%15.6f' % stress[0]) \n" + ret+= " f.write('%15.6f' % stress[1]) \n" + ret+= " f.write('%15.6f' % stress[2]) \n" + ret+= " f.write('%15.6f' % stress[3]) \n" + ret+= " f.write('%15.6f' % stress[4]) \n" + ret+= " f.write('%15.6f' % stress[5]) \n" + ret+= " f.write('\\n') \n" + ret+= " ext_pressure = np.sum(stress[0] + stress[1] + stress[2])/3.0 - pstress \n" + ret+= " f.write('external pressure = %20.6f kB Pullay stress = %20.6f kB\\n'% (ext_pressure, pstress))\n" + ret+= " f.write('volume of cell : %20.6f\\n' % volume) \n" + ret+= " f.write('direct lattice vectors\\n') \n" + ret+= " for i in range(3): \n" + ret+= " f.write('%10.6f %10.6f %10.6f\\n' % tuple(lat[i])) \n" + ret+= " f.write('POSITION TOTAL-FORCE(eV/Angst)\\n') \n" + ret+= " f.write('-------------------------------------------------------------------\\n') \n" + ret+= " na = sum(ele.values()) \n" + ret+= " for i in range(na): \n" + ret+= " f.write('%15.6f %15.6f %15.6f' % tuple(pos[i])) \n" + ret+= " f.write('%15.6f %15.6f %15.6f\\n' % tuple(force[i])) \n" + ret+= " f.write('-------------------------------------------------------------------\\n') \n" + ret+= " f.write('energy without entropy= %20.6f %20.6f\\n' % (ene, ene)) \n" + ret+= " enthalpy = ene + pstress * volume / 1602.17733 \n" + ret+= " f.write('enthalpy is TOTEN = %20.6f %20.6f\\n' % (enthalpy, enthalpy)) \n" + ret+= " \n" + ret+= "def check(): \n" + ret+= " \n" + ret+= " from deepmd.calculator import DP \n" + ret+= " from ase.io import read \n" + ret+= " model_path = sys.argv[1] \n" + ret+= " Model_List = glob.glob('%s/graph*pb'%model_path) \n" + ret+= " calc = DP(model='%s'%(Model_List[0])) # init the model before iteration \n" + ret+= " \n" + ret+= " to_be_opti = read('POSCAR') \n" + ret+= " to_be_opti.calc = calc \n" + ret+= " # --------------------------------- \n" + ret+= " # for failed outcar \n" + ret+= " atoms_symbols_f = to_be_opti.get_chemical_symbols() \n" + ret+= " element_f, ele_f = Get_Element_Num(atoms_symbols_f) \n" + ret+= " atoms_vol_f = to_be_opti.get_volume() \n" + ret+= " atoms_stress_f = to_be_opti.get_stress() \n" + ret+= " atoms_stress_f = atoms_stress_f/(0.01*0.6242) \n" + ret+= " atoms_lat_f = to_be_opti.cell \n" + ret+= " atoms_pos_f = to_be_opti.positions \n" + ret+= " atoms_force_f = to_be_opti.get_forces() \n" + ret+= " atoms_ene_f = 610612509 \n" + ret+= " # --------------------------------- \n" + ret+= " Write_Contcar(element_f, ele_f, atoms_lat_f, atoms_pos_f) \n" + ret+= " Write_Outcar(element_f, ele_f, atoms_vol_f, atoms_lat_f, atoms_pos_f,atoms_ene_f, atoms_force_f, atoms_stress_f * -10.0, 0) \n" + ret+= " \n" + ret+= "cwd = os.getcwd() \n" + ret+= "if not os.path.exists(os.path.join(cwd,'OUTCAR')): \n" + ret+= " check() \n" + return ret + + +def make_calypso_input(nameofatoms,numberofatoms, + numberofformula,volume, + distanceofion,psoratio,popsize, + maxstep,icode,split,vsc, + maxnumatom,ctrlrange,pstress,fmax): + assert len(distanceofion) == len(nameofatoms) #"check distance of ions and the number of atoms" + assert len(distanceofion[0]) == len(nameofatoms) + ret = "################################ The Basic Parameters of CALYPSO ################################\n" + ret+= "# A string of one or several words contain a descriptive name of the system (max. 40 characters).\n" + ret+= "SystemName = %s\n"%(''.join(nameofatoms)) + ret+= "# Number of different atomic species in the simulation.\n" + ret+= "NumberOfSpecies = %d\n"%(len(nameofatoms)) + ret+= "# Element symbols of the different chemical species.\n" + ret+= "NameOfAtoms = %s\n"%(' '.join(nameofatoms)) + ret+= "# Number of atoms for each chemical species in one formula unit. \n" + ret+= "NumberOfAtoms = %s\n"%(' '.join(list(map(str,numberofatoms)))) + ret+= "# The range of formula unit per cell in your simulation. \n" + ret+= "NumberOfFormula = %s\n"%(' '.join(list(map(str,numberofformula)))) + ret+= "# The volume per formula unit. Unit is in angstrom^3.\n" + ret+= "Volume = %s\n"%(volume[0]) + ret+= "# Minimal distance between atoms of each chemical species. Unit is in angstrom.\n" + ret+= "@DistanceOfIon \n" + for temp in distanceofion: + ret+="%4s \n"%(' '.join(list(map(str,temp)))) + ret+= "@End\n" + ret+= "# It determines which algorithm should be adopted in the simulation.\n" + ret+= "Ialgo = 2\n" + ret+= "# Ialgo = 1 for Global PSO\n" + ret+= "# Ialgo = 2 for Local PSO (default value)\n" + ret+= "# The proportion of the structures generated by PSO.\n" + ret+= "PsoRatio = %s\n"%(psoratio[0]) + ret+= "# The population size. Normally, it has a larger number for larger systems.\n" + ret+= "PopSize = %d\n"%(popsize[0]) + ret+= "# The Max step for iteration\n" + ret+= "MaxStep = %d\n"%(maxstep[0]) + ret+= "#It determines which method should be adopted in generation the random structure. \n" + ret+= "GenType= 1 \n" + ret+= "# 1 under symmetric constraints\n" + ret+= "# 2 grid method for large system\n" + ret+= "# 3 and 4 core grow method \n" + ret+= "# 0 combination of all method\n" + ret+= "# If GenType=3 or 4, it determined the small unit to grow the whole structure\n" + ret+= "# It determines which local optimization method should be interfaced in the simulation.\n" + ret+= "ICode= %d\n"%(icode[0]) + ret+= "# ICode= 1 interfaced with VASP\n" + ret+= "# ICode= 2 interfaced with SIESTA\n" + ret+= "# ICode= 3 interfaced with GULP\n" + ret+= "# The number of lbest for local PSO\n" + ret+= "NumberOfLbest=4\n" + ret+= "# The Number of local optimization for each structure.\n" + ret+= "NumberOfLocalOptim= 3\n" + ret+= "# The command to perform local optimiztion calculation (e.g., VASP, SIESTA) on your computer.\n" + ret+= "Command = sh submit.sh\n" + ret+= "MaxTime = 9000 \n" + ret+= "# If True, a previous calculation will be continued.\n" + ret+= "PickUp = F\n" + ret+= "# At which step will the previous calculation be picked up.\n" + ret+= "PickStep = 1\n" + ret+= "# If True, the local optimizations performed by parallel\n" + ret+= "Parallel = F\n" + ret+= "# The number node for parallel \n" + ret+= "NumberOfParallel = 4\n" + ret+= "Split = %s\n"%(split) + ret+= "PSTRESS = %s\n"%(str(pstress[0])) + ret+= "fmax = %s\n"%(str(fmax[0])) + ret+= "################################ End of The Basic Parameters of CALYPSO #######################\n" + if vsc == 'T': + assert len(ctrlrange) == len(nameofatoms) #'check distance of ions and the number of atoms' + ret+= "##### The Parameters For Variational Stoichiometry ##############\n" + ret+= "## If True, Variational Stoichiometry structure prediction is performed\n" + ret+= "VSC = %s\n"%(vsc) + ret+= "# The Max Number of Atoms in unit cell\n" + ret+= "MaxNumAtom = %s\n"%(maxnumatom[0]) + ret+= "# The Variation Range for each type atom \n" + ret+= "@CtrlRange\n" + for ttemp in ctrlrange: + ret+="%4s \n"%(' '.join(list(map(str,ttemp)))) + ret+= "@end\n" + ret+= "###################End Parameters for VSC ##########################\n" + return ret diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index 1c4733037..b420e9505 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -41,6 +41,7 @@ from dpgen.generator.lib.utils import log_task from dpgen.generator.lib.utils import symlink_user_forward_files from dpgen.generator.lib.lammps import make_lammps_input, get_dumped_forces +from dpgen.generator.lib.calypso import make_run_opt_script,make_check_outcar_script,make_calypso_input from dpgen.generator.lib.vasp import write_incar_dict from dpgen.generator.lib.vasp import make_vasp_incar_user_dict from dpgen.generator.lib.vasp import incar_upper @@ -79,6 +80,9 @@ fp_name = '02.fp' fp_task_fmt = data_system_fmt + '.%06d' cvasp_file=os.path.join(ROOT_PATH,'generator/lib/cvasp.py') +# for calypso local optimization +calypso_run_opt_name = 'GenStruAnaly' +calypso_model_devi_name = 'Model_Devi_Results' def get_job_names(jdata) : jobkeys = [] @@ -230,11 +234,12 @@ def make_train (iter_index, "By default, the picking probability of data from one system or one iter is proportional to the number of batches (the number of frames divided by batch_size) of that systems or iter.\n" \ "Detailed discussion about init-model (in Chinese) please see https://mp.weixin.qq.com/s/qsKMZ0j270YhQKvwXUiFvQ") + model_devi_engine = jdata.get('model_devi_engine', "lammps") if iter_index > 0 and _check_empty_iter(iter_index-1, fp_task_min) : log_task('prev data is empty, copy prev model') copy_model(numb_models, iter_index-1, iter_index) return - elif iter_index > 0 and _check_skip_train(model_devi_jobs[iter_index-1]): + elif model_devi_engine != 'calypso' and iter_index > 0 and _check_skip_train(model_devi_jobs[iter_index-1]): log_task('skip training at step %d ' % (iter_index-1)) copy_model(numb_models, iter_index-1, iter_index) return @@ -272,7 +277,10 @@ def make_train (iter_index, if 'sys_batch_size' in jdata: sys_batch_size = jdata['sys_batch_size'] else: - sys_batch_size = ["auto" for aa in range(len(jdata['sys_configs']))] + if model_devi_engine != 'calypso': + sys_batch_size = ["auto" for aa in range(len(jdata['sys_configs']))] + else: + sys_batch_size = ["auto" for aa in range(3000)] # make sure all init_data_sys has the batch size -- for the following `zip` assert (len(init_data_sys_) <= len(init_batch_size_)) @@ -767,101 +775,254 @@ def revise_by_keys(lmp_lines, keys, values): lmp_lines[ii] = lmp_lines[ii].replace(kk, str(vv)) return lmp_lines +def _parse_calypso_input(var,input_path): + try: + f = open(input_path,'r') + except: + f = open(os.path.join(input_path,'input.dat'),'r') + lines = f.readlines() + + for line in lines: + if var in line: + variable = line.split('=')[1].strip() + return variable + +def _parse_calypso_dis_mtx(numberofspecies,input_path): + try: + f = open(input_path,'r') + except: + f = open(os.path.join(input_path,'input.dat'),'r') + while True: + line = f.readline() + if len(line) == 0: + break + if '@DistanceOfIon' in line: + dis = [] + for i in range(int(numberofspecies)): + line = f.readline() + dis.append(line.split()) + break + dis = np.array(dis) + dis = dis.reshape((1,int(numberofspecies)**2)) + return dis[0][np.argmin(dis)] + + +def make_calypso_model_devi(iter_index,jdata,mdata): + ''' for calypso ''' -def make_model_devi (iter_index, - jdata, - mdata) : - # The MD engine to perform model deviation - # Default is lammps - model_devi_engine = jdata.get('model_devi_engine', "lammps") model_devi_jobs = jdata['model_devi_jobs'] - if (iter_index >= len(model_devi_jobs)) : + try: + maxiter = max(model_devi_jobs[-1].get('times')) + except Exception as e: + maxiter = jdata.get('model_devi_max_iter',0) + + if (iter_index > maxiter) : return False - cur_job = model_devi_jobs[iter_index] - if "sys_configs_prefix" in jdata: - sys_configs = [] - for sys_list in jdata["sys_configs"]: - #assert (isinstance(sys_list, list) ), "Currently only support type list for sys in 'sys_conifgs' " - temp_sys_list = [os.path.join(jdata["sys_configs_prefix"], sys) for sys in sys_list] - sys_configs.append(temp_sys_list) - else: - sys_configs = jdata['sys_configs'] - shuffle_poscar = jdata['shuffle_poscar'] - - sys_idx = expand_idx(cur_job['sys_idx']) - if (len(sys_idx) != len(list(set(sys_idx)))) : - raise RuntimeError("system index should be uniq") - conf_systems = [] - for idx in sys_idx : - cur_systems = [] - ss = sys_configs[idx] - for ii in ss : - cur_systems += glob.glob(ii) - cur_systems.sort() - cur_systems = [os.path.abspath(ii) for ii in cur_systems] - conf_systems.append (cur_systems) + calypso_path = jdata.get('calypso_path') iter_name = make_iter_name(iter_index) train_path = os.path.join(iter_name, train_name) train_path = os.path.abspath(train_path) - models = sorted(glob.glob(os.path.join(train_path, "graph*pb"))) + models = glob.glob(os.path.join(train_path, "graph*pb")) work_path = os.path.join(iter_name, model_devi_name) + calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) + calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) + create_path(work_path) + create_path(calypso_run_opt_path) + create_path(calypso_model_devi_path) + + # the model for mm in models : model_name = os.path.basename(mm) - os.symlink(mm, os.path.join(work_path, model_name)) - with open(os.path.join(work_path, 'cur_job.json'), 'w') as outfile: - json.dump(cur_job, outfile, indent = 4) + os.symlink(mm, os.path.join(calypso_run_opt_path, model_name)) - conf_path = os.path.join(work_path, 'confs') - create_path(conf_path) - sys_counter = 0 - for ss in conf_systems: - conf_counter = 0 - for cc in ss : - if model_devi_engine == "lammps": - conf_name = make_model_devi_conf_name(sys_idx[sys_counter], conf_counter) - orig_poscar_name = conf_name + '.orig.poscar' - poscar_name = conf_name + '.poscar' - lmp_name = conf_name + '.lmp' - if shuffle_poscar : - os.symlink(cc, os.path.join(conf_path, orig_poscar_name)) - poscar_shuffle(os.path.join(conf_path, orig_poscar_name), - os.path.join(conf_path, poscar_name)) - else : - os.symlink(cc, os.path.join(conf_path, poscar_name)) - if 'sys_format' in jdata: - fmt = jdata['sys_format'] - else: - fmt = 'vasp/poscar' - system = dpdata.System(os.path.join(conf_path, poscar_name), fmt = fmt, type_map = jdata['type_map']) - if jdata.get('model_devi_nopbc', False): - system.remove_pbc() - system.to_lammps_lmp(os.path.join(conf_path, lmp_name)) - elif model_devi_engine == "gromacs": - pass - conf_counter += 1 - sys_counter += 1 input_mode = "native" - if "template" in cur_job: - input_mode = "revise_template" - use_plm = jdata.get('model_devi_plumed', False) - use_plm_path = jdata.get('model_devi_plumed_path', False) + if "calypso_input_path" in jdata: + input_mode = "buffet" + if input_mode == "native": - if model_devi_engine == "lammps": - _make_model_devi_native(iter_index, jdata, mdata, conf_systems) - elif model_devi_engine == "gromacs": - _make_model_devi_native_gromacs(iter_index, jdata, mdata, conf_systems) - else: - raise RuntimeError("unknown model_devi engine", model_devi_engine) + for iiidx, jobbs in enumerate(model_devi_jobs): + if iter_index in jobbs.get('times'): + cur_job = model_devi_jobs[iiidx] + + _make_model_devi_native_calypso(jdata, cur_job, calypso_run_opt_path) + + run_opt_script = os.path.join(calypso_run_opt_path,'run_opt.py') + with open(run_opt_script,'w') as ffff: + ffff.write(make_run_opt_script(cur_job.get('fmax',0.01))) + + # ----------for check the nan situation ------------- + check_outcar_script = os.path.join(calypso_run_opt_path,'check_outcar.py') + with open(check_outcar_script,'w') as ffff: + ffff.write(make_check_outcar_script()) + # ----------for check the nan situation ------------- + + return True elif input_mode == "revise_template": - _make_model_devi_revmat(iter_index, jdata, mdata, conf_systems) + pass + elif input_mode == "buffet": + calypso_input_path = jdata.get('calypso_input_path') + shutil.copyfile(os.path.join(calypso_input_path,'input.dat'),os.path.join(calypso_run_opt_path, 'input.dat')) + popsize = _parse_calypso_input('PopSize',calypso_run_opt_path+'/input.dat') + run_opt_path = calypso_run_opt_path + + run_opt_script = os.path.join(calypso_run_opt_path,'run_opt.py') + with open(run_opt_script,'w') as ffff: + ffff.write(make_run_opt_script(jdata.get('fmax',0.01))) + + # ----------for check the nan situation ------------- + check_outcar_script = os.path.join(calypso_run_opt_path,'check_outcar.py') + with open(check_outcar_script,'w') as ffff: + ffff.write(make_check_outcar_script()) + # ----------for check the nan situation ------------- + return True + else: raise RuntimeError('unknown model_devi input mode', input_mode) - #Copy user defined forward_files - symlink_user_forward_files(mdata=mdata, task_type="model_devi", work_path=work_path) - return True + + +def _make_model_devi_native_calypso(jdata, cur_job, calypso_run_opt_path): + + # Crystal Parameters + nameofatoms = cur_job.get('NameOfAtoms') + numberofatoms = cur_job.get('NumberOfAtoms') + numberofformula = cur_job.get('NumberOfFormula',[1,1]) + volume = cur_job.get('Volume') + distanceofion = cur_job.get('DistanceOfIon') + psoratio = cur_job.get('PsoRatio') + popsize = cur_job.get('PopSize') + maxstep = cur_job.get('MaxStep') + icode = cur_job.get('ICode',[1]) + split = cur_job.get('Split','T') + # VSC Control + maxnumatom = None + ctrlrange = None + vsc = cur_job.get('VSC','F') + if vsc == 'T': + maxnumatom = cur_job.get('MaxNumAtom') + ctrlrange = cur_job.get('CtrlRange') + + # Optimization + pstress = cur_job.get('PSTRESS',[0.001]) + fmax = cur_job.get('fmax',[0.01]) + + # Cluster + + # 2D + + file_c = make_calypso_input(nameofatoms,numberofatoms, + numberofformula,volume, + distanceofion,psoratio,popsize, + maxstep,icode,split,vsc, + maxnumatom,ctrlrange,pstress,fmax) + with open(os.path.join(calypso_run_opt_path, 'input.dat'), 'w') as cin : + cin.write(file_c) + + +def make_model_devi (iter_index, + jdata, + mdata) : + # The MD engine to perform model deviation + # Default is lammps + model_devi_engine = jdata.get('model_devi_engine', "lammps") + if model_devi_engine == 'calypso': + Lcalymake = make_calypso_model_devi(iter_index,jdata,mdata) + if Lcalymake == True: + return True + else: + print('calypso make model devi wrong') + return False + else: + model_devi_jobs = jdata['model_devi_jobs'] + if (iter_index >= len(model_devi_jobs)) : + return False + cur_job = model_devi_jobs[iter_index] + if "sys_configs_prefix" in jdata: + sys_configs = [] + for sys_list in jdata["sys_configs"]: + #assert (isinstance(sys_list, list) ), "Currently only support type list for sys in 'sys_conifgs' " + temp_sys_list = [os.path.join(jdata["sys_configs_prefix"], sys) for sys in sys_list] + sys_configs.append(temp_sys_list) + else: + sys_configs = jdata['sys_configs'] + shuffle_poscar = jdata['shuffle_poscar'] + + sys_idx = expand_idx(cur_job['sys_idx']) + if (len(sys_idx) != len(list(set(sys_idx)))) : + raise RuntimeError("system index should be uniq") + conf_systems = [] + for idx in sys_idx : + cur_systems = [] + ss = sys_configs[idx] + for ii in ss : + cur_systems += glob.glob(ii) + cur_systems.sort() + cur_systems = [os.path.abspath(ii) for ii in cur_systems] + conf_systems.append (cur_systems) + + iter_name = make_iter_name(iter_index) + train_path = os.path.join(iter_name, train_name) + train_path = os.path.abspath(train_path) + models = sorted(glob.glob(os.path.join(train_path, "graph*pb"))) + work_path = os.path.join(iter_name, model_devi_name) + create_path(work_path) + for mm in models : + model_name = os.path.basename(mm) + os.symlink(mm, os.path.join(work_path, model_name)) + with open(os.path.join(work_path, 'cur_job.json'), 'w') as outfile: + json.dump(cur_job, outfile, indent = 4) + + conf_path = os.path.join(work_path, 'confs') + create_path(conf_path) + sys_counter = 0 + for ss in conf_systems: + conf_counter = 0 + for cc in ss : + if model_devi_engine == "lammps": + conf_name = make_model_devi_conf_name(sys_idx[sys_counter], conf_counter) + orig_poscar_name = conf_name + '.orig.poscar' + poscar_name = conf_name + '.poscar' + lmp_name = conf_name + '.lmp' + if shuffle_poscar : + os.symlink(cc, os.path.join(conf_path, orig_poscar_name)) + poscar_shuffle(os.path.join(conf_path, orig_poscar_name), + os.path.join(conf_path, poscar_name)) + else : + os.symlink(cc, os.path.join(conf_path, poscar_name)) + if 'sys_format' in jdata: + fmt = jdata['sys_format'] + else: + fmt = 'vasp/poscar' + system = dpdata.System(os.path.join(conf_path, poscar_name), fmt = fmt, type_map = jdata['type_map']) + if jdata.get('model_devi_nopbc', False): + system.remove_pbc() + system.to_lammps_lmp(os.path.join(conf_path, lmp_name)) + elif model_devi_engine == "gromacs": + pass + conf_counter += 1 + sys_counter += 1 + + input_mode = "native" + if "template" in cur_job: + input_mode = "revise_template" + use_plm = jdata.get('model_devi_plumed', False) + use_plm_path = jdata.get('model_devi_plumed_path', False) + if input_mode == "native": + if model_devi_engine == "lammps": + _make_model_devi_native(iter_index, jdata, mdata, conf_systems) + elif model_devi_engine == "gromacs": + _make_model_devi_native_gromacs(iter_index, jdata, mdata, conf_systems) + else: + raise RuntimeError("unknown model_devi engine", model_devi_engine) + elif input_mode == "revise_template": + _make_model_devi_revmat(iter_index, jdata, mdata, conf_systems) + else: + raise RuntimeError('unknown model_devi input mode', input_mode) + #Copy user defined forward_files + symlink_user_forward_files(mdata=mdata, task_type="model_devi", work_path=work_path) + return True def _make_model_devi_revmat(iter_index, jdata, mdata, conf_systems): @@ -1206,104 +1367,248 @@ def _make_model_devi_native_gromacs(iter_index, jdata, mdata, conf_systems): conf_counter += 1 sys_counter += 1 - - -def run_model_devi (iter_index, - jdata, - mdata) : - #rmdlog.info("This module has been run !") - model_devi_exec = mdata['model_devi_command'] +def write_model_devi_out(devi, fname): + assert devi.shape[1] == 8 + #assert devi.shape[1] == 7 + header = "%5s" % "step" + for item in 'vf': + header += "%16s%16s%16s" % (f"max_devi_{item}", f"min_devi_{item}",f"avg_devi_{item}") + header += "%16s"%str('min_dis') + np.savetxt(fname, + devi, + fmt=['%5d'] + ['%17.6e' for _ in range(7)], + delimiter='', + header=header) + return devi + +def GenVscStructures(iter_index,jdata,mdata): + + # run calypso + import sys,os,shutil + from pathlib import Path model_devi_group_size = mdata['model_devi_group_size'] model_devi_resources = mdata['model_devi_resources'] - use_plm = jdata.get('model_devi_plumed', False) - use_plm_path = jdata.get('model_devi_plumed_path', False) + iter_name = make_iter_name(iter_index) work_path = os.path.join(iter_name, model_devi_name) assert(os.path.isdir(work_path)) + # + calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) + calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) + # + + calypso_path = jdata.get('calypso_path') + #calypso_input_path = jdata.get('calypso_input_path') + popsize = int(_parse_calypso_input('PopSize',calypso_run_opt_path)) + maxstep = int(_parse_calypso_input('MaxStep',calypso_run_opt_path)) + + all_models = glob.glob(os.path.join(calypso_run_opt_path, 'graph*pb')) + model_names = [os.path.basename(ii) for ii in all_models] + + deepmdkit_python = mdata.get('deepmdkit_python') + command = "%s run_opt.py %s 1>> model_devi.log 2>> model_devi.log" % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) + command += " || %s check_outcar.py %s " % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) + commands = [command] - all_task = glob.glob(os.path.join(work_path, "task.*")) + cwd = os.getcwd() + os.chdir(calypso_run_opt_path) + + forward_files = ['POSCAR', 'run_opt.py','check_outcar.py'] + backward_files = ['OUTCAR','CONTCAR','traj.traj','model_devi.log'] + + run_calypso = calypso_path+'/calypso.x | tee log' + #os.system('%s'%run_calypso) + + calypso_run = run_calypso + + # -------------------------------------------------------------- + component = ['Mg','Al','Cu','MgAl','MgCu','AlCu','MgAlCu'] + + for idx,com in enumerate(component): + pwd = os.getcwd() + os.mkdir(str(idx)) + shutil.copyfile(os.path.join(cwd,'calypso_input','input.dat.%s'%com),os.path.join(str(idx),'input.dat')) + os.chdir(str(idx)) + os.system(calypso_run) + os.chdir(pwd) + + name_list = Path('.').glob('*/POSCAR_*') + for idx,name in enumerate(name_list): + shutil.copyfile(name,'POSCAR_%s'%(idx+1)) + # -------------------------------------------------------------- + try: + os.mkdir('task.%04d'%(idx+1)) + except: + shutil.rmtree('task.%04d'%(idx+1)) + os.mkdir('task.%04d'%(idx+1)) + shutil.copyfile('run_opt.py',os.path.join('task.%04d'%(idx+1),'run_opt.py')) + shutil.copyfile('check_outcar.py',os.path.join('task.%04d'%(idx+1),'check_outcar.py')) + shutil.copyfile('POSCAR_%s'%str(idx+1),os.path.join('task.%04d'%(idx+1),'POSCAR')) + + dlog.info('##################### copy file done #####################') + + all_task = glob.glob( "task.*") all_task.sort() - fp = open (os.path.join(work_path, 'cur_job.json'), 'r') - cur_job = json.load (fp) run_tasks_ = all_task - # for ii in all_task: - # fres = os.path.join(ii, 'model_devi.out') - # if os.path.isfile(fres) : - # nlines = np.loadtxt(fres).shape[0] - # if nframes != nlines : - # run_tasks_.append(ii) - # else : - # run_tasks_.append(ii) run_tasks = [os.path.basename(ii) for ii in run_tasks_] - #dlog.info("all_task is ", all_task) - #dlog.info("run_tasks in run_model_deviation",run_tasks_) - all_models = glob.glob(os.path.join(work_path, 'graph*pb')) - model_names = [os.path.basename(ii) for ii in all_models] - model_devi_engine = jdata.get("model_devi_engine", "lammps") - if model_devi_engine == "lammps": - command = "{ if [ ! -f dpgen.restart.10000 ]; then %s -i input.lammps -v restart 0; else %s -i input.lammps -v restart 1; fi }" % (model_devi_exec, model_devi_exec) - command = "/bin/sh -c '%s'" % command - commands = [command] - forward_files = ['conf.lmp', 'input.lammps', 'traj'] - backward_files = ['model_devi.out', 'model_devi.log', 'traj'] - if use_plm: - forward_files += ['input.plumed'] - # backward_files += ['output.plumed'] - backward_files += ['output.plumed','COLVAR'] - if use_plm_path: - forward_files += ['plmpath.pdb'] - elif model_devi_engine == "gromacs": - - gromacs_settings = jdata.get("gromacs_settings", {}) - mdp_filename = gromacs_settings.get("mdp_filename", "md.mdp") - topol_filename = gromacs_settings.get("topol_filename", "processed.top") - conf_filename = gromacs_settings.get("conf_filename", "conf.gro") - index_filename = gromacs_settings.get("index_filename", "index.raw") - type_filename = gromacs_settings.get("type_filename", "type.raw") - ndx_filename = gromacs_settings.get("ndx_filename", "") - # Initial reference to process pbc condition. - # Default is em.tpr - ref_filename = gromacs_settings.get("ref_filename", "em.tpr") - deffnm = gromacs_settings.get("deffnm", "deepmd") - maxwarn = gromacs_settings.get("maxwarn", 1) - traj_filename = gromacs_settings.get("traj_filename", "deepmd_traj.gro") - grp_name = gromacs_settings.get("group_name", "Other") - trj_freq = cur_job.get("trj_freq", 10) - - command = "%s grompp -f %s -p %s -c %s -o %s -maxwarn %d" % (model_devi_exec, mdp_filename, topol_filename, conf_filename, deffnm, maxwarn) - command += "&& %s mdrun -deffnm %s -cpi" %(model_devi_exec, deffnm) - if ndx_filename: - command += f"&& echo -e \"{grp_name}\\n{grp_name}\\n\" | {model_devi_exec} trjconv -s {ref_filename} -f {deffnm}.trr -n {ndx_filename} -o {traj_filename} -pbc mol -ur compact -center" - else: - command += "&& echo -e \"%s\\n%s\\n\" | %s trjconv -s %s -f %s.trr -o %s -pbc mol -ur compact -center" % (grp_name, grp_name, model_devi_exec, ref_filename, deffnm, traj_filename) - command += "&& if [ ! -d traj ]; then \n mkdir traj; fi\n" - command += f"python -c \"import dpdata;system = dpdata.System('{traj_filename}', fmt='gromacs/gro'); [system.to_gromacs_gro('traj/%d.gromacstrj' % (i * {trj_freq}), frame_idx=i) for i in range(system.get_nframes())]; system.to_deepmd_npy('traj_deepmd')\"" - command += f"&& dp model-devi -m ../graph.000.pb ../graph.001.pb ../graph.002.pb ../graph.003.pb -s traj_deepmd -o model_devi.out -f {trj_freq}" - commands = [command] + dlog.info('$$$$$$$$$$$$$$$ dispatcher running $$$$$$$$$$$$$$$$$$') + dispatcher=make_dispatcher(mdata['model_devi_machine'],mdata['model_devi_resources'],'./', run_tasks, model_devi_group_size) + #print(dispatcher) + dispatcher.run_jobs(mdata['model_devi_resources'], + commands, + './', + run_tasks, + model_devi_group_size, + model_names, + forward_files, + backward_files, + outlog = 'model_devi.log', + errlog = 'model_devi.log') + dlog.info('$$$$$$$$$$$$$$$ dispatcher $$$$$$$$$$$$$$$$$$') + + os.mkdir('opt') + if not os.path.exists('traj'): + os.mkdir('traj') + # ------------------------------------------------- + + for jjj in range(len(all_task)): + # to opt directory + shutil.copyfile('POSCAR_%s'%str(jjj+1),os.path.join('opt','POSCAR_%s'%str(jjj+1)),) + shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'OUTCAR'),os.path.join('opt','OUTCAR_%s'%str(jjj+1)),) + shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'CONTCAR'),os.path.join('opt','CONTCAR_%s'%str(jjj+1)),) + # to run calypso directory + shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'OUTCAR'),'OUTCAR_%s'%str(jjj+1),) + shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'CONTCAR'),'CONTCAR_%s'%str(jjj+1),) + # to traj + shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'traj.traj'),os.path.join('traj','%s.traj'%str(jjj+1)),) + - forward_files = [mdp_filename, topol_filename, conf_filename, index_filename, ref_filename, type_filename, "input.json", "job.json" ] - if ndx_filename: forward_files.append(ndx_filename) - backward_files = ["%s.tpr" % deffnm, "%s.log" %deffnm , traj_filename, 'model_devi.out', "traj", "traj_deepmd" ] + dlog.info('##################### copy file back done #####################') + #os.rename('jr.json','jr_%s.json'%(str(ii))) + + tlist = glob.glob('task.*') + for t in tlist: + shutil.rmtree(t) + + os.chdir(cwd) + os.chdir(work_path) + f = open('record.calypso','a+') + f.write('2\n') + f.close() + os.chdir(cwd) + + + +def GenStructures(iter_index,jdata,mdata): + + # run calypso + # vsc means generate elemental, binary and ternary at the same time + vsc = jdata.get('vsc',False) + if vsc: + GenVscStructures(iter_index,jdata,mdata) + return + + model_devi_group_size = mdata['model_devi_group_size'] + model_devi_resources = mdata['model_devi_resources'] + + + iter_name = make_iter_name(iter_index) + work_path = os.path.join(iter_name, model_devi_name) + assert(os.path.isdir(work_path)) + # + calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) + calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) + # + + calypso_path = jdata.get('calypso_path') + #calypso_input_path = jdata.get('calypso_input_path') + popsize = int(_parse_calypso_input('PopSize',calypso_run_opt_path)) + maxstep = int(_parse_calypso_input('MaxStep',calypso_run_opt_path)) + + all_models = glob.glob(os.path.join(calypso_run_opt_path, 'graph*pb')) + model_names = [os.path.basename(ii) for ii in all_models] + + deepmdkit_python = mdata.get('deepmdkit_python') + command = "%s run_opt.py %s 1>> model_devi.log 2>> model_devi.log" % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) + command += " || %s check_outcar.py %s " % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) + commands = [command] cwd = os.getcwd() + os.chdir(calypso_run_opt_path) + + forward_files = ['POSCAR', 'run_opt.py','check_outcar.py'] + backward_files = ['OUTCAR','CONTCAR','traj.traj','model_devi.log'] + + Lpickup = _parse_calypso_input('PickUp',calypso_run_opt_path) + PickUpStep = _parse_calypso_input('PickStep',calypso_run_opt_path) + if os.path.exists('tag_pickup_%s'%(str(PickUpStep))): + dlog.info('caution! tag_pickup_%s exists!'%str(PickUpStep)) + Lpickup = 'F' + if Lpickup == 'T': + ftag = open('tag_pickup_%s'%(str(PickUpStep)),'w') + ftag.close() + os.remove('step') + fstep = open('step','w') + fstep.write('%12s'%str(PickUpStep)) + fstep.close() + else: + PickUpStep = 1 + try: + os.mkdir('opt') + except: + pass + #shutil.rmtree('opt') + #os.mkdir('opt') - user_forward_files = mdata.get("model_devi" + "_user_forward_files", []) - forward_files += [os.path.basename(file) for file in user_forward_files] - backward_files += mdata.get("model_devi" + "_user_backward_files", []) - api_version = mdata.get('api_version', '0.9') - if LooseVersion(api_version) < LooseVersion('1.0'): - warnings.warn(f"the dpdispatcher will be updated to new version." - f"And the interface may be changed. Please check the documents for more details") - dispatcher = make_dispatcher(mdata['model_devi_machine'], mdata['model_devi_resources'], work_path, run_tasks, model_devi_group_size) + + run_calypso = calypso_path+'/calypso.x | tee log' + for ii in range(int(PickUpStep)-1,maxstep+1): + dlog.info('$$$$$$$$$$$$$$$$$$$$ step %s $$$$$$$$$$$$$$$$$'%ii) + if ii == maxstep : + dlog.info('##################### counting OUTCAR #####################') + while True: + if len(glob.glob('OUTCAR_*')) == popsize: + break + dlog.info('##################### done counting OUTCAR #####################') + os.system('%s'%run_calypso) + break + # run calypso + + os.system('%s'%(run_calypso)) + + for pop in range(ii*int(popsize),(ii+1)*int(popsize)): + try: + os.mkdir('task.%03d'%pop) + except: + shutil.rmtree('task.%03d'%pop) + os.mkdir('task.%03d'%pop) + shutil.copyfile('run_opt.py',os.path.join('task.%03d'%pop,'run_opt.py')) + shutil.copyfile('check_outcar.py',os.path.join('task.%03d'%pop,'check_outcar.py')) + shutil.copyfile('POSCAR_%s'%str(pop-ii*int(popsize)+1),os.path.join('task.%03d'%(pop),'POSCAR')) + #for iii in range(1,popsize+1): + # shutil.copyfile('POSCAR_%s'%str(iii),os.path.join('task.%03d'%(iii-1),'POSCAR')) + dlog.info('##################### copy file done #####################') + + all_task = glob.glob( "task.*") + all_task.sort() + + run_tasks_ = all_task + + run_tasks = [os.path.basename(ii) for ii in run_tasks_] + #print(run_tasks) + + dlog.info('$$$$$$$$$$$$$$$ dispatcher running $$$$$$$$$$$$$$$$$$') + dispatcher=make_dispatcher(mdata['model_devi_machine'],mdata['model_devi_resources'],'./', run_tasks, model_devi_group_size) + #print(dispatcher) dispatcher.run_jobs(mdata['model_devi_resources'], commands, - work_path, + './', run_tasks, model_devi_group_size, model_names, @@ -1311,21 +1616,597 @@ def run_model_devi (iter_index, backward_files, outlog = 'model_devi.log', errlog = 'model_devi.log') + dlog.info('$$$$$$$$$$$$$$$ dispatcher $$$$$$$$$$$$$$$$$$') + + sstep = os.path.join('opt',str(ii)) + os.mkdir(sstep) + if not os.path.exists('traj'): + os.mkdir('traj') + + for jjj in range(ii*int(popsize),(ii+1)*int(popsize)): + # to opt directory + shutil.copyfile('POSCAR_%s'%str(jjj+1-ii*int(popsize)),os.path.join(sstep,'POSCAR_%s'%str(jjj+1-ii*int(popsize))),) + shutil.copyfile(os.path.join('task.%03d'%(jjj),'OUTCAR'),os.path.join(sstep,'OUTCAR_%s'%str(jjj+1-ii*int(popsize))),) + shutil.copyfile(os.path.join('task.%03d'%(jjj),'CONTCAR'),os.path.join(sstep,'CONTCAR_%s'%str(jjj+1-ii*int(popsize))),) + # to run calypso directory + shutil.copyfile(os.path.join('task.%03d'%(jjj),'OUTCAR'),'OUTCAR_%s'%str(jjj+1-ii*int(popsize)),) + shutil.copyfile(os.path.join('task.%03d'%(jjj),'CONTCAR'),'CONTCAR_%s'%str(jjj+1-ii*int(popsize)),) + # to traj + shutil.copyfile(os.path.join('task.%03d'%(jjj),'traj.traj'),os.path.join('traj','%s.traj'%str(jjj+1)),) + + + dlog.info('##################### copy file back done #####################') + + os.rename('jr.json','jr_%s.json'%(str(ii))) + + tlist = glob.glob('task.*') + for t in tlist: + shutil.rmtree(t) + + os.chdir(cwd) + os.chdir(work_path) + f = open('record.calypso','a+') + f.write('2\n') + f.close() + os.chdir(cwd) + +def VscAnalysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): + + # Analysis + + from ase.io.vasp import write_vasp + from ase.io.trajectory import Trajectory + import math + + #print('vscAnalysis') + ms = dpdata.MultiSystems() + + # trajs to be model devi + traj_path = os.path.join(calypso_run_opt_path,'traj') + traj_list = glob.glob(traj_path+'/*.traj') + + dlog.info('len(traj_list) %s'%str(len(traj_list))) + + record_traj_num = 0 + #print(traj_list) + for traj_name in traj_list: + traj_num = os.path.basename(traj_name).split('.')[0] + #print(traj_name) + trajs_origin = Trajectory(traj_name) + #print(trajs_origin) + + record_traj_num += len(trajs_origin) + #print(len(trajs_origin)) + if len(trajs_origin) >= 20 : + + trajs = [trajs_origin[iii] for iii in [0,4,-10,-5,-1]] + #print('1',trajs) + elif 5<=len(trajs_origin)<20: + trajs = [trajs_origin[random.randint(1,len(trajs_origin)-1)] for iii in range(3)] + trajs.append(trajs[0]) + trajs.append(trajs[-1]) + #print('2',trajs) + elif 3<= len(trajs_origin) <5: + trajs = [trajs_origin[round((len(trajs_origin)-1)/2)] ] + trajs.append(trajs[0]) + trajs.append(trajs[-1]) + elif len(trajs_origin) == 1: + trajs = [trajs_origin[0] ] + elif len(trajs_origin) == 2: + trajs = [trajs_origin[0],trajs_origin[-1] ] + else: + pass + + #print(trajs) + for idx, traj in enumerate(trajs): + write_vasp(os.path.join( + traj_path,'%04d.%03d.poscar' % ( + int(traj_num), int(idx) + ) + ), + traj) + + traj_pos_list = glob.glob(traj_path+'/*.poscar') + + dlog.info('traj_num %s'%str(len(traj_pos_list))) + dlog.info('total_traj_num %s'%str(record_traj_num)) + + for npos in traj_pos_list: + try: + ms.append(dpdata.System(npos, type_map = jdata['type_map'])) + except Exception as e: + print(npos,'failed : ',e) + + if len(ms) == 0: + print('too little confs, ') + return + + result_path = os.path.join(calypso_run_opt_path,'results') + + if os.path.exists(os.path.join(result_path,'deepmd')): + shutil.rmtree(os.path.join(result_path,'deepmd')) + ms.to_deepmd_raw(os.path.join(result_path,'deepmd')) + ms.to_deepmd_npy(os.path.join(result_path,'deepmd')) + + + split_list = glob.glob(os.path.join(result_path,'deepmd','*')) + for i in range(len(split_list)): + ss = dpdata.System(split_list[i],fmt='deepmd') + for j in range(ss.get_nframes()): + ss.to('vasp/poscar',os.path.join(split_list[i],'%03d.%03d.poscar'%(i,j)),frame_idx=j) + strus_path = os.path.join(calypso_model_devi_path,'%03d.structures'%i) + if not os.path.exists(strus_path): + shutil.copytree(split_list[i],strus_path) + else: + shutil.rmtree(strus_path) + shutil.copytree(split_list[i],strus_path) + + iter_name = make_iter_name(iter_index) + work_path = os.path.join(iter_name, model_devi_name) + cwd = os.getcwd() + os.chdir(cwd) + os.chdir(work_path) + f = open('record.calypso','a+') + f.write('3\n') + f.close() + os.chdir(cwd) + +def Analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): + + # vsc means generate elemental, binary and ternary at the same time + vsc = jdata.get('vsc',False) + if vsc: + VscAnalysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path) + return + + # Analysis + + from ase.io.vasp import write_vasp + from ase.io.trajectory import Trajectory + import math + + dlog.info('$$$$$$$$$$$$$$$ Analysis Started $$$$$$$$$$$$$$$$$$') + + iter_name = make_iter_name(iter_index) + work_path = os.path.join(iter_name, model_devi_name) + + cwd = os.getcwd() + result_path = os.path.join(calypso_run_opt_path,'results') + os.chdir(result_path) + # may need to delete + os.system('cak.py --vasp -a ') + lll = glob.glob('dir*') + if len(lll)== 0: + return + + dlog.info('cak.py run successfully') + + dlog.info('$$$$$$$$$$$$$$$ Analysis Done $$$$$$$$$$$$$$$$$$') + + os.chdir(cwd) + dir_list = glob.glob(result_path+'/dir*') + sys_num = len(dir_list) + ms = dpdata.MultiSystems() + for num in range(sys_num): + + if sys_num > 1: + temp_path = os.path.join(dir_list[num],'dir_0.01') + os.chdir(temp_path) + os.system('rename .vasp .poscar *') + os.chdir(cwd) + confs = glob.glob(os.path.join(temp_path,'*.poscar')) + elif sys_num == 1: + os.chdir(dir_list[num]) + os.system('rename .vasp .poscar *') + os.chdir(cwd) + confs = glob.glob(os.path.join(dir_list[num],'*.poscar')) + if confs == []: + if sys_num == 1: + dlog.info('dir empty') + return + else: + continue + else: + for cc in confs: + try: + ms.append(dpdata.System(cc, type_map = jdata['type_map'])) + except: + dlog.info(cc,'convert failed') + + # trajs to be model devi + traj_path = os.path.join(calypso_run_opt_path,'traj') + traj_list = glob.glob(traj_path+'/*.traj') + + dlog.info('len(traj_list) %s'%str(len(traj_list))) + + record_traj_num = 0 + for traj_name in traj_list: + traj_num = os.path.basename(traj_name).split('.')[0] + trajs_origin = Trajectory(traj_name) + + record_traj_num += len(trajs_origin) + if len(trajs_origin) >= 20 : + + trajs = [trajs_origin[iii] for iii in [4,-10,-5]] + #print('1',trajs) + elif 5<=len(trajs_origin)<20: + trajs = [trajs_origin[random.randint(1,len(trajs_origin)-1)] for iii in range(3)] + #print('2',trajs) + elif 3<= len(trajs_origin) <5: + trajs = [trajs_origin[round((len(trajs_origin)-1)/2)] ] + elif len(trajs_origin) == 1: + trajs = [trajs_origin[0] ] + else: + pass + + for idx, traj in enumerate(trajs): + write_vasp(os.path.join( + traj_path,'%03d.%03d.poscar' % ( + int(traj_num), int(idx) + ) + ), + traj) + + traj_pos_list = glob.glob(traj_path+'/*.poscar') + + dlog.info('traj_num %s'%str(len(traj_pos_list))) + dlog.info('total_traj_num %s'%str(record_traj_num)) + + for npos in traj_pos_list: + try: + ms.append(dpdata.System(npos, type_map = jdata['type_map'])) + except Exception as e: + print(npos,'failed : ',e) + + if len(ms) == 0: + print('too little confs, ') + return + + if os.path.exists(os.path.join(result_path,'deepmd')): + shutil.rmtree(os.path.join(result_path,'deepmd')) + ms.to_deepmd_raw(os.path.join(result_path,'deepmd')) + ms.to_deepmd_npy(os.path.join(result_path,'deepmd')) + + + result_path = os.path.join(calypso_run_opt_path,'results') + split_list = glob.glob(os.path.join(result_path,'deepmd','*')) + for i in range(len(split_list)): + ss = dpdata.System(split_list[i],fmt='deepmd') + for j in range(ss.get_nframes()): + ss.to('vasp/poscar',os.path.join(split_list[i],'%03d.%03d.poscar'%(i,j)),frame_idx=j) + strus_path = os.path.join(calypso_model_devi_path,'%03d.structures'%i) + if not os.path.exists(strus_path): + shutil.copytree(split_list[i],strus_path) + else: + shutil.rmtree(strus_path) + shutil.copytree(split_list[i],strus_path) + + os.chdir(cwd) + os.chdir(work_path) + f = open('record.calypso','a+') + f.write('3\n') + f.close() + os.chdir(cwd) + + +def Modd(iter_index,calypso_model_devi_path,all_models,jdata): + + # Model Devi + + + import re + import math + import copy + from deepmd.infer import calc_model_devi + from deepmd.infer import DeepPot as DP + + dlog.info('$$$$$$$$$$$$$$$ Model Devi Start $$$$$$$$$$$$$$$$$$') + + iter_name = make_iter_name(iter_index) + work_path = os.path.join(iter_name, model_devi_name) + + cwd = os.getcwd() + Devis = [] + pcount = 0 + strus_list = glob.glob(os.path.join(calypso_model_devi_path,'*.structures')) + for num in range(len(strus_list)): + strus_path = strus_list[num] + os.chdir(strus_path) + os.system('rename .vasp .poscar *') + os.chdir(cwd) + + structure_list = glob.glob(os.path.join(strus_path,'*.poscar')) + #structure_list = sorted(structure_list_temp, key = lambda i:int(re.match(r'(\d+)',i).group())) + #print(structure_list) + + + if len(structure_list) == 0: + continue + else: + num_per_task = math.ceil(len(structure_list)/500) + graphs = [DP(model) for model in all_models] + for temp in range(num_per_task): + #for temp in range(len(structure_list)): + task_name = os.path.join(calypso_model_devi_path,'task.%03d.%03d'%(num,temp)) + put_poscar = os.path.join(task_name,'traj') + if not os.path.exists(task_name): + os.mkdir(task_name) + os.mkdir(put_poscar) + else: + shutil.rmtree(task_name) + os.mkdir(task_name) + os.mkdir(put_poscar) + devis = [] + try: + temp_sl = structure_list[temp*500:(temp+1)*500] + except Exception as err: + dlog.info('err %s'%str(err)) + temp_sl = structure_list[temp*500:] + + #print(temp_sl) + new_index = 0 + for index,sname in enumerate(temp_sl): + #print(index,sname) + shutil.copyfile(sname,os.path.join(put_poscar,'%s.poscar'%str(index))) + os.chdir(put_poscar) + pdata = dpdata.System('%s.poscar'%str(index),type_map = jdata['type_map']) + nopbc = pdata.nopbc + coord = pdata.data['coords'] + cell = pdata.data['cells'] + atom_types = pdata.data['atom_types'] + devi = calc_model_devi(coord,cell,atom_types,graphs,nopbc=nopbc) + # ------------------------------------------------------------------------------------ + #print(devi) + #print(devi.shape) + dis_temp = dpdata.System('%s.poscar'%str(index)) + dis = dis_temp.to_ase_structure()[0].get_all_distances(mic=True) + row,col = np.diag_indices_from(dis) + dis[row,col] = 10000 + min_dis = np.nanmin(dis) + devi = np.append(devi[0],min_dis) + #devi.reshape((1,8)) + t = [devi] + devi = np.array(t) + # ------------------------------------------------------------------------------------ + #devi[0] = index + temp_d = copy.deepcopy(devi) + temp_D = copy.deepcopy(devi) + devis.append(temp_d) + Devis.append(temp_D) + devis[index][0][0] = np.array(index) + Devis[pcount][0][0] = np.array(pcount) + #print(devis[0][0][0]) + #print(devis) + #print(Devis) + pcount += 1 + new_index += 1 + os.chdir(cwd) + # a problem occur when the cycle is done + # the first number always become 18 and don't know why + # use deep copy to solve this soluation + os.chdir(task_name) + #os.chdir(calypso_model_devi_path) + devis = np.vstack(devis) + write_model_devi_out(devis,'model_devi.out') + os.chdir(cwd) + + os.chdir(calypso_model_devi_path) + #Devis = Devis[np.argsort(Devis[:,0])] + Devis = np.vstack(Devis) + write_model_devi_out(Devis,'Model_Devi.out') + os.chdir(cwd) + + os.chdir(work_path) + f = open('record.calypso','a+') + f.write('4\n') + f.close() + os.chdir(cwd) + + + #else: + # dlog.info('wrong') + # return False + +def run_model_devi_calypso (iter_index, + jdata, + mdata) : + + print('############## run calypso model devi #######################') + + + iter_name = make_iter_name(iter_index) + work_path = os.path.join(iter_name, model_devi_name) + assert(os.path.isdir(work_path)) + + calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) + calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) + + all_models = glob.glob(os.path.join(calypso_run_opt_path, 'graph*pb')) + + cwd = os.getcwd() + + api_version = mdata.get('api_version', '0.9') + if LooseVersion(api_version) < LooseVersion('1.0'): + warnings.warn(f"the dpdispatcher will be updated to new version." + f"And the interface may be changed. Please check the documents for more details") + record_calypso_path = os.path.join(work_path,'record.calypso') + while True: + if not os.path.exists(record_calypso_path): + f = open(record_calypso_path,'w') + f.write('1\n') + lines = '1' + f.close() + else: + f = open(record_calypso_path,'r') + lines = f.readlines() + f.close() + + if lines[-1].strip().strip('\n') == '1': + # Gen Structures + GenStructures(iter_index,jdata,mdata) + + elif lines[-1].strip().strip('\n') == '2': + # Analysis & to deepmd/raw + Analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path) + + elif lines[-1].strip().strip('\n') == '3': + # Model Devi + Modd(iter_index,calypso_model_devi_path,all_models,jdata) + + elif lines[-1].strip().strip('\n') == '4': + # Model Devi + print('Model Devi is done.') + break elif LooseVersion(api_version) >= LooseVersion('1.0'): - submission = make_submission( - mdata['model_devi_machine'], - mdata['model_devi_resources'], - commands=commands, - work_path=work_path, - run_tasks=run_tasks, - group_size=model_devi_group_size, - forward_common_files=model_names, - forward_files=forward_files, - backward_files=backward_files, - outlog = 'model_devi.log', - errlog = 'model_devi.log') - submission.run_submission() + + record_calypso_path = os.path.join(work_path,'record.calypso') + if not os.path.exists(record_calypso_path): + f = open(record_calypso_path,'w') + f.write('1\n') + lines = '1' + f.close() + else: + f = open(record.calypso_path,'r') + lines = f.readlines() + f.close() + + if lines[-1].strip().strip('\n') == '1': + # Gen Structures + GenStructures(iter_index,jdata,mdata) + + elif lines[-1].strip().strip('\n') == '2': + # Analysis & to deepmd/raw + Analysis(calypso_run_opt_path,calypso_model_devi_path) + + elif lines[-1].strip().strip('\n') == '3': + # Model Devi + Modd(calypso_model_devi_path,all_models,jdata) + + +def run_model_devi (iter_index, + jdata, + mdata) : + + model_devi_engine = jdata.get("model_devi_engine", "lammps") + if model_devi_engine == "calypso": + run_model_devi_calypso(iter_index,jdata,mdata) + else: + #rmdlog.info("This module has been run !") + model_devi_exec = mdata['model_devi_command'] + + model_devi_group_size = mdata['model_devi_group_size'] + model_devi_resources = mdata['model_devi_resources'] + use_plm = jdata.get('model_devi_plumed', False) + use_plm_path = jdata.get('model_devi_plumed_path', False) + + iter_name = make_iter_name(iter_index) + work_path = os.path.join(iter_name, model_devi_name) + assert(os.path.isdir(work_path)) + + all_task = glob.glob(os.path.join(work_path, "task.*")) + all_task.sort() + fp = open (os.path.join(work_path, 'cur_job.json'), 'r') + cur_job = json.load (fp) + + run_tasks_ = all_task + # for ii in all_task: + # fres = os.path.join(ii, 'model_devi.out') + # if os.path.isfile(fres) : + # nlines = np.loadtxt(fres).shape[0] + # if nframes != nlines : + # run_tasks_.append(ii) + # else : + # run_tasks_.append(ii) + + run_tasks = [os.path.basename(ii) for ii in run_tasks_] + #dlog.info("all_task is ", all_task) + #dlog.info("run_tasks in run_model_deviation",run_tasks_) + all_models = glob.glob(os.path.join(work_path, 'graph*pb')) + model_names = [os.path.basename(ii) for ii in all_models] + + model_devi_engine = jdata.get("model_devi_engine", "lammps") + if model_devi_engine == "lammps": + command = "{ if [ ! -f dpgen.restart.10000 ]; then %s -i input.lammps -v restart 0; else %s -i input.lammps -v restart 1; fi }" % (model_devi_exec, model_devi_exec) + command = "/bin/sh -c '%s'" % command + commands = [command] + forward_files = ['conf.lmp', 'input.lammps', 'traj'] + backward_files = ['model_devi.out', 'model_devi.log', 'traj'] + if use_plm: + forward_files += ['input.plumed'] + # backward_files += ['output.plumed'] + backward_files += ['output.plumed','COLVAR'] + if use_plm_path: + forward_files += ['plmpath.pdb'] + elif model_devi_engine == "gromacs": + + gromacs_settings = jdata.get("gromacs_settings", {}) + mdp_filename = gromacs_settings.get("mdp_filename", "md.mdp") + topol_filename = gromacs_settings.get("topol_filename", "processed.top") + conf_filename = gromacs_settings.get("conf_filename", "conf.gro") + index_filename = gromacs_settings.get("index_filename", "index.raw") + type_filename = gromacs_settings.get("type_filename", "type.raw") + ndx_filename = gromacs_settings.get("ndx_filename", "") + # Initial reference to process pbc condition. + # Default is em.tpr + ref_filename = gromacs_settings.get("ref_filename", "em.tpr") + deffnm = gromacs_settings.get("deffnm", "deepmd") + maxwarn = gromacs_settings.get("maxwarn", 1) + traj_filename = gromacs_settings.get("traj_filename", "deepmd_traj.gro") + grp_name = gromacs_settings.get("group_name", "Other") + trj_freq = cur_job.get("trj_freq", 10) + + command = "%s grompp -f %s -p %s -c %s -o %s -maxwarn %d" % (model_devi_exec, mdp_filename, topol_filename, conf_filename, deffnm, maxwarn) + command += "&& %s mdrun -deffnm %s -cpi" %(model_devi_exec, deffnm) + if ndx_filename: + command += f"&& echo -e \"{grp_name}\\n{grp_name}\\n\" | {model_devi_exec} trjconv -s {ref_filename} -f {deffnm}.trr -n {ndx_filename} -o {traj_filename} -pbc mol -ur compact -center" + else: + command += "&& echo -e \"%s\\n%s\\n\" | %s trjconv -s %s -f %s.trr -o %s -pbc mol -ur compact -center" % (grp_name, grp_name, model_devi_exec, ref_filename, deffnm, traj_filename) + command += "&& if [ ! -d traj ]; then \n mkdir traj; fi\n" + command += f"python -c \"import dpdata;system = dpdata.System('{traj_filename}', fmt='gromacs/gro'); [system.to_gromacs_gro('traj/%d.gromacstrj' % (i * {trj_freq}), frame_idx=i) for i in range(system.get_nframes())]; system.to_deepmd_npy('traj_deepmd')\"" + command += f"&& dp model-devi -m ../graph.000.pb ../graph.001.pb ../graph.002.pb ../graph.003.pb -s traj_deepmd -o model_devi.out -f {trj_freq}" + commands = [command] + + forward_files = [mdp_filename, topol_filename, conf_filename, index_filename, ref_filename, type_filename, "input.json", "job.json" ] + if ndx_filename: forward_files.append(ndx_filename) + backward_files = ["%s.tpr" % deffnm, "%s.log" %deffnm , traj_filename, 'model_devi.out', "traj", "traj_deepmd" ] + + + cwd = os.getcwd() + + user_forward_files = mdata.get("model_devi" + "_user_forward_files", []) + forward_files += [os.path.basename(file) for file in user_forward_files] + backward_files += mdata.get("model_devi" + "_user_backward_files", []) + api_version = mdata.get('api_version', '0.9') + if LooseVersion(api_version) < LooseVersion('1.0'): + warnings.warn(f"the dpdispatcher will be updated to new version." + f"And the interface may be changed. Please check the documents for more details") + dispatcher = make_dispatcher(mdata['model_devi_machine'], mdata['model_devi_resources'], work_path, run_tasks, model_devi_group_size) + dispatcher.run_jobs(mdata['model_devi_resources'], + commands, + work_path, + run_tasks, + model_devi_group_size, + model_names, + forward_files, + backward_files, + outlog = 'model_devi.log', + errlog = 'model_devi.log') + + elif LooseVersion(api_version) >= LooseVersion('1.0'): + submission = make_submission( + mdata['model_devi_machine'], + mdata['model_devi_resources'], + commands=commands, + work_path=work_path, + run_tasks=run_tasks, + group_size=model_devi_group_size, + forward_common_files=model_names, + forward_files=forward_files, + backward_files=backward_files, + outlog = 'model_devi.log', + errlog = 'model_devi.log') + submission.run_submission() def post_model_devi (iter_index, jdata, @@ -1383,6 +2264,15 @@ def check_bad_box(conf_name, ratio = np.max(lengths) / np.min(dists) if ratio > float(value): is_bad = True + # + elif key == 'wrap_ratio': + ratio=[sys['cells'][0][1][0]/sys['cells'][0][0][0],sys['cells'][0][2][1]/sys['cells'][0][1][1],sys['cells'][0][2][0]/sys['cells'][0][0][0]] + if np.max(np.abs(ratio)) > float(value): + is_bad = True + elif key == 'tilt_ratio': + ratio=[sys['cells'][0][1][0]/sys['cells'][0][1][1],sys['cells'][0][2][1]/sys['cells'][0][2][2],sys['cells'][0][2][0]/sys['cells'][0][2][2]] + if np.max(np.abs(ratio)) > float(value): + is_bad= True else: raise RuntimeError('unknow key', key) return is_bad @@ -1417,6 +2307,8 @@ def _select_by_model_devi_standard( model_devi_f_avg_relative : bool = False, detailed_report_make_fp : bool = True, ): + numofspecies = _parse_calypso_input('NumberOfSpecies',calypso_run_opt_path) + min_dis = _parse_calypso_dis_mtx(numofspecies,calypso_run_opt_path) fp_candidate = [] if detailed_report_make_fp: fp_rest_accurate = [] @@ -1430,25 +2322,49 @@ def _select_by_model_devi_standard( with warnings.catch_warnings(): warnings.simplefilter("ignore") all_conf = _read_model_devi_file(tt, model_devi_f_avg_relative) + + model_devi_engine = jdata.get('model_devi_engine','lammps') + if all_conf.shape == (7,): + all_conf = all_conf.reshape(1,all_conf.shape[0]) + elif model_devi_engine == 'calypso' and all_conf.shape == (8,): + all_conf = all_conf.reshape(1,all_conf.shape[0]) for ii in range(all_conf.shape[0]) : if all_conf[ii][0] < model_devi_skip : continue cc = int(all_conf[ii][0]) if cluster_cutoff is None: - if (all_conf[ii][1] < v_trust_hi and all_conf[ii][1] >= v_trust_lo) or \ - (all_conf[ii][4] < f_trust_hi and all_conf[ii][4] >= f_trust_lo) : - fp_candidate.append([tt, cc]) - counter['candidate'] += 1 - elif (all_conf[ii][1] >= v_trust_hi ) or (all_conf[ii][4] >= f_trust_hi ): - if detailed_report_make_fp: - fp_rest_failed.append([tt, cc]) - counter['failed'] += 1 - elif (all_conf[ii][1] < v_trust_lo and all_conf[ii][4] < f_trust_lo ): - if detailed_report_make_fp: - fp_rest_accurate.append([tt, cc]) - counter['accurate'] += 1 - else : - raise RuntimeError('md traj %s frame %d with f devi %f does not belong to either accurate, candidiate and failed, it should not happen' % (tt, ii, all_conf[ii][4])) + if model_devi_engine != 'calypso': + if (all_conf[ii][1] < v_trust_hi and all_conf[ii][1] >= v_trust_lo) or \ + (all_conf[ii][4] < f_trust_hi and all_conf[ii][4] >= f_trust_lo) : + fp_candidate.append([tt, cc]) + counter['candidate'] += 1 + elif (all_conf[ii][1] >= v_trust_hi ) or (all_conf[ii][4] >= f_trust_hi ): + if detailed_report_make_fp: + fp_rest_failed.append([tt, cc]) + counter['failed'] += 1 + elif (all_conf[ii][1] < v_trust_lo and all_conf[ii][4] < f_trust_lo ): + if detailed_report_make_fp: + fp_rest_accurate.append([tt, cc]) + counter['accurate'] += 1 + else : + raise RuntimeError('md traj %s frame %d with f devi %f does not belong to either accurate, candidiate and failed, it should not happen' % (tt, ii, all_conf[ii][4])) + elif model_devi_engine == 'calypso': + if (all_conf[ii][1] < v_trust_hi and all_conf[ii][1] >= v_trust_lo and float(all_conf[ii][-1]) > float(min_dis) or \ + (all_conf[ii][4] < f_trust_hi and all_conf[ii][4] >= f_trust_lo) and float(all_conf[ii][-1]) > float(min_dis)): + fp_candidate.append([tt, cc]) + counter['candidate'] += 1 + elif (all_conf[ii][1] >= v_trust_hi ) or (all_conf[ii][4] >= f_trust_hi ) or (float(all_conf[ii][-1]) <= float(min_dis)): + if detailed_report_make_fp: + fp_rest_failed.append([tt, cc]) + counter['failed'] += 1 + elif (all_conf[ii][1] < v_trust_lo and all_conf[ii][4] < f_trust_lo and float(all_conf[ii][-1]) > float(min_dis) ): + if detailed_report_make_fp: + fp_rest_accurate.append([tt, cc]) + counter['accurate'] += 1 + else : + dlog.info('ase opt traj %s frame %d with f devi %f does not belong to either accurate, candidiate and failed ' \ + % (tt, ii, all_conf[ii][4])) + pass else: idx_candidate = np.where(np.logical_and(all_conf[ii][7:] < f_trust_hi, all_conf[ii][7:] >= f_trust_lo))[0] for jj in idx_candidate: @@ -1586,6 +2502,28 @@ def _make_fp_vasp_inner (modd_path, fp_params map parameters for fp """ + # -------------------------------------------------------------------------------------------------------------------------------------- + model_devi_engine = jdata.get('model_devi_engine', 'lammps') + if model_devi_engine == 'calypso': + calypso_total_fp_num = 300 + modd_path = os.path.join(modd_path,calypso_model_devi_name) + model_devi_skip = -1 + with open(os.path.join(modd_path,'Model_Devi.out'),'r') as summfile: + summary = np.loadtxt(summfile) + summaryfmax = summary[:,-4] + sort_summary = np.sort(summaryfmax) + acc = np.where(sort_summary>f_trust_lo ) + fail = np.where(sort_summary 1 and candi_num <= calypso_total_fp_num): + numb_task = min(this_fp_task_max, len(fp_candidate)) + if (numb_task < fp_task_min): + numb_task = 0 + elif model_devi_engine == 'calypso' and len(jdata.get('type_map')) > 1 and candi_num > calypso_total_fp_num): + numb_task = calypso_intend_fp_num + if (len(fp_candidate) < numb_task): + numb_task = 0 + # ---------------------------------------------------------------------------- dlog.info("system {0:s} accurate_ratio: {1:8.4f} thresholds: {2:6.4f} and {3:6.4f} eff. task min and max {4:4d} {5:4d} number of fp tasks: {6:6d}".format(ss, accurate_ratio, fp_accurate_soft_threshold, fp_accurate_threshold, fp_task_min, this_fp_task_max, numb_task)) # make fp tasks model_devi_engine = jdata.get("model_devi_engine", "lammps") @@ -1698,13 +2659,18 @@ def _trust_limitation_check(sys_idx, lim): conf_name = os.path.join(tt, "traj") if model_devi_engine == "lammps": conf_name = os.path.join(conf_name, str(ii) + '.lammpstrj') + ffmt = 'lammps/dump' elif model_devi_engine == "gromacs": conf_name = os.path.join(conf_name, str(ii) + '.gromacstrj') + ffmt = 'lammps/dump' + elif model_devi_engine == "calypso": + conf_name = os.path.join(conf_name, str(ii) + '.poscar') + ffmt = 'vasp/poscar' else: raise RuntimeError("unknown model_devi engine", model_devi_engine) conf_name = os.path.abspath(conf_name) if skip_bad_box is not None: - skip = check_bad_box(conf_name, skip_bad_box) + skip = check_bad_box(conf_name, skip_bad_box, fmt=ffmt) if skip: count_bad_box += 1 continue @@ -1716,9 +2682,10 @@ def _trust_limitation_check(sys_idx, lim): count_bad_cluster +=1 continue - # link job.json - job_name = os.path.join(tt, "job.json") - job_name = os.path.abspath(job_name) + if model_devi_engine != 'calypso': + # link job.json + job_name = os.path.join(tt, "job.json") + job_name = os.path.abspath(job_name) if cluster_cutoff is not None: # take clusters @@ -1735,8 +2702,11 @@ def _trust_limitation_check(sys_idx, lim): cwd = os.getcwd() os.chdir(fp_task_path) if cluster_cutoff is None: - os.symlink(os.path.relpath(conf_name), 'conf.dump') - os.symlink(os.path.relpath(job_name), 'job.json') + if model_devi_engine != 'calypso': + os.symlink(os.path.relpath(conf_name), 'conf.dump') + os.symlink(os.path.relpath(job_name), 'job.json') + else: + os.symlink(os.path.relpath(conf_name), 'POSCAR') else: os.symlink(os.path.relpath(poscar_name), 'POSCAR') np.save("atom_pref", new_system.data["atom_pref"]) @@ -1747,6 +2717,7 @@ def _trust_limitation_check(sys_idx, lim): dlog.info("system {0:s} skipped {1:6d} confs with bad box, {2:6d} remains".format(ss, count_bad_box, numb_task - count_bad_box)) if count_bad_cluster > 0: dlog.info("system {0:s} skipped {1:6d} confs with bad cluster, {2:6d} remains".format(ss, count_bad_cluster, numb_task - count_bad_cluster)) + dlog.info("summary accurate_ratio: {0:8.4f}% candidata_ratio: {1:8.4f}% failed_ratio: {2:8.4f}% in {3:d} structures".format( acc_num*100/tot,candi_num*100/tot,fail_num*100/tot,tot )) if cluster_cutoff is None: cwd = os.getcwd() for idx, task in enumerate(fp_tasks): @@ -1761,6 +2732,8 @@ def _trust_limitation_check(sys_idx, lim): dump_to_deepmd_raw('conf.dump', 'deepmd.raw', type_map, fmt='gromacs/gro', charge=charges_recorder[idx]) else: dump_to_deepmd_raw('conf.dump', 'deepmd.raw', type_map, fmt='gromacs/gro', charge=None) + elif model_devi_engine == 'calypso': + pass else: raise RuntimeError("unknown model_devi engine", model_devi_engine) os.chdir(cwd) @@ -2494,8 +3467,10 @@ def post_fp_vasp (iter_index, rfailed=None): ratio_failed = rfailed if rfailed else jdata.get('ratio_failed',0.05) - model_devi_jobs = jdata['model_devi_jobs'] - assert (iter_index < len(model_devi_jobs)) + model_devi_engine = jdata.get('model_devi_engine', "lammps") + if model_devi_engine != 'calypso': + model_devi_jobs = jdata['model_devi_jobs'] + assert (iter_index < len(model_devi_jobs)) use_ele_temp = jdata.get('use_ele_temp', 0) iter_name = make_iter_name(iter_index) @@ -2537,14 +3512,19 @@ def post_fp_vasp (iter_index, if all_sys is None: all_sys = _sys else: - all_sys.append(_sys) + try: + all_sys.append(_sys) + except: + dlog.info('%s has different formula, so pass in line 3518'%oo) + pass # save ele_temp, if any - with open(oo.replace('OUTCAR', 'job.json')) as fp: - job_data = json.load(fp) - if 'ele_temp' in job_data: - assert(use_ele_temp) - ele_temp = job_data['ele_temp'] - all_te.append(ele_temp) + if model_devi_engine != 'calypso': + with open(oo.replace('OUTCAR', 'job.json')) as fp: + job_data = json.load(fp) + if 'ele_temp' in job_data: + assert(use_ele_temp) + ele_temp = job_data['ele_temp'] + all_te.append(ele_temp) else: icount+=1 all_te = np.array(all_te) @@ -2552,25 +3532,28 @@ def post_fp_vasp (iter_index, sys_data_path = os.path.join(work_path, 'data.%s'%ss) all_sys.to_deepmd_raw(sys_data_path) all_sys.to_deepmd_npy(sys_data_path, set_size = len(sys_outcars)) - if all_te.size > 0: - assert(len(all_sys) == all_sys.get_nframes()) - assert(len(all_sys) == all_te.size) - all_te = np.reshape(all_te, [-1,1]) - if use_ele_temp == 0: - raise RuntimeError('should not get ele temp at setting: use_ele_temp == 0') - elif use_ele_temp == 1: - np.savetxt(os.path.join(sys_data_path, 'fparam.raw'), all_te) - np.save(os.path.join(sys_data_path, 'set.000', 'fparam.npy'), all_te) - elif use_ele_temp == 2: - tile_te = np.tile(all_te, [1, all_sys.get_natoms()]) - np.savetxt(os.path.join(sys_data_path, 'aparam.raw'), tile_te) - np.save(os.path.join(sys_data_path, 'set.000', 'aparam.npy'), tile_te) - else: - raise RuntimeError('invalid setting of use_ele_temp ' + str(use_ele_temp)) + if model_devi_engine != 'calypso': + if all_te.size > 0: + assert(len(all_sys) == all_sys.get_nframes()) + assert(len(all_sys) == all_te.size) + all_te = np.reshape(all_te, [-1,1]) + if use_ele_temp == 0: + raise RuntimeError('should not get ele temp at setting: use_ele_temp == 0') + elif use_ele_temp == 1: + np.savetxt(os.path.join(sys_data_path, 'fparam.raw'), all_te) + np.save(os.path.join(sys_data_path, 'set.000', 'fparam.npy'), all_te) + elif use_ele_temp == 2: + tile_te = np.tile(all_te, [1, all_sys.get_natoms()]) + np.savetxt(os.path.join(sys_data_path, 'aparam.raw'), tile_te) + np.save(os.path.join(sys_data_path, 'set.000', 'aparam.npy'), tile_te) + else: + raise RuntimeError('invalid setting of use_ele_temp ' + str(use_ele_temp)) rfail=float(icount)/float(tcount) dlog.info("failed frame: %6d in %6d %6.2f %% " % (icount, tcount, rfail * 100.)) + if model_devi_engine == 'calypso': + ratio_failed = 0.1 if rfail>ratio_failed: raise RuntimeError("find too many unsuccessfully terminated jobs. Too many FP tasks are not converged. Please check your input parameters (e.g. INCAR) or configuration (e.g. POSCAR) in directories \'iter.*.*/02.fp/task.*.*/.\'") From 3b26eb9e0843422ea862b0ef1967b09882ebaa65 Mon Sep 17 00:00:00 2001 From: zhenyu Date: Tue, 22 Feb 2022 13:55:46 +0800 Subject: [PATCH 02/46] fix:fix syntax bug --- dpgen/generator/run.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index b420e9505..b4703222f 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -1545,8 +1545,8 @@ def GenStructures(iter_index,jdata,mdata): forward_files = ['POSCAR', 'run_opt.py','check_outcar.py'] backward_files = ['OUTCAR','CONTCAR','traj.traj','model_devi.log'] - Lpickup = _parse_calypso_input('PickUp',calypso_run_opt_path) - PickUpStep = _parse_calypso_input('PickStep',calypso_run_opt_path) + Lpickup = _parse_calypso_input('PickUp','.') + PickUpStep = _parse_calypso_input('PickStep','.') if os.path.exists('tag_pickup_%s'%(str(PickUpStep))): dlog.info('caution! tag_pickup_%s exists!'%str(PickUpStep)) Lpickup = 'F' @@ -2364,7 +2364,7 @@ def _select_by_model_devi_standard( else : dlog.info('ase opt traj %s frame %d with f devi %f does not belong to either accurate, candidiate and failed ' \ % (tt, ii, all_conf[ii][4])) - pass + pass else: idx_candidate = np.where(np.logical_and(all_conf[ii][7:] < f_trust_hi, all_conf[ii][7:] >= f_trust_lo))[0] for jj in idx_candidate: @@ -2642,7 +2642,7 @@ def _trust_limitation_check(sys_idx, lim): numb_task = min(this_fp_task_max, len(fp_candidate)) if (numb_task < fp_task_min): numb_task = 0 - elif model_devi_engine == 'calypso' and len(jdata.get('type_map')) > 1 and candi_num > calypso_total_fp_num): + elif (model_devi_engine == 'calypso' and len(jdata.get('type_map')) > 1 and candi_num > calypso_total_fp_num): numb_task = calypso_intend_fp_num if (len(fp_candidate) < numb_task): numb_task = 0 From 729d8bd985cf7f72289e3b5ea284189ee3e34071 Mon Sep 17 00:00:00 2001 From: zhenyu Date: Tue, 22 Feb 2022 16:19:26 +0800 Subject: [PATCH 03/46] fix summary ratio bug --- dpgen/generator/run.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index b4703222f..7d0350f2d 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -2504,6 +2504,8 @@ def _make_fp_vasp_inner (modd_path, # -------------------------------------------------------------------------------------------------------------------------------------- model_devi_engine = jdata.get('model_devi_engine', 'lammps') + numofspecies = _parse_calypso_input('NumberOfSpecies',calypso_run_opt_path) + min_dis = _parse_calypso_dis_mtx(numofspecies,calypso_run_opt_path) if model_devi_engine == 'calypso': calypso_total_fp_num = 300 modd_path = os.path.join(modd_path,calypso_model_devi_name) @@ -2511,17 +2513,18 @@ def _make_fp_vasp_inner (modd_path, with open(os.path.join(modd_path,'Model_Devi.out'),'r') as summfile: summary = np.loadtxt(summfile) summaryfmax = summary[:,-4] - sort_summary = np.sort(summaryfmax) - acc = np.where(sort_summary>f_trust_lo ) - fail = np.where(sort_summary min_dis)) + fail = np.where((summaryfmax > f_trust_hi) | (dis <= min_dis)) + nnan = np.where(np.isnan(summaryfmax)) + + acc_num = len(acc[0]) + fail_num = len(fail[0]) + nan_num = len(nnan[0]) + tot = len(summaryfmax) - nan_num + candi_num = tot - acc_num - fail_num dlog.info("summary accurate_ratio: {0:8.4f}% candidata_ratio: {1:8.4f}% failed_ratio: {2:8.4f}% in {3:d} structures".format( - acc_num*100/tot,candi_num*100/tot,fail_num*100/tot,tot )) + acc_num*100/tot,candi_num*100/tot,fail_num*100/tot,tot )) # -------------------------------------------------------------------------------------------------------------------------------------- modd_task = glob.glob(os.path.join(modd_path, "task.*")) From e08b4792abfe782fdf317819e9d9f936478b936e Mon Sep 17 00:00:00 2001 From: zhenyu Date: Thu, 24 Feb 2022 18:48:30 +0800 Subject: [PATCH 04/46] fix: fix bug of var(calypso_run_opt_path) not defined --- dpgen/generator/run.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index 7d0350f2d..e22ef0606 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -2307,6 +2307,9 @@ def _select_by_model_devi_standard( model_devi_f_avg_relative : bool = False, detailed_report_make_fp : bool = True, ): + iter_name = modd_system_task[0].split('/')[0] + _work_path = os.path.join(iter_name, model_devi_name) + calypso_run_opt_path = os.path.join(_work_path,calypso_run_opt_name) numofspecies = _parse_calypso_input('NumberOfSpecies',calypso_run_opt_path) min_dis = _parse_calypso_dis_mtx(numofspecies,calypso_run_opt_path) fp_candidate = [] @@ -2504,6 +2507,9 @@ def _make_fp_vasp_inner (modd_path, # -------------------------------------------------------------------------------------------------------------------------------------- model_devi_engine = jdata.get('model_devi_engine', 'lammps') + iter_name = work_path.split('/')[0] + _work_path = os.path.join(iter_name, model_devi_name) + calypso_run_opt_path = os.path.join(_work_path,calypso_run_opt_name) numofspecies = _parse_calypso_input('NumberOfSpecies',calypso_run_opt_path) min_dis = _parse_calypso_dis_mtx(numofspecies,calypso_run_opt_path) if model_devi_engine == 'calypso': @@ -2514,8 +2520,8 @@ def _make_fp_vasp_inner (modd_path, summary = np.loadtxt(summfile) summaryfmax = summary[:,-4] dis = summary[:,-1] - acc = np.where((summaryfmax <= f_trust_lo) & (dis > min_dis)) - fail = np.where((summaryfmax > f_trust_hi) | (dis <= min_dis)) + acc = np.where((summaryfmax <= f_trust_lo) & (dis > float(min_dis))) + fail = np.where((summaryfmax > f_trust_hi) | (dis <= float(min_dis))) nnan = np.where(np.isnan(summaryfmax)) acc_num = len(acc[0]) From 7d30fc1348e8894a85626993eea9e9594ffbc0cd Mon Sep 17 00:00:00 2001 From: zhenyu Date: Thu, 3 Mar 2022 14:29:48 +0800 Subject: [PATCH 05/46] reorganize code v1 --- dpgen/generator/lib/calypso.py | 55 ++ dpgen/generator/lib/modd_calypso.py | 423 +++++++++++++++ dpgen/generator/lib/parse_calypso.py | 33 ++ dpgen/generator/run.py | 782 ++------------------------- 4 files changed, 552 insertions(+), 741 deletions(-) create mode 100644 dpgen/generator/lib/modd_calypso.py create mode 100644 dpgen/generator/lib/parse_calypso.py diff --git a/dpgen/generator/lib/calypso.py b/dpgen/generator/lib/calypso.py index d3d7bd83a..2aefe085d 100644 --- a/dpgen/generator/lib/calypso.py +++ b/dpgen/generator/lib/calypso.py @@ -1,4 +1,6 @@ #!/usr/bin/env python3 +import os +import numpy as np def make_run_opt_script(fmax ) : @@ -350,3 +352,56 @@ def make_calypso_input(nameofatoms,numberofatoms, ret+= "@end\n" ret+= "###################End Parameters for VSC ##########################\n" return ret + + +def _make_model_devi_native_calypso(jdata, cur_job, calypso_run_opt_path): + + # Crystal Parameters + nameofatoms = cur_job.get('NameOfAtoms') + numberofatoms = cur_job.get('NumberOfAtoms') + numberofformula = cur_job.get('NumberOfFormula',[1,1]) + volume = cur_job.get('Volume') + distanceofion = cur_job.get('DistanceOfIon') + psoratio = cur_job.get('PsoRatio') + popsize = cur_job.get('PopSize') + maxstep = cur_job.get('MaxStep') + icode = cur_job.get('ICode',[1]) + split = cur_job.get('Split','T') + # VSC Control + maxnumatom = None + ctrlrange = None + vsc = cur_job.get('VSC','F') + if vsc == 'T': + maxnumatom = cur_job.get('MaxNumAtom') + ctrlrange = cur_job.get('CtrlRange') + + # Optimization + pstress = cur_job.get('PSTRESS',[0.001]) + fmax = cur_job.get('fmax',[0.01]) + + # Cluster + + # 2D + + file_c = make_calypso_input(nameofatoms,numberofatoms, + numberofformula,volume, + distanceofion,psoratio,popsize, + maxstep,icode,split,vsc, + maxnumatom,ctrlrange,pstress,fmax) + with open(os.path.join(calypso_run_opt_path, 'input.dat'), 'w') as cin : + cin.write(file_c) + +def write_model_devi_out(devi, fname): + assert devi.shape[1] == 8 + #assert devi.shape[1] == 7 + header = "%5s" % "step" + for item in 'vf': + header += "%16s%16s%16s" % (f"max_devi_{item}", f"min_devi_{item}",f"avg_devi_{item}") + header += "%16s"%str('min_dis') + np.savetxt(fname, + devi, + fmt=['%5d'] + ['%17.6e' for _ in range(7)], + delimiter='', + header=header) + return devi + diff --git a/dpgen/generator/lib/modd_calypso.py b/dpgen/generator/lib/modd_calypso.py new file mode 100644 index 000000000..c4838cd8f --- /dev/null +++ b/dpgen/generator/lib/modd_calypso.py @@ -0,0 +1,423 @@ +""" +calypso model devi: + 1. GenStructures + 2. Analysis + 3. Modd +""" + +import copy +import dpdata +import math +import numpy as np +import os +import random +import re +import glob +import shutil +from ase.io.vasp import write_vasp +from ase.io.trajectory import Trajectory +from deepmd.infer import calc_model_devi +from deepmd.infer import DeepPot as DP +from dpgen import dlog +from dpgen.generator.lib.utils import make_iter_name +from dpgen.generator.lib.calypso import write_model_devi_out +from dpgen.generator.lib.parse_calypso import _parse_calypso_input,_parse_calypso_dis_mtx +from dpgen.dispatcher.Dispatcher import Dispatcher, _split_tasks, make_dispatcher, make_submission + +train_name = '00.train' +model_devi_name = '01.model_devi' +fp_name = '02.fp' +calypso_run_opt_name = 'gen_stru_analy' +calypso_model_devi_name = 'model_devi_results' + +def GenStructures(iter_index,jdata,mdata): + + # run calypso + # vsc means generate elemental, binary and ternary at the same time + vsc = jdata.get('vsc',False) # take CALYPSO as confs generator + + model_devi_group_size = mdata['model_devi_group_size'] + model_devi_resources = mdata['model_devi_resources'] + + + iter_name = make_iter_name(iter_index) + work_path = os.path.join(iter_name, model_devi_name) + assert(os.path.isdir(work_path)) + # + calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) + calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) + # + + calypso_path = jdata.get('calypso_path') + #calypso_input_path = jdata.get('calypso_input_path') + popsize = int(_parse_calypso_input('PopSize',calypso_run_opt_path)) + maxstep = int(_parse_calypso_input('MaxStep',calypso_run_opt_path)) + + all_models = glob.glob(os.path.join(calypso_run_opt_path, 'graph*pb')) + model_names = [os.path.basename(ii) for ii in all_models] + + deepmdkit_python = mdata.get('deepmdkit_python') + command = "%s run_opt.py %s 1>> model_devi.log 2>> model_devi.log" % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) + command += " || %s check_outcar.py %s " % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) + commands = [command] + + cwd = os.getcwd() + os.chdir(calypso_run_opt_path) + + forward_files = ['POSCAR', 'run_opt.py','check_outcar.py'] + backward_files = ['OUTCAR','CONTCAR','traj.traj','model_devi.log'] + + run_calypso = calypso_path+'/calypso.x | tee log' + if not vsc: + Lpickup = _parse_calypso_input('PickUp','.') + PickUpStep = _parse_calypso_input('PickStep','.') + if os.path.exists('tag_pickup_%s'%(str(PickUpStep))): + dlog.info('caution! tag_pickup_%s exists!'%str(PickUpStep)) + Lpickup = 'F' + if Lpickup == 'T': + ftag = open('tag_pickup_%s'%(str(PickUpStep)),'w') + ftag.close() + os.remove('step') + fstep = open('step','w') + fstep.write('%12s'%str(PickUpStep)) + fstep.close() + else: + PickUpStep = 1 + try: + os.mkdir('opt') + except: + pass + #shutil.rmtree('opt') + #os.mkdir('opt') + + + for ii in range(int(PickUpStep)-1,maxstep+1): + dlog.info('$$$$$$$$$$$$$$$$$$$$ step %s $$$$$$$$$$$$$$$$$'%ii) + if ii == maxstep : + dlog.info('##################### counting OUTCAR #####################') + while True: + if len(glob.glob('OUTCAR_*')) == popsize: + break + dlog.info('##################### done counting OUTCAR #####################') + os.system('%s'%run_calypso) + break + # run calypso + + os.system('%s'%(run_calypso)) + + for pop in range(ii*int(popsize),(ii+1)*int(popsize)): + try: + os.mkdir('task.%03d'%pop) + except: + shutil.rmtree('task.%03d'%pop) + os.mkdir('task.%03d'%pop) + shutil.copyfile('run_opt.py',os.path.join('task.%03d'%pop,'run_opt.py')) + shutil.copyfile('check_outcar.py',os.path.join('task.%03d'%pop,'check_outcar.py')) + shutil.copyfile('POSCAR_%s'%str(pop-ii*int(popsize)+1),os.path.join('task.%03d'%(pop),'POSCAR')) + #for iii in range(1,popsize+1): + # shutil.copyfile('POSCAR_%s'%str(iii),os.path.join('task.%03d'%(iii-1),'POSCAR')) + dlog.info('##################### copy file done #####################') + + all_task = glob.glob( "task.*") + all_task.sort() + + run_tasks_ = all_task + + run_tasks = [os.path.basename(ii) for ii in run_tasks_] + + dlog.info('$$$$$$$$$$$$$$$ dispatcher running $$$$$$$$$$$$$$$$$$') + dispatcher=make_dispatcher(mdata['model_devi_machine'],mdata['model_devi_resources'],'./', run_tasks, model_devi_group_size) + dispatcher.run_jobs(mdata['model_devi_resources'], + commands, + './', + run_tasks, + model_devi_group_size, + model_names, + forward_files, + backward_files, + outlog = 'model_devi.log', + errlog = 'model_devi.log') + dlog.info('$$$$$$$$$$$$$$$ dispatcher $$$$$$$$$$$$$$$$$$') + + sstep = os.path.join('opt',str(ii)) + os.mkdir(sstep) + if not os.path.exists('traj'): + os.mkdir('traj') + + for jjj in range(ii*int(popsize),(ii+1)*int(popsize)): + # to opt directory + shutil.copyfile('POSCAR_%s'%str(jjj+1-ii*int(popsize)),os.path.join(sstep,'POSCAR_%s'%str(jjj+1-ii*int(popsize))),) + shutil.copyfile(os.path.join('task.%03d'%(jjj),'OUTCAR'),os.path.join(sstep,'OUTCAR_%s'%str(jjj+1-ii*int(popsize))),) + shutil.copyfile(os.path.join('task.%03d'%(jjj),'CONTCAR'),os.path.join(sstep,'CONTCAR_%s'%str(jjj+1-ii*int(popsize))),) + # to run calypso directory + shutil.copyfile(os.path.join('task.%03d'%(jjj),'OUTCAR'),'OUTCAR_%s'%str(jjj+1-ii*int(popsize)),) + shutil.copyfile(os.path.join('task.%03d'%(jjj),'CONTCAR'),'CONTCAR_%s'%str(jjj+1-ii*int(popsize)),) + # to traj + shutil.copyfile(os.path.join('task.%03d'%(jjj),'traj.traj'),os.path.join('traj','%s.traj'%str(jjj+1)),) + + + dlog.info('##################### copy file back done #####################') + + os.rename('jr.json','jr_%s.json'%(str(ii))) + + tlist = glob.glob('task.*') + for t in tlist: + shutil.rmtree(t) + + else: + # -------------------------------------------------------------- + component = ['Mg','Al','Cu','MgAl','MgCu','AlCu','MgAlCu'] + + for idx,com in enumerate(component): + pwd = os.getcwd() + os.mkdir(str(idx)) + shutil.copyfile(os.path.join(cwd,'calypso_input','input.dat.%s'%com),os.path.join(str(idx),'input.dat')) + os.chdir(str(idx)) + os.system(run_calypso) + os.chdir(pwd) + + name_list = Path('.').glob('*/POSCAR_*') + for idx,name in enumerate(name_list): + shutil.copyfile(name,'POSCAR_%s'%(idx+1)) + try: + os.mkdir('task.%04d'%(idx+1)) + except: + shutil.rmtree('task.%04d'%(idx+1)) + os.mkdir('task.%04d'%(idx+1)) + shutil.copyfile('run_opt.py',os.path.join('task.%04d'%(idx+1),'run_opt.py')) + shutil.copyfile('check_outcar.py',os.path.join('task.%04d'%(idx+1),'check_outcar.py')) + shutil.copyfile('POSCAR_%s'%str(idx+1),os.path.join('task.%04d'%(idx+1),'POSCAR')) + + dlog.info('##################### copy file done #####################') + + all_task = glob.glob( "task.*") + all_task.sort() + + run_tasks_ = all_task + + run_tasks = [os.path.basename(ii) for ii in run_tasks_] + + dlog.info('$$$$$$$$$$$$$$$ dispatcher running $$$$$$$$$$$$$$$$$$') + dispatcher=make_dispatcher(mdata['model_devi_machine'],mdata['model_devi_resources'],'./', run_tasks, model_devi_group_size) + #print(dispatcher) + dispatcher.run_jobs(mdata['model_devi_resources'], + commands, + './', + run_tasks, + model_devi_group_size, + model_names, + forward_files, + backward_files, + outlog = 'model_devi.log', + errlog = 'model_devi.log') + dlog.info('$$$$$$$$$$$$$$$ dispatcher $$$$$$$$$$$$$$$$$$') + + os.mkdir('opt') + if not os.path.exists('traj'): + os.mkdir('traj') + for jjj in range(len(all_task)): + # to opt directory + shutil.copyfile('POSCAR_%s'%str(jjj+1),os.path.join('opt','POSCAR_%s'%str(jjj+1)),) + shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'OUTCAR'),os.path.join('opt','OUTCAR_%s'%str(jjj+1)),) + shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'CONTCAR'),os.path.join('opt','CONTCAR_%s'%str(jjj+1)),) + # to run calypso directory + shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'OUTCAR'),'OUTCAR_%s'%str(jjj+1),) + shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'CONTCAR'),'CONTCAR_%s'%str(jjj+1),) + # to traj + shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'traj.traj'),os.path.join('traj','%s.traj'%str(jjj+1)),) + + dlog.info('##################### copy file back done #####################') + + tlist = glob.glob('task.*') + for t in tlist: + shutil.rmtree(t) + # -------------------------------------------------------------- + + os.chdir(cwd) + os.chdir(work_path) + f = open('record.calypso','a+') + f.write('2\n') + f.close() + os.chdir(cwd) + +def Analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): + + # Analysis + + + dlog.info('$$$$$$$$$$$$$$$ Analysis Started $$$$$$$$$$$$$$$$$$') + + ms = dpdata.MultiSystems() + + cwd = os.getcwd() + iter_name = make_iter_name(iter_index) + work_path = os.path.join(iter_name, model_devi_name) + result_path = os.path.join(calypso_run_opt_path,'results') + + # trajs to be model devi + traj_path = os.path.join(calypso_run_opt_path,'traj') + traj_list = glob.glob(traj_path+'/*.traj') + + dlog.info('len(traj_list) %s'%str(len(traj_list))) + + # read confs from traj + record_traj_num = 0 + for traj_name in traj_list: + traj_num = os.path.basename(traj_name).split('.')[0] + trajs_origin = Trajectory(traj_name) + + record_traj_num += len(trajs_origin) + if len(trajs_origin) >= 20 : + trajs = [trajs_origin[iii] for iii in [4,9,-10,-5,-1]] + elif 5<=len(trajs_origin)<20: + trajs = [trajs_origin[random.randint(1,len(trajs_origin)-1)] for iii in range(4)] + trajs.append(trajs[-1]) + elif 3<= len(trajs_origin) <5: + trajs = [trajs_origin[round((len(trajs_origin)-1)/2)] ] + trajs.append(trajs[-1]) + elif len(trajs_origin) == 2: + trajs = [trajs_origin[0],trajs_origin[-1] ] + elif len(trajs_origin) == 1: + trajs = [trajs_origin[0] ] + else: + pass + + for idx, traj in enumerate(trajs): + write_vasp(os.path.join( + traj_path,'%03d.%03d.poscar' % ( + int(traj_num), int(idx) + ) + ), + traj) + + traj_pos_list = glob.glob(traj_path+'/*.poscar') + + dlog.info('traj_num %s'%str(len(traj_pos_list))) + dlog.info('total_traj_num %s'%str(record_traj_num)) + + for npos in traj_pos_list: + try: + ms.append(dpdata.System(npos, type_map = jdata['type_map'])) + except Exception as e: + dlog.info(npos,'failed : ',e) + + if len(ms) == 0: + print('too little confs, ') + return + + if os.path.exists(os.path.join(result_path,'deepmd')): + shutil.rmtree(os.path.join(result_path,'deepmd')) + ms.to_deepmd_raw(os.path.join(result_path,'deepmd')) + ms.to_deepmd_npy(os.path.join(result_path,'deepmd')) + + + split_lists = glob.glob(os.path.join(result_path,'deepmd','*')) + for i,split_list in enumerate(split_lists): + ss = dpdata.System(split_list,fmt='deepmd') + for j in range(ss.get_nframes()): + ss.to('vasp/poscar',os.path.join(split_list,'%03d.%03d.poscar'%(i,j)),frame_idx=j) + strus_path = os.path.join(calypso_model_devi_path,'%03d.structures'%i) + if not os.path.exists(strus_path): + shutil.copytree(split_list,strus_path) + else: + shutil.rmtree(strus_path) + shutil.copytree(split_list,strus_path) + + os.chdir(cwd) + os.chdir(work_path) + f = open('record.calypso','a+') + f.write('3\n') + f.close() + os.chdir(cwd) + + +def Modd(iter_index,calypso_model_devi_path,all_models,jdata): + + # Model Devi + + + dlog.info('$$$$$$$$$$$$$$$ Model Devi Start $$$$$$$$$$$$$$$$$$') + + iter_name = make_iter_name(iter_index) + work_path = os.path.join(iter_name, model_devi_name) + + cwd = os.getcwd() + Devis = [] + pcount = 0 + strus_lists = glob.glob(os.path.join(calypso_model_devi_path,'*.structures')) + for num, strus_path in enumerate(strus_lists): + os.chdir(strus_path) + os.system('rename .vasp .poscar *') + os.chdir(cwd) + + structure_list = glob.glob(os.path.join(strus_path,'*.poscar')) + + # every 500 confs in one task dir + if len(structure_list) == 0: + continue + else: + num_per_task = math.ceil(len(structure_list)/500) + graphs = [DP(model) for model in all_models] + for temp in range(num_per_task): + task_name = os.path.join(calypso_model_devi_path,'task.%03d.%03d'%(num,temp)) + put_poscar = os.path.join(task_name,'traj') + if not os.path.exists(task_name): + os.mkdir(task_name) + os.mkdir(put_poscar) + else: + shutil.rmtree(task_name) + os.mkdir(task_name) + os.mkdir(put_poscar) + devis = [] + try: + temp_sl = structure_list[temp*500:(temp+1)*500] + except Exception as err: + dlog.info('err %s'%str(err)) + temp_sl = structure_list[temp*500:] + + new_index = 0 + for index,sname in enumerate(temp_sl): + shutil.copyfile(sname,os.path.join(put_poscar,'%s.poscar'%str(index))) + os.chdir(put_poscar) + pdata = dpdata.System('%s.poscar'%str(index),type_map = jdata['type_map']) + nopbc = pdata.nopbc + coord = pdata.data['coords'] + cell = pdata.data['cells'] + atom_types = pdata.data['atom_types'] + devi = calc_model_devi(coord,cell,atom_types,graphs,nopbc=nopbc) + # ------------------------------------------------------------------------------------ + # append min-distance in devi list + dis_temp = dpdata.System('%s.poscar'%str(index)) + dis = dis_temp.to_ase_structure()[0].get_all_distances(mic=True) + row,col = np.diag_indices_from(dis) + dis[row,col] = 10000 + min_dis = np.nanmin(dis) + devi = np.append(devi[0],min_dis) + t = [devi] + devi = np.array(t) + # ------------------------------------------------------------------------------------ + temp_d = copy.deepcopy(devi) + temp_D = copy.deepcopy(devi) + devis.append(temp_d) + Devis.append(temp_D) + devis[index][0][0] = np.array(index) + Devis[pcount][0][0] = np.array(pcount) + pcount += 1 + new_index += 1 + os.chdir(cwd) + os.chdir(task_name) + devis = np.vstack(devis) + write_model_devi_out(devis,'model_devi.out') + os.chdir(cwd) + + os.chdir(calypso_model_devi_path) + Devis = np.vstack(Devis) + write_model_devi_out(Devis,'Model_Devi.out') + os.chdir(cwd) + + os.chdir(work_path) + f = open('record.calypso','a+') + f.write('4\n') + f.close() + os.chdir(cwd) + diff --git a/dpgen/generator/lib/parse_calypso.py b/dpgen/generator/lib/parse_calypso.py new file mode 100644 index 000000000..d8983c066 --- /dev/null +++ b/dpgen/generator/lib/parse_calypso.py @@ -0,0 +1,33 @@ +import numpy as np +import os + +def _parse_calypso_input(var,input_path): + try: + f = open(input_path,'r') + except: + f = open(os.path.join(input_path,'input.dat'),'r') + lines = f.readlines() + + for line in lines: + if var in line: + variable = line.split('=')[1].strip() + return variable + +def _parse_calypso_dis_mtx(numberofspecies,input_path): + try: + f = open(input_path,'r') + except: + f = open(os.path.join(input_path,'input.dat'),'r') + while True: + line = f.readline() + if len(line) == 0: + break + if '@DistanceOfIon' in line: + dis = [] + for i in range(int(numberofspecies)): + line = f.readline() + dis.append(line.split()) + break + dis = np.array(dis) + dis = dis.reshape((1,int(numberofspecies)**2)) + return dis[0][np.argmin(dis)] diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index e22ef0606..dbb36ad2c 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -41,7 +41,10 @@ from dpgen.generator.lib.utils import log_task from dpgen.generator.lib.utils import symlink_user_forward_files from dpgen.generator.lib.lammps import make_lammps_input, get_dumped_forces -from dpgen.generator.lib.calypso import make_run_opt_script,make_check_outcar_script,make_calypso_input +from dpgen.generator.lib.calypso import make_run_opt_script,make_check_outcar_script +from dpgen.generator.lib.calypso import _make_model_devi_native_calypso,write_model_devi_out +from dpgen.generator.lib.modd_calypso import GenStructures,Analysis,Modd +from dpgen.generator.lib.parse_calypso import _parse_calypso_input,_parse_calypso_dis_mtx from dpgen.generator.lib.vasp import write_incar_dict from dpgen.generator.lib.vasp import make_vasp_incar_user_dict from dpgen.generator.lib.vasp import incar_upper @@ -80,9 +83,9 @@ fp_name = '02.fp' fp_task_fmt = data_system_fmt + '.%06d' cvasp_file=os.path.join(ROOT_PATH,'generator/lib/cvasp.py') -# for calypso local optimization -calypso_run_opt_name = 'GenStruAnaly' -calypso_model_devi_name = 'Model_Devi_Results' +# for calypso +calypso_run_opt_name = 'gen_stru_analy' +calypso_model_devi_name = 'model_devi_results' def get_job_names(jdata) : jobkeys = [] @@ -775,37 +778,6 @@ def revise_by_keys(lmp_lines, keys, values): lmp_lines[ii] = lmp_lines[ii].replace(kk, str(vv)) return lmp_lines -def _parse_calypso_input(var,input_path): - try: - f = open(input_path,'r') - except: - f = open(os.path.join(input_path,'input.dat'),'r') - lines = f.readlines() - - for line in lines: - if var in line: - variable = line.split('=')[1].strip() - return variable - -def _parse_calypso_dis_mtx(numberofspecies,input_path): - try: - f = open(input_path,'r') - except: - f = open(os.path.join(input_path,'input.dat'),'r') - while True: - line = f.readline() - if len(line) == 0: - break - if '@DistanceOfIon' in line: - dis = [] - for i in range(int(numberofspecies)): - line = f.readline() - dis.append(line.split()) - break - dis = np.array(dis) - dis = dis.reshape((1,int(numberofspecies)**2)) - return dis[0][np.argmin(dis)] - def make_calypso_model_devi(iter_index,jdata,mdata): ''' for calypso ''' @@ -842,6 +814,7 @@ def make_calypso_model_devi(iter_index,jdata,mdata): if "calypso_input_path" in jdata: input_mode = "buffet" + # generate input.dat automatic in each iter if input_mode == "native": for iiidx, jobbs in enumerate(model_devi_jobs): if iter_index in jobbs.get('times'): @@ -858,10 +831,9 @@ def make_calypso_model_devi(iter_index,jdata,mdata): with open(check_outcar_script,'w') as ffff: ffff.write(make_check_outcar_script()) # ----------for check the nan situation ------------- - return True - elif input_mode == "revise_template": - pass + + # generate confs according to the input.dat provided elif input_mode == "buffet": calypso_input_path = jdata.get('calypso_input_path') shutil.copyfile(os.path.join(calypso_input_path,'input.dat'),os.path.join(calypso_run_opt_path, 'input.dat')) @@ -880,45 +852,7 @@ def make_calypso_model_devi(iter_index,jdata,mdata): return True else: - raise RuntimeError('unknown model_devi input mode', input_mode) - - -def _make_model_devi_native_calypso(jdata, cur_job, calypso_run_opt_path): - - # Crystal Parameters - nameofatoms = cur_job.get('NameOfAtoms') - numberofatoms = cur_job.get('NumberOfAtoms') - numberofformula = cur_job.get('NumberOfFormula',[1,1]) - volume = cur_job.get('Volume') - distanceofion = cur_job.get('DistanceOfIon') - psoratio = cur_job.get('PsoRatio') - popsize = cur_job.get('PopSize') - maxstep = cur_job.get('MaxStep') - icode = cur_job.get('ICode',[1]) - split = cur_job.get('Split','T') - # VSC Control - maxnumatom = None - ctrlrange = None - vsc = cur_job.get('VSC','F') - if vsc == 'T': - maxnumatom = cur_job.get('MaxNumAtom') - ctrlrange = cur_job.get('CtrlRange') - - # Optimization - pstress = cur_job.get('PSTRESS',[0.001]) - fmax = cur_job.get('fmax',[0.01]) - - # Cluster - - # 2D - - file_c = make_calypso_input(nameofatoms,numberofatoms, - numberofformula,volume, - distanceofion,psoratio,popsize, - maxstep,icode,split,vsc, - maxnumatom,ctrlrange,pstress,fmax) - with open(os.path.join(calypso_run_opt_path, 'input.dat'), 'w') as cin : - cin.write(file_c) + raise RuntimeError('do not found `calypso_input_path` or `model_devi_jobs`', input_mode) def make_model_devi (iter_index, @@ -1367,645 +1301,6 @@ def _make_model_devi_native_gromacs(iter_index, jdata, mdata, conf_systems): conf_counter += 1 sys_counter += 1 -def write_model_devi_out(devi, fname): - assert devi.shape[1] == 8 - #assert devi.shape[1] == 7 - header = "%5s" % "step" - for item in 'vf': - header += "%16s%16s%16s" % (f"max_devi_{item}", f"min_devi_{item}",f"avg_devi_{item}") - header += "%16s"%str('min_dis') - np.savetxt(fname, - devi, - fmt=['%5d'] + ['%17.6e' for _ in range(7)], - delimiter='', - header=header) - return devi - -def GenVscStructures(iter_index,jdata,mdata): - - # run calypso - import sys,os,shutil - from pathlib import Path - - model_devi_group_size = mdata['model_devi_group_size'] - model_devi_resources = mdata['model_devi_resources'] - - - iter_name = make_iter_name(iter_index) - work_path = os.path.join(iter_name, model_devi_name) - assert(os.path.isdir(work_path)) - # - calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) - calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) - # - - calypso_path = jdata.get('calypso_path') - #calypso_input_path = jdata.get('calypso_input_path') - popsize = int(_parse_calypso_input('PopSize',calypso_run_opt_path)) - maxstep = int(_parse_calypso_input('MaxStep',calypso_run_opt_path)) - - all_models = glob.glob(os.path.join(calypso_run_opt_path, 'graph*pb')) - model_names = [os.path.basename(ii) for ii in all_models] - - deepmdkit_python = mdata.get('deepmdkit_python') - command = "%s run_opt.py %s 1>> model_devi.log 2>> model_devi.log" % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) - command += " || %s check_outcar.py %s " % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) - commands = [command] - - cwd = os.getcwd() - os.chdir(calypso_run_opt_path) - - forward_files = ['POSCAR', 'run_opt.py','check_outcar.py'] - backward_files = ['OUTCAR','CONTCAR','traj.traj','model_devi.log'] - - run_calypso = calypso_path+'/calypso.x | tee log' - #os.system('%s'%run_calypso) - - calypso_run = run_calypso - - # -------------------------------------------------------------- - component = ['Mg','Al','Cu','MgAl','MgCu','AlCu','MgAlCu'] - - for idx,com in enumerate(component): - pwd = os.getcwd() - os.mkdir(str(idx)) - shutil.copyfile(os.path.join(cwd,'calypso_input','input.dat.%s'%com),os.path.join(str(idx),'input.dat')) - os.chdir(str(idx)) - os.system(calypso_run) - os.chdir(pwd) - - name_list = Path('.').glob('*/POSCAR_*') - for idx,name in enumerate(name_list): - shutil.copyfile(name,'POSCAR_%s'%(idx+1)) - # -------------------------------------------------------------- - try: - os.mkdir('task.%04d'%(idx+1)) - except: - shutil.rmtree('task.%04d'%(idx+1)) - os.mkdir('task.%04d'%(idx+1)) - shutil.copyfile('run_opt.py',os.path.join('task.%04d'%(idx+1),'run_opt.py')) - shutil.copyfile('check_outcar.py',os.path.join('task.%04d'%(idx+1),'check_outcar.py')) - shutil.copyfile('POSCAR_%s'%str(idx+1),os.path.join('task.%04d'%(idx+1),'POSCAR')) - - dlog.info('##################### copy file done #####################') - - all_task = glob.glob( "task.*") - all_task.sort() - - run_tasks_ = all_task - - run_tasks = [os.path.basename(ii) for ii in run_tasks_] - - dlog.info('$$$$$$$$$$$$$$$ dispatcher running $$$$$$$$$$$$$$$$$$') - dispatcher=make_dispatcher(mdata['model_devi_machine'],mdata['model_devi_resources'],'./', run_tasks, model_devi_group_size) - #print(dispatcher) - dispatcher.run_jobs(mdata['model_devi_resources'], - commands, - './', - run_tasks, - model_devi_group_size, - model_names, - forward_files, - backward_files, - outlog = 'model_devi.log', - errlog = 'model_devi.log') - dlog.info('$$$$$$$$$$$$$$$ dispatcher $$$$$$$$$$$$$$$$$$') - - os.mkdir('opt') - if not os.path.exists('traj'): - os.mkdir('traj') - # ------------------------------------------------- - - for jjj in range(len(all_task)): - # to opt directory - shutil.copyfile('POSCAR_%s'%str(jjj+1),os.path.join('opt','POSCAR_%s'%str(jjj+1)),) - shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'OUTCAR'),os.path.join('opt','OUTCAR_%s'%str(jjj+1)),) - shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'CONTCAR'),os.path.join('opt','CONTCAR_%s'%str(jjj+1)),) - # to run calypso directory - shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'OUTCAR'),'OUTCAR_%s'%str(jjj+1),) - shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'CONTCAR'),'CONTCAR_%s'%str(jjj+1),) - # to traj - shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'traj.traj'),os.path.join('traj','%s.traj'%str(jjj+1)),) - - - dlog.info('##################### copy file back done #####################') - - #os.rename('jr.json','jr_%s.json'%(str(ii))) - - tlist = glob.glob('task.*') - for t in tlist: - shutil.rmtree(t) - - os.chdir(cwd) - os.chdir(work_path) - f = open('record.calypso','a+') - f.write('2\n') - f.close() - os.chdir(cwd) - - - -def GenStructures(iter_index,jdata,mdata): - - # run calypso - # vsc means generate elemental, binary and ternary at the same time - vsc = jdata.get('vsc',False) - if vsc: - GenVscStructures(iter_index,jdata,mdata) - return - - model_devi_group_size = mdata['model_devi_group_size'] - model_devi_resources = mdata['model_devi_resources'] - - - iter_name = make_iter_name(iter_index) - work_path = os.path.join(iter_name, model_devi_name) - assert(os.path.isdir(work_path)) - # - calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) - calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) - # - - calypso_path = jdata.get('calypso_path') - #calypso_input_path = jdata.get('calypso_input_path') - popsize = int(_parse_calypso_input('PopSize',calypso_run_opt_path)) - maxstep = int(_parse_calypso_input('MaxStep',calypso_run_opt_path)) - - all_models = glob.glob(os.path.join(calypso_run_opt_path, 'graph*pb')) - model_names = [os.path.basename(ii) for ii in all_models] - - deepmdkit_python = mdata.get('deepmdkit_python') - command = "%s run_opt.py %s 1>> model_devi.log 2>> model_devi.log" % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) - command += " || %s check_outcar.py %s " % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) - commands = [command] - - cwd = os.getcwd() - os.chdir(calypso_run_opt_path) - - forward_files = ['POSCAR', 'run_opt.py','check_outcar.py'] - backward_files = ['OUTCAR','CONTCAR','traj.traj','model_devi.log'] - - Lpickup = _parse_calypso_input('PickUp','.') - PickUpStep = _parse_calypso_input('PickStep','.') - if os.path.exists('tag_pickup_%s'%(str(PickUpStep))): - dlog.info('caution! tag_pickup_%s exists!'%str(PickUpStep)) - Lpickup = 'F' - if Lpickup == 'T': - ftag = open('tag_pickup_%s'%(str(PickUpStep)),'w') - ftag.close() - os.remove('step') - fstep = open('step','w') - fstep.write('%12s'%str(PickUpStep)) - fstep.close() - else: - PickUpStep = 1 - try: - os.mkdir('opt') - except: - pass - #shutil.rmtree('opt') - #os.mkdir('opt') - - - run_calypso = calypso_path+'/calypso.x | tee log' - for ii in range(int(PickUpStep)-1,maxstep+1): - dlog.info('$$$$$$$$$$$$$$$$$$$$ step %s $$$$$$$$$$$$$$$$$'%ii) - if ii == maxstep : - dlog.info('##################### counting OUTCAR #####################') - while True: - if len(glob.glob('OUTCAR_*')) == popsize: - break - dlog.info('##################### done counting OUTCAR #####################') - os.system('%s'%run_calypso) - break - # run calypso - - os.system('%s'%(run_calypso)) - - for pop in range(ii*int(popsize),(ii+1)*int(popsize)): - try: - os.mkdir('task.%03d'%pop) - except: - shutil.rmtree('task.%03d'%pop) - os.mkdir('task.%03d'%pop) - shutil.copyfile('run_opt.py',os.path.join('task.%03d'%pop,'run_opt.py')) - shutil.copyfile('check_outcar.py',os.path.join('task.%03d'%pop,'check_outcar.py')) - shutil.copyfile('POSCAR_%s'%str(pop-ii*int(popsize)+1),os.path.join('task.%03d'%(pop),'POSCAR')) - #for iii in range(1,popsize+1): - # shutil.copyfile('POSCAR_%s'%str(iii),os.path.join('task.%03d'%(iii-1),'POSCAR')) - dlog.info('##################### copy file done #####################') - - all_task = glob.glob( "task.*") - all_task.sort() - - run_tasks_ = all_task - - run_tasks = [os.path.basename(ii) for ii in run_tasks_] - #print(run_tasks) - - dlog.info('$$$$$$$$$$$$$$$ dispatcher running $$$$$$$$$$$$$$$$$$') - dispatcher=make_dispatcher(mdata['model_devi_machine'],mdata['model_devi_resources'],'./', run_tasks, model_devi_group_size) - #print(dispatcher) - dispatcher.run_jobs(mdata['model_devi_resources'], - commands, - './', - run_tasks, - model_devi_group_size, - model_names, - forward_files, - backward_files, - outlog = 'model_devi.log', - errlog = 'model_devi.log') - dlog.info('$$$$$$$$$$$$$$$ dispatcher $$$$$$$$$$$$$$$$$$') - - sstep = os.path.join('opt',str(ii)) - os.mkdir(sstep) - if not os.path.exists('traj'): - os.mkdir('traj') - - for jjj in range(ii*int(popsize),(ii+1)*int(popsize)): - # to opt directory - shutil.copyfile('POSCAR_%s'%str(jjj+1-ii*int(popsize)),os.path.join(sstep,'POSCAR_%s'%str(jjj+1-ii*int(popsize))),) - shutil.copyfile(os.path.join('task.%03d'%(jjj),'OUTCAR'),os.path.join(sstep,'OUTCAR_%s'%str(jjj+1-ii*int(popsize))),) - shutil.copyfile(os.path.join('task.%03d'%(jjj),'CONTCAR'),os.path.join(sstep,'CONTCAR_%s'%str(jjj+1-ii*int(popsize))),) - # to run calypso directory - shutil.copyfile(os.path.join('task.%03d'%(jjj),'OUTCAR'),'OUTCAR_%s'%str(jjj+1-ii*int(popsize)),) - shutil.copyfile(os.path.join('task.%03d'%(jjj),'CONTCAR'),'CONTCAR_%s'%str(jjj+1-ii*int(popsize)),) - # to traj - shutil.copyfile(os.path.join('task.%03d'%(jjj),'traj.traj'),os.path.join('traj','%s.traj'%str(jjj+1)),) - - - dlog.info('##################### copy file back done #####################') - - os.rename('jr.json','jr_%s.json'%(str(ii))) - - tlist = glob.glob('task.*') - for t in tlist: - shutil.rmtree(t) - - os.chdir(cwd) - os.chdir(work_path) - f = open('record.calypso','a+') - f.write('2\n') - f.close() - os.chdir(cwd) - -def VscAnalysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): - - # Analysis - - from ase.io.vasp import write_vasp - from ase.io.trajectory import Trajectory - import math - - #print('vscAnalysis') - ms = dpdata.MultiSystems() - - # trajs to be model devi - traj_path = os.path.join(calypso_run_opt_path,'traj') - traj_list = glob.glob(traj_path+'/*.traj') - - dlog.info('len(traj_list) %s'%str(len(traj_list))) - - record_traj_num = 0 - #print(traj_list) - for traj_name in traj_list: - traj_num = os.path.basename(traj_name).split('.')[0] - #print(traj_name) - trajs_origin = Trajectory(traj_name) - #print(trajs_origin) - - record_traj_num += len(trajs_origin) - #print(len(trajs_origin)) - if len(trajs_origin) >= 20 : - - trajs = [trajs_origin[iii] for iii in [0,4,-10,-5,-1]] - #print('1',trajs) - elif 5<=len(trajs_origin)<20: - trajs = [trajs_origin[random.randint(1,len(trajs_origin)-1)] for iii in range(3)] - trajs.append(trajs[0]) - trajs.append(trajs[-1]) - #print('2',trajs) - elif 3<= len(trajs_origin) <5: - trajs = [trajs_origin[round((len(trajs_origin)-1)/2)] ] - trajs.append(trajs[0]) - trajs.append(trajs[-1]) - elif len(trajs_origin) == 1: - trajs = [trajs_origin[0] ] - elif len(trajs_origin) == 2: - trajs = [trajs_origin[0],trajs_origin[-1] ] - else: - pass - - #print(trajs) - for idx, traj in enumerate(trajs): - write_vasp(os.path.join( - traj_path,'%04d.%03d.poscar' % ( - int(traj_num), int(idx) - ) - ), - traj) - - traj_pos_list = glob.glob(traj_path+'/*.poscar') - - dlog.info('traj_num %s'%str(len(traj_pos_list))) - dlog.info('total_traj_num %s'%str(record_traj_num)) - - for npos in traj_pos_list: - try: - ms.append(dpdata.System(npos, type_map = jdata['type_map'])) - except Exception as e: - print(npos,'failed : ',e) - - if len(ms) == 0: - print('too little confs, ') - return - - result_path = os.path.join(calypso_run_opt_path,'results') - - if os.path.exists(os.path.join(result_path,'deepmd')): - shutil.rmtree(os.path.join(result_path,'deepmd')) - ms.to_deepmd_raw(os.path.join(result_path,'deepmd')) - ms.to_deepmd_npy(os.path.join(result_path,'deepmd')) - - - split_list = glob.glob(os.path.join(result_path,'deepmd','*')) - for i in range(len(split_list)): - ss = dpdata.System(split_list[i],fmt='deepmd') - for j in range(ss.get_nframes()): - ss.to('vasp/poscar',os.path.join(split_list[i],'%03d.%03d.poscar'%(i,j)),frame_idx=j) - strus_path = os.path.join(calypso_model_devi_path,'%03d.structures'%i) - if not os.path.exists(strus_path): - shutil.copytree(split_list[i],strus_path) - else: - shutil.rmtree(strus_path) - shutil.copytree(split_list[i],strus_path) - - iter_name = make_iter_name(iter_index) - work_path = os.path.join(iter_name, model_devi_name) - cwd = os.getcwd() - os.chdir(cwd) - os.chdir(work_path) - f = open('record.calypso','a+') - f.write('3\n') - f.close() - os.chdir(cwd) - -def Analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): - - # vsc means generate elemental, binary and ternary at the same time - vsc = jdata.get('vsc',False) - if vsc: - VscAnalysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path) - return - - # Analysis - - from ase.io.vasp import write_vasp - from ase.io.trajectory import Trajectory - import math - - dlog.info('$$$$$$$$$$$$$$$ Analysis Started $$$$$$$$$$$$$$$$$$') - - iter_name = make_iter_name(iter_index) - work_path = os.path.join(iter_name, model_devi_name) - - cwd = os.getcwd() - result_path = os.path.join(calypso_run_opt_path,'results') - os.chdir(result_path) - # may need to delete - os.system('cak.py --vasp -a ') - lll = glob.glob('dir*') - if len(lll)== 0: - return - - dlog.info('cak.py run successfully') - - dlog.info('$$$$$$$$$$$$$$$ Analysis Done $$$$$$$$$$$$$$$$$$') - - os.chdir(cwd) - dir_list = glob.glob(result_path+'/dir*') - sys_num = len(dir_list) - ms = dpdata.MultiSystems() - for num in range(sys_num): - - if sys_num > 1: - temp_path = os.path.join(dir_list[num],'dir_0.01') - os.chdir(temp_path) - os.system('rename .vasp .poscar *') - os.chdir(cwd) - confs = glob.glob(os.path.join(temp_path,'*.poscar')) - elif sys_num == 1: - os.chdir(dir_list[num]) - os.system('rename .vasp .poscar *') - os.chdir(cwd) - confs = glob.glob(os.path.join(dir_list[num],'*.poscar')) - if confs == []: - if sys_num == 1: - dlog.info('dir empty') - return - else: - continue - else: - for cc in confs: - try: - ms.append(dpdata.System(cc, type_map = jdata['type_map'])) - except: - dlog.info(cc,'convert failed') - - # trajs to be model devi - traj_path = os.path.join(calypso_run_opt_path,'traj') - traj_list = glob.glob(traj_path+'/*.traj') - - dlog.info('len(traj_list) %s'%str(len(traj_list))) - - record_traj_num = 0 - for traj_name in traj_list: - traj_num = os.path.basename(traj_name).split('.')[0] - trajs_origin = Trajectory(traj_name) - - record_traj_num += len(trajs_origin) - if len(trajs_origin) >= 20 : - - trajs = [trajs_origin[iii] for iii in [4,-10,-5]] - #print('1',trajs) - elif 5<=len(trajs_origin)<20: - trajs = [trajs_origin[random.randint(1,len(trajs_origin)-1)] for iii in range(3)] - #print('2',trajs) - elif 3<= len(trajs_origin) <5: - trajs = [trajs_origin[round((len(trajs_origin)-1)/2)] ] - elif len(trajs_origin) == 1: - trajs = [trajs_origin[0] ] - else: - pass - - for idx, traj in enumerate(trajs): - write_vasp(os.path.join( - traj_path,'%03d.%03d.poscar' % ( - int(traj_num), int(idx) - ) - ), - traj) - - traj_pos_list = glob.glob(traj_path+'/*.poscar') - - dlog.info('traj_num %s'%str(len(traj_pos_list))) - dlog.info('total_traj_num %s'%str(record_traj_num)) - - for npos in traj_pos_list: - try: - ms.append(dpdata.System(npos, type_map = jdata['type_map'])) - except Exception as e: - print(npos,'failed : ',e) - - if len(ms) == 0: - print('too little confs, ') - return - - if os.path.exists(os.path.join(result_path,'deepmd')): - shutil.rmtree(os.path.join(result_path,'deepmd')) - ms.to_deepmd_raw(os.path.join(result_path,'deepmd')) - ms.to_deepmd_npy(os.path.join(result_path,'deepmd')) - - - result_path = os.path.join(calypso_run_opt_path,'results') - split_list = glob.glob(os.path.join(result_path,'deepmd','*')) - for i in range(len(split_list)): - ss = dpdata.System(split_list[i],fmt='deepmd') - for j in range(ss.get_nframes()): - ss.to('vasp/poscar',os.path.join(split_list[i],'%03d.%03d.poscar'%(i,j)),frame_idx=j) - strus_path = os.path.join(calypso_model_devi_path,'%03d.structures'%i) - if not os.path.exists(strus_path): - shutil.copytree(split_list[i],strus_path) - else: - shutil.rmtree(strus_path) - shutil.copytree(split_list[i],strus_path) - - os.chdir(cwd) - os.chdir(work_path) - f = open('record.calypso','a+') - f.write('3\n') - f.close() - os.chdir(cwd) - - -def Modd(iter_index,calypso_model_devi_path,all_models,jdata): - - # Model Devi - - - import re - import math - import copy - from deepmd.infer import calc_model_devi - from deepmd.infer import DeepPot as DP - - dlog.info('$$$$$$$$$$$$$$$ Model Devi Start $$$$$$$$$$$$$$$$$$') - - iter_name = make_iter_name(iter_index) - work_path = os.path.join(iter_name, model_devi_name) - - cwd = os.getcwd() - Devis = [] - pcount = 0 - strus_list = glob.glob(os.path.join(calypso_model_devi_path,'*.structures')) - for num in range(len(strus_list)): - strus_path = strus_list[num] - os.chdir(strus_path) - os.system('rename .vasp .poscar *') - os.chdir(cwd) - - structure_list = glob.glob(os.path.join(strus_path,'*.poscar')) - #structure_list = sorted(structure_list_temp, key = lambda i:int(re.match(r'(\d+)',i).group())) - #print(structure_list) - - - if len(structure_list) == 0: - continue - else: - num_per_task = math.ceil(len(structure_list)/500) - graphs = [DP(model) for model in all_models] - for temp in range(num_per_task): - #for temp in range(len(structure_list)): - task_name = os.path.join(calypso_model_devi_path,'task.%03d.%03d'%(num,temp)) - put_poscar = os.path.join(task_name,'traj') - if not os.path.exists(task_name): - os.mkdir(task_name) - os.mkdir(put_poscar) - else: - shutil.rmtree(task_name) - os.mkdir(task_name) - os.mkdir(put_poscar) - devis = [] - try: - temp_sl = structure_list[temp*500:(temp+1)*500] - except Exception as err: - dlog.info('err %s'%str(err)) - temp_sl = structure_list[temp*500:] - - #print(temp_sl) - new_index = 0 - for index,sname in enumerate(temp_sl): - #print(index,sname) - shutil.copyfile(sname,os.path.join(put_poscar,'%s.poscar'%str(index))) - os.chdir(put_poscar) - pdata = dpdata.System('%s.poscar'%str(index),type_map = jdata['type_map']) - nopbc = pdata.nopbc - coord = pdata.data['coords'] - cell = pdata.data['cells'] - atom_types = pdata.data['atom_types'] - devi = calc_model_devi(coord,cell,atom_types,graphs,nopbc=nopbc) - # ------------------------------------------------------------------------------------ - #print(devi) - #print(devi.shape) - dis_temp = dpdata.System('%s.poscar'%str(index)) - dis = dis_temp.to_ase_structure()[0].get_all_distances(mic=True) - row,col = np.diag_indices_from(dis) - dis[row,col] = 10000 - min_dis = np.nanmin(dis) - devi = np.append(devi[0],min_dis) - #devi.reshape((1,8)) - t = [devi] - devi = np.array(t) - # ------------------------------------------------------------------------------------ - #devi[0] = index - temp_d = copy.deepcopy(devi) - temp_D = copy.deepcopy(devi) - devis.append(temp_d) - Devis.append(temp_D) - devis[index][0][0] = np.array(index) - Devis[pcount][0][0] = np.array(pcount) - #print(devis[0][0][0]) - #print(devis) - #print(Devis) - pcount += 1 - new_index += 1 - os.chdir(cwd) - # a problem occur when the cycle is done - # the first number always become 18 and don't know why - # use deep copy to solve this soluation - os.chdir(task_name) - #os.chdir(calypso_model_devi_path) - devis = np.vstack(devis) - write_model_devi_out(devis,'model_devi.out') - os.chdir(cwd) - - os.chdir(calypso_model_devi_path) - #Devis = Devis[np.argsort(Devis[:,0])] - Devis = np.vstack(Devis) - write_model_devi_out(Devis,'Model_Devi.out') - os.chdir(cwd) - - os.chdir(work_path) - f = open('record.calypso','a+') - f.write('4\n') - f.close() - os.chdir(cwd) - - - #else: - # dlog.info('wrong') - # return False def run_model_devi_calypso (iter_index, jdata, @@ -2054,35 +1349,38 @@ def run_model_devi_calypso (iter_index, Modd(iter_index,calypso_model_devi_path,all_models,jdata) elif lines[-1].strip().strip('\n') == '4': - # Model Devi print('Model Devi is done.') break elif LooseVersion(api_version) >= LooseVersion('1.0'): record_calypso_path = os.path.join(work_path,'record.calypso') - if not os.path.exists(record_calypso_path): - f = open(record_calypso_path,'w') - f.write('1\n') - lines = '1' - f.close() - else: - f = open(record.calypso_path,'r') - lines = f.readlines() - f.close() + while True: + if not os.path.exists(record_calypso_path): + f = open(record_calypso_path,'w') + f.write('1\n') + lines = '1' + f.close() + else: + f = open(record_calypso_path,'r') + lines = f.readlines() + f.close() - if lines[-1].strip().strip('\n') == '1': - # Gen Structures - GenStructures(iter_index,jdata,mdata) + if lines[-1].strip().strip('\n') == '1': + # Gen Structures + GenStructures(iter_index,jdata,mdata) - elif lines[-1].strip().strip('\n') == '2': - # Analysis & to deepmd/raw - Analysis(calypso_run_opt_path,calypso_model_devi_path) + elif lines[-1].strip().strip('\n') == '2': + # Analysis & to deepmd/raw + Analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path) - elif lines[-1].strip().strip('\n') == '3': - # Model Devi - Modd(calypso_model_devi_path,all_models,jdata) + elif lines[-1].strip().strip('\n') == '3': + # Model Devi + Modd(iter_index,calypso_model_devi_path,all_models,jdata) + elif lines[-1].strip().strip('\n') == '4': + print('Model Devi is done.') + break def run_model_devi (iter_index, jdata, @@ -2303,15 +1601,17 @@ def _select_by_model_devi_standard( v_trust_lo : float, v_trust_hi : float, cluster_cutoff : float, + model_devi_engine : str, model_devi_skip : int = 0, model_devi_f_avg_relative : bool = False, detailed_report_make_fp : bool = True, ): - iter_name = modd_system_task[0].split('/')[0] - _work_path = os.path.join(iter_name, model_devi_name) - calypso_run_opt_path = os.path.join(_work_path,calypso_run_opt_name) - numofspecies = _parse_calypso_input('NumberOfSpecies',calypso_run_opt_path) - min_dis = _parse_calypso_dis_mtx(numofspecies,calypso_run_opt_path) + if model_devi_engine == 'calypso': + iter_name = modd_system_task[0].split('/')[0] + _work_path = os.path.join(iter_name, model_devi_name) + calypso_run_opt_path = os.path.join(_work_path,calypso_run_opt_name) + numofspecies = _parse_calypso_input('NumberOfSpecies',calypso_run_opt_path) + min_dis = _parse_calypso_dis_mtx(numofspecies,calypso_run_opt_path) fp_candidate = [] if detailed_report_make_fp: fp_rest_accurate = [] @@ -2326,7 +1626,6 @@ def _select_by_model_devi_standard( warnings.simplefilter("ignore") all_conf = _read_model_devi_file(tt, model_devi_f_avg_relative) - model_devi_engine = jdata.get('model_devi_engine','lammps') if all_conf.shape == (7,): all_conf = all_conf.reshape(1,all_conf.shape[0]) elif model_devi_engine == 'calypso' and all_conf.shape == (8,): @@ -2583,6 +1882,7 @@ def _trust_limitation_check(sys_idx, lim): f_trust_lo_sys, f_trust_hi_sys, v_trust_lo_sys, v_trust_hi_sys, cluster_cutoff, + model_devi_engine, model_devi_skip, model_devi_f_avg_relative = model_devi_f_avg_relative, detailed_report_make_fp = detailed_report_make_fp, From 7c68fe907e249653a466c77ee77c8edabb0ffa83 Mon Sep 17 00:00:00 2001 From: zhenyu Date: Thu, 3 Mar 2022 21:23:26 +0800 Subject: [PATCH 06/46] add: add unittest --- .gitignore | 1 + dpgen/generator/lib/calypso.py | 22 ++++-- dpgen/generator/lib/parse_calypso.py | 2 + dpgen/generator/run.py | 2 +- tests/generator/calypso_data/input.dat | 72 +++++++++++++++++++ tests/generator/context.py | 3 + tests/generator/test_calypso.py | 99 ++++++++++++++++++++++++++ 7 files changed, 194 insertions(+), 7 deletions(-) create mode 100644 tests/generator/calypso_data/input.dat create mode 100644 tests/generator/test_calypso.py diff --git a/.gitignore b/.gitignore index f833e149e..1aa647f54 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ dbconfig.json .vscode/* .idea/* _build +tests/generator/calypso_test_path diff --git a/dpgen/generator/lib/calypso.py b/dpgen/generator/lib/calypso.py index 2aefe085d..6d5a50548 100644 --- a/dpgen/generator/lib/calypso.py +++ b/dpgen/generator/lib/calypso.py @@ -110,7 +110,7 @@ def make_run_opt_script(fmax ) : ret+=" try:\n" ret+=" model_path = sys.argv[1] \n" ret+=" Model_List = glob.glob('%s/graph*pb'%model_path) \n" - ret+=" calc = DP(model=Model_List[1]) # init the model before iteration \n" + ret+=" calc = DP(model=Model_List[0]) # init the model before iteration \n" ret+=" except:\n" ret+=" assert os.path.exists('graph.pb'), 'did not found graph.pb in this directory %s, or did you forget to add args?'%(os.getcwd())\n" ret+=" calc = DP(model='graph.pb') # init the model before iteration \n" @@ -277,22 +277,25 @@ def make_calypso_input(nameofatoms,numberofatoms, distanceofion,psoratio,popsize, maxstep,icode,split,vsc, maxnumatom,ctrlrange,pstress,fmax): - assert len(distanceofion) == len(nameofatoms) #"check distance of ions and the number of atoms" - assert len(distanceofion[0]) == len(nameofatoms) ret = "################################ The Basic Parameters of CALYPSO ################################\n" ret+= "# A string of one or several words contain a descriptive name of the system (max. 40 characters).\n" + assert (nameofatoms != None ) ret+= "SystemName = %s\n"%(''.join(nameofatoms)) ret+= "# Number of different atomic species in the simulation.\n" ret+= "NumberOfSpecies = %d\n"%(len(nameofatoms)) ret+= "# Element symbols of the different chemical species.\n" ret+= "NameOfAtoms = %s\n"%(' '.join(nameofatoms)) ret+= "# Number of atoms for each chemical species in one formula unit. \n" + assert numberofatoms != None or len(numberofatoms) == len(nameofatoms) ret+= "NumberOfAtoms = %s\n"%(' '.join(list(map(str,numberofatoms)))) ret+= "# The range of formula unit per cell in your simulation. \n" + assert numberofformula != None or len(numberofformula) == 2 or type(numberofformula) == type([1,2]) ret+= "NumberOfFormula = %s\n"%(' '.join(list(map(str,numberofformula)))) ret+= "# The volume per formula unit. Unit is in angstrom^3.\n" ret+= "Volume = %s\n"%(volume[0]) ret+= "# Minimal distance between atoms of each chemical species. Unit is in angstrom.\n" + assert len(distanceofion) == len(nameofatoms) #"check distance of ions and the number of atoms" + assert len(distanceofion[0]) == len(nameofatoms) ret+= "@DistanceOfIon \n" for temp in distanceofion: ret+="%4s \n"%(' '.join(list(map(str,temp)))) @@ -302,9 +305,12 @@ def make_calypso_input(nameofatoms,numberofatoms, ret+= "# Ialgo = 1 for Global PSO\n" ret+= "# Ialgo = 2 for Local PSO (default value)\n" ret+= "# The proportion of the structures generated by PSO.\n" + assert (0 <= psoratio[0] <= 1 ) ret+= "PsoRatio = %s\n"%(psoratio[0]) ret+= "# The population size. Normally, it has a larger number for larger systems.\n" + assert popsize[0] != None or popsize!=None or type(popsize) == type([0,1]) or type(popsize[0]) == type(0) ret+= "PopSize = %d\n"%(popsize[0]) + assert maxstep[0] != None or maxstep!=None or type(maxstep) == type([0,1]) or type(maxstep[0]) == type(0) ret+= "# The Max step for iteration\n" ret+= "MaxStep = %d\n"%(maxstep[0]) ret+= "#It determines which method should be adopted in generation the random structure. \n" @@ -315,6 +321,7 @@ def make_calypso_input(nameofatoms,numberofatoms, ret+= "# 0 combination of all method\n" ret+= "# If GenType=3 or 4, it determined the small unit to grow the whole structure\n" ret+= "# It determines which local optimization method should be interfaced in the simulation.\n" + assert icode != None or type(icode) == type([0,1]) or type(icode[0]) == type(0) ret+= "ICode= %d\n"%(icode[0]) ret+= "# ICode= 1 interfaced with VASP\n" ret+= "# ICode= 2 interfaced with SIESTA\n" @@ -334,9 +341,12 @@ def make_calypso_input(nameofatoms,numberofatoms, ret+= "Parallel = F\n" ret+= "# The number node for parallel \n" ret+= "NumberOfParallel = 4\n" + assert split != None ret+= "Split = %s\n"%(split) - ret+= "PSTRESS = %s\n"%(str(pstress[0])) - ret+= "fmax = %s\n"%(str(fmax[0])) + assert pstress != None or type(pstress) == type([200]) + ret+= "PSTRESS = %f\n"%(pstress[0]) + assert fmax != None or type(fmax) == type([200]) + ret+= "fmax = %f\n"%(fmax[0]) ret+= "################################ End of The Basic Parameters of CALYPSO #######################\n" if vsc == 'T': assert len(ctrlrange) == len(nameofatoms) #'check distance of ions and the number of atoms' @@ -354,7 +364,7 @@ def make_calypso_input(nameofatoms,numberofatoms, return ret -def _make_model_devi_native_calypso(jdata, cur_job, calypso_run_opt_path): +def _make_model_devi_native_calypso(cur_job, calypso_run_opt_path): # Crystal Parameters nameofatoms = cur_job.get('NameOfAtoms') diff --git a/dpgen/generator/lib/parse_calypso.py b/dpgen/generator/lib/parse_calypso.py index d8983c066..123a77e94 100644 --- a/dpgen/generator/lib/parse_calypso.py +++ b/dpgen/generator/lib/parse_calypso.py @@ -7,6 +7,7 @@ def _parse_calypso_input(var,input_path): except: f = open(os.path.join(input_path,'input.dat'),'r') lines = f.readlines() + f.close() for line in lines: if var in line: @@ -27,6 +28,7 @@ def _parse_calypso_dis_mtx(numberofspecies,input_path): for i in range(int(numberofspecies)): line = f.readline() dis.append(line.split()) + f.close() break dis = np.array(dis) dis = dis.reshape((1,int(numberofspecies)**2)) diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index dbb36ad2c..62265467b 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -820,7 +820,7 @@ def make_calypso_model_devi(iter_index,jdata,mdata): if iter_index in jobbs.get('times'): cur_job = model_devi_jobs[iiidx] - _make_model_devi_native_calypso(jdata, cur_job, calypso_run_opt_path) + _make_model_devi_native_calypso(cur_job, calypso_run_opt_path) run_opt_script = os.path.join(calypso_run_opt_path,'run_opt.py') with open(run_opt_script,'w') as ffff: diff --git a/tests/generator/calypso_data/input.dat b/tests/generator/calypso_data/input.dat new file mode 100644 index 000000000..0a22afbed --- /dev/null +++ b/tests/generator/calypso_data/input.dat @@ -0,0 +1,72 @@ +################################ The Basic Parameters of CALYPSO ################################ +# A string of one or several words contain a descriptive name of the system (max. 40 characters). +SystemName = MgAlCu +# Number of different atomic species in the simulation. +NumberOfSpecies = 3 +# Element symbols of the different chemical species. +NameOfAtoms = Mg Al Cu +# Number of atoms for each chemical species in one formula unit. +NumberOfAtoms = 1 1 1 +# The range of formula unit per cell in your simulation. +NumberOfFormula = 1 4 +# The volume per formula unit. Unit is in angstrom^3. +Volume = 30 +# Minimal distance between atoms of each chemical species. Unit is in angstrom. +@DistanceOfIon +1.48 1.44 1.59 +1.44 1.41 1.56 +1.59 1.56 1.7 +@End +# It determines which algorithm should be adopted in the simulation. +Ialgo = 2 +# Ialgo = 1 for Global PSO +# Ialgo = 2 for Local PSO (default value) +# The proportion of the structures generated by PSO. +PsoRatio = 0.6 +# The population size. Normally, it has a larger number for larger systems. +PopSize = 5 +# The Max step for iteration +MaxStep = 3 +#It determines which method should be adopted in generation the random structure. +GenType= 1 +# 1 under symmetric constraints +# 2 grid method for large system +# 3 and 4 core grow method +# 0 combination of all method +# If GenType=3 or 4, it determined the small unit to grow the whole structure +# It determines which local optimization method should be interfaced in the simulation. +ICode= 13 +# ICode= 1 interfaced with VASP +# ICode= 2 interfaced with SIESTA +# ICode= 3 interfaced with GULP +# The number of lbest for local PSO +NumberOfLbest=4 +# The Number of local optimization for each structure. +NumberOfLocalOptim= 3 +# The command to perform local optimiztion calculation (e.g., VASP, SIESTA) on your computer. +Command = sh submit.sh +MaxTime = 9000 +# If True, a previous calculation will be continued. +PickUp = F +# At which step will the previous calculation be picked up. +PickStep = 1 +# If True, the local optimizations performed by parallel +Parallel = F +# The number node for parallel +NumberOfParallel = 4 +Split = T +PSTRESS = 0.000000 +fmax = 0.010000 +################################ End of The Basic Parameters of CALYPSO ####################### +##### The Parameters For Variational Stoichiometry ############## +## If True, Variational Stoichiometry structure prediction is performed +VSC = T +# The Max Number of Atoms in unit cell +MaxNumAtom = 31 +# The Variation Range for each type atom +@CtrlRange +1 10 +1 10 +1 10 +@end +###################End Parameters for VSC ########################## diff --git a/tests/generator/context.py b/tests/generator/context.py index a943bc895..2c804751a 100644 --- a/tests/generator/context.py +++ b/tests/generator/context.py @@ -5,6 +5,9 @@ from dpgen.generator.lib.gaussian import detect_multiplicity from dpgen.generator.lib.ele_temp import NBandsEsti from dpgen.generator.lib.lammps import get_dumped_forces +from dpgen.generator.lib.calypso import make_run_opt_script,make_check_outcar_script +from dpgen.generator.lib.calypso import write_model_devi_out,make_calypso_input +from dpgen.generator.lib.parse_calypso import _parse_calypso_input,_parse_calypso_dis_mtx param_file = 'param-mg-vasp.json' param_file_v1 = 'param-mg-vasp-v1.json' diff --git a/tests/generator/test_calypso.py b/tests/generator/test_calypso.py new file mode 100644 index 000000000..de250addc --- /dev/null +++ b/tests/generator/test_calypso.py @@ -0,0 +1,99 @@ +import os,sys,json,glob,shutil,textwrap +import dpdata +import numpy as np +import unittest +from pathlib import Path + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +__package__ = 'generator' + +from .context import make_run_opt_script +from .context import make_check_outcar_script +from .context import make_calypso_input +from .context import write_model_devi_out +from .context import _parse_calypso_input +from .context import _parse_calypso_dis_mtx + +calypso_data = Path('.').joinpath('calypso_data') + +#assert os.path.exists(calypso_data) +#assert os.path.exists(calypso_path) +#assert os.path.exists(graph_path) + +# temp dir +test_path = Path('.').joinpath('calypso_test_path') +test_poscar_path = Path('.').joinpath('calypso_test_path','POSCAR') +test_path.mkdir(parents=True,exist_ok=True) +os.system('rm calypso_test_path/*') +fmax = 0.01 +cwd = os.getcwd() + +model_devi = np.array([[0.000000e+00, 2.328491e-02, 5.476687e-09, 1.009454e-02, + 3.279617e-02, 4.053224e-03, 1.869795e-02, 2.184905e+00], + [1.000000e+00, 3.668334e-02, 8.200870e-09, 1.706517e-02, + 2.844074e-02, 7.093109e-03, 1.623275e-02, 2.424708e+00], + [2.000000e+00, 2.832296e-02, 4.828951e-08, 1.573961e-02, + 2.443331e-02, 2.871548e-03, 1.489787e-02, 2.564113e+00]]) + +model_devi_jobs = {"model_devi_jobs": {"times":[4],"NameOfAtoms":["Mg","Al","Cu"],"NumberOfAtoms":[1,1,1],"NumberOfFormula":[1,4],"Volume":[30],"DistanceOfIon":[[1.48,1.44,1.59],[1.44,1.41,1.56],[1.59,1.56,1.70]],"PsoRatio":[0.6],"PopSize":[5],"MaxStep":[3],"ICode":[13],"Split":"T","VSC":"T","MaxNumAtom":[31], + "CtrlRange":[[1,10],[1,10],[1,10]],"PSTRESS":[0],"fmax":[0.01]}} + + +class TestCALYPSOScript(unittest.TestCase): + def setUp(self): + pass + + def tearDown(self): + pass + + def test_write_run_opt_script(self): + ret = make_run_opt_script(fmax) + for temp in ret.split('\n'): + if 'opt.run' in temp: + self.assertEqual(temp.strip(),'opt.run(fmax=0.01,steps=200)') + + def test_write_check_outcar_script(self): + ret = make_check_outcar_script() + with open('calypso_test_path/check_outcar.py','w') as fc: + fc.write(ret) + self.assertTrue(os.path.exists('calypso_test_path/check_outcar.py')) + + def test_write_model_devi_out(self): + devi = write_model_devi_out(model_devi, 'calypso_test_path/model_devi.out') + ndevi = np.loadtxt('calypso_test_path/model_devi.out') + self.assertEqual(ndevi[2,4],model_devi[2,4]) + + def test_make_calypso_input(self): + ret = make_calypso_input(["Mg","Al","Cu"],[1,1,1],[1,4],[30],[ + [1.48,1.44,1.59],[1.44,1.41,1.56],[1.59,1.56,1.70] + ],[0.6],[5],[3],[13],"T","T",[31],[[1,10],[1,10],[1,10]],[0],[0.01]) + with open('calypso_test_path/input.dat','w') as fin: + fin.write(ret) + f = open('calypso_test_path/input.dat') + lines = f.readlines() + f.close() + for line in lines : + if line[0] == '#': + continue + if 'PopSize' in line: + temp_1 = line.split('=')[1].strip() + self.assertEqual(int(temp_1),5) + if 'MaxStep' in line: + temp_2 = line.split('=')[1].strip() + self.assertEqual(int(temp_2),3) + break + + def test_parse_calypso_input(self): + formula = _parse_calypso_input('NumberOfFormula',calypso_data).split() + formula = list(map(int,formula)) + self.assertEqual(formula,model_devi_jobs.get('model_devi_jobs').get('NumberOfFormula')) + + nameofatoms = _parse_calypso_input('NameOfAtoms',calypso_data).split() + self.assertEqual(nameofatoms,model_devi_jobs.get('model_devi_jobs').get('NameOfAtoms')) + + min_dis = _parse_calypso_dis_mtx(len(nameofatoms),calypso_data) + self.assertEqual(float(min_dis),np.nanmin(model_devi_jobs.get('model_devi_jobs').get('DistanceOfIon'))) + + +if __name__ == '__main__': + unittest.main(verbosity=2) From 420df0dc6bf02f6c49901afdf6c2b370e4ce294a Mon Sep 17 00:00:00 2001 From: zhenyu Date: Fri, 4 Mar 2022 16:32:12 +0800 Subject: [PATCH 07/46] add: add some explainations of dp+calypso in README.md --- README.md | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c9bc9494b..5d31f3d91 100644 --- a/README.md +++ b/README.md @@ -378,7 +378,7 @@ In each iteration, there are three stages of work, namely, `00.train 01.model_d + 00.train: DP-GEN will train several (default 4) models based on initial and generated data. The only difference between these models is the random seed for neural network initialization. -+ 01.model_devi : represent for model-deviation. DP-GEN will use models obtained from 00.train to run Molecular Dynamics(default LAMMPS). Larger deviation for structure properties (default is force of atoms) means less accuracy of the models. Using this criterion, a few fructures will be selected and put into next stage `02.fp` for more accurate calculation based on First Principles. ++ 01.model_devi : represent for model-deviation. Model-deviation engine in `01.model_devi` can be chosen between Molecular Dynamics(default LAMMPS) or Structures Prediction(default CALYPSO). DP-GEN will use models obtained from 00.train to run Molecular Dynamics or to run structures optimization with ASE in CALYPSO. Larger deviation for structure properties (default is force of atoms) means less accuracy of the models. Using this criterion, a few fructures will be selected and put into next stage `02.fp` for more accurate calculation based on First Principles. + 02.fp : Selected structures will be calculated by first principles methods(default VASP). DP-GEN will obtain some new data and put them together with initial data and data generated in previous iterations. After that a new training will be set up and DP-GEN will enter next iteration! @@ -615,6 +615,36 @@ The bold notation of key (such aas **type_map**) means that it's a necessary key | **user_fp_params** | Dict | |Parameters for cp2k calculation. find detail in manual.cp2k.org. only the kind section must be set before use. we assume that you have basic knowledge for cp2k input. | **external_input_path** | String | | Conflict with key:user_fp_params, use the template input provided by user, some rules should be followed, read the following text in detail. +One can choose the model-deviation engine by specifying the key `model_devi_engine`. If `model_devi_engine` is not specified, the default model-deviation engine will be LAMMPS. + +There are some new keys needed to be added into `param` and `machine` if CALYPSO as model-deviation engine. + +The bold notation of key (such as **calypso_path**) means that it's a necessary key. + + Key | Type | Example | Discription | +| :---------------- | :--------------------- | :-------------------------------------- | :-------------------------------------------------------------| +| *in param file* +| **model_devi_engine** | string | "calypso" | CALYPSO as model-deviation engine.| +| **calypso_path** | string | "/home/zhenyu/workplace/debug" | The absolute path of calypso.x.| +| **calypso_input_path** | string | "/home/zhenyu/workplace/debug" | The absolute path of CALYPSO input file named input.dat, when this keys exists, all the iters will use the same CALYPSO input file until reach the number of max iter specified by **model_devi_max_iter** and **model_devi_jobs** key will not work.| +| **model_devi_max_iter** | int | 10 | The max iter number code can run, it works when **calypso_input_path** exists.| +| **fmax** | float | 0.01 | The convergence criterion is that the force on all individual atoms should be less than *fmax*, it works when **calypso_input_path** exists.| +| **model_devi_jobs** | List of Dict | [{ "times":[3],"NameOfAtoms":["Al","Cu"],"NumberOfAtoms":[1,10],"NumberOfFormula":[1,2],"Volume":[300],"DistanceOfIon":[[ 1.48,1.44],[ 1.44,1.41]],"PsoRatio":[0.6],"PopSize":[5],"MaxStep":[3],"ICode":[1],"Split":"T"},...] | Settings for exploration in `01.model_devi`. Different number in `times` List means different iteration index and iterations mentioned in List wil use same CALYPSO parameters.| +| **model_devi_jobs["times"]** | List of int | [0,1,2] | Different number in `times` List means different iteration index and iterations mentioned in List wil use same CALYPSO parameters.| +| **model_devi_jobs["NameOfAtoms"]** | List of string |["Al","Cu"] | Parameter of CALYPSO input file, means the element species of structures to be generated. | +| **model_devi_jobs["NumberOfAtoms"]** | List of int |[1,10] | Parameter of CALYPSO input file, means the number of atoms for each chemical species in one formula unit. | +| **model_devi_jobs["NumberOfFormula"]** | List of int |[1,2] | Parameter of CALYPSO input file, means the range of formula unit per cell. | +| **model_devi_jobs["Volume"]** | List of int |[300] | Parameter of CALYPSO input file, means the colume per formula unit(angstrom^3). | +| **model_devi_jobs["DistanceOfIon"]** | List of float |[[ 1.48,1.44],[ 1.44,1.41]] | Parameter of CALYPSO input file, means minimal distance between atoms of each chemical species. Unit is in angstrom. | +| **model_devi_jobs["PsoRatio"]** | List of float |[0.6] | Parameter of CALYPSO input file, means the proportion of the structures generated by PSO. | +| **model_devi_jobs["PopSize"]** | List of int |[5] | Parameter of CALYPSO input file, means the number of structures to be generated in one step in CALYPSO. | +| **model_devi_jobs["MaxStep"]** | List of int |[3] | Parameter of CALYPSO input file, means the number of max step in CALYPSO.| +| **model_devi_jobs["ICode"]** | List of int |[13] | Parameter of CALYPSO input file, means the chosen of local optimization, 1 is vasp and 13 is ASE with dp. | +| **model_devi_jobs["Split"]** | String |"T" | Parameter of CALYPSO input file, means that generating structures and optimizing structures are split into two parts, in dpgen workflow, Split must be T. | +| **model_devi_jobs["PSTRESS"]** | List of float |[0.001] | Same as PSTRESS in INCAR. | +| **model_devi_jobs["fmax"]** | List of float |[0.01] | The convergence criterion of local optimization. | +| *in machine file* +| **deepmdkit_python** | String | "/home/zhenyu/soft/deepmd-kit/bin/python" | A python path with deepmd package. | #### Rules for cp2k input at dictionary form Converting cp2k input is very simple as dictionary used to dpgen input. You just need follow some simple rule: @@ -1352,7 +1382,7 @@ mem_limit | Interger | 16 | Maximal memory permitted to apply for the job. | group_size | Integer | 5 | DP-GEN will put these jobs together in one submitting script. | user_forward_files | List of str | ["/path_to/vdw_kernel.bindat"] | These files will be uploaded in each calculation task. You should make sure provide the path exists. | user_backward_files | List of str | ["HILLS"] | Besides DP-GEN's normal output, these files will be downloaded after each calculation. You should make sure these files can be generated. - + ## Troubleshooting 1. The most common problem is whether two settings correspond with each other, including: - The order of elements in `type_map` and `mass_map` and **`fp_pp_files`**. From 3277c26b677817640d43fcaadbe2e863dd92eb34 Mon Sep 17 00:00:00 2001 From: zhenyu Date: Fri, 4 Mar 2022 16:41:16 +0800 Subject: [PATCH 08/46] change a little bit in readme.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5d31f3d91..9bd5d056a 100644 --- a/README.md +++ b/README.md @@ -378,7 +378,7 @@ In each iteration, there are three stages of work, namely, `00.train 01.model_d + 00.train: DP-GEN will train several (default 4) models based on initial and generated data. The only difference between these models is the random seed for neural network initialization. -+ 01.model_devi : represent for model-deviation. Model-deviation engine in `01.model_devi` can be chosen between Molecular Dynamics(default LAMMPS) or Structures Prediction(default CALYPSO). DP-GEN will use models obtained from 00.train to run Molecular Dynamics or to run structures optimization with ASE in CALYPSO. Larger deviation for structure properties (default is force of atoms) means less accuracy of the models. Using this criterion, a few fructures will be selected and put into next stage `02.fp` for more accurate calculation based on First Principles. ++ 01.model_devi : represent for model-deviation. Model-deviation engine in `01.model_devi` can be chosen between Molecular Dynamics(default LAMMPS) or Structures Prediction(CALYPSO). DP-GEN will use models obtained from 00.train to run Molecular Dynamics or to run structure optimization with ASE in CALYPSO. Larger deviation for structure properties (default is force of atoms) means less accuracy of the models. Using this criterion, a few fructures will be selected and put into next stage `02.fp` for more accurate calculation based on First Principles. + 02.fp : Selected structures will be calculated by first principles methods(default VASP). DP-GEN will obtain some new data and put them together with initial data and data generated in previous iterations. After that a new training will be set up and DP-GEN will enter next iteration! From 09be22388643f2030c2f6382062b0cd5ae546752 Mon Sep 17 00:00:00 2001 From: zhenyu Date: Fri, 4 Mar 2022 16:50:22 +0800 Subject: [PATCH 09/46] change position of function --- dpgen/generator/run.py | 311 ++++++++++++++++++++--------------------- 1 file changed, 155 insertions(+), 156 deletions(-) diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index 62265467b..3a1aea3b5 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -778,83 +778,6 @@ def revise_by_keys(lmp_lines, keys, values): lmp_lines[ii] = lmp_lines[ii].replace(kk, str(vv)) return lmp_lines - -def make_calypso_model_devi(iter_index,jdata,mdata): - ''' for calypso ''' - - model_devi_jobs = jdata['model_devi_jobs'] - try: - maxiter = max(model_devi_jobs[-1].get('times')) - except Exception as e: - maxiter = jdata.get('model_devi_max_iter',0) - - if (iter_index > maxiter) : - return False - - calypso_path = jdata.get('calypso_path') - iter_name = make_iter_name(iter_index) - train_path = os.path.join(iter_name, train_name) - train_path = os.path.abspath(train_path) - models = glob.glob(os.path.join(train_path, "graph*pb")) - work_path = os.path.join(iter_name, model_devi_name) - calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) - calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) - - create_path(work_path) - create_path(calypso_run_opt_path) - create_path(calypso_model_devi_path) - - # the model - for mm in models : - model_name = os.path.basename(mm) - os.symlink(mm, os.path.join(calypso_run_opt_path, model_name)) - - - input_mode = "native" - if "calypso_input_path" in jdata: - input_mode = "buffet" - - # generate input.dat automatic in each iter - if input_mode == "native": - for iiidx, jobbs in enumerate(model_devi_jobs): - if iter_index in jobbs.get('times'): - cur_job = model_devi_jobs[iiidx] - - _make_model_devi_native_calypso(cur_job, calypso_run_opt_path) - - run_opt_script = os.path.join(calypso_run_opt_path,'run_opt.py') - with open(run_opt_script,'w') as ffff: - ffff.write(make_run_opt_script(cur_job.get('fmax',0.01))) - - # ----------for check the nan situation ------------- - check_outcar_script = os.path.join(calypso_run_opt_path,'check_outcar.py') - with open(check_outcar_script,'w') as ffff: - ffff.write(make_check_outcar_script()) - # ----------for check the nan situation ------------- - return True - - # generate confs according to the input.dat provided - elif input_mode == "buffet": - calypso_input_path = jdata.get('calypso_input_path') - shutil.copyfile(os.path.join(calypso_input_path,'input.dat'),os.path.join(calypso_run_opt_path, 'input.dat')) - popsize = _parse_calypso_input('PopSize',calypso_run_opt_path+'/input.dat') - run_opt_path = calypso_run_opt_path - - run_opt_script = os.path.join(calypso_run_opt_path,'run_opt.py') - with open(run_opt_script,'w') as ffff: - ffff.write(make_run_opt_script(jdata.get('fmax',0.01))) - - # ----------for check the nan situation ------------- - check_outcar_script = os.path.join(calypso_run_opt_path,'check_outcar.py') - with open(check_outcar_script,'w') as ffff: - ffff.write(make_check_outcar_script()) - # ----------for check the nan situation ------------- - return True - - else: - raise RuntimeError('do not found `calypso_input_path` or `model_devi_jobs`', input_mode) - - def make_model_devi (iter_index, jdata, mdata) : @@ -958,6 +881,80 @@ def make_model_devi (iter_index, symlink_user_forward_files(mdata=mdata, task_type="model_devi", work_path=work_path) return True +def make_calypso_model_devi(iter_index,jdata,mdata): + ''' for calypso ''' + + model_devi_jobs = jdata['model_devi_jobs'] + try: + maxiter = max(model_devi_jobs[-1].get('times')) + except Exception as e: + maxiter = jdata.get('model_devi_max_iter',0) + + if (iter_index > maxiter) : + return False + + calypso_path = jdata.get('calypso_path') + iter_name = make_iter_name(iter_index) + train_path = os.path.join(iter_name, train_name) + train_path = os.path.abspath(train_path) + models = glob.glob(os.path.join(train_path, "graph*pb")) + work_path = os.path.join(iter_name, model_devi_name) + calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) + calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) + + create_path(work_path) + create_path(calypso_run_opt_path) + create_path(calypso_model_devi_path) + + # the model + for mm in models : + model_name = os.path.basename(mm) + os.symlink(mm, os.path.join(calypso_run_opt_path, model_name)) + + + input_mode = "native" + if "calypso_input_path" in jdata: + input_mode = "buffet" + + # generate input.dat automatic in each iter + if input_mode == "native": + for iiidx, jobbs in enumerate(model_devi_jobs): + if iter_index in jobbs.get('times'): + cur_job = model_devi_jobs[iiidx] + + _make_model_devi_native_calypso(cur_job, calypso_run_opt_path) + + run_opt_script = os.path.join(calypso_run_opt_path,'run_opt.py') + with open(run_opt_script,'w') as ffff: + ffff.write(make_run_opt_script(cur_job.get('fmax',0.01))) + + # ----------for check the nan situation ------------- + check_outcar_script = os.path.join(calypso_run_opt_path,'check_outcar.py') + with open(check_outcar_script,'w') as ffff: + ffff.write(make_check_outcar_script()) + # ----------for check the nan situation ------------- + return True + + # generate confs according to the input.dat provided + elif input_mode == "buffet": + calypso_input_path = jdata.get('calypso_input_path') + shutil.copyfile(os.path.join(calypso_input_path,'input.dat'),os.path.join(calypso_run_opt_path, 'input.dat')) + popsize = _parse_calypso_input('PopSize',calypso_run_opt_path+'/input.dat') + run_opt_path = calypso_run_opt_path + + run_opt_script = os.path.join(calypso_run_opt_path,'run_opt.py') + with open(run_opt_script,'w') as ffff: + ffff.write(make_run_opt_script(jdata.get('fmax',0.01))) + + # ----------for check the nan situation ------------- + check_outcar_script = os.path.join(calypso_run_opt_path,'check_outcar.py') + with open(check_outcar_script,'w') as ffff: + ffff.write(make_check_outcar_script()) + # ----------for check the nan situation ------------- + return True + + else: + raise RuntimeError('do not found `calypso_input_path` or `model_devi_jobs`', input_mode) def _make_model_devi_revmat(iter_index, jdata, mdata, conf_systems): model_devi_jobs = jdata['model_devi_jobs'] @@ -1302,85 +1299,6 @@ def _make_model_devi_native_gromacs(iter_index, jdata, mdata, conf_systems): sys_counter += 1 -def run_model_devi_calypso (iter_index, - jdata, - mdata) : - - print('############## run calypso model devi #######################') - - - iter_name = make_iter_name(iter_index) - work_path = os.path.join(iter_name, model_devi_name) - assert(os.path.isdir(work_path)) - - calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) - calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) - - all_models = glob.glob(os.path.join(calypso_run_opt_path, 'graph*pb')) - - cwd = os.getcwd() - - api_version = mdata.get('api_version', '0.9') - if LooseVersion(api_version) < LooseVersion('1.0'): - warnings.warn(f"the dpdispatcher will be updated to new version." - f"And the interface may be changed. Please check the documents for more details") - record_calypso_path = os.path.join(work_path,'record.calypso') - while True: - if not os.path.exists(record_calypso_path): - f = open(record_calypso_path,'w') - f.write('1\n') - lines = '1' - f.close() - else: - f = open(record_calypso_path,'r') - lines = f.readlines() - f.close() - - if lines[-1].strip().strip('\n') == '1': - # Gen Structures - GenStructures(iter_index,jdata,mdata) - - elif lines[-1].strip().strip('\n') == '2': - # Analysis & to deepmd/raw - Analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path) - - elif lines[-1].strip().strip('\n') == '3': - # Model Devi - Modd(iter_index,calypso_model_devi_path,all_models,jdata) - - elif lines[-1].strip().strip('\n') == '4': - print('Model Devi is done.') - break - - elif LooseVersion(api_version) >= LooseVersion('1.0'): - - record_calypso_path = os.path.join(work_path,'record.calypso') - while True: - if not os.path.exists(record_calypso_path): - f = open(record_calypso_path,'w') - f.write('1\n') - lines = '1' - f.close() - else: - f = open(record_calypso_path,'r') - lines = f.readlines() - f.close() - - if lines[-1].strip().strip('\n') == '1': - # Gen Structures - GenStructures(iter_index,jdata,mdata) - - elif lines[-1].strip().strip('\n') == '2': - # Analysis & to deepmd/raw - Analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path) - - elif lines[-1].strip().strip('\n') == '3': - # Model Devi - Modd(iter_index,calypso_model_devi_path,all_models,jdata) - - elif lines[-1].strip().strip('\n') == '4': - print('Model Devi is done.') - break def run_model_devi (iter_index, jdata, @@ -1506,6 +1424,87 @@ def run_model_devi (iter_index, errlog = 'model_devi.log') submission.run_submission() + +def run_model_devi_calypso (iter_index, + jdata, + mdata) : + + print('############## run calypso model devi #######################') + + + iter_name = make_iter_name(iter_index) + work_path = os.path.join(iter_name, model_devi_name) + assert(os.path.isdir(work_path)) + + calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) + calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) + + all_models = glob.glob(os.path.join(calypso_run_opt_path, 'graph*pb')) + + cwd = os.getcwd() + + api_version = mdata.get('api_version', '0.9') + if LooseVersion(api_version) < LooseVersion('1.0'): + warnings.warn(f"the dpdispatcher will be updated to new version." + f"And the interface may be changed. Please check the documents for more details") + record_calypso_path = os.path.join(work_path,'record.calypso') + while True: + if not os.path.exists(record_calypso_path): + f = open(record_calypso_path,'w') + f.write('1\n') + lines = '1' + f.close() + else: + f = open(record_calypso_path,'r') + lines = f.readlines() + f.close() + + if lines[-1].strip().strip('\n') == '1': + # Gen Structures + GenStructures(iter_index,jdata,mdata) + + elif lines[-1].strip().strip('\n') == '2': + # Analysis & to deepmd/raw + Analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path) + + elif lines[-1].strip().strip('\n') == '3': + # Model Devi + Modd(iter_index,calypso_model_devi_path,all_models,jdata) + + elif lines[-1].strip().strip('\n') == '4': + print('Model Devi is done.') + break + + elif LooseVersion(api_version) >= LooseVersion('1.0'): + + record_calypso_path = os.path.join(work_path,'record.calypso') + while True: + if not os.path.exists(record_calypso_path): + f = open(record_calypso_path,'w') + f.write('1\n') + lines = '1' + f.close() + else: + f = open(record_calypso_path,'r') + lines = f.readlines() + f.close() + + if lines[-1].strip().strip('\n') == '1': + # Gen Structures + GenStructures(iter_index,jdata,mdata) + + elif lines[-1].strip().strip('\n') == '2': + # Analysis & to deepmd/raw + Analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path) + + elif lines[-1].strip().strip('\n') == '3': + # Model Devi + Modd(iter_index,calypso_model_devi_path,all_models,jdata) + + elif lines[-1].strip().strip('\n') == '4': + print('Model Devi is done.') + break + def post_model_devi (iter_index, jdata, mdata) : From 224b62d940977538a5408e85f86afe5bfd38bd6c Mon Sep 17 00:00:00 2001 From: zhenyu Date: Fri, 4 Mar 2022 19:17:00 +0800 Subject: [PATCH 10/46] add deepmd-kit>=2.0.1 install_require in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e051389bf..2bc2fc212 100755 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ with open(path.join('dpgen', '_date.py'), 'w') as fp : fp.write('date = \'%s\'' % today) -install_requires=['numpy>=1.14.3', 'dpdata>=0.2.4', 'pymatgen>=2019.1.13', 'ase', 'monty>2.0.0', 'paramiko', 'custodian','GromacsWrapper>=0.8.0', 'dpdispatcher>=0.3.11'] +install_requires=['numpy>=1.14.3', 'dpdata>=0.2.4', 'pymatgen>=2019.1.13', 'ase', 'monty>2.0.0', 'paramiko', 'custodian','GromacsWrapper>=0.8.0', 'dpdispatcher>=0.3.11','deepmd-kit>=2.0.1'] setuptools.setup( name=NAME, From 8af2b2ae91d7c613e70325b20c6b82f324a64bed Mon Sep 17 00:00:00 2001 From: zhenyu Date: Fri, 4 Mar 2022 19:39:30 +0800 Subject: [PATCH 11/46] test failed cause no tensorflow module so add tensorflow in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2bc2fc212..c2158728c 100755 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ with open(path.join('dpgen', '_date.py'), 'w') as fp : fp.write('date = \'%s\'' % today) -install_requires=['numpy>=1.14.3', 'dpdata>=0.2.4', 'pymatgen>=2019.1.13', 'ase', 'monty>2.0.0', 'paramiko', 'custodian','GromacsWrapper>=0.8.0', 'dpdispatcher>=0.3.11','deepmd-kit>=2.0.1'] +install_requires=['numpy>=1.14.3', 'dpdata>=0.2.4', 'pymatgen>=2019.1.13', 'ase', 'monty>2.0.0', 'paramiko', 'custodian','GromacsWrapper>=0.8.0', 'dpdispatcher>=0.3.11','deepmd-kit>=2.0.1','tensorflow'] setuptools.setup( name=NAME, From 239765a3c9aa3bd32fc12ace4e14372198e030c5 Mon Sep 17 00:00:00 2001 From: zhenyu Date: Wed, 9 Mar 2022 10:41:23 +0800 Subject: [PATCH 12/46] calc model devi in command line --- dpgen/generator/lib/modd.py | 123 +++++++++++++++++++ dpgen/generator/lib/modd_calypso.py | 182 ++++++++++++++-------------- dpgen/generator/lib/write_modd.py | 127 +++++++++++++++++++ dpgen/generator/run.py | 23 +++- setup.py | 2 +- 5 files changed, 362 insertions(+), 95 deletions(-) create mode 100644 dpgen/generator/lib/modd.py create mode 100644 dpgen/generator/lib/write_modd.py diff --git a/dpgen/generator/lib/modd.py b/dpgen/generator/lib/modd.py new file mode 100644 index 000000000..7d619b865 --- /dev/null +++ b/dpgen/generator/lib/modd.py @@ -0,0 +1,123 @@ +import argparse +import copy +import dpdata +import glob +import math +import numpy as np +import os +import sys +import shutil + + +def write_model_devi_out(devi, fname): + assert devi.shape[1] == 8 + #assert devi.shape[1] == 7 + header = "%5s" % "step" + for item in 'vf': + header += "%16s%16s%16s" % (f"max_devi_{item}", f"min_devi_{item}",f"avg_devi_{item}") + header += "%16s"%str('min_dis') + np.savetxt(fname, + devi, + fmt=['%5d'] + ['%17.6e' for _ in range(7)], + delimiter='', + header=header) + return devi + + +def Modd(all_models,type_map): + from deepmd.infer import calc_model_devi + from deepmd.infer import DeepPot as DP + + # Model Devi + + cwd = os.getcwd() + graphs = [DP(model) for model in all_models] + + Devis = [] + pcount = 0 + strus_lists = glob.glob(os.path.join(cwd,'*.structures')) + for num, strus_path in enumerate(strus_lists): + + structures_data = dpdata.System(strus_path,'deepmd/npy',type_map=type_map) + + # every 500 confs in one task dir + nnum = structures_data.get_nframes() + if nnum == 0: + continue + else: + num_per_task = math.ceil(nnum/500) + + + for temp in range(num_per_task): + task_name = os.path.join(cwd,'task.%03d.%03d'%(num,temp)) + put_poscar = os.path.join(task_name,'traj') + if not os.path.exists(task_name): + os.mkdir(task_name) + os.mkdir(put_poscar) + else: + shutil.rmtree(task_name) + os.mkdir(task_name) + os.mkdir(put_poscar) + devis = [] + if (nnum - (temp+1)*500) >= 0: + temp_sl = range(temp*500,(temp+1)*500) + else: + temp_sl = range(temp*500,nnum) + + new_index = 0 + for index,frameid in enumerate(temp_sl): + pdata = structures_data[frameid] + pdata.to_vasp_poscar(os.path.join(put_poscar,'%s.poscar'%str(index))) + nopbc = pdata.nopbc + coord = pdata.data['coords'] + cell = pdata.data['cells'] + atom_types = pdata.data['atom_types'] + devi = calc_model_devi(coord,cell,atom_types,graphs,nopbc=nopbc) + # ------------------------------------------------------------------------------------ + # append min-distance in devi list + dis = pdata.to_ase_structure()[0].get_all_distances(mic=True) + row,col = np.diag_indices_from(dis) + dis[row,col] = 10000 + min_dis = np.nanmin(dis) + devi = np.append(devi[0],min_dis) + t = [devi] + devi = np.array(t) + # ------------------------------------------------------------------------------------ + temp_d = copy.deepcopy(devi) + temp_D = copy.deepcopy(devi) + devis.append(temp_d) + Devis.append(temp_D) + devis[index][0][0] = np.array(index) + Devis[pcount][0][0] = np.array(pcount) + pcount += 1 + new_index += 1 + devis = np.vstack(devis) + write_model_devi_out(devis,os.path.join(task_name, 'model_devi.out')) + + Devis = np.vstack(Devis) + write_model_devi_out(Devis,os.path.join(cwd,'Model_Devi.out')) + + f = open(os.path.join(os.path.abspath(os.path.join(cwd,os.pardir)),'record.calypso'),'a+') + f.write('4\n') + f.close() + +if __name__ == "__main__": + + cwd = os.getcwd() + model_path = os.path.join(os.path.abspath(os.path.join(cwd,os.pardir)),'gen_stru_analy') + parser = argparse.ArgumentParser(description='calc model-devi by `all_models` and `type_map`') + parser.add_argument( + "--all_models", + type=str, + nargs='+', + default=model_path, + help="the path of models which will be used to do model-deviation", + ) + parser.add_argument( + "--type_map", + type=list, + help="the type map of models which will be used to do model-deviation", + ) + args = parser.parse_args() + print(vars(args)) + Modd(args.all_models,args.type_map) diff --git a/dpgen/generator/lib/modd_calypso.py b/dpgen/generator/lib/modd_calypso.py index c4838cd8f..2968d34dd 100644 --- a/dpgen/generator/lib/modd_calypso.py +++ b/dpgen/generator/lib/modd_calypso.py @@ -16,8 +16,8 @@ import shutil from ase.io.vasp import write_vasp from ase.io.trajectory import Trajectory -from deepmd.infer import calc_model_devi -from deepmd.infer import DeepPot as DP +#from deepmd.infer import calc_model_devi +#from deepmd.infer import DeepPot as DP from dpgen import dlog from dpgen.generator.lib.utils import make_iter_name from dpgen.generator.lib.calypso import write_model_devi_out @@ -331,93 +331,93 @@ def Analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): os.chdir(cwd) -def Modd(iter_index,calypso_model_devi_path,all_models,jdata): - - # Model Devi - - - dlog.info('$$$$$$$$$$$$$$$ Model Devi Start $$$$$$$$$$$$$$$$$$') - - iter_name = make_iter_name(iter_index) - work_path = os.path.join(iter_name, model_devi_name) - - cwd = os.getcwd() - Devis = [] - pcount = 0 - strus_lists = glob.glob(os.path.join(calypso_model_devi_path,'*.structures')) - for num, strus_path in enumerate(strus_lists): - os.chdir(strus_path) - os.system('rename .vasp .poscar *') - os.chdir(cwd) - - structure_list = glob.glob(os.path.join(strus_path,'*.poscar')) - - # every 500 confs in one task dir - if len(structure_list) == 0: - continue - else: - num_per_task = math.ceil(len(structure_list)/500) - graphs = [DP(model) for model in all_models] - for temp in range(num_per_task): - task_name = os.path.join(calypso_model_devi_path,'task.%03d.%03d'%(num,temp)) - put_poscar = os.path.join(task_name,'traj') - if not os.path.exists(task_name): - os.mkdir(task_name) - os.mkdir(put_poscar) - else: - shutil.rmtree(task_name) - os.mkdir(task_name) - os.mkdir(put_poscar) - devis = [] - try: - temp_sl = structure_list[temp*500:(temp+1)*500] - except Exception as err: - dlog.info('err %s'%str(err)) - temp_sl = structure_list[temp*500:] - - new_index = 0 - for index,sname in enumerate(temp_sl): - shutil.copyfile(sname,os.path.join(put_poscar,'%s.poscar'%str(index))) - os.chdir(put_poscar) - pdata = dpdata.System('%s.poscar'%str(index),type_map = jdata['type_map']) - nopbc = pdata.nopbc - coord = pdata.data['coords'] - cell = pdata.data['cells'] - atom_types = pdata.data['atom_types'] - devi = calc_model_devi(coord,cell,atom_types,graphs,nopbc=nopbc) - # ------------------------------------------------------------------------------------ - # append min-distance in devi list - dis_temp = dpdata.System('%s.poscar'%str(index)) - dis = dis_temp.to_ase_structure()[0].get_all_distances(mic=True) - row,col = np.diag_indices_from(dis) - dis[row,col] = 10000 - min_dis = np.nanmin(dis) - devi = np.append(devi[0],min_dis) - t = [devi] - devi = np.array(t) - # ------------------------------------------------------------------------------------ - temp_d = copy.deepcopy(devi) - temp_D = copy.deepcopy(devi) - devis.append(temp_d) - Devis.append(temp_D) - devis[index][0][0] = np.array(index) - Devis[pcount][0][0] = np.array(pcount) - pcount += 1 - new_index += 1 - os.chdir(cwd) - os.chdir(task_name) - devis = np.vstack(devis) - write_model_devi_out(devis,'model_devi.out') - os.chdir(cwd) - - os.chdir(calypso_model_devi_path) - Devis = np.vstack(Devis) - write_model_devi_out(Devis,'Model_Devi.out') - os.chdir(cwd) - - os.chdir(work_path) - f = open('record.calypso','a+') - f.write('4\n') - f.close() - os.chdir(cwd) +#def Modd(iter_index,calypso_model_devi_path,all_models,jdata): +# +# # Model Devi +# +# +# dlog.info('$$$$$$$$$$$$$$$ Model Devi Start $$$$$$$$$$$$$$$$$$') +# +# iter_name = make_iter_name(iter_index) +# work_path = os.path.join(iter_name, model_devi_name) +# +# cwd = os.getcwd() +# Devis = [] +# pcount = 0 +# strus_lists = glob.glob(os.path.join(calypso_model_devi_path,'*.structures')) +# for num, strus_path in enumerate(strus_lists): +# os.chdir(strus_path) +# os.system('rename .vasp .poscar *') +# os.chdir(cwd) +# +# structure_list = glob.glob(os.path.join(strus_path,'*.poscar')) +# +# # every 500 confs in one task dir +# if len(structure_list) == 0: +# continue +# else: +# num_per_task = math.ceil(len(structure_list)/500) +# graphs = [DP(model) for model in all_models] +# for temp in range(num_per_task): +# task_name = os.path.join(calypso_model_devi_path,'task.%03d.%03d'%(num,temp)) +# put_poscar = os.path.join(task_name,'traj') +# if not os.path.exists(task_name): +# os.mkdir(task_name) +# os.mkdir(put_poscar) +# else: +# shutil.rmtree(task_name) +# os.mkdir(task_name) +# os.mkdir(put_poscar) +# devis = [] +# try: +# temp_sl = structure_list[temp*500:(temp+1)*500] +# except Exception as err: +# dlog.info('err %s'%str(err)) +# temp_sl = structure_list[temp*500:] +# +# new_index = 0 +# for index,sname in enumerate(temp_sl): +# shutil.copyfile(sname,os.path.join(put_poscar,'%s.poscar'%str(index))) +# os.chdir(put_poscar) +# pdata = dpdata.System('%s.poscar'%str(index),type_map = jdata['type_map']) +# nopbc = pdata.nopbc +# coord = pdata.data['coords'] +# cell = pdata.data['cells'] +# atom_types = pdata.data['atom_types'] +# devi = calc_model_devi(coord,cell,atom_types,graphs,nopbc=nopbc) +# # ------------------------------------------------------------------------------------ +# # append min-distance in devi list +# dis_temp = dpdata.System('%s.poscar'%str(index)) +# dis = dis_temp.to_ase_structure()[0].get_all_distances(mic=True) +# row,col = np.diag_indices_from(dis) +# dis[row,col] = 10000 +# min_dis = np.nanmin(dis) +# devi = np.append(devi[0],min_dis) +# t = [devi] +# devi = np.array(t) +# # ------------------------------------------------------------------------------------ +# temp_d = copy.deepcopy(devi) +# temp_D = copy.deepcopy(devi) +# devis.append(temp_d) +# Devis.append(temp_D) +# devis[index][0][0] = np.array(index) +# Devis[pcount][0][0] = np.array(pcount) +# pcount += 1 +# new_index += 1 +# os.chdir(cwd) +# os.chdir(task_name) +# devis = np.vstack(devis) +# write_model_devi_out(devis,'model_devi.out') +# os.chdir(cwd) +# +# os.chdir(calypso_model_devi_path) +# Devis = np.vstack(Devis) +# write_model_devi_out(Devis,'Model_Devi.out') +# os.chdir(cwd) +# +# os.chdir(work_path) +# f = open('record.calypso','a+') +# f.write('4\n') +# f.close() +# os.chdir(cwd) diff --git a/dpgen/generator/lib/write_modd.py b/dpgen/generator/lib/write_modd.py new file mode 100644 index 000000000..b8e114c53 --- /dev/null +++ b/dpgen/generator/lib/write_modd.py @@ -0,0 +1,127 @@ +def write_modd(): + + ret ="import argparse" + ret+="import copy" + ret+="import dpdata" + ret+="import glob" + ret+="import math" + ret+="import numpy as np" + ret+="import os" + ret+="import sys" + ret+="import shutil" + ret+="" + ret+="" + ret+="def write_model_devi_out(devi, fname):" + ret+=" assert devi.shape[1] == 8" + ret+=" #assert devi.shape[1] == 7" + ret+=" header = '%5s' % 'step'" + ret+=" for item in 'vf':" + ret+=" header += '%16s%16s%16s' % (f'max_devi_{item}', f'min_devi_{item}',f'avg_devi_{item}')" + ret+=" header += '%16s'%str('min_dis')" + ret+=" np.savetxt(fname," + ret+=" devi," + ret+=" fmt=['%5d'] + ['%17.6e' for _ in range(7)]," + ret+=" delimiter=''," + ret+=" header=header)" + ret+=" return devi" + ret+="" + ret+="" + ret+="def Modd(all_models,type_map):" + ret+=" from deepmd.infer import calc_model_devi" + ret+=" from deepmd.infer import DeepPot as DP" + ret+="" + ret+=" # Model Devi " + ret+="" + ret+=" cwd = os.getcwd()" + ret+=" graphs = [DP(model) for model in all_models]" + ret+="" + ret+=" Devis = []" + ret+=" pcount = 0" + ret+=" strus_lists = glob.glob(os.path.join(cwd,'*.structures'))" + ret+=" for num, strus_path in enumerate(strus_lists):" + ret+="" + ret+=" structures_data = dpdata.System(strus_path,'deepmd/npy',type_map=type_map)" + ret+="" + ret+=" # every 500 confs in one task dir" + ret+=" nnum = structures_data.get_nframes()" + ret+=" if nnum == 0:" + ret+=" continue" + ret+=" else:" + ret+=" num_per_task = math.ceil(nnum/500)" + ret+=" " + ret+="" + ret+=" for temp in range(num_per_task):" + ret+=" task_name = os.path.join(cwd,'task.%03d.%03d'%(num,temp)) " + ret+=" put_poscar = os.path.join(task_name,'traj')" + ret+=" if not os.path.exists(task_name):" + ret+=" os.mkdir(task_name)" + ret+=" os.mkdir(put_poscar)" + ret+=" else:" + ret+=" shutil.rmtree(task_name)" + ret+=" os.mkdir(task_name)" + ret+=" os.mkdir(put_poscar)" + ret+=" devis = []" + ret+=" if (nnum - (temp+1)*500) >= 0:" + ret+=" temp_sl = range(temp*500,(temp+1)*500)" + ret+=" else:" + ret+=" temp_sl = range(temp*500,nnum)" + ret+=" " + ret+=" new_index = 0" + ret+=" for index,frameid in enumerate(temp_sl):" + ret+=" pdata = structures_data[frameid]" + ret+=" pdata.to_vasp_poscar(os.path.join(put_poscar,'%s.poscar'%str(index)))" + ret+=" nopbc = pdata.nopbc" + ret+=" coord = pdata.data['coords']" + ret+=" cell = pdata.data['cells']" + ret+=" atom_types = pdata.data['atom_types']" + ret+=" devi = calc_model_devi(coord,cell,atom_types,graphs,nopbc=nopbc)" + ret+=" # ------------------------------------------------------------------------------------" + ret+=" # append min-distance in devi list" + ret+=" dis = pdata.to_ase_structure()[0].get_all_distances(mic=True)" + ret+=" row,col = np.diag_indices_from(dis)" + ret+=" dis[row,col] = 10000" + ret+=" min_dis = np.nanmin(dis)" + ret+=" devi = np.append(devi[0],min_dis) " + ret+=" t = [devi]" + ret+=" devi = np.array(t)" + ret+=" # ------------------------------------------------------------------------------------" + ret+=" temp_d = copy.deepcopy(devi)" + ret+=" temp_D = copy.deepcopy(devi)" + ret+=" devis.append(temp_d)" + ret+=" Devis.append(temp_D)" + ret+=" devis[index][0][0] = np.array(index)" + ret+=" Devis[pcount][0][0] = np.array(pcount)" + ret+=" pcount += 1" + ret+=" new_index += 1" + ret+=" devis = np.vstack(devis)" + ret+=" write_model_devi_out(devis,os.path.join(task_name, 'model_devi.out'))" + ret+="" + ret+=" Devis = np.vstack(Devis)" + ret+=" write_model_devi_out(Devis,os.path.join(cwd,'Model_Devi.out'))" + ret+="" + ret+=" f = open(os.path.join(os.path.abspath(os.path.join(cwd,os.pardir)),'record.calypso'),'a+')" + ret+=" f.write('4\n')" + ret+=" f.close()" + ret+="" + ret+="if __name__ == '__main__':" + ret+="" + ret+=" cwd = os.getcwd()" + ret+=" model_path = os.path.join(os.path.abspath(os.path.join(cwd,os.pardir)),'gen_stru_analy')" + ret+=" parser = argparse.ArgumentParser(description='calc model-devi by `all_models` and `type_map`')" + ret+=" parser.add_argument(" + ret+=" '--all_models'," + ret+=" type=str," + ret+=" nargs='+'," + ret+=" default=model_path," + ret+=" help='the path of models which will be used to do model-deviation'," + ret+=" )" + ret+=" parser.add_argument(" + ret+=" '--type_map'," + ret+=" type=list," + ret+=" help='the type map of models which will be used to do model-deviation'," + ret+=" )" + ret+=" args = parser.parse_args()" + ret+=" print(vars(args))" + ret+=" #Modd(args.all_models,args.type_map)" + ret+=" #Modd(sys.argv[1],sys.argv[2])" + return ret diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index 3a1aea3b5..23b6d9972 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -43,7 +43,8 @@ from dpgen.generator.lib.lammps import make_lammps_input, get_dumped_forces from dpgen.generator.lib.calypso import make_run_opt_script,make_check_outcar_script from dpgen.generator.lib.calypso import _make_model_devi_native_calypso,write_model_devi_out -from dpgen.generator.lib.modd_calypso import GenStructures,Analysis,Modd +from dpgen.generator.lib.modd_calypso import GenStructures,Analysis +from dpgen.generator.lib.write_modd import write_modd from dpgen.generator.lib.parse_calypso import _parse_calypso_input,_parse_calypso_dis_mtx from dpgen.generator.lib.vasp import write_incar_dict from dpgen.generator.lib.vasp import make_vasp_incar_user_dict @@ -911,6 +912,10 @@ def make_calypso_model_devi(iter_index,jdata,mdata): model_name = os.path.basename(mm) os.symlink(mm, os.path.join(calypso_run_opt_path, model_name)) + # modd script + modd_script = os.path.join(calypso_model_devi_path,'modd.py') + with open(modd_script,'w') as fm: + fm.write(write_moddwrite_modd()) input_mode = "native" if "calypso_input_path" in jdata: @@ -1469,7 +1474,13 @@ def run_model_devi_calypso (iter_index, elif lines[-1].strip().strip('\n') == '3': # Model Devi - Modd(iter_index,calypso_model_devi_path,all_models,jdata) + cwd = os.getcwd() + os.chdir(calypso_model_devi_path) + args = ' '.join(['modd.py', '--all_models',' '.join(all_models),'--type_map',' '.join(jdata.get('type_map'))]) + deepmdkit_python = mdata.get('deepmdkit_python') + os.system(f'{deepmdkit_python} {args} ') + #Modd(iter_index,calypso_model_devi_path,all_models,jdata) + os.chdir(cwd) elif lines[-1].strip().strip('\n') == '4': print('Model Devi is done.') @@ -1499,7 +1510,13 @@ def run_model_devi_calypso (iter_index, elif lines[-1].strip().strip('\n') == '3': # Model Devi - Modd(iter_index,calypso_model_devi_path,all_models,jdata) + cwd = os.getcwd() + os.chdir(calypso_model_devi_path) + args = ' '.join(['modd.py', '--all_models',' '.join(all_models),'--type_map',' '.join(jdata.get('type_map'))]) + deepmdkit_python = mdata.get('deepmdkit_python') + os.system(f'{deepmdkit_python} {args} ') + #Modd(iter_index,calypso_model_devi_path,all_models,jdata) + os.chdir(cwd) elif lines[-1].strip().strip('\n') == '4': print('Model Devi is done.') diff --git a/setup.py b/setup.py index c2158728c..e051389bf 100755 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ with open(path.join('dpgen', '_date.py'), 'w') as fp : fp.write('date = \'%s\'' % today) -install_requires=['numpy>=1.14.3', 'dpdata>=0.2.4', 'pymatgen>=2019.1.13', 'ase', 'monty>2.0.0', 'paramiko', 'custodian','GromacsWrapper>=0.8.0', 'dpdispatcher>=0.3.11','deepmd-kit>=2.0.1','tensorflow'] +install_requires=['numpy>=1.14.3', 'dpdata>=0.2.4', 'pymatgen>=2019.1.13', 'ase', 'monty>2.0.0', 'paramiko', 'custodian','GromacsWrapper>=0.8.0', 'dpdispatcher>=0.3.11'] setuptools.setup( name=NAME, From 0bf1473ead34752092ab17b2e29cd001f7e028da Mon Sep 17 00:00:00 2001 From: zhenyu Date: Wed, 9 Mar 2022 19:16:31 +0800 Subject: [PATCH 13/46] fix some small bugs of modd and PBS script of cpu --- dpgen/dispatcher/PBS.py | 2 +- dpgen/generator/lib/modd.py | 3 +- dpgen/generator/lib/write_modd.py | 248 +++++++++++++++--------------- dpgen/generator/run.py | 5 +- 4 files changed, 129 insertions(+), 129 deletions(-) diff --git a/dpgen/dispatcher/PBS.py b/dpgen/dispatcher/PBS.py index 0fed5a888..d6362de2b 100644 --- a/dpgen/dispatcher/PBS.py +++ b/dpgen/dispatcher/PBS.py @@ -94,7 +94,7 @@ def sub_script_head(self, res): if res['numb_gpu'] == 0: ret += '#PBS -l nodes=%d:ppn=%d\n' % (res['numb_node'], res['task_per_node']) else : - ret += '#PBS -l nodes=%d:ppn=%d:gpus=%d\n' % (res['numb_node'], res['task_per_node'], res['numb_gpu']) + ret += '#PBS -l select=%d:ncpus=%d:ngpus=%d\n' % (res['numb_node'], res['task_per_node'], res['numb_gpu']) ret += '#PBS -l walltime=%s\n' % (res['time_limit']) if res['mem_limit'] > 0 : ret += "#PBS -l mem=%dG \n" % res['mem_limit'] diff --git a/dpgen/generator/lib/modd.py b/dpgen/generator/lib/modd.py index 7d619b865..577ebde7d 100644 --- a/dpgen/generator/lib/modd.py +++ b/dpgen/generator/lib/modd.py @@ -108,14 +108,13 @@ def Modd(all_models,type_map): parser = argparse.ArgumentParser(description='calc model-devi by `all_models` and `type_map`') parser.add_argument( "--all_models", - type=str, nargs='+', default=model_path, help="the path of models which will be used to do model-deviation", ) parser.add_argument( "--type_map", - type=list, + nargs='+', help="the type map of models which will be used to do model-deviation", ) args = parser.parse_args() diff --git a/dpgen/generator/lib/write_modd.py b/dpgen/generator/lib/write_modd.py index b8e114c53..61637065f 100644 --- a/dpgen/generator/lib/write_modd.py +++ b/dpgen/generator/lib/write_modd.py @@ -1,127 +1,127 @@ def write_modd(): - ret ="import argparse" - ret+="import copy" - ret+="import dpdata" - ret+="import glob" - ret+="import math" - ret+="import numpy as np" - ret+="import os" - ret+="import sys" - ret+="import shutil" - ret+="" - ret+="" - ret+="def write_model_devi_out(devi, fname):" - ret+=" assert devi.shape[1] == 8" - ret+=" #assert devi.shape[1] == 7" - ret+=" header = '%5s' % 'step'" - ret+=" for item in 'vf':" - ret+=" header += '%16s%16s%16s' % (f'max_devi_{item}', f'min_devi_{item}',f'avg_devi_{item}')" - ret+=" header += '%16s'%str('min_dis')" - ret+=" np.savetxt(fname," - ret+=" devi," - ret+=" fmt=['%5d'] + ['%17.6e' for _ in range(7)]," - ret+=" delimiter=''," - ret+=" header=header)" - ret+=" return devi" - ret+="" - ret+="" - ret+="def Modd(all_models,type_map):" - ret+=" from deepmd.infer import calc_model_devi" - ret+=" from deepmd.infer import DeepPot as DP" - ret+="" - ret+=" # Model Devi " - ret+="" - ret+=" cwd = os.getcwd()" - ret+=" graphs = [DP(model) for model in all_models]" - ret+="" - ret+=" Devis = []" - ret+=" pcount = 0" - ret+=" strus_lists = glob.glob(os.path.join(cwd,'*.structures'))" - ret+=" for num, strus_path in enumerate(strus_lists):" - ret+="" - ret+=" structures_data = dpdata.System(strus_path,'deepmd/npy',type_map=type_map)" - ret+="" - ret+=" # every 500 confs in one task dir" - ret+=" nnum = structures_data.get_nframes()" - ret+=" if nnum == 0:" - ret+=" continue" - ret+=" else:" - ret+=" num_per_task = math.ceil(nnum/500)" - ret+=" " - ret+="" - ret+=" for temp in range(num_per_task):" - ret+=" task_name = os.path.join(cwd,'task.%03d.%03d'%(num,temp)) " - ret+=" put_poscar = os.path.join(task_name,'traj')" - ret+=" if not os.path.exists(task_name):" - ret+=" os.mkdir(task_name)" - ret+=" os.mkdir(put_poscar)" - ret+=" else:" - ret+=" shutil.rmtree(task_name)" - ret+=" os.mkdir(task_name)" - ret+=" os.mkdir(put_poscar)" - ret+=" devis = []" - ret+=" if (nnum - (temp+1)*500) >= 0:" - ret+=" temp_sl = range(temp*500,(temp+1)*500)" - ret+=" else:" - ret+=" temp_sl = range(temp*500,nnum)" - ret+=" " - ret+=" new_index = 0" - ret+=" for index,frameid in enumerate(temp_sl):" - ret+=" pdata = structures_data[frameid]" - ret+=" pdata.to_vasp_poscar(os.path.join(put_poscar,'%s.poscar'%str(index)))" - ret+=" nopbc = pdata.nopbc" - ret+=" coord = pdata.data['coords']" - ret+=" cell = pdata.data['cells']" - ret+=" atom_types = pdata.data['atom_types']" - ret+=" devi = calc_model_devi(coord,cell,atom_types,graphs,nopbc=nopbc)" - ret+=" # ------------------------------------------------------------------------------------" - ret+=" # append min-distance in devi list" - ret+=" dis = pdata.to_ase_structure()[0].get_all_distances(mic=True)" - ret+=" row,col = np.diag_indices_from(dis)" - ret+=" dis[row,col] = 10000" - ret+=" min_dis = np.nanmin(dis)" - ret+=" devi = np.append(devi[0],min_dis) " - ret+=" t = [devi]" - ret+=" devi = np.array(t)" - ret+=" # ------------------------------------------------------------------------------------" - ret+=" temp_d = copy.deepcopy(devi)" - ret+=" temp_D = copy.deepcopy(devi)" - ret+=" devis.append(temp_d)" - ret+=" Devis.append(temp_D)" - ret+=" devis[index][0][0] = np.array(index)" - ret+=" Devis[pcount][0][0] = np.array(pcount)" - ret+=" pcount += 1" - ret+=" new_index += 1" - ret+=" devis = np.vstack(devis)" - ret+=" write_model_devi_out(devis,os.path.join(task_name, 'model_devi.out'))" - ret+="" - ret+=" Devis = np.vstack(Devis)" - ret+=" write_model_devi_out(Devis,os.path.join(cwd,'Model_Devi.out'))" - ret+="" - ret+=" f = open(os.path.join(os.path.abspath(os.path.join(cwd,os.pardir)),'record.calypso'),'a+')" - ret+=" f.write('4\n')" - ret+=" f.close()" - ret+="" - ret+="if __name__ == '__main__':" - ret+="" - ret+=" cwd = os.getcwd()" - ret+=" model_path = os.path.join(os.path.abspath(os.path.join(cwd,os.pardir)),'gen_stru_analy')" - ret+=" parser = argparse.ArgumentParser(description='calc model-devi by `all_models` and `type_map`')" - ret+=" parser.add_argument(" - ret+=" '--all_models'," - ret+=" type=str," - ret+=" nargs='+'," - ret+=" default=model_path," - ret+=" help='the path of models which will be used to do model-deviation'," - ret+=" )" - ret+=" parser.add_argument(" - ret+=" '--type_map'," - ret+=" type=list," - ret+=" help='the type map of models which will be used to do model-deviation'," - ret+=" )" - ret+=" args = parser.parse_args()" - ret+=" print(vars(args))" - ret+=" #Modd(args.all_models,args.type_map)" - ret+=" #Modd(sys.argv[1],sys.argv[2])" + ret ="import argparse\n" + ret+="import copy\n" + ret+="import dpdata\n" + ret+="import glob\n" + ret+="import math\n" + ret+="import numpy as np\n" + ret+="import os\n" + ret+="import sys\n" + ret+="import shutil\n" + ret+="\n" + ret+="\n" + ret+="def write_model_devi_out(devi, fname):\n" + ret+=" assert devi.shape[1] == 8\n" + ret+=" #assert devi.shape[1] == 7\n" + ret+=" header = '%5s' % 'step'\n" + ret+=" for item in 'vf':\n" + ret+=" header += '%16s%16s%16s' % (f'max_devi_{item}', f'min_devi_{item}',f'avg_devi_{item}')\n" + ret+=" header += '%16s'%str('min_dis')\n" + ret+=" np.savetxt(fname,\n" + ret+=" devi,\n" + ret+=" fmt=['%5d'] + ['%17.6e' for _ in range(7)],\n" + ret+=" delimiter='',\n" + ret+=" header=header)\n" + ret+=" return devi\n" + ret+="\n" + ret+="\n" + ret+="def Modd(all_models,type_map):\n" + ret+=" from deepmd.infer import calc_model_devi\n" + ret+=" from deepmd.infer import DeepPot as DP\n" + ret+="\n" + ret+=" # Model Devi \n" + ret+="\n" + ret+=" cwd = os.getcwd()\n" + ret+=" graphs = [DP(model) for model in all_models]\n" + ret+="\n" + ret+=" Devis = []\n" + ret+=" pcount = 0\n" + ret+=" strus_lists = glob.glob(os.path.join(cwd,'*.structures'))\n" + ret+=" for num, strus_path in enumerate(strus_lists):\n" + ret+="\n" + ret+=" structures_data = dpdata.System(strus_path,'deepmd/npy',type_map=type_map)\n" + ret+="\n" + ret+=" # every 500 confs in one task dir\n" + ret+=" nnum = structures_data.get_nframes()\n" + ret+=" if nnum == 0:\n" + ret+=" continue\n" + ret+=" else:\n" + ret+=" num_per_task = math.ceil(nnum/500)\n" + ret+=" \n" + ret+="\n" + ret+=" for temp in range(num_per_task):\n" + ret+=" task_name = os.path.join(cwd,'task.%03d.%03d'%(num,temp)) \n" + ret+=" put_poscar = os.path.join(task_name,'traj')\n" + ret+=" if not os.path.exists(task_name):\n" + ret+=" os.mkdir(task_name)\n" + ret+=" os.mkdir(put_poscar)\n" + ret+=" else:\n" + ret+=" shutil.rmtree(task_name)\n" + ret+=" os.mkdir(task_name)\n" + ret+=" os.mkdir(put_poscar)\n" + ret+=" devis = []\n" + ret+=" if (nnum - (temp+1)*500) >= 0:\n" + ret+=" temp_sl = range(temp*500,(temp+1)*500)\n" + ret+=" else:\n" + ret+=" temp_sl = range(temp*500,nnum)\n" + ret+=" \n" + ret+=" new_index = 0\n" + ret+=" for index,frameid in enumerate(temp_sl):\n" + ret+=" pdata = structures_data[frameid]\n" + ret+=" pdata.to_vasp_poscar(os.path.join(put_poscar,'%s.poscar'%str(index)))\n" + ret+=" nopbc = pdata.nopbc\n" + ret+=" coord = pdata.data['coords']\n" + ret+=" cell = pdata.data['cells']\n" + ret+=" atom_types = pdata.data['atom_types']\n" + ret+=" devi = calc_model_devi(coord,cell,atom_types,graphs,nopbc=nopbc)\n" + ret+=" # ------------------------------------------------------------------------------------\n" + ret+=" # append min-distance in devi list\n" + ret+=" dis = pdata.to_ase_structure()[0].get_all_distances(mic=True)\n" + ret+=" row,col = np.diag_indices_from(dis)\n" + ret+=" dis[row,col] = 10000\n" + ret+=" min_dis = np.nanmin(dis)\n" + ret+=" devi = np.append(devi[0],min_dis)\n " + ret+=" t = [devi]\n" + ret+=" devi = np.array(t)\n" + ret+=" # ------------------------------------------------------------------------------------\n" + ret+=" temp_d = copy.deepcopy(devi)\n" + ret+=" temp_D = copy.deepcopy(devi)\n" + ret+=" devis.append(temp_d)\n" + ret+=" Devis.append(temp_D)\n" + ret+=" devis[index][0][0] = np.array(index)\n" + ret+=" Devis[pcount][0][0] = np.array(pcount)\n" + ret+=" pcount += 1\n" + ret+=" new_index += 1\n" + ret+=" devis = np.vstack(devis)\n" + ret+=" write_model_devi_out(devis,os.path.join(task_name, 'model_devi.out'))\n" + ret+="\n" + ret+=" Devis = np.vstack(Devis)\n" + ret+=" write_model_devi_out(Devis,os.path.join(cwd,'Model_Devi.out'))\n" + ret+="\n" + ret+=" f = open(os.path.join(os.path.abspath(os.path.join(cwd,os.pardir)),'record.calypso'),'a+')\n" + ret+=" f.write('4\n')\n" + ret+=" f.close()\n" + ret+="\n" + ret+="if __name__ == '__main__':\n" + ret+="\n" + ret+=" cwd = os.getcwd()\n" + ret+=" model_path = os.path.join(os.path.abspath(os.path.join(cwd,os.pardir)),'gen_stru_analy')\n" + ret+=" parser = argparse.ArgumentParser(description='calc model-devi by `all_models` and `type_map`')\n" + ret+=" parser.add_argument(\n" + ret+=" '--all_models',\n" + ret+=" type=str,\n" + ret+=" nargs='+',\n" + ret+=" default=model_path,\n" + ret+=" help='the path of models which will be used to do model-deviation',\n" + ret+=" )\n" + ret+=" parser.add_argument(\n" + ret+=" '--type_map',\n" + ret+=" nargs='+',\n" + ret+=" help='the type map of models which will be used to do model-deviation',\n" + ret+=" )\n" + ret+=" args = parser.parse_args()\n" + ret+=" print(vars(args))\n" + ret+=" #Modd(args.all_models,args.type_map)\n" + ret+=" #Modd(sys.argv[1],sys.argv[2])\n" return ret diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index 23b6d9972..50543f2f6 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -915,7 +915,7 @@ def make_calypso_model_devi(iter_index,jdata,mdata): # modd script modd_script = os.path.join(calypso_model_devi_path,'modd.py') with open(modd_script,'w') as fm: - fm.write(write_moddwrite_modd()) + fm.write(write_modd()) input_mode = "native" if "calypso_input_path" in jdata: @@ -1444,7 +1444,6 @@ def run_model_devi_calypso (iter_index, calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) - all_models = glob.glob(os.path.join(calypso_run_opt_path, 'graph*pb')) cwd = os.getcwd() @@ -1474,6 +1473,8 @@ def run_model_devi_calypso (iter_index, elif lines[-1].strip().strip('\n') == '3': # Model Devi + _calypso_run_opt_path = os.path.abspath(calypso_run_opt_path) + all_models = glob.glob(os.path.join(calypso_run_opt_path, 'graph*pb')) cwd = os.getcwd() os.chdir(calypso_model_devi_path) args = ' '.join(['modd.py', '--all_models',' '.join(all_models),'--type_map',' '.join(jdata.get('type_map'))]) From 7b0db3f78945efb5197e18e8947a52886c44da4a Mon Sep 17 00:00:00 2001 From: zhenyu Date: Wed, 9 Mar 2022 20:36:12 +0800 Subject: [PATCH 14/46] fix unittest --- .../input.dat => parse_input.dat} | 0 tests/generator/test_calypso.py | 32 +++++++++++++------ 2 files changed, 22 insertions(+), 10 deletions(-) rename tests/generator/{calypso_data/input.dat => parse_input.dat} (100%) diff --git a/tests/generator/calypso_data/input.dat b/tests/generator/parse_input.dat similarity index 100% rename from tests/generator/calypso_data/input.dat rename to tests/generator/parse_input.dat diff --git a/tests/generator/test_calypso.py b/tests/generator/test_calypso.py index de250addc..cfdd75785 100644 --- a/tests/generator/test_calypso.py +++ b/tests/generator/test_calypso.py @@ -14,7 +14,7 @@ from .context import _parse_calypso_input from .context import _parse_calypso_dis_mtx -calypso_data = Path('.').joinpath('calypso_data') +#calypso_data = Path('.').joinpath('calypso_data') #assert os.path.exists(calypso_data) #assert os.path.exists(calypso_path) @@ -54,22 +54,30 @@ def test_write_run_opt_script(self): def test_write_check_outcar_script(self): ret = make_check_outcar_script() - with open('calypso_test_path/check_outcar.py','w') as fc: + #with open('calypso_test_path/check_outcar.py','w') as fc: + with open('check_outcar.py','w') as fc: fc.write(ret) - self.assertTrue(os.path.exists('calypso_test_path/check_outcar.py')) + #self.assertTrue(os.path.exists('calypso_test_path/check_outcar.py')) + self.assertTrue(os.path.exists('check_outcar.py')) + os.remove('check_outcar.py') def test_write_model_devi_out(self): - devi = write_model_devi_out(model_devi, 'calypso_test_path/model_devi.out') - ndevi = np.loadtxt('calypso_test_path/model_devi.out') + #devi = write_model_devi_out(model_devi, 'calypso_test_path/model_devi.out') + #ndevi = np.loadtxt('calypso_test_path/model_devi.out') + devi = write_model_devi_out(model_devi, 'model_devi.out') + ndevi = np.loadtxt('model_devi.out') self.assertEqual(ndevi[2,4],model_devi[2,4]) + os.remove('model_devi.out') def test_make_calypso_input(self): ret = make_calypso_input(["Mg","Al","Cu"],[1,1,1],[1,4],[30],[ [1.48,1.44,1.59],[1.44,1.41,1.56],[1.59,1.56,1.70] ],[0.6],[5],[3],[13],"T","T",[31],[[1,10],[1,10],[1,10]],[0],[0.01]) - with open('calypso_test_path/input.dat','w') as fin: + #with open('calypso_test_path/input.dat','w') as fin: + with open('input.dat','w') as fin: fin.write(ret) - f = open('calypso_test_path/input.dat') + f = open('input.dat') + #f = open('calypso_test_path/input.dat') lines = f.readlines() f.close() for line in lines : @@ -81,17 +89,21 @@ def test_make_calypso_input(self): if 'MaxStep' in line: temp_2 = line.split('=')[1].strip() self.assertEqual(int(temp_2),3) + os.remove('input.dat') break def test_parse_calypso_input(self): - formula = _parse_calypso_input('NumberOfFormula',calypso_data).split() + formula = _parse_calypso_input('NumberOfFormula','parse_input.dat').split() + #formula = _parse_calypso_input('NumberOfFormula',calypso_data).split() formula = list(map(int,formula)) self.assertEqual(formula,model_devi_jobs.get('model_devi_jobs').get('NumberOfFormula')) - nameofatoms = _parse_calypso_input('NameOfAtoms',calypso_data).split() + nameofatoms = _parse_calypso_input('NameOfAtoms','parse_input.dat').split() + #nameofatoms = _parse_calypso_input('NameOfAtoms',calypso_data).split() self.assertEqual(nameofatoms,model_devi_jobs.get('model_devi_jobs').get('NameOfAtoms')) - min_dis = _parse_calypso_dis_mtx(len(nameofatoms),calypso_data) + min_dis = _parse_calypso_dis_mtx(len(nameofatoms),'parse_input.dat') + #min_dis = _parse_calypso_dis_mtx(len(nameofatoms),calypso_data) self.assertEqual(float(min_dis),np.nanmin(model_devi_jobs.get('model_devi_jobs').get('DistanceOfIon'))) From 15d130a2c1280148c46afdc9991b37bb3d75813f Mon Sep 17 00:00:00 2001 From: zhenyu Date: Wed, 9 Mar 2022 21:20:31 +0800 Subject: [PATCH 15/46] delete parse_input.dat --- tests/generator/parse_input.dat | 72 --------------------------------- tests/generator/test_calypso.py | 13 ++++-- 2 files changed, 10 insertions(+), 75 deletions(-) delete mode 100644 tests/generator/parse_input.dat diff --git a/tests/generator/parse_input.dat b/tests/generator/parse_input.dat deleted file mode 100644 index 0a22afbed..000000000 --- a/tests/generator/parse_input.dat +++ /dev/null @@ -1,72 +0,0 @@ -################################ The Basic Parameters of CALYPSO ################################ -# A string of one or several words contain a descriptive name of the system (max. 40 characters). -SystemName = MgAlCu -# Number of different atomic species in the simulation. -NumberOfSpecies = 3 -# Element symbols of the different chemical species. -NameOfAtoms = Mg Al Cu -# Number of atoms for each chemical species in one formula unit. -NumberOfAtoms = 1 1 1 -# The range of formula unit per cell in your simulation. -NumberOfFormula = 1 4 -# The volume per formula unit. Unit is in angstrom^3. -Volume = 30 -# Minimal distance between atoms of each chemical species. Unit is in angstrom. -@DistanceOfIon -1.48 1.44 1.59 -1.44 1.41 1.56 -1.59 1.56 1.7 -@End -# It determines which algorithm should be adopted in the simulation. -Ialgo = 2 -# Ialgo = 1 for Global PSO -# Ialgo = 2 for Local PSO (default value) -# The proportion of the structures generated by PSO. -PsoRatio = 0.6 -# The population size. Normally, it has a larger number for larger systems. -PopSize = 5 -# The Max step for iteration -MaxStep = 3 -#It determines which method should be adopted in generation the random structure. -GenType= 1 -# 1 under symmetric constraints -# 2 grid method for large system -# 3 and 4 core grow method -# 0 combination of all method -# If GenType=3 or 4, it determined the small unit to grow the whole structure -# It determines which local optimization method should be interfaced in the simulation. -ICode= 13 -# ICode= 1 interfaced with VASP -# ICode= 2 interfaced with SIESTA -# ICode= 3 interfaced with GULP -# The number of lbest for local PSO -NumberOfLbest=4 -# The Number of local optimization for each structure. -NumberOfLocalOptim= 3 -# The command to perform local optimiztion calculation (e.g., VASP, SIESTA) on your computer. -Command = sh submit.sh -MaxTime = 9000 -# If True, a previous calculation will be continued. -PickUp = F -# At which step will the previous calculation be picked up. -PickStep = 1 -# If True, the local optimizations performed by parallel -Parallel = F -# The number node for parallel -NumberOfParallel = 4 -Split = T -PSTRESS = 0.000000 -fmax = 0.010000 -################################ End of The Basic Parameters of CALYPSO ####################### -##### The Parameters For Variational Stoichiometry ############## -## If True, Variational Stoichiometry structure prediction is performed -VSC = T -# The Max Number of Atoms in unit cell -MaxNumAtom = 31 -# The Variation Range for each type atom -@CtrlRange -1 10 -1 10 -1 10 -@end -###################End Parameters for VSC ########################## diff --git a/tests/generator/test_calypso.py b/tests/generator/test_calypso.py index cfdd75785..f997c463f 100644 --- a/tests/generator/test_calypso.py +++ b/tests/generator/test_calypso.py @@ -93,18 +93,25 @@ def test_make_calypso_input(self): break def test_parse_calypso_input(self): - formula = _parse_calypso_input('NumberOfFormula','parse_input.dat').split() + ret = make_calypso_input(["Mg","Al","Cu"],[1,1,1],[1,4],[30],[ + [1.48,1.44,1.59],[1.44,1.41,1.56],[1.59,1.56,1.70] + ],[0.6],[5],[3],[13],"T","T",[31],[[1,10],[1,10],[1,10]],[0],[0.01]) + #with open('calypso_test_path/input.dat','w') as fin: + with open('input.dat','w') as fin: + fin.write(ret) + formula = _parse_calypso_input('NumberOfFormula','input.dat').split() #formula = _parse_calypso_input('NumberOfFormula',calypso_data).split() formula = list(map(int,formula)) self.assertEqual(formula,model_devi_jobs.get('model_devi_jobs').get('NumberOfFormula')) - nameofatoms = _parse_calypso_input('NameOfAtoms','parse_input.dat').split() + nameofatoms = _parse_calypso_input('NameOfAtoms','input.dat').split() #nameofatoms = _parse_calypso_input('NameOfAtoms',calypso_data).split() self.assertEqual(nameofatoms,model_devi_jobs.get('model_devi_jobs').get('NameOfAtoms')) - min_dis = _parse_calypso_dis_mtx(len(nameofatoms),'parse_input.dat') + min_dis = _parse_calypso_dis_mtx(len(nameofatoms),'input.dat') #min_dis = _parse_calypso_dis_mtx(len(nameofatoms),calypso_data) self.assertEqual(float(min_dis),np.nanmin(model_devi_jobs.get('model_devi_jobs').get('DistanceOfIon'))) + os.remove('input.dat') if __name__ == '__main__': From d1e73c964f5c83fbf750f86ac9e57cd75b41b0ae Mon Sep 17 00:00:00 2001 From: zhenyu Date: Thu, 10 Mar 2022 10:19:39 +0800 Subject: [PATCH 16/46] fix unittest --- dpgen/generator/run.py | 25 ++++++++++++++----------- tests/generator/test_make_fp.py | 1 + 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index 50543f2f6..de51f4005 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -1823,12 +1823,13 @@ def _make_fp_vasp_inner (modd_path, # -------------------------------------------------------------------------------------------------------------------------------------- model_devi_engine = jdata.get('model_devi_engine', 'lammps') - iter_name = work_path.split('/')[0] - _work_path = os.path.join(iter_name, model_devi_name) - calypso_run_opt_path = os.path.join(_work_path,calypso_run_opt_name) - numofspecies = _parse_calypso_input('NumberOfSpecies',calypso_run_opt_path) - min_dis = _parse_calypso_dis_mtx(numofspecies,calypso_run_opt_path) if model_devi_engine == 'calypso': + iter_name = work_path.split('/')[0] + _work_path = os.path.join(iter_name, model_devi_name) + calypso_run_opt_path = os.path.join(_work_path,calypso_run_opt_name) + numofspecies = _parse_calypso_input('NumberOfSpecies',calypso_run_opt_path) + min_dis = _parse_calypso_dis_mtx(numofspecies,calypso_run_opt_path) + calypso_total_fp_num = 300 modd_path = os.path.join(modd_path,calypso_model_devi_name) model_devi_skip = -1 @@ -1952,11 +1953,12 @@ def _trust_limitation_check(sys_idx, lim): else: this_fp_task_max = 0 # ---------------------------------------------------------------------------- - calypso_intend_fp_num_temp = (len(fp_candidate)/candi_num)*calypso_total_fp_num - if calypso_intend_fp_num_temp < 1: - calypso_intend_fp_num = 1 - else: - calypso_intend_fp_num = int(calypso_intend_fp_num_temp) + if model_devi_engine == 'calypso': + calypso_intend_fp_num_temp = (len(fp_candidate)/candi_num)*calypso_total_fp_num + if calypso_intend_fp_num_temp < 1: + calypso_intend_fp_num = 1 + else: + calypso_intend_fp_num = int(calypso_intend_fp_num_temp) # ---------------------------------------------------------------------------- numb_task = min(this_fp_task_max, len(fp_candidate)) if (numb_task < fp_task_min): @@ -2043,7 +2045,8 @@ def _trust_limitation_check(sys_idx, lim): dlog.info("system {0:s} skipped {1:6d} confs with bad box, {2:6d} remains".format(ss, count_bad_box, numb_task - count_bad_box)) if count_bad_cluster > 0: dlog.info("system {0:s} skipped {1:6d} confs with bad cluster, {2:6d} remains".format(ss, count_bad_cluster, numb_task - count_bad_cluster)) - dlog.info("summary accurate_ratio: {0:8.4f}% candidata_ratio: {1:8.4f}% failed_ratio: {2:8.4f}% in {3:d} structures".format( acc_num*100/tot,candi_num*100/tot,fail_num*100/tot,tot )) + if model_devi_engine == 'calypso': + dlog.info("summary accurate_ratio: {0:8.4f}% candidata_ratio: {1:8.4f}% failed_ratio: {2:8.4f}% in {3:d} structures".format( acc_num*100/tot,candi_num*100/tot,fail_num*100/tot,tot )) if cluster_cutoff is None: cwd = os.getcwd() for idx, task in enumerate(fp_tasks): diff --git a/tests/generator/test_make_fp.py b/tests/generator/test_make_fp.py index 1687b695d..57789fee9 100644 --- a/tests/generator/test_make_fp.py +++ b/tests/generator/test_make_fp.py @@ -578,6 +578,7 @@ def test_make_fp_abacus(self): make_fp(0, jdata, {}) _check_sel(self, 0, jdata['fp_task_max'], jdata['model_devi_f_trust_lo'], jdata['model_devi_f_trust_hi']) _check_poscars(self, 0, jdata['fp_task_max'], jdata['type_map']) + _check_poscars(self, 0, jdata['fp_task_max'], jdata['type_map']) _check_abacus_input(self, 0) _check_abacus_kpt(self, 0) _check_potcar(self, 0, jdata['fp_pp_path'], jdata['fp_pp_files']) From 361546e1b135e23b33ab889c00a31f5d7291fa9d Mon Sep 17 00:00:00 2001 From: zhenyu Date: Sun, 13 Mar 2022 13:03:52 +0800 Subject: [PATCH 17/46] first reorganization --- dpgen/generator/run.py | 281 ++++++++++++++++++++--------------------- 1 file changed, 138 insertions(+), 143 deletions(-) diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index de51f4005..8cc3ba6ca 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -1305,132 +1305,136 @@ def _make_model_devi_native_gromacs(iter_index, jdata, mdata, conf_systems): -def run_model_devi (iter_index, +def run_single_model_devi (iter_index, jdata, mdata) : + #rmdlog.info("This module has been run !") + model_devi_exec = mdata['model_devi_command'] + + model_devi_group_size = mdata['model_devi_group_size'] + model_devi_resources = mdata['model_devi_resources'] + use_plm = jdata.get('model_devi_plumed', False) + use_plm_path = jdata.get('model_devi_plumed_path', False) + + iter_name = make_iter_name(iter_index) + work_path = os.path.join(iter_name, model_devi_name) + assert(os.path.isdir(work_path)) + + all_task = glob.glob(os.path.join(work_path, "task.*")) + all_task.sort() + fp = open (os.path.join(work_path, 'cur_job.json'), 'r') + cur_job = json.load (fp) + + run_tasks_ = all_task + # for ii in all_task: + # fres = os.path.join(ii, 'model_devi.out') + # if os.path.isfile(fres) : + # nlines = np.loadtxt(fres).shape[0] + # if nframes != nlines : + # run_tasks_.append(ii) + # else : + # run_tasks_.append(ii) + + run_tasks = [os.path.basename(ii) for ii in run_tasks_] + #dlog.info("all_task is ", all_task) + #dlog.info("run_tasks in run_model_deviation",run_tasks_) + all_models = glob.glob(os.path.join(work_path, 'graph*pb')) + model_names = [os.path.basename(ii) for ii in all_models] + model_devi_engine = jdata.get("model_devi_engine", "lammps") - if model_devi_engine == "calypso": - run_model_devi_calypso(iter_index,jdata,mdata) - else: - #rmdlog.info("This module has been run !") - model_devi_exec = mdata['model_devi_command'] + if model_devi_engine == "lammps": + command = "{ if [ ! -f dpgen.restart.10000 ]; then %s -i input.lammps -v restart 0; else %s -i input.lammps -v restart 1; fi }" % (model_devi_exec, model_devi_exec) + command = "/bin/sh -c '%s'" % command + commands = [command] + forward_files = ['conf.lmp', 'input.lammps', 'traj'] + backward_files = ['model_devi.out', 'model_devi.log', 'traj'] + if use_plm: + forward_files += ['input.plumed'] + # backward_files += ['output.plumed'] + backward_files += ['output.plumed','COLVAR'] + if use_plm_path: + forward_files += ['plmpath.pdb'] + elif model_devi_engine == "gromacs": + + gromacs_settings = jdata.get("gromacs_settings", {}) + mdp_filename = gromacs_settings.get("mdp_filename", "md.mdp") + topol_filename = gromacs_settings.get("topol_filename", "processed.top") + conf_filename = gromacs_settings.get("conf_filename", "conf.gro") + index_filename = gromacs_settings.get("index_filename", "index.raw") + type_filename = gromacs_settings.get("type_filename", "type.raw") + ndx_filename = gromacs_settings.get("ndx_filename", "") + # Initial reference to process pbc condition. + # Default is em.tpr + ref_filename = gromacs_settings.get("ref_filename", "em.tpr") + deffnm = gromacs_settings.get("deffnm", "deepmd") + maxwarn = gromacs_settings.get("maxwarn", 1) + traj_filename = gromacs_settings.get("traj_filename", "deepmd_traj.gro") + grp_name = gromacs_settings.get("group_name", "Other") + trj_freq = cur_job.get("trj_freq", 10) + + command = "%s grompp -f %s -p %s -c %s -o %s -maxwarn %d" % (model_devi_exec, mdp_filename, topol_filename, conf_filename, deffnm, maxwarn) + command += "&& %s mdrun -deffnm %s -cpi" %(model_devi_exec, deffnm) + if ndx_filename: + command += f"&& echo -e \"{grp_name}\\n{grp_name}\\n\" | {model_devi_exec} trjconv -s {ref_filename} -f {deffnm}.trr -n {ndx_filename} -o {traj_filename} -pbc mol -ur compact -center" + else: + command += "&& echo -e \"%s\\n%s\\n\" | %s trjconv -s %s -f %s.trr -o %s -pbc mol -ur compact -center" % (grp_name, grp_name, model_devi_exec, ref_filename, deffnm, traj_filename) + command += "&& if [ ! -d traj ]; then \n mkdir traj; fi\n" + command += f"python -c \"import dpdata;system = dpdata.System('{traj_filename}', fmt='gromacs/gro'); [system.to_gromacs_gro('traj/%d.gromacstrj' % (i * {trj_freq}), frame_idx=i) for i in range(system.get_nframes())]; system.to_deepmd_npy('traj_deepmd')\"" + command += f"&& dp model-devi -m ../graph.000.pb ../graph.001.pb ../graph.002.pb ../graph.003.pb -s traj_deepmd -o model_devi.out -f {trj_freq}" + commands = [command] - model_devi_group_size = mdata['model_devi_group_size'] - model_devi_resources = mdata['model_devi_resources'] - use_plm = jdata.get('model_devi_plumed', False) - use_plm_path = jdata.get('model_devi_plumed_path', False) + forward_files = [mdp_filename, topol_filename, conf_filename, index_filename, ref_filename, type_filename, "input.json", "job.json" ] + if ndx_filename: forward_files.append(ndx_filename) + backward_files = ["%s.tpr" % deffnm, "%s.log" %deffnm , traj_filename, 'model_devi.out', "traj", "traj_deepmd" ] - iter_name = make_iter_name(iter_index) - work_path = os.path.join(iter_name, model_devi_name) - assert(os.path.isdir(work_path)) - - all_task = glob.glob(os.path.join(work_path, "task.*")) - all_task.sort() - fp = open (os.path.join(work_path, 'cur_job.json'), 'r') - cur_job = json.load (fp) - - run_tasks_ = all_task - # for ii in all_task: - # fres = os.path.join(ii, 'model_devi.out') - # if os.path.isfile(fres) : - # nlines = np.loadtxt(fres).shape[0] - # if nframes != nlines : - # run_tasks_.append(ii) - # else : - # run_tasks_.append(ii) - - run_tasks = [os.path.basename(ii) for ii in run_tasks_] - #dlog.info("all_task is ", all_task) - #dlog.info("run_tasks in run_model_deviation",run_tasks_) - all_models = glob.glob(os.path.join(work_path, 'graph*pb')) - model_names = [os.path.basename(ii) for ii in all_models] - model_devi_engine = jdata.get("model_devi_engine", "lammps") - if model_devi_engine == "lammps": - command = "{ if [ ! -f dpgen.restart.10000 ]; then %s -i input.lammps -v restart 0; else %s -i input.lammps -v restart 1; fi }" % (model_devi_exec, model_devi_exec) - command = "/bin/sh -c '%s'" % command - commands = [command] - forward_files = ['conf.lmp', 'input.lammps', 'traj'] - backward_files = ['model_devi.out', 'model_devi.log', 'traj'] - if use_plm: - forward_files += ['input.plumed'] - # backward_files += ['output.plumed'] - backward_files += ['output.plumed','COLVAR'] - if use_plm_path: - forward_files += ['plmpath.pdb'] - elif model_devi_engine == "gromacs": - - gromacs_settings = jdata.get("gromacs_settings", {}) - mdp_filename = gromacs_settings.get("mdp_filename", "md.mdp") - topol_filename = gromacs_settings.get("topol_filename", "processed.top") - conf_filename = gromacs_settings.get("conf_filename", "conf.gro") - index_filename = gromacs_settings.get("index_filename", "index.raw") - type_filename = gromacs_settings.get("type_filename", "type.raw") - ndx_filename = gromacs_settings.get("ndx_filename", "") - # Initial reference to process pbc condition. - # Default is em.tpr - ref_filename = gromacs_settings.get("ref_filename", "em.tpr") - deffnm = gromacs_settings.get("deffnm", "deepmd") - maxwarn = gromacs_settings.get("maxwarn", 1) - traj_filename = gromacs_settings.get("traj_filename", "deepmd_traj.gro") - grp_name = gromacs_settings.get("group_name", "Other") - trj_freq = cur_job.get("trj_freq", 10) - - command = "%s grompp -f %s -p %s -c %s -o %s -maxwarn %d" % (model_devi_exec, mdp_filename, topol_filename, conf_filename, deffnm, maxwarn) - command += "&& %s mdrun -deffnm %s -cpi" %(model_devi_exec, deffnm) - if ndx_filename: - command += f"&& echo -e \"{grp_name}\\n{grp_name}\\n\" | {model_devi_exec} trjconv -s {ref_filename} -f {deffnm}.trr -n {ndx_filename} -o {traj_filename} -pbc mol -ur compact -center" - else: - command += "&& echo -e \"%s\\n%s\\n\" | %s trjconv -s %s -f %s.trr -o %s -pbc mol -ur compact -center" % (grp_name, grp_name, model_devi_exec, ref_filename, deffnm, traj_filename) - command += "&& if [ ! -d traj ]; then \n mkdir traj; fi\n" - command += f"python -c \"import dpdata;system = dpdata.System('{traj_filename}', fmt='gromacs/gro'); [system.to_gromacs_gro('traj/%d.gromacstrj' % (i * {trj_freq}), frame_idx=i) for i in range(system.get_nframes())]; system.to_deepmd_npy('traj_deepmd')\"" - command += f"&& dp model-devi -m ../graph.000.pb ../graph.001.pb ../graph.002.pb ../graph.003.pb -s traj_deepmd -o model_devi.out -f {trj_freq}" - commands = [command] + cwd = os.getcwd() + + user_forward_files = mdata.get("model_devi" + "_user_forward_files", []) + forward_files += [os.path.basename(file) for file in user_forward_files] + backward_files += mdata.get("model_devi" + "_user_backward_files", []) + api_version = mdata.get('api_version', '0.9') + if LooseVersion(api_version) < LooseVersion('1.0'): + warnings.warn(f"the dpdispatcher will be updated to new version." + f"And the interface may be changed. Please check the documents for more details") + dispatcher = make_dispatcher(mdata['model_devi_machine'], mdata['model_devi_resources'], work_path, run_tasks, model_devi_group_size) + dispatcher.run_jobs(mdata['model_devi_resources'], + commands, + work_path, + run_tasks, + model_devi_group_size, + model_names, + forward_files, + backward_files, + outlog = 'model_devi.log', + errlog = 'model_devi.log') - forward_files = [mdp_filename, topol_filename, conf_filename, index_filename, ref_filename, type_filename, "input.json", "job.json" ] - if ndx_filename: forward_files.append(ndx_filename) - backward_files = ["%s.tpr" % deffnm, "%s.log" %deffnm , traj_filename, 'model_devi.out', "traj", "traj_deepmd" ] + elif LooseVersion(api_version) >= LooseVersion('1.0'): + submission = make_submission( + mdata['model_devi_machine'], + mdata['model_devi_resources'], + commands=commands, + work_path=work_path, + run_tasks=run_tasks, + group_size=model_devi_group_size, + forward_common_files=model_names, + forward_files=forward_files, + backward_files=backward_files, + outlog = 'model_devi.log', + errlog = 'model_devi.log') + submission.run_submission() +def run_model_devi(iter_index,jdata,mdata): - cwd = os.getcwd() + model_devi_engine = jdata.get("model_devi_engine", "lammps") + if model_devi_engine == "calypso": + run_multi_model_devi(iter_index,jdata,mdata) + else: + run_single_model_devi(iter_index,jdata,mdata) + - user_forward_files = mdata.get("model_devi" + "_user_forward_files", []) - forward_files += [os.path.basename(file) for file in user_forward_files] - backward_files += mdata.get("model_devi" + "_user_backward_files", []) - api_version = mdata.get('api_version', '0.9') - if LooseVersion(api_version) < LooseVersion('1.0'): - warnings.warn(f"the dpdispatcher will be updated to new version." - f"And the interface may be changed. Please check the documents for more details") - dispatcher = make_dispatcher(mdata['model_devi_machine'], mdata['model_devi_resources'], work_path, run_tasks, model_devi_group_size) - dispatcher.run_jobs(mdata['model_devi_resources'], - commands, - work_path, - run_tasks, - model_devi_group_size, - model_names, - forward_files, - backward_files, - outlog = 'model_devi.log', - errlog = 'model_devi.log') - - elif LooseVersion(api_version) >= LooseVersion('1.0'): - submission = make_submission( - mdata['model_devi_machine'], - mdata['model_devi_resources'], - commands=commands, - work_path=work_path, - run_tasks=run_tasks, - group_size=model_devi_group_size, - forward_common_files=model_names, - forward_files=forward_files, - backward_files=backward_files, - outlog = 'model_devi.log', - errlog = 'model_devi.log') - submission.run_submission() - - -def run_model_devi_calypso (iter_index, +def run_multi_model_devi (iter_index, jdata, mdata) : @@ -1652,38 +1656,29 @@ def _select_by_model_devi_standard( continue cc = int(all_conf[ii][0]) if cluster_cutoff is None: - if model_devi_engine != 'calypso': - if (all_conf[ii][1] < v_trust_hi and all_conf[ii][1] >= v_trust_lo) or \ - (all_conf[ii][4] < f_trust_hi and all_conf[ii][4] >= f_trust_lo) : - fp_candidate.append([tt, cc]) - counter['candidate'] += 1 - elif (all_conf[ii][1] >= v_trust_hi ) or (all_conf[ii][4] >= f_trust_hi ): + if model_devi_engine == 'calypso': + if float(all_conf[ii][-1]) <= float(min_dis): if detailed_report_make_fp: fp_rest_failed.append([tt, cc]) counter['failed'] += 1 - elif (all_conf[ii][1] < v_trust_lo and all_conf[ii][4] < f_trust_lo ): - if detailed_report_make_fp: - fp_rest_accurate.append([tt, cc]) - counter['accurate'] += 1 - else : + continue + if (all_conf[ii][1] < v_trust_hi and all_conf[ii][1] >= v_trust_lo) or \ + (all_conf[ii][4] < f_trust_hi and all_conf[ii][4] >= f_trust_lo) : + fp_candidate.append([tt, cc]) + counter['candidate'] += 1 + elif (all_conf[ii][1] >= v_trust_hi ) or (all_conf[ii][4] >= f_trust_hi ): + if detailed_report_make_fp: + fp_rest_failed.append([tt, cc]) + counter['failed'] += 1 + elif (all_conf[ii][1] < v_trust_lo and all_conf[ii][4] < f_trust_lo ): + if detailed_report_make_fp: + fp_rest_accurate.append([tt, cc]) + counter['accurate'] += 1 + else : + if model_devi_engine == 'calypso': + dlog.info('ase opt traj %s frame %d with f devi %f does not belong to either accurate, candidiate and failed '% (tt, ii, all_conf[ii][4])) + else: raise RuntimeError('md traj %s frame %d with f devi %f does not belong to either accurate, candidiate and failed, it should not happen' % (tt, ii, all_conf[ii][4])) - elif model_devi_engine == 'calypso': - if (all_conf[ii][1] < v_trust_hi and all_conf[ii][1] >= v_trust_lo and float(all_conf[ii][-1]) > float(min_dis) or \ - (all_conf[ii][4] < f_trust_hi and all_conf[ii][4] >= f_trust_lo) and float(all_conf[ii][-1]) > float(min_dis)): - fp_candidate.append([tt, cc]) - counter['candidate'] += 1 - elif (all_conf[ii][1] >= v_trust_hi ) or (all_conf[ii][4] >= f_trust_hi ) or (float(all_conf[ii][-1]) <= float(min_dis)): - if detailed_report_make_fp: - fp_rest_failed.append([tt, cc]) - counter['failed'] += 1 - elif (all_conf[ii][1] < v_trust_lo and all_conf[ii][4] < f_trust_lo and float(all_conf[ii][-1]) > float(min_dis) ): - if detailed_report_make_fp: - fp_rest_accurate.append([tt, cc]) - counter['accurate'] += 1 - else : - dlog.info('ase opt traj %s frame %d with f devi %f does not belong to either accurate, candidiate and failed ' \ - % (tt, ii, all_conf[ii][4])) - pass else: idx_candidate = np.where(np.logical_and(all_conf[ii][7:] < f_trust_hi, all_conf[ii][7:] >= f_trust_lo))[0] for jj in idx_candidate: From a8ca7f84d416a54f9e3051c347fb19a3f9b882f1 Mon Sep 17 00:00:00 2001 From: zhenyu Date: Sun, 13 Mar 2022 22:42:19 +0800 Subject: [PATCH 18/46] secord reorganization --- dpgen/generator/lib/calypso.py | 21 ++- dpgen/generator/run.py | 313 ++++++++++++++------------------- 2 files changed, 151 insertions(+), 183 deletions(-) diff --git a/dpgen/generator/lib/calypso.py b/dpgen/generator/lib/calypso.py index 6d5a50548..56dfd200a 100644 --- a/dpgen/generator/lib/calypso.py +++ b/dpgen/generator/lib/calypso.py @@ -363,8 +363,27 @@ def make_calypso_input(nameofatoms,numberofatoms, ret+= "###################End Parameters for VSC ##########################\n" return ret +def _make_model_devi_buffet(jdata,calypso_run_opt_path): + calypso_input_path = jdata.get('calypso_input_path') + shutil.copyfile(os.path.join(calypso_input_path,'input.dat'),os.path.join(calypso_run_opt_path, 'input.dat')) + # run opt script + run_opt_script = os.path.join(calypso_run_opt_path,'run_opt.py') + with open(run_opt_script,'w') as ffff: + ffff.write(make_run_opt_script(jdata.get('fmax',0.01))) -def _make_model_devi_native_calypso(cur_job, calypso_run_opt_path): +def _make_model_devi_native_calypso(model_devi_jobs, calypso_run_opt_path): + + for iiidx, jobbs in enumerate(model_devi_jobs): + if iter_index in jobbs.get('times'): + cur_job = model_devi_jobs[iiidx] + + work_path = os.path.dirname(calypso_run_opt_path) + with open(os.path.join(work_path, 'cur_job.json'), 'w') as outfile: + json.dump(cur_job, outfile, indent = 4) + + run_opt_script = os.path.join(calypso_run_opt_path,'run_opt.py') + with open(run_opt_script,'w') as ffff: + ffff.write(make_run_opt_script(cur_job.get('fmax',0.01))) # Crystal Parameters nameofatoms = cur_job.get('NameOfAtoms') diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index 8cc3ba6ca..15a70def5 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -243,7 +243,7 @@ def make_train (iter_index, log_task('prev data is empty, copy prev model') copy_model(numb_models, iter_index-1, iter_index) return - elif model_devi_engine != 'calypso' and iter_index > 0 and _check_skip_train(model_devi_jobs[iter_index-1]): + elif iter_index > 0 and _check_skip_train(model_devi_jobs[iter_index-1]): log_task('skip training at step %d ' % (iter_index-1)) copy_model(numb_models, iter_index-1, iter_index) return @@ -785,181 +785,132 @@ def make_model_devi (iter_index, # The MD engine to perform model deviation # Default is lammps model_devi_engine = jdata.get('model_devi_engine', "lammps") - if model_devi_engine == 'calypso': - Lcalymake = make_calypso_model_devi(iter_index,jdata,mdata) - if Lcalymake == True: - return True - else: - print('calypso make model devi wrong') + + model_devi_jobs = jdata['model_devi_jobs'] + if model_devi_engine != 'calypso': + if (iter_index >= len(model_devi_jobs)) : return False else: - model_devi_jobs = jdata['model_devi_jobs'] - if (iter_index >= len(model_devi_jobs)) : + try: + maxiter = max(model_devi_jobs[-1].get('times')) + except Exception as e: + maxiter = jdata.get('model_devi_max_iter',0) + if (iter_index > maxiter) : + print('please check `times` in model_devi_jobs or `model_devi_max_iter` in input.json') return False - cur_job = model_devi_jobs[iter_index] - if "sys_configs_prefix" in jdata: - sys_configs = [] - for sys_list in jdata["sys_configs"]: - #assert (isinstance(sys_list, list) ), "Currently only support type list for sys in 'sys_conifgs' " - temp_sys_list = [os.path.join(jdata["sys_configs_prefix"], sys) for sys in sys_list] - sys_configs.append(temp_sys_list) - else: - sys_configs = jdata['sys_configs'] - shuffle_poscar = jdata['shuffle_poscar'] - sys_idx = expand_idx(cur_job['sys_idx']) - if (len(sys_idx) != len(list(set(sys_idx)))) : - raise RuntimeError("system index should be uniq") - conf_systems = [] - for idx in sys_idx : - cur_systems = [] - ss = sys_configs[idx] - for ii in ss : - cur_systems += glob.glob(ii) - cur_systems.sort() - cur_systems = [os.path.abspath(ii) for ii in cur_systems] - conf_systems.append (cur_systems) - - iter_name = make_iter_name(iter_index) - train_path = os.path.join(iter_name, train_name) - train_path = os.path.abspath(train_path) - models = sorted(glob.glob(os.path.join(train_path, "graph*pb"))) - work_path = os.path.join(iter_name, model_devi_name) - create_path(work_path) - for mm in models : - model_name = os.path.basename(mm) - os.symlink(mm, os.path.join(work_path, model_name)) - with open(os.path.join(work_path, 'cur_job.json'), 'w') as outfile: - json.dump(cur_job, outfile, indent = 4) - - conf_path = os.path.join(work_path, 'confs') - create_path(conf_path) - sys_counter = 0 - for ss in conf_systems: - conf_counter = 0 - for cc in ss : - if model_devi_engine == "lammps": - conf_name = make_model_devi_conf_name(sys_idx[sys_counter], conf_counter) - orig_poscar_name = conf_name + '.orig.poscar' - poscar_name = conf_name + '.poscar' - lmp_name = conf_name + '.lmp' - if shuffle_poscar : - os.symlink(cc, os.path.join(conf_path, orig_poscar_name)) - poscar_shuffle(os.path.join(conf_path, orig_poscar_name), - os.path.join(conf_path, poscar_name)) - else : - os.symlink(cc, os.path.join(conf_path, poscar_name)) - if 'sys_format' in jdata: - fmt = jdata['sys_format'] - else: - fmt = 'vasp/poscar' - system = dpdata.System(os.path.join(conf_path, poscar_name), fmt = fmt, type_map = jdata['type_map']) - if jdata.get('model_devi_nopbc', False): - system.remove_pbc() - system.to_lammps_lmp(os.path.join(conf_path, lmp_name)) - elif model_devi_engine == "gromacs": - pass - conf_counter += 1 - sys_counter += 1 - - input_mode = "native" - if "template" in cur_job: - input_mode = "revise_template" - use_plm = jdata.get('model_devi_plumed', False) - use_plm_path = jdata.get('model_devi_plumed_path', False) - if input_mode == "native": - if model_devi_engine == "lammps": - _make_model_devi_native(iter_index, jdata, mdata, conf_systems) - elif model_devi_engine == "gromacs": - _make_model_devi_native_gromacs(iter_index, jdata, mdata, conf_systems) - else: - raise RuntimeError("unknown model_devi engine", model_devi_engine) - elif input_mode == "revise_template": - _make_model_devi_revmat(iter_index, jdata, mdata, conf_systems) - else: - raise RuntimeError('unknown model_devi input mode', input_mode) - #Copy user defined forward_files - symlink_user_forward_files(mdata=mdata, task_type="model_devi", work_path=work_path) - return True + if "sys_configs_prefix" in jdata: + sys_configs = [] + for sys_list in jdata["sys_configs"]: + #assert (isinstance(sys_list, list) ), "Currently only support type list for sys in 'sys_conifgs' " + temp_sys_list = [os.path.join(jdata["sys_configs_prefix"], sys) for sys in sys_list] + sys_configs.append(temp_sys_list) + else: + sys_configs = jdata['sys_configs'] + shuffle_poscar = jdata['shuffle_poscar'] -def make_calypso_model_devi(iter_index,jdata,mdata): - ''' for calypso ''' + if model_devi_engine != 'calypso': + cur_job = model_devi_jobs[iter_index] + sys_idx = expand_idx(cur_job['sys_idx']) + else: + cur_job = [] + sys_idx = [] - model_devi_jobs = jdata['model_devi_jobs'] - try: - maxiter = max(model_devi_jobs[-1].get('times')) - except Exception as e: - maxiter = jdata.get('model_devi_max_iter',0) - - if (iter_index > maxiter) : - return False + if (len(sys_idx) != len(list(set(sys_idx)))) : + raise RuntimeError("system index should be uniq") + conf_systems = [] + for idx in sys_idx : + cur_systems = [] + ss = sys_configs[idx] + for ii in ss : + cur_systems += glob.glob(ii) + cur_systems.sort() + cur_systems = [os.path.abspath(ii) for ii in cur_systems] + conf_systems.append (cur_systems) - calypso_path = jdata.get('calypso_path') iter_name = make_iter_name(iter_index) train_path = os.path.join(iter_name, train_name) train_path = os.path.abspath(train_path) - models = glob.glob(os.path.join(train_path, "graph*pb")) + models = sorted(glob.glob(os.path.join(train_path, "graph*pb"))) work_path = os.path.join(iter_name, model_devi_name) - calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) - calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) - create_path(work_path) - create_path(calypso_run_opt_path) - create_path(calypso_model_devi_path) - - # the model + if model_devi_engine == 'calypso': + calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) + calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) + create_path(calypso_run_opt_path) + create_path(calypso_model_devi_path) + # modd script + modd_script = os.path.join(calypso_model_devi_path,'modd.py') + with open(modd_script,'w') as fm: + fm.write(write_modd()) + # check outcar script + check_outcar_script = os.path.join(calypso_run_opt_path,'check_outcar.py') + with open(check_outcar_script,'w') as ffff: + ffff.write(make_check_outcar_script()) for mm in models : model_name = os.path.basename(mm) - os.symlink(mm, os.path.join(calypso_run_opt_path, model_name)) + if model_devi_engine != 'calypso': + os.symlink(mm, os.path.join(work_path, model_name)) + else: + os.symlink(mm, os.path.join(calypso_run_opt_path, model_name)) + with open(os.path.join(work_path, 'cur_job.json'), 'w') as outfile: + json.dump(cur_job, outfile, indent = 4) - # modd script - modd_script = os.path.join(calypso_model_devi_path,'modd.py') - with open(modd_script,'w') as fm: - fm.write(write_modd()) + conf_path = os.path.join(work_path, 'confs') + create_path(conf_path) + sys_counter = 0 + for ss in conf_systems: + conf_counter = 0 + for cc in ss : + if model_devi_engine == "lammps": + conf_name = make_model_devi_conf_name(sys_idx[sys_counter], conf_counter) + orig_poscar_name = conf_name + '.orig.poscar' + poscar_name = conf_name + '.poscar' + lmp_name = conf_name + '.lmp' + if shuffle_poscar : + os.symlink(cc, os.path.join(conf_path, orig_poscar_name)) + poscar_shuffle(os.path.join(conf_path, orig_poscar_name), + os.path.join(conf_path, poscar_name)) + else : + os.symlink(cc, os.path.join(conf_path, poscar_name)) + if 'sys_format' in jdata: + fmt = jdata['sys_format'] + else: + fmt = 'vasp/poscar' + system = dpdata.System(os.path.join(conf_path, poscar_name), fmt = fmt, type_map = jdata['type_map']) + if jdata.get('model_devi_nopbc', False): + system.remove_pbc() + system.to_lammps_lmp(os.path.join(conf_path, lmp_name)) + elif model_devi_engine == "gromacs": + pass + conf_counter += 1 + sys_counter += 1 input_mode = "native" if "calypso_input_path" in jdata: input_mode = "buffet" - - # generate input.dat automatic in each iter + if "template" in cur_job: + input_mode = "revise_template" + use_plm = jdata.get('model_devi_plumed', False) + use_plm_path = jdata.get('model_devi_plumed_path', False) if input_mode == "native": - for iiidx, jobbs in enumerate(model_devi_jobs): - if iter_index in jobbs.get('times'): - cur_job = model_devi_jobs[iiidx] - - _make_model_devi_native_calypso(cur_job, calypso_run_opt_path) - - run_opt_script = os.path.join(calypso_run_opt_path,'run_opt.py') - with open(run_opt_script,'w') as ffff: - ffff.write(make_run_opt_script(cur_job.get('fmax',0.01))) - - # ----------for check the nan situation ------------- - check_outcar_script = os.path.join(calypso_run_opt_path,'check_outcar.py') - with open(check_outcar_script,'w') as ffff: - ffff.write(make_check_outcar_script()) - # ----------for check the nan situation ------------- - return True - - # generate confs according to the input.dat provided + if model_devi_engine == "lammps": + _make_model_devi_native(iter_index, jdata, mdata, conf_systems) + elif model_devi_engine == "gromacs": + _make_model_devi_native_gromacs(iter_index, jdata, mdata, conf_systems) + elif model_devi_engine == "calypso": + _make_model_devi_native_calypso(model_devi_jobs, calypso_run_opt_path) # generate input.dat automatic in each iter + else: + raise RuntimeError("unknown model_devi engine", model_devi_engine) + elif input_mode == "revise_template": + _make_model_devi_revmat(iter_index, jdata, mdata, conf_systems) elif input_mode == "buffet": - calypso_input_path = jdata.get('calypso_input_path') - shutil.copyfile(os.path.join(calypso_input_path,'input.dat'),os.path.join(calypso_run_opt_path, 'input.dat')) - popsize = _parse_calypso_input('PopSize',calypso_run_opt_path+'/input.dat') - run_opt_path = calypso_run_opt_path - - run_opt_script = os.path.join(calypso_run_opt_path,'run_opt.py') - with open(run_opt_script,'w') as ffff: - ffff.write(make_run_opt_script(jdata.get('fmax',0.01))) - - # ----------for check the nan situation ------------- - check_outcar_script = os.path.join(calypso_run_opt_path,'check_outcar.py') - with open(check_outcar_script,'w') as ffff: - ffff.write(make_check_outcar_script()) - # ----------for check the nan situation ------------- - return True - + _make_model_devi_buffet(jdata,calypso_run_opt_path) # generate confs according to the input.dat provided else: - raise RuntimeError('do not found `calypso_input_path` or `model_devi_jobs`', input_mode) + raise RuntimeError('unknown model_devi input mode', input_mode) + #Copy user defined forward_files + symlink_user_forward_files(mdata=mdata, task_type="model_devi", work_path=work_path) + return True def _make_model_devi_revmat(iter_index, jdata, mdata, conf_systems): model_devi_jobs = jdata['model_devi_jobs'] @@ -1428,10 +1379,10 @@ def run_single_model_devi (iter_index, def run_model_devi(iter_index,jdata,mdata): model_devi_engine = jdata.get("model_devi_engine", "lammps") - if model_devi_engine == "calypso": - run_multi_model_devi(iter_index,jdata,mdata) - else: + if model_devi_engine != "calypso": run_single_model_devi(iter_index,jdata,mdata) + else: + run_multi_model_devi(iter_index,jdata,mdata) def run_multi_model_devi (iter_index, @@ -2030,6 +1981,7 @@ def _trust_limitation_check(sys_idx, lim): os.symlink(os.path.relpath(job_name), 'job.json') else: os.symlink(os.path.relpath(conf_name), 'POSCAR') + os.system('touch job.json') else: os.symlink(os.path.relpath(poscar_name), 'POSCAR') np.save("atom_pref", new_system.data["atom_pref"]) @@ -2842,13 +2794,13 @@ def post_fp_vasp (iter_index, dlog.info('%s has different formula, so pass in line 3518'%oo) pass # save ele_temp, if any - if model_devi_engine != 'calypso': - with open(oo.replace('OUTCAR', 'job.json')) as fp: - job_data = json.load(fp) - if 'ele_temp' in job_data: - assert(use_ele_temp) - ele_temp = job_data['ele_temp'] - all_te.append(ele_temp) + # need to check + with open(oo.replace('OUTCAR', 'job.json')) as fp: + job_data = json.load(fp) + if 'ele_temp' in job_data: + assert(use_ele_temp) + ele_temp = job_data['ele_temp'] + all_te.append(ele_temp) else: icount+=1 all_te = np.array(all_te) @@ -2856,28 +2808,25 @@ def post_fp_vasp (iter_index, sys_data_path = os.path.join(work_path, 'data.%s'%ss) all_sys.to_deepmd_raw(sys_data_path) all_sys.to_deepmd_npy(sys_data_path, set_size = len(sys_outcars)) - if model_devi_engine != 'calypso': - if all_te.size > 0: - assert(len(all_sys) == all_sys.get_nframes()) - assert(len(all_sys) == all_te.size) - all_te = np.reshape(all_te, [-1,1]) - if use_ele_temp == 0: - raise RuntimeError('should not get ele temp at setting: use_ele_temp == 0') - elif use_ele_temp == 1: - np.savetxt(os.path.join(sys_data_path, 'fparam.raw'), all_te) - np.save(os.path.join(sys_data_path, 'set.000', 'fparam.npy'), all_te) - elif use_ele_temp == 2: - tile_te = np.tile(all_te, [1, all_sys.get_natoms()]) - np.savetxt(os.path.join(sys_data_path, 'aparam.raw'), tile_te) - np.save(os.path.join(sys_data_path, 'set.000', 'aparam.npy'), tile_te) - else: - raise RuntimeError('invalid setting of use_ele_temp ' + str(use_ele_temp)) + if all_te.size > 0: + assert(len(all_sys) == all_sys.get_nframes()) + assert(len(all_sys) == all_te.size) + all_te = np.reshape(all_te, [-1,1]) + if use_ele_temp == 0: + raise RuntimeError('should not get ele temp at setting: use_ele_temp == 0') + elif use_ele_temp == 1: + np.savetxt(os.path.join(sys_data_path, 'fparam.raw'), all_te) + np.save(os.path.join(sys_data_path, 'set.000', 'fparam.npy'), all_te) + elif use_ele_temp == 2: + tile_te = np.tile(all_te, [1, all_sys.get_natoms()]) + np.savetxt(os.path.join(sys_data_path, 'aparam.raw'), tile_te) + np.save(os.path.join(sys_data_path, 'set.000', 'aparam.npy'), tile_te) + else: + raise RuntimeError('invalid setting of use_ele_temp ' + str(use_ele_temp)) rfail=float(icount)/float(tcount) dlog.info("failed frame: %6d in %6d %6.2f %% " % (icount, tcount, rfail * 100.)) - if model_devi_engine == 'calypso': - ratio_failed = 0.1 if rfail>ratio_failed: raise RuntimeError("find too many unsuccessfully terminated jobs. Too many FP tasks are not converged. Please check your input parameters (e.g. INCAR) or configuration (e.g. POSCAR) in directories \'iter.*.*/02.fp/task.*.*/.\'") From 2ac1efe929c66186312b8f34db428665a73c074b Mon Sep 17 00:00:00 2001 From: zhenyu Date: Tue, 15 Mar 2022 11:11:20 +0800 Subject: [PATCH 19/46] fix bugs --- dpgen/generator/lib/calypso.py | 4 +++- dpgen/generator/run.py | 11 +++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/dpgen/generator/lib/calypso.py b/dpgen/generator/lib/calypso.py index 56dfd200a..56ece6bd7 100644 --- a/dpgen/generator/lib/calypso.py +++ b/dpgen/generator/lib/calypso.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 import os +import shutil +import json import numpy as np def make_run_opt_script(fmax ) : @@ -371,7 +373,7 @@ def _make_model_devi_buffet(jdata,calypso_run_opt_path): with open(run_opt_script,'w') as ffff: ffff.write(make_run_opt_script(jdata.get('fmax',0.01))) -def _make_model_devi_native_calypso(model_devi_jobs, calypso_run_opt_path): +def _make_model_devi_native_calypso(iter_index,model_devi_jobs, calypso_run_opt_path): for iiidx, jobbs in enumerate(model_devi_jobs): if iter_index in jobbs.get('times'): diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index 15a70def5..0172522f6 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -42,7 +42,7 @@ from dpgen.generator.lib.utils import symlink_user_forward_files from dpgen.generator.lib.lammps import make_lammps_input, get_dumped_forces from dpgen.generator.lib.calypso import make_run_opt_script,make_check_outcar_script -from dpgen.generator.lib.calypso import _make_model_devi_native_calypso,write_model_devi_out +from dpgen.generator.lib.calypso import _make_model_devi_native_calypso,write_model_devi_out,_make_model_devi_buffet from dpgen.generator.lib.modd_calypso import GenStructures,Analysis from dpgen.generator.lib.write_modd import write_modd from dpgen.generator.lib.parse_calypso import _parse_calypso_input,_parse_calypso_dis_mtx @@ -899,7 +899,7 @@ def make_model_devi (iter_index, elif model_devi_engine == "gromacs": _make_model_devi_native_gromacs(iter_index, jdata, mdata, conf_systems) elif model_devi_engine == "calypso": - _make_model_devi_native_calypso(model_devi_jobs, calypso_run_opt_path) # generate input.dat automatic in each iter + _make_model_devi_native_calypso(iter_index,model_devi_jobs, calypso_run_opt_path) # generate input.dat automatic in each iter else: raise RuntimeError("unknown model_devi engine", model_devi_engine) elif input_mode == "revise_template": @@ -1429,7 +1429,7 @@ def run_multi_model_devi (iter_index, elif lines[-1].strip().strip('\n') == '3': # Model Devi _calypso_run_opt_path = os.path.abspath(calypso_run_opt_path) - all_models = glob.glob(os.path.join(calypso_run_opt_path, 'graph*pb')) + all_models = glob.glob(os.path.join(_calypso_run_opt_path, 'graph*pb')) cwd = os.getcwd() os.chdir(calypso_model_devi_path) args = ' '.join(['modd.py', '--all_models',' '.join(all_models),'--type_map',' '.join(jdata.get('type_map'))]) @@ -1981,7 +1981,10 @@ def _trust_limitation_check(sys_idx, lim): os.symlink(os.path.relpath(job_name), 'job.json') else: os.symlink(os.path.relpath(conf_name), 'POSCAR') - os.system('touch job.json') + fjob = open('job.json','w+') + fjob.write('{"model_devi_engine":"calypso"}') + fjob.close() + #os.system('touch job.json') else: os.symlink(os.path.relpath(poscar_name), 'POSCAR') np.save("atom_pref", new_system.data["atom_pref"]) From f1ae450971b8c27d1bf5c06d03ad03b8fe419234 Mon Sep 17 00:00:00 2001 From: zhenyu Date: Sat, 19 Mar 2022 18:29:45 +0800 Subject: [PATCH 20/46] fix: fix bugs in modd_calypso.py write_modd.py and run.py --- dpgen/generator/lib/modd_calypso.py | 1 + dpgen/generator/lib/write_modd.py | 6 +++--- dpgen/generator/run.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dpgen/generator/lib/modd_calypso.py b/dpgen/generator/lib/modd_calypso.py index 2968d34dd..d86066557 100644 --- a/dpgen/generator/lib/modd_calypso.py +++ b/dpgen/generator/lib/modd_calypso.py @@ -18,6 +18,7 @@ from ase.io.trajectory import Trajectory #from deepmd.infer import calc_model_devi #from deepmd.infer import DeepPot as DP +from pathlib import Path from dpgen import dlog from dpgen.generator.lib.utils import make_iter_name from dpgen.generator.lib.calypso import write_model_devi_out diff --git a/dpgen/generator/lib/write_modd.py b/dpgen/generator/lib/write_modd.py index 61637065f..31f7e4f9a 100644 --- a/dpgen/generator/lib/write_modd.py +++ b/dpgen/generator/lib/write_modd.py @@ -81,7 +81,7 @@ def write_modd(): ret+=" row,col = np.diag_indices_from(dis)\n" ret+=" dis[row,col] = 10000\n" ret+=" min_dis = np.nanmin(dis)\n" - ret+=" devi = np.append(devi[0],min_dis)\n " + ret+=" devi = np.append(devi[0],min_dis)\n" ret+=" t = [devi]\n" ret+=" devi = np.array(t)\n" ret+=" # ------------------------------------------------------------------------------------\n" @@ -100,7 +100,7 @@ def write_modd(): ret+=" write_model_devi_out(Devis,os.path.join(cwd,'Model_Devi.out'))\n" ret+="\n" ret+=" f = open(os.path.join(os.path.abspath(os.path.join(cwd,os.pardir)),'record.calypso'),'a+')\n" - ret+=" f.write('4\n')\n" + ret+=" f.write('4')\n" ret+=" f.close()\n" ret+="\n" ret+="if __name__ == '__main__':\n" @@ -122,6 +122,6 @@ def write_modd(): ret+=" )\n" ret+=" args = parser.parse_args()\n" ret+=" print(vars(args))\n" - ret+=" #Modd(args.all_models,args.type_map)\n" + ret+=" Modd(args.all_models,args.type_map)\n" ret+=" #Modd(sys.argv[1],sys.argv[2])\n" return ret diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index 0172522f6..11f59122b 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -243,7 +243,7 @@ def make_train (iter_index, log_task('prev data is empty, copy prev model') copy_model(numb_models, iter_index-1, iter_index) return - elif iter_index > 0 and _check_skip_train(model_devi_jobs[iter_index-1]): + elif model_devi_engine != 'calypso' and iter_index > 0 and _check_skip_train(model_devi_jobs[iter_index-1]): log_task('skip training at step %d ' % (iter_index-1)) copy_model(numb_models, iter_index-1, iter_index) return From a6048fb8d92a6444b7ef70d0bfba877c0e7d9dcc Mon Sep 17 00:00:00 2001 From: zhenyu Date: Tue, 22 Mar 2022 15:21:05 +0800 Subject: [PATCH 21/46] change functions name and fix bugs --- dpgen/generator/lib/modd.py | 122 ------ dpgen/generator/lib/model_devi_calypso.py | 452 ++++++++++++++++++++++ dpgen/generator/lib/write_modd.py | 2 +- dpgen/generator/run.py | 149 +++---- 4 files changed, 512 insertions(+), 213 deletions(-) delete mode 100644 dpgen/generator/lib/modd.py create mode 100644 dpgen/generator/lib/model_devi_calypso.py diff --git a/dpgen/generator/lib/modd.py b/dpgen/generator/lib/modd.py deleted file mode 100644 index 577ebde7d..000000000 --- a/dpgen/generator/lib/modd.py +++ /dev/null @@ -1,122 +0,0 @@ -import argparse -import copy -import dpdata -import glob -import math -import numpy as np -import os -import sys -import shutil - - -def write_model_devi_out(devi, fname): - assert devi.shape[1] == 8 - #assert devi.shape[1] == 7 - header = "%5s" % "step" - for item in 'vf': - header += "%16s%16s%16s" % (f"max_devi_{item}", f"min_devi_{item}",f"avg_devi_{item}") - header += "%16s"%str('min_dis') - np.savetxt(fname, - devi, - fmt=['%5d'] + ['%17.6e' for _ in range(7)], - delimiter='', - header=header) - return devi - - -def Modd(all_models,type_map): - from deepmd.infer import calc_model_devi - from deepmd.infer import DeepPot as DP - - # Model Devi - - cwd = os.getcwd() - graphs = [DP(model) for model in all_models] - - Devis = [] - pcount = 0 - strus_lists = glob.glob(os.path.join(cwd,'*.structures')) - for num, strus_path in enumerate(strus_lists): - - structures_data = dpdata.System(strus_path,'deepmd/npy',type_map=type_map) - - # every 500 confs in one task dir - nnum = structures_data.get_nframes() - if nnum == 0: - continue - else: - num_per_task = math.ceil(nnum/500) - - - for temp in range(num_per_task): - task_name = os.path.join(cwd,'task.%03d.%03d'%(num,temp)) - put_poscar = os.path.join(task_name,'traj') - if not os.path.exists(task_name): - os.mkdir(task_name) - os.mkdir(put_poscar) - else: - shutil.rmtree(task_name) - os.mkdir(task_name) - os.mkdir(put_poscar) - devis = [] - if (nnum - (temp+1)*500) >= 0: - temp_sl = range(temp*500,(temp+1)*500) - else: - temp_sl = range(temp*500,nnum) - - new_index = 0 - for index,frameid in enumerate(temp_sl): - pdata = structures_data[frameid] - pdata.to_vasp_poscar(os.path.join(put_poscar,'%s.poscar'%str(index))) - nopbc = pdata.nopbc - coord = pdata.data['coords'] - cell = pdata.data['cells'] - atom_types = pdata.data['atom_types'] - devi = calc_model_devi(coord,cell,atom_types,graphs,nopbc=nopbc) - # ------------------------------------------------------------------------------------ - # append min-distance in devi list - dis = pdata.to_ase_structure()[0].get_all_distances(mic=True) - row,col = np.diag_indices_from(dis) - dis[row,col] = 10000 - min_dis = np.nanmin(dis) - devi = np.append(devi[0],min_dis) - t = [devi] - devi = np.array(t) - # ------------------------------------------------------------------------------------ - temp_d = copy.deepcopy(devi) - temp_D = copy.deepcopy(devi) - devis.append(temp_d) - Devis.append(temp_D) - devis[index][0][0] = np.array(index) - Devis[pcount][0][0] = np.array(pcount) - pcount += 1 - new_index += 1 - devis = np.vstack(devis) - write_model_devi_out(devis,os.path.join(task_name, 'model_devi.out')) - - Devis = np.vstack(Devis) - write_model_devi_out(Devis,os.path.join(cwd,'Model_Devi.out')) - - f = open(os.path.join(os.path.abspath(os.path.join(cwd,os.pardir)),'record.calypso'),'a+') - f.write('4\n') - f.close() - -if __name__ == "__main__": - - cwd = os.getcwd() - model_path = os.path.join(os.path.abspath(os.path.join(cwd,os.pardir)),'gen_stru_analy') - parser = argparse.ArgumentParser(description='calc model-devi by `all_models` and `type_map`') - parser.add_argument( - "--all_models", - nargs='+', - default=model_path, - help="the path of models which will be used to do model-deviation", - ) - parser.add_argument( - "--type_map", - nargs='+', - help="the type map of models which will be used to do model-deviation", - ) - args = parser.parse_args() - print(vars(args)) - Modd(args.all_models,args.type_map) diff --git a/dpgen/generator/lib/model_devi_calypso.py b/dpgen/generator/lib/model_devi_calypso.py new file mode 100644 index 000000000..21b6df4f4 --- /dev/null +++ b/dpgen/generator/lib/model_devi_calypso.py @@ -0,0 +1,452 @@ +""" +calypso model devi: + 1. gen_structures + 2. analysis + 3. model devi +""" + +import copy +import dpdata +import math +import numpy as np +import os +import random +import re +import glob +import shutil +from ase.io.vasp import write_vasp +from ase.io.trajectory import Trajectory +#from deepmd.infer import calc_model_devi +#from deepmd.infer import DeepPot as DP +from pathlib import Path +from distutils.version import LooseVersion +from dpgen import dlog +from dpgen.generator.lib.utils import make_iter_name +from dpgen.generator.lib.calypso import write_model_devi_out +from dpgen.generator.lib.parse_calypso import _parse_calypso_input,_parse_calypso_dis_mtx +from dpgen.dispatcher.Dispatcher import Dispatcher, _split_tasks, make_dispatcher, make_submission + +train_name = '00.train' +model_devi_name = '01.model_devi' +fp_name = '02.fp' +calypso_run_opt_name = 'gen_stru_analy' +calypso_model_devi_name = 'model_devi_results' + +def gen_structures(iter_index,jdata,mdata): + + # run calypso + # vsc means generate elemental, binary and ternary at the same time + vsc = jdata.get('vsc',False) # take CALYPSO as confs generator + + model_devi_group_size = mdata['model_devi_group_size'] + model_devi_resources = mdata['model_devi_resources'] + + + iter_name = make_iter_name(iter_index) + work_path = os.path.join(iter_name, model_devi_name) + assert(os.path.isdir(work_path)) + # + calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) + calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) + # + + calypso_path = jdata.get('calypso_path') + #calypso_input_path = jdata.get('calypso_input_path') + popsize = int(_parse_calypso_input('PopSize',calypso_run_opt_path)) + maxstep = int(_parse_calypso_input('MaxStep',calypso_run_opt_path)) + + all_models = glob.glob(os.path.join(calypso_run_opt_path, 'graph*pb')) + model_names = [os.path.basename(ii) for ii in all_models] + + deepmdkit_python = mdata.get('deepmdkit_python') + command = "%s run_opt.py %s 1>> model_devi.log 2>> model_devi.log" % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) + command += " || %s check_outcar.py %s " % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) + commands = [command] + + cwd = os.getcwd() + os.chdir(calypso_run_opt_path) + + forward_files = ['POSCAR', 'run_opt.py','check_outcar.py','input.dat'] + backward_files = ['OUTCAR','CONTCAR','traj.traj','model_devi.log'] + + run_calypso = calypso_path+'/calypso.x | tee log' + if not vsc: + Lpickup = _parse_calypso_input('PickUp','.') + PickUpStep = _parse_calypso_input('PickStep','.') + if os.path.exists('tag_pickup_%s'%(str(PickUpStep))): + dlog.info('caution! tag_pickup_%s exists!'%str(PickUpStep)) + Lpickup = 'F' + if Lpickup == 'T': + ftag = open('tag_pickup_%s'%(str(PickUpStep)),'w') + ftag.close() + os.remove('step') + fstep = open('step','w') + fstep.write('%12s'%str(PickUpStep)) + fstep.close() + else: + PickUpStep = 1 + try: + os.mkdir('opt') + except: + pass + #shutil.rmtree('opt') + #os.mkdir('opt') + + + for ii in range(int(PickUpStep)-1,maxstep+1): + dlog.info('$$$$$$$$$$$$$$$$$$$$ step %s $$$$$$$$$$$$$$$$$'%ii) + if ii == maxstep : + while True: + if len(glob.glob('OUTCAR_*')) == popsize: + break + os.system('%s'%run_calypso) + break + # run calypso + + os.system('%s'%(run_calypso)) + + for pop in range(ii*int(popsize),(ii+1)*int(popsize)): + try: + os.mkdir('task.%03d'%pop) + except: + shutil.rmtree('task.%03d'%pop) + os.mkdir('task.%03d'%pop) + shutil.copyfile('run_opt.py',os.path.join('task.%03d'%pop,'run_opt.py')) + shutil.copyfile('check_outcar.py',os.path.join('task.%03d'%pop,'check_outcar.py')) + shutil.copyfile('POSCAR_%s'%str(pop-ii*int(popsize)+1),os.path.join('task.%03d'%(pop),'POSCAR')) + shutil.copyfile('input.dat',os.path.join('task.%03d'%pop,'input.dat')) + #for iii in range(1,popsize+1): + # shutil.copyfile('POSCAR_%s'%str(iii),os.path.join('task.%03d'%(iii-1),'POSCAR')) + + all_task = glob.glob( "task.*") + all_task.sort() + + run_tasks_ = all_task + + run_tasks = [os.path.basename(ii) for ii in run_tasks_] + + api_version = mdata.get('api_version', '0.9') + if LooseVersion(api_version) < LooseVersion('1.0'): + warnings.warn(f"the dpdispatcher will be updated to new version." + f"And the interface may be changed. Please check the documents for more details") + dispatcher=make_dispatcher(mdata['model_devi_machine'],mdata['model_devi_resources'],'./', run_tasks, model_devi_group_size) + dispatcher.run_jobs(mdata['model_devi_resources'], + commands, + './', + run_tasks, + model_devi_group_size, + model_names, + forward_files, + backward_files, + outlog = 'model_devi.log', + errlog = 'model_devi.log') + elif LooseVersion(api_version) >= LooseVersion('1.0'): + os.chdir(cwd) + submission = make_submission( + mdata['model_devi_machine'], + mdata['model_devi_resources'], + commands=commands, + work_path=calypso_run_opt_path, + run_tasks=run_tasks, + group_size=model_devi_group_size, + forward_common_files=model_names, + forward_files=forward_files, + backward_files=backward_files, + outlog = 'model_devi.log', + errlog = 'model_devi.log') + submission.run_submission() + os.chdir(calypso_run_opt_path) + + sstep = os.path.join('opt',str(ii)) + os.mkdir(sstep) + if not os.path.exists('traj'): + os.mkdir('traj') + + for jjj in range(ii*int(popsize),(ii+1)*int(popsize)): + # to opt directory + shutil.copyfile('POSCAR_%s'%str(jjj+1-ii*int(popsize)),os.path.join(sstep,'POSCAR_%s'%str(jjj+1-ii*int(popsize))),) + shutil.copyfile(os.path.join('task.%03d'%(jjj),'OUTCAR'),os.path.join(sstep,'OUTCAR_%s'%str(jjj+1-ii*int(popsize))),) + shutil.copyfile(os.path.join('task.%03d'%(jjj),'CONTCAR'),os.path.join(sstep,'CONTCAR_%s'%str(jjj+1-ii*int(popsize))),) + # to run calypso directory + shutil.copyfile(os.path.join('task.%03d'%(jjj),'OUTCAR'),'OUTCAR_%s'%str(jjj+1-ii*int(popsize)),) + shutil.copyfile(os.path.join('task.%03d'%(jjj),'CONTCAR'),'CONTCAR_%s'%str(jjj+1-ii*int(popsize)),) + # to traj + shutil.copyfile(os.path.join('task.%03d'%(jjj),'traj.traj'),os.path.join('traj','%s.traj'%str(jjj+1)),) + + if LooseVersion(api_version) < LooseVersion('1.0'): + os.rename('jr.json','jr_%s.json'%(str(ii))) + + tlist = glob.glob('task.*') + for t in tlist: + shutil.rmtree(t) + + else: + # -------------------------------------------------------------- + component = ['Mg','Al','Cu','MgAl','MgCu','AlCu','MgAlCu'] + + for idx,com in enumerate(component): + pwd = os.getcwd() + os.mkdir(str(idx)) + shutil.copyfile(os.path.join(cwd,'calypso_input','input.dat.%s'%com),os.path.join(str(idx),'input.dat')) + os.chdir(str(idx)) + os.system(run_calypso) + os.chdir(pwd) + + name_list = Path('.').glob('*/POSCAR_*') + for idx,name in enumerate(name_list): + shutil.copyfile(name,'POSCAR_%s'%(idx+1)) + try: + os.mkdir('task.%04d'%(idx+1)) + except: + shutil.rmtree('task.%04d'%(idx+1)) + os.mkdir('task.%04d'%(idx+1)) + shutil.copyfile('run_opt.py',os.path.join('task.%04d'%(idx+1),'run_opt.py')) + shutil.copyfile('check_outcar.py',os.path.join('task.%04d'%(idx+1),'check_outcar.py')) + shutil.copyfile('POSCAR_%s'%str(idx+1),os.path.join('task.%04d'%(idx+1),'POSCAR')) + shutil.copyfile('input.dat',os.path.join('task.%04d'%(idx+1),'input.dat')) + + all_task = glob.glob( "task.*") + all_task.sort() + + run_tasks_ = all_task + + run_tasks = [os.path.basename(ii) for ii in run_tasks_] + + if LooseVersion(api_version) < LooseVersion('1.0'): + warnings.warn(f"the dpdispatcher will be updated to new version." + f"And the interface may be changed. Please check the documents for more details") + dispatcher=make_dispatcher(mdata['model_devi_machine'],mdata['model_devi_resources'],'./', run_tasks, model_devi_group_size) + dispatcher.run_jobs(mdata['model_devi_resources'], + commands, + './', + run_tasks, + model_devi_group_size, + model_names, + forward_files, + backward_files, + outlog = 'model_devi.log', + errlog = 'model_devi.log') + elif LooseVersion(api_version) >= LooseVersion('1.0'): + os.chdir(cwd) + submission = make_submission( + mdata['model_devi_machine'], + mdata['model_devi_resources'], + commands=commands, + work_path=calypso_run_opt_path, + run_tasks=run_tasks, + group_size=model_devi_group_size, + forward_common_files=model_names, + forward_files=forward_files, + backward_files=backward_files, + outlog = 'model_devi.log', + errlog = 'model_devi.log') + submission.run_submission() + os.chdir(calypso_run_opt_path) + + os.mkdir('opt') + if not os.path.exists('traj'): + os.mkdir('traj') + for jjj in range(len(all_task)): + # to opt directory + shutil.copyfile('POSCAR_%s'%str(jjj+1),os.path.join('opt','POSCAR_%s'%str(jjj+1)),) + shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'OUTCAR'),os.path.join('opt','OUTCAR_%s'%str(jjj+1)),) + shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'CONTCAR'),os.path.join('opt','CONTCAR_%s'%str(jjj+1)),) + # to run calypso directory + shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'OUTCAR'),'OUTCAR_%s'%str(jjj+1),) + shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'CONTCAR'),'CONTCAR_%s'%str(jjj+1),) + # to traj + shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'traj.traj'),os.path.join('traj','%s.traj'%str(jjj+1)),) + + tlist = glob.glob('task.*') + for t in tlist: + shutil.rmtree(t) + # -------------------------------------------------------------- + + os.chdir(cwd) + os.chdir(work_path) + f = open('record.calypso','a+') + f.write('2\n') + f.close() + os.chdir(cwd) + +def analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): + + # Analysis + + + dlog.info('$$$$$$$$$$$$$$$ Analysis Started $$$$$$$$$$$$$$$$$$') + + ms = dpdata.MultiSystems() + + cwd = os.getcwd() + iter_name = make_iter_name(iter_index) + work_path = os.path.join(iter_name, model_devi_name) + result_path = os.path.join(calypso_run_opt_path,'results') + + # trajs to be model devi + traj_path = os.path.join(calypso_run_opt_path,'traj') + traj_list = glob.glob(traj_path+'/*.traj') + + dlog.info('len(traj_list) %s'%str(len(traj_list))) + + # read confs from traj + record_traj_num = 0 + for traj_name in traj_list: + traj_num = os.path.basename(traj_name).split('.')[0] + trajs_origin = Trajectory(traj_name) + + record_traj_num += len(trajs_origin) + if len(trajs_origin) >= 20 : + trajs = [trajs_origin[iii] for iii in [4,9,-10,-5,-1]] + elif 5<=len(trajs_origin)<20: + trajs = [trajs_origin[random.randint(1,len(trajs_origin)-1)] for iii in range(4)] + trajs.append(trajs[-1]) + elif 3<= len(trajs_origin) <5: + trajs = [trajs_origin[round((len(trajs_origin)-1)/2)] ] + trajs.append(trajs[-1]) + elif len(trajs_origin) == 2: + trajs = [trajs_origin[0],trajs_origin[-1] ] + elif len(trajs_origin) == 1: + trajs = [trajs_origin[0] ] + else: + pass + + for idx, traj in enumerate(trajs): + write_vasp(os.path.join( + traj_path,'%03d.%03d.poscar' % ( + int(traj_num), int(idx) + ) + ), + traj) + + traj_pos_list = glob.glob(traj_path+'/*.poscar') + + #dlog.info('traj_num %s'%str(len(traj_pos_list))) + #dlog.info('total_traj_num %s'%str(record_traj_num)) + + for npos in traj_pos_list: + try: + ms.append(dpdata.System(npos, type_map = jdata['type_map'])) + except Exception as e: + dlog.info(npos,'failed : ',e) + + if len(ms) == 0: + print('too little confs, ') + return + + if os.path.exists(os.path.join(result_path,'deepmd')): + shutil.rmtree(os.path.join(result_path,'deepmd')) + ms.to_deepmd_raw(os.path.join(result_path,'deepmd')) + ms.to_deepmd_npy(os.path.join(result_path,'deepmd')) + + + split_lists = glob.glob(os.path.join(result_path,'deepmd','*')) + for i,split_list in enumerate(split_lists): + #ss = dpdata.System(split_list,fmt='deepmd') + #for j in range(ss.get_nframes()): + # ss.to('vasp/poscar',os.path.join(split_list,'%03d.%03d.poscar'%(i,j)),frame_idx=j) + strus_path = os.path.join(calypso_model_devi_path,'%03d.structures'%i) + if not os.path.exists(strus_path): + shutil.copytree(split_list,strus_path) + else: + shutil.rmtree(strus_path) + shutil.copytree(split_list,strus_path) + + os.chdir(cwd) + os.chdir(work_path) + f = open('record.calypso','a+') + f.write('3\n') + f.close() + os.chdir(cwd) + + +#def model_devi(iter_index,calypso_model_devi_path,all_models,jdata): +# +# # Model Devi +# +# +# dlog.info('$$$$$$$$$$$$$$$ Model Devi Start $$$$$$$$$$$$$$$$$$') +# +# iter_name = make_iter_name(iter_index) +# work_path = os.path.join(iter_name, model_devi_name) +# +# cwd = os.getcwd() +# Devis = [] +# pcount = 0 +# strus_lists = glob.glob(os.path.join(calypso_model_devi_path,'*.structures')) +# for num, strus_path in enumerate(strus_lists): +# os.chdir(strus_path) +# os.system('rename .vasp .poscar *') +# os.chdir(cwd) +# +# structure_list = glob.glob(os.path.join(strus_path,'*.poscar')) +# +# # every 500 confs in one task dir +# if len(structure_list) == 0: +# continue +# else: +# num_per_task = math.ceil(len(structure_list)/500) +# graphs = [DP(model) for model in all_models] +# for temp in range(num_per_task): +# task_name = os.path.join(calypso_model_devi_path,'task.%03d.%03d'%(num,temp)) +# put_poscar = os.path.join(task_name,'traj') +# if not os.path.exists(task_name): +# os.mkdir(task_name) +# os.mkdir(put_poscar) +# else: +# shutil.rmtree(task_name) +# os.mkdir(task_name) +# os.mkdir(put_poscar) +# devis = [] +# try: +# temp_sl = structure_list[temp*500:(temp+1)*500] +# except Exception as err: +# dlog.info('err %s'%str(err)) +# temp_sl = structure_list[temp*500:] +# +# new_index = 0 +# for index,sname in enumerate(temp_sl): +# shutil.copyfile(sname,os.path.join(put_poscar,'%s.poscar'%str(index))) +# os.chdir(put_poscar) +# pdata = dpdata.System('%s.poscar'%str(index),type_map = jdata['type_map']) +# nopbc = pdata.nopbc +# coord = pdata.data['coords'] +# cell = pdata.data['cells'] +# atom_types = pdata.data['atom_types'] +# devi = calc_model_devi(coord,cell,atom_types,graphs,nopbc=nopbc) +# # ------------------------------------------------------------------------------------ +# # append min-distance in devi list +# dis_temp = dpdata.System('%s.poscar'%str(index)) +# dis = dis_temp.to_ase_structure()[0].get_all_distances(mic=True) +# row,col = np.diag_indices_from(dis) +# dis[row,col] = 10000 +# min_dis = np.nanmin(dis) +# devi = np.append(devi[0],min_dis) +# t = [devi] +# devi = np.array(t) +# # ------------------------------------------------------------------------------------ +# temp_d = copy.deepcopy(devi) +# temp_D = copy.deepcopy(devi) +# devis.append(temp_d) +# Devis.append(temp_D) +# devis[index][0][0] = np.array(index) +# Devis[pcount][0][0] = np.array(pcount) +# pcount += 1 +# new_index += 1 +# os.chdir(cwd) +# os.chdir(task_name) +# devis = np.vstack(devis) +# write_model_devi_out(devis,'model_devi.out') +# os.chdir(cwd) +# +# os.chdir(calypso_model_devi_path) +# Devis = np.vstack(Devis) +# write_model_devi_out(Devis,'Model_Devi.out') +# os.chdir(cwd) +# +# os.chdir(work_path) +# f = open('record.calypso','a+') +# f.write('4\n') +# f.close() +# os.chdir(cwd) + diff --git a/dpgen/generator/lib/write_modd.py b/dpgen/generator/lib/write_modd.py index 31f7e4f9a..44f38d88b 100644 --- a/dpgen/generator/lib/write_modd.py +++ b/dpgen/generator/lib/write_modd.py @@ -121,7 +121,7 @@ def write_modd(): ret+=" help='the type map of models which will be used to do model-deviation',\n" ret+=" )\n" ret+=" args = parser.parse_args()\n" - ret+=" print(vars(args))\n" + ret+=" #print(vars(args))\n" ret+=" Modd(args.all_models,args.type_map)\n" ret+=" #Modd(sys.argv[1],sys.argv[2])\n" return ret diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index 11f59122b..19280df92 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -43,7 +43,7 @@ from dpgen.generator.lib.lammps import make_lammps_input, get_dumped_forces from dpgen.generator.lib.calypso import make_run_opt_script,make_check_outcar_script from dpgen.generator.lib.calypso import _make_model_devi_native_calypso,write_model_devi_out,_make_model_devi_buffet -from dpgen.generator.lib.modd_calypso import GenStructures,Analysis +from dpgen.generator.lib.model_devi_calypso import gen_structures,analysis from dpgen.generator.lib.write_modd import write_modd from dpgen.generator.lib.parse_calypso import _parse_calypso_input,_parse_calypso_dis_mtx from dpgen.generator.lib.vasp import write_incar_dict @@ -281,10 +281,7 @@ def make_train (iter_index, if 'sys_batch_size' in jdata: sys_batch_size = jdata['sys_batch_size'] else: - if model_devi_engine != 'calypso': - sys_batch_size = ["auto" for aa in range(len(jdata['sys_configs']))] - else: - sys_batch_size = ["auto" for aa in range(3000)] + sys_batch_size = ["auto" for aa in range(len(jdata['sys_configs']))] # make sure all init_data_sys has the batch size -- for the following `zip` assert (len(init_data_sys_) <= len(init_batch_size_)) @@ -303,6 +300,10 @@ def make_train (iter_index, old_range = len(init_data_sys) fp_path = os.path.join(make_iter_name(ii), fp_name) fp_data_sys = glob.glob(os.path.join(fp_path, "data.*")) + if model_devi_engine == 'calypso': + _modd_path = os.path.join(make_iter_name(ii), model_devi_name, calypso_model_devi_name) + sys_list = glob.glob(os.path.join(_modd_path, "*.structures")) + sys_batch_size = ["auto" for aa in range(len(sys_list))] for jj in fp_data_sys : sys_idx = int(jj.split('.')[-1]) if jdata.get('use_clusters', False): @@ -791,12 +792,20 @@ def make_model_devi (iter_index, if (iter_index >= len(model_devi_jobs)) : return False else: - try: - maxiter = max(model_devi_jobs[-1].get('times')) - except Exception as e: - maxiter = jdata.get('model_devi_max_iter',0) + # mode 1: generate structures according to the user-provided input.dat file, so calypso_input_path and model_devi_max_iter and fmax is needed + if "calypso_input_path" in jdata: + try: + maxiter = jdata.get('model_devi_max_iter') + except KeyError: + raise KeyError('calypso_input_path key exists so you should provide model_devi_max_iter key to control the max iter number') + # mode 2: control each iteration to generate structures in specific way by providing model_devi_jobs key + else: + try: + maxiter = max(model_devi_jobs[-1].get('times')) + except KeyError: + raise KeyError('did not find model_devi_jobs["times"] key') if (iter_index > maxiter) : - print('please check `times` in model_devi_jobs or `model_devi_max_iter` in input.json') + print(f'iter_index is {iter_index} and maxiter is {maxiter}') return False if "sys_configs_prefix" in jdata: @@ -1389,7 +1398,7 @@ def run_multi_model_devi (iter_index, jdata, mdata) : - print('############## run calypso model devi #######################') + dlog.info('############## run calypso model devi #######################') iter_name = make_iter_name(iter_index) @@ -1402,81 +1411,41 @@ def run_multi_model_devi (iter_index, cwd = os.getcwd() - api_version = mdata.get('api_version', '0.9') - if LooseVersion(api_version) < LooseVersion('1.0'): - warnings.warn(f"the dpdispatcher will be updated to new version." - f"And the interface may be changed. Please check the documents for more details") - record_calypso_path = os.path.join(work_path,'record.calypso') - while True: - if not os.path.exists(record_calypso_path): - f = open(record_calypso_path,'w') - f.write('1\n') - lines = '1' - f.close() - else: - f = open(record_calypso_path,'r') - lines = f.readlines() - f.close() - - if lines[-1].strip().strip('\n') == '1': - # Gen Structures - GenStructures(iter_index,jdata,mdata) - - elif lines[-1].strip().strip('\n') == '2': - # Analysis & to deepmd/raw - Analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path) - - elif lines[-1].strip().strip('\n') == '3': - # Model Devi - _calypso_run_opt_path = os.path.abspath(calypso_run_opt_path) - all_models = glob.glob(os.path.join(_calypso_run_opt_path, 'graph*pb')) - cwd = os.getcwd() - os.chdir(calypso_model_devi_path) - args = ' '.join(['modd.py', '--all_models',' '.join(all_models),'--type_map',' '.join(jdata.get('type_map'))]) - deepmdkit_python = mdata.get('deepmdkit_python') - os.system(f'{deepmdkit_python} {args} ') - #Modd(iter_index,calypso_model_devi_path,all_models,jdata) - os.chdir(cwd) - - elif lines[-1].strip().strip('\n') == '4': - print('Model Devi is done.') - break - - elif LooseVersion(api_version) >= LooseVersion('1.0'): + record_calypso_path = os.path.join(work_path,'record.calypso') + while True: + if not os.path.exists(record_calypso_path): + f = open(record_calypso_path,'w') + f.write('1\n') + lines = '1' + f.close() + else: + f = open(record_calypso_path,'r') + lines = f.readlines() + f.close() + + if lines[-1].strip().strip('\n') == '1': + # Gen Structures + gen_structures(iter_index,jdata,mdata) + + elif lines[-1].strip().strip('\n') == '2': + # Analysis & to deepmd/raw + analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path) + + elif lines[-1].strip().strip('\n') == '3': + # Model Devi + _calypso_run_opt_path = os.path.abspath(calypso_run_opt_path) + all_models = glob.glob(os.path.join(_calypso_run_opt_path, 'graph*pb')) + cwd = os.getcwd() + os.chdir(calypso_model_devi_path) + args = ' '.join(['modd.py', '--all_models',' '.join(all_models),'--type_map',' '.join(jdata.get('type_map'))]) + deepmdkit_python = mdata.get('deepmdkit_python') + os.system(f'{deepmdkit_python} {args} ') + #Modd(iter_index,calypso_model_devi_path,all_models,jdata) + os.chdir(cwd) - record_calypso_path = os.path.join(work_path,'record.calypso') - while True: - if not os.path.exists(record_calypso_path): - f = open(record_calypso_path,'w') - f.write('1\n') - lines = '1' - f.close() - else: - f = open(record_calypso_path,'r') - lines = f.readlines() - f.close() - - if lines[-1].strip().strip('\n') == '1': - # Gen Structures - GenStructures(iter_index,jdata,mdata) - - elif lines[-1].strip().strip('\n') == '2': - # Analysis & to deepmd/raw - Analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path) - - elif lines[-1].strip().strip('\n') == '3': - # Model Devi - cwd = os.getcwd() - os.chdir(calypso_model_devi_path) - args = ' '.join(['modd.py', '--all_models',' '.join(all_models),'--type_map',' '.join(jdata.get('type_map'))]) - deepmdkit_python = mdata.get('deepmdkit_python') - os.system(f'{deepmdkit_python} {args} ') - #Modd(iter_index,calypso_model_devi_path,all_models,jdata) - os.chdir(cwd) - - elif lines[-1].strip().strip('\n') == '4': - print('Model Devi is done.') - break + elif lines[-1].strip().strip('\n') == '4': + dlog.info('Model Devi is done.') + break def post_model_devi (iter_index, jdata, @@ -2791,11 +2760,11 @@ def post_fp_vasp (iter_index, if all_sys is None: all_sys = _sys else: - try: - all_sys.append(_sys) - except: - dlog.info('%s has different formula, so pass in line 3518'%oo) - pass + #try: + all_sys.append(_sys) + #except RuntimeError: + # dlog.info('%s has different formula, so pass '%oo) + # pass # save ele_temp, if any # need to check with open(oo.replace('OUTCAR', 'job.json')) as fp: From d651e36ae375a49acad5736977a1557600580176 Mon Sep 17 00:00:00 2001 From: zhenyu Date: Tue, 22 Mar 2022 15:25:41 +0800 Subject: [PATCH 22/46] remove modd_calypso.py --- dpgen/generator/lib/modd_calypso.py | 128 +++++++++++++++++----------- 1 file changed, 78 insertions(+), 50 deletions(-) diff --git a/dpgen/generator/lib/modd_calypso.py b/dpgen/generator/lib/modd_calypso.py index d86066557..21b6df4f4 100644 --- a/dpgen/generator/lib/modd_calypso.py +++ b/dpgen/generator/lib/modd_calypso.py @@ -1,8 +1,8 @@ """ calypso model devi: - 1. GenStructures - 2. Analysis - 3. Modd + 1. gen_structures + 2. analysis + 3. model devi """ import copy @@ -19,6 +19,7 @@ #from deepmd.infer import calc_model_devi #from deepmd.infer import DeepPot as DP from pathlib import Path +from distutils.version import LooseVersion from dpgen import dlog from dpgen.generator.lib.utils import make_iter_name from dpgen.generator.lib.calypso import write_model_devi_out @@ -31,7 +32,7 @@ calypso_run_opt_name = 'gen_stru_analy' calypso_model_devi_name = 'model_devi_results' -def GenStructures(iter_index,jdata,mdata): +def gen_structures(iter_index,jdata,mdata): # run calypso # vsc means generate elemental, binary and ternary at the same time @@ -65,7 +66,7 @@ def GenStructures(iter_index,jdata,mdata): cwd = os.getcwd() os.chdir(calypso_run_opt_path) - forward_files = ['POSCAR', 'run_opt.py','check_outcar.py'] + forward_files = ['POSCAR', 'run_opt.py','check_outcar.py','input.dat'] backward_files = ['OUTCAR','CONTCAR','traj.traj','model_devi.log'] run_calypso = calypso_path+'/calypso.x | tee log' @@ -95,11 +96,9 @@ def GenStructures(iter_index,jdata,mdata): for ii in range(int(PickUpStep)-1,maxstep+1): dlog.info('$$$$$$$$$$$$$$$$$$$$ step %s $$$$$$$$$$$$$$$$$'%ii) if ii == maxstep : - dlog.info('##################### counting OUTCAR #####################') while True: if len(glob.glob('OUTCAR_*')) == popsize: break - dlog.info('##################### done counting OUTCAR #####################') os.system('%s'%run_calypso) break # run calypso @@ -115,9 +114,9 @@ def GenStructures(iter_index,jdata,mdata): shutil.copyfile('run_opt.py',os.path.join('task.%03d'%pop,'run_opt.py')) shutil.copyfile('check_outcar.py',os.path.join('task.%03d'%pop,'check_outcar.py')) shutil.copyfile('POSCAR_%s'%str(pop-ii*int(popsize)+1),os.path.join('task.%03d'%(pop),'POSCAR')) + shutil.copyfile('input.dat',os.path.join('task.%03d'%pop,'input.dat')) #for iii in range(1,popsize+1): # shutil.copyfile('POSCAR_%s'%str(iii),os.path.join('task.%03d'%(iii-1),'POSCAR')) - dlog.info('##################### copy file done #####################') all_task = glob.glob( "task.*") all_task.sort() @@ -126,19 +125,37 @@ def GenStructures(iter_index,jdata,mdata): run_tasks = [os.path.basename(ii) for ii in run_tasks_] - dlog.info('$$$$$$$$$$$$$$$ dispatcher running $$$$$$$$$$$$$$$$$$') - dispatcher=make_dispatcher(mdata['model_devi_machine'],mdata['model_devi_resources'],'./', run_tasks, model_devi_group_size) - dispatcher.run_jobs(mdata['model_devi_resources'], - commands, - './', - run_tasks, - model_devi_group_size, - model_names, - forward_files, - backward_files, - outlog = 'model_devi.log', - errlog = 'model_devi.log') - dlog.info('$$$$$$$$$$$$$$$ dispatcher $$$$$$$$$$$$$$$$$$') + api_version = mdata.get('api_version', '0.9') + if LooseVersion(api_version) < LooseVersion('1.0'): + warnings.warn(f"the dpdispatcher will be updated to new version." + f"And the interface may be changed. Please check the documents for more details") + dispatcher=make_dispatcher(mdata['model_devi_machine'],mdata['model_devi_resources'],'./', run_tasks, model_devi_group_size) + dispatcher.run_jobs(mdata['model_devi_resources'], + commands, + './', + run_tasks, + model_devi_group_size, + model_names, + forward_files, + backward_files, + outlog = 'model_devi.log', + errlog = 'model_devi.log') + elif LooseVersion(api_version) >= LooseVersion('1.0'): + os.chdir(cwd) + submission = make_submission( + mdata['model_devi_machine'], + mdata['model_devi_resources'], + commands=commands, + work_path=calypso_run_opt_path, + run_tasks=run_tasks, + group_size=model_devi_group_size, + forward_common_files=model_names, + forward_files=forward_files, + backward_files=backward_files, + outlog = 'model_devi.log', + errlog = 'model_devi.log') + submission.run_submission() + os.chdir(calypso_run_opt_path) sstep = os.path.join('opt',str(ii)) os.mkdir(sstep) @@ -156,10 +173,8 @@ def GenStructures(iter_index,jdata,mdata): # to traj shutil.copyfile(os.path.join('task.%03d'%(jjj),'traj.traj'),os.path.join('traj','%s.traj'%str(jjj+1)),) - - dlog.info('##################### copy file back done #####################') - - os.rename('jr.json','jr_%s.json'%(str(ii))) + if LooseVersion(api_version) < LooseVersion('1.0'): + os.rename('jr.json','jr_%s.json'%(str(ii))) tlist = glob.glob('task.*') for t in tlist: @@ -188,8 +203,7 @@ def GenStructures(iter_index,jdata,mdata): shutil.copyfile('run_opt.py',os.path.join('task.%04d'%(idx+1),'run_opt.py')) shutil.copyfile('check_outcar.py',os.path.join('task.%04d'%(idx+1),'check_outcar.py')) shutil.copyfile('POSCAR_%s'%str(idx+1),os.path.join('task.%04d'%(idx+1),'POSCAR')) - - dlog.info('##################### copy file done #####################') + shutil.copyfile('input.dat',os.path.join('task.%04d'%(idx+1),'input.dat')) all_task = glob.glob( "task.*") all_task.sort() @@ -198,20 +212,36 @@ def GenStructures(iter_index,jdata,mdata): run_tasks = [os.path.basename(ii) for ii in run_tasks_] - dlog.info('$$$$$$$$$$$$$$$ dispatcher running $$$$$$$$$$$$$$$$$$') - dispatcher=make_dispatcher(mdata['model_devi_machine'],mdata['model_devi_resources'],'./', run_tasks, model_devi_group_size) - #print(dispatcher) - dispatcher.run_jobs(mdata['model_devi_resources'], - commands, - './', - run_tasks, - model_devi_group_size, - model_names, - forward_files, - backward_files, - outlog = 'model_devi.log', - errlog = 'model_devi.log') - dlog.info('$$$$$$$$$$$$$$$ dispatcher $$$$$$$$$$$$$$$$$$') + if LooseVersion(api_version) < LooseVersion('1.0'): + warnings.warn(f"the dpdispatcher will be updated to new version." + f"And the interface may be changed. Please check the documents for more details") + dispatcher=make_dispatcher(mdata['model_devi_machine'],mdata['model_devi_resources'],'./', run_tasks, model_devi_group_size) + dispatcher.run_jobs(mdata['model_devi_resources'], + commands, + './', + run_tasks, + model_devi_group_size, + model_names, + forward_files, + backward_files, + outlog = 'model_devi.log', + errlog = 'model_devi.log') + elif LooseVersion(api_version) >= LooseVersion('1.0'): + os.chdir(cwd) + submission = make_submission( + mdata['model_devi_machine'], + mdata['model_devi_resources'], + commands=commands, + work_path=calypso_run_opt_path, + run_tasks=run_tasks, + group_size=model_devi_group_size, + forward_common_files=model_names, + forward_files=forward_files, + backward_files=backward_files, + outlog = 'model_devi.log', + errlog = 'model_devi.log') + submission.run_submission() + os.chdir(calypso_run_opt_path) os.mkdir('opt') if not os.path.exists('traj'): @@ -227,8 +257,6 @@ def GenStructures(iter_index,jdata,mdata): # to traj shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'traj.traj'),os.path.join('traj','%s.traj'%str(jjj+1)),) - dlog.info('##################### copy file back done #####################') - tlist = glob.glob('task.*') for t in tlist: shutil.rmtree(t) @@ -241,7 +269,7 @@ def GenStructures(iter_index,jdata,mdata): f.close() os.chdir(cwd) -def Analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): +def analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): # Analysis @@ -293,8 +321,8 @@ def Analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): traj_pos_list = glob.glob(traj_path+'/*.poscar') - dlog.info('traj_num %s'%str(len(traj_pos_list))) - dlog.info('total_traj_num %s'%str(record_traj_num)) + #dlog.info('traj_num %s'%str(len(traj_pos_list))) + #dlog.info('total_traj_num %s'%str(record_traj_num)) for npos in traj_pos_list: try: @@ -314,9 +342,9 @@ def Analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): split_lists = glob.glob(os.path.join(result_path,'deepmd','*')) for i,split_list in enumerate(split_lists): - ss = dpdata.System(split_list,fmt='deepmd') - for j in range(ss.get_nframes()): - ss.to('vasp/poscar',os.path.join(split_list,'%03d.%03d.poscar'%(i,j)),frame_idx=j) + #ss = dpdata.System(split_list,fmt='deepmd') + #for j in range(ss.get_nframes()): + # ss.to('vasp/poscar',os.path.join(split_list,'%03d.%03d.poscar'%(i,j)),frame_idx=j) strus_path = os.path.join(calypso_model_devi_path,'%03d.structures'%i) if not os.path.exists(strus_path): shutil.copytree(split_list,strus_path) @@ -332,7 +360,7 @@ def Analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): os.chdir(cwd) -#def Modd(iter_index,calypso_model_devi_path,all_models,jdata): +#def model_devi(iter_index,calypso_model_devi_path,all_models,jdata): # # # Model Devi # From 1c1b67003f72fa1689bce6874906583bf2b277d8 Mon Sep 17 00:00:00 2001 From: zhenyu Date: Tue, 22 Mar 2022 15:26:15 +0800 Subject: [PATCH 23/46] remove: remove modd_calypso.py --- dpgen/generator/lib/modd_calypso.py | 452 ---------------------------- 1 file changed, 452 deletions(-) delete mode 100644 dpgen/generator/lib/modd_calypso.py diff --git a/dpgen/generator/lib/modd_calypso.py b/dpgen/generator/lib/modd_calypso.py deleted file mode 100644 index 21b6df4f4..000000000 --- a/dpgen/generator/lib/modd_calypso.py +++ /dev/null @@ -1,452 +0,0 @@ -""" -calypso model devi: - 1. gen_structures - 2. analysis - 3. model devi -""" - -import copy -import dpdata -import math -import numpy as np -import os -import random -import re -import glob -import shutil -from ase.io.vasp import write_vasp -from ase.io.trajectory import Trajectory -#from deepmd.infer import calc_model_devi -#from deepmd.infer import DeepPot as DP -from pathlib import Path -from distutils.version import LooseVersion -from dpgen import dlog -from dpgen.generator.lib.utils import make_iter_name -from dpgen.generator.lib.calypso import write_model_devi_out -from dpgen.generator.lib.parse_calypso import _parse_calypso_input,_parse_calypso_dis_mtx -from dpgen.dispatcher.Dispatcher import Dispatcher, _split_tasks, make_dispatcher, make_submission - -train_name = '00.train' -model_devi_name = '01.model_devi' -fp_name = '02.fp' -calypso_run_opt_name = 'gen_stru_analy' -calypso_model_devi_name = 'model_devi_results' - -def gen_structures(iter_index,jdata,mdata): - - # run calypso - # vsc means generate elemental, binary and ternary at the same time - vsc = jdata.get('vsc',False) # take CALYPSO as confs generator - - model_devi_group_size = mdata['model_devi_group_size'] - model_devi_resources = mdata['model_devi_resources'] - - - iter_name = make_iter_name(iter_index) - work_path = os.path.join(iter_name, model_devi_name) - assert(os.path.isdir(work_path)) - # - calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) - calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) - # - - calypso_path = jdata.get('calypso_path') - #calypso_input_path = jdata.get('calypso_input_path') - popsize = int(_parse_calypso_input('PopSize',calypso_run_opt_path)) - maxstep = int(_parse_calypso_input('MaxStep',calypso_run_opt_path)) - - all_models = glob.glob(os.path.join(calypso_run_opt_path, 'graph*pb')) - model_names = [os.path.basename(ii) for ii in all_models] - - deepmdkit_python = mdata.get('deepmdkit_python') - command = "%s run_opt.py %s 1>> model_devi.log 2>> model_devi.log" % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) - command += " || %s check_outcar.py %s " % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) - commands = [command] - - cwd = os.getcwd() - os.chdir(calypso_run_opt_path) - - forward_files = ['POSCAR', 'run_opt.py','check_outcar.py','input.dat'] - backward_files = ['OUTCAR','CONTCAR','traj.traj','model_devi.log'] - - run_calypso = calypso_path+'/calypso.x | tee log' - if not vsc: - Lpickup = _parse_calypso_input('PickUp','.') - PickUpStep = _parse_calypso_input('PickStep','.') - if os.path.exists('tag_pickup_%s'%(str(PickUpStep))): - dlog.info('caution! tag_pickup_%s exists!'%str(PickUpStep)) - Lpickup = 'F' - if Lpickup == 'T': - ftag = open('tag_pickup_%s'%(str(PickUpStep)),'w') - ftag.close() - os.remove('step') - fstep = open('step','w') - fstep.write('%12s'%str(PickUpStep)) - fstep.close() - else: - PickUpStep = 1 - try: - os.mkdir('opt') - except: - pass - #shutil.rmtree('opt') - #os.mkdir('opt') - - - for ii in range(int(PickUpStep)-1,maxstep+1): - dlog.info('$$$$$$$$$$$$$$$$$$$$ step %s $$$$$$$$$$$$$$$$$'%ii) - if ii == maxstep : - while True: - if len(glob.glob('OUTCAR_*')) == popsize: - break - os.system('%s'%run_calypso) - break - # run calypso - - os.system('%s'%(run_calypso)) - - for pop in range(ii*int(popsize),(ii+1)*int(popsize)): - try: - os.mkdir('task.%03d'%pop) - except: - shutil.rmtree('task.%03d'%pop) - os.mkdir('task.%03d'%pop) - shutil.copyfile('run_opt.py',os.path.join('task.%03d'%pop,'run_opt.py')) - shutil.copyfile('check_outcar.py',os.path.join('task.%03d'%pop,'check_outcar.py')) - shutil.copyfile('POSCAR_%s'%str(pop-ii*int(popsize)+1),os.path.join('task.%03d'%(pop),'POSCAR')) - shutil.copyfile('input.dat',os.path.join('task.%03d'%pop,'input.dat')) - #for iii in range(1,popsize+1): - # shutil.copyfile('POSCAR_%s'%str(iii),os.path.join('task.%03d'%(iii-1),'POSCAR')) - - all_task = glob.glob( "task.*") - all_task.sort() - - run_tasks_ = all_task - - run_tasks = [os.path.basename(ii) for ii in run_tasks_] - - api_version = mdata.get('api_version', '0.9') - if LooseVersion(api_version) < LooseVersion('1.0'): - warnings.warn(f"the dpdispatcher will be updated to new version." - f"And the interface may be changed. Please check the documents for more details") - dispatcher=make_dispatcher(mdata['model_devi_machine'],mdata['model_devi_resources'],'./', run_tasks, model_devi_group_size) - dispatcher.run_jobs(mdata['model_devi_resources'], - commands, - './', - run_tasks, - model_devi_group_size, - model_names, - forward_files, - backward_files, - outlog = 'model_devi.log', - errlog = 'model_devi.log') - elif LooseVersion(api_version) >= LooseVersion('1.0'): - os.chdir(cwd) - submission = make_submission( - mdata['model_devi_machine'], - mdata['model_devi_resources'], - commands=commands, - work_path=calypso_run_opt_path, - run_tasks=run_tasks, - group_size=model_devi_group_size, - forward_common_files=model_names, - forward_files=forward_files, - backward_files=backward_files, - outlog = 'model_devi.log', - errlog = 'model_devi.log') - submission.run_submission() - os.chdir(calypso_run_opt_path) - - sstep = os.path.join('opt',str(ii)) - os.mkdir(sstep) - if not os.path.exists('traj'): - os.mkdir('traj') - - for jjj in range(ii*int(popsize),(ii+1)*int(popsize)): - # to opt directory - shutil.copyfile('POSCAR_%s'%str(jjj+1-ii*int(popsize)),os.path.join(sstep,'POSCAR_%s'%str(jjj+1-ii*int(popsize))),) - shutil.copyfile(os.path.join('task.%03d'%(jjj),'OUTCAR'),os.path.join(sstep,'OUTCAR_%s'%str(jjj+1-ii*int(popsize))),) - shutil.copyfile(os.path.join('task.%03d'%(jjj),'CONTCAR'),os.path.join(sstep,'CONTCAR_%s'%str(jjj+1-ii*int(popsize))),) - # to run calypso directory - shutil.copyfile(os.path.join('task.%03d'%(jjj),'OUTCAR'),'OUTCAR_%s'%str(jjj+1-ii*int(popsize)),) - shutil.copyfile(os.path.join('task.%03d'%(jjj),'CONTCAR'),'CONTCAR_%s'%str(jjj+1-ii*int(popsize)),) - # to traj - shutil.copyfile(os.path.join('task.%03d'%(jjj),'traj.traj'),os.path.join('traj','%s.traj'%str(jjj+1)),) - - if LooseVersion(api_version) < LooseVersion('1.0'): - os.rename('jr.json','jr_%s.json'%(str(ii))) - - tlist = glob.glob('task.*') - for t in tlist: - shutil.rmtree(t) - - else: - # -------------------------------------------------------------- - component = ['Mg','Al','Cu','MgAl','MgCu','AlCu','MgAlCu'] - - for idx,com in enumerate(component): - pwd = os.getcwd() - os.mkdir(str(idx)) - shutil.copyfile(os.path.join(cwd,'calypso_input','input.dat.%s'%com),os.path.join(str(idx),'input.dat')) - os.chdir(str(idx)) - os.system(run_calypso) - os.chdir(pwd) - - name_list = Path('.').glob('*/POSCAR_*') - for idx,name in enumerate(name_list): - shutil.copyfile(name,'POSCAR_%s'%(idx+1)) - try: - os.mkdir('task.%04d'%(idx+1)) - except: - shutil.rmtree('task.%04d'%(idx+1)) - os.mkdir('task.%04d'%(idx+1)) - shutil.copyfile('run_opt.py',os.path.join('task.%04d'%(idx+1),'run_opt.py')) - shutil.copyfile('check_outcar.py',os.path.join('task.%04d'%(idx+1),'check_outcar.py')) - shutil.copyfile('POSCAR_%s'%str(idx+1),os.path.join('task.%04d'%(idx+1),'POSCAR')) - shutil.copyfile('input.dat',os.path.join('task.%04d'%(idx+1),'input.dat')) - - all_task = glob.glob( "task.*") - all_task.sort() - - run_tasks_ = all_task - - run_tasks = [os.path.basename(ii) for ii in run_tasks_] - - if LooseVersion(api_version) < LooseVersion('1.0'): - warnings.warn(f"the dpdispatcher will be updated to new version." - f"And the interface may be changed. Please check the documents for more details") - dispatcher=make_dispatcher(mdata['model_devi_machine'],mdata['model_devi_resources'],'./', run_tasks, model_devi_group_size) - dispatcher.run_jobs(mdata['model_devi_resources'], - commands, - './', - run_tasks, - model_devi_group_size, - model_names, - forward_files, - backward_files, - outlog = 'model_devi.log', - errlog = 'model_devi.log') - elif LooseVersion(api_version) >= LooseVersion('1.0'): - os.chdir(cwd) - submission = make_submission( - mdata['model_devi_machine'], - mdata['model_devi_resources'], - commands=commands, - work_path=calypso_run_opt_path, - run_tasks=run_tasks, - group_size=model_devi_group_size, - forward_common_files=model_names, - forward_files=forward_files, - backward_files=backward_files, - outlog = 'model_devi.log', - errlog = 'model_devi.log') - submission.run_submission() - os.chdir(calypso_run_opt_path) - - os.mkdir('opt') - if not os.path.exists('traj'): - os.mkdir('traj') - for jjj in range(len(all_task)): - # to opt directory - shutil.copyfile('POSCAR_%s'%str(jjj+1),os.path.join('opt','POSCAR_%s'%str(jjj+1)),) - shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'OUTCAR'),os.path.join('opt','OUTCAR_%s'%str(jjj+1)),) - shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'CONTCAR'),os.path.join('opt','CONTCAR_%s'%str(jjj+1)),) - # to run calypso directory - shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'OUTCAR'),'OUTCAR_%s'%str(jjj+1),) - shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'CONTCAR'),'CONTCAR_%s'%str(jjj+1),) - # to traj - shutil.copyfile(os.path.join('task.%04d'%(jjj+1),'traj.traj'),os.path.join('traj','%s.traj'%str(jjj+1)),) - - tlist = glob.glob('task.*') - for t in tlist: - shutil.rmtree(t) - # -------------------------------------------------------------- - - os.chdir(cwd) - os.chdir(work_path) - f = open('record.calypso','a+') - f.write('2\n') - f.close() - os.chdir(cwd) - -def analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): - - # Analysis - - - dlog.info('$$$$$$$$$$$$$$$ Analysis Started $$$$$$$$$$$$$$$$$$') - - ms = dpdata.MultiSystems() - - cwd = os.getcwd() - iter_name = make_iter_name(iter_index) - work_path = os.path.join(iter_name, model_devi_name) - result_path = os.path.join(calypso_run_opt_path,'results') - - # trajs to be model devi - traj_path = os.path.join(calypso_run_opt_path,'traj') - traj_list = glob.glob(traj_path+'/*.traj') - - dlog.info('len(traj_list) %s'%str(len(traj_list))) - - # read confs from traj - record_traj_num = 0 - for traj_name in traj_list: - traj_num = os.path.basename(traj_name).split('.')[0] - trajs_origin = Trajectory(traj_name) - - record_traj_num += len(trajs_origin) - if len(trajs_origin) >= 20 : - trajs = [trajs_origin[iii] for iii in [4,9,-10,-5,-1]] - elif 5<=len(trajs_origin)<20: - trajs = [trajs_origin[random.randint(1,len(trajs_origin)-1)] for iii in range(4)] - trajs.append(trajs[-1]) - elif 3<= len(trajs_origin) <5: - trajs = [trajs_origin[round((len(trajs_origin)-1)/2)] ] - trajs.append(trajs[-1]) - elif len(trajs_origin) == 2: - trajs = [trajs_origin[0],trajs_origin[-1] ] - elif len(trajs_origin) == 1: - trajs = [trajs_origin[0] ] - else: - pass - - for idx, traj in enumerate(trajs): - write_vasp(os.path.join( - traj_path,'%03d.%03d.poscar' % ( - int(traj_num), int(idx) - ) - ), - traj) - - traj_pos_list = glob.glob(traj_path+'/*.poscar') - - #dlog.info('traj_num %s'%str(len(traj_pos_list))) - #dlog.info('total_traj_num %s'%str(record_traj_num)) - - for npos in traj_pos_list: - try: - ms.append(dpdata.System(npos, type_map = jdata['type_map'])) - except Exception as e: - dlog.info(npos,'failed : ',e) - - if len(ms) == 0: - print('too little confs, ') - return - - if os.path.exists(os.path.join(result_path,'deepmd')): - shutil.rmtree(os.path.join(result_path,'deepmd')) - ms.to_deepmd_raw(os.path.join(result_path,'deepmd')) - ms.to_deepmd_npy(os.path.join(result_path,'deepmd')) - - - split_lists = glob.glob(os.path.join(result_path,'deepmd','*')) - for i,split_list in enumerate(split_lists): - #ss = dpdata.System(split_list,fmt='deepmd') - #for j in range(ss.get_nframes()): - # ss.to('vasp/poscar',os.path.join(split_list,'%03d.%03d.poscar'%(i,j)),frame_idx=j) - strus_path = os.path.join(calypso_model_devi_path,'%03d.structures'%i) - if not os.path.exists(strus_path): - shutil.copytree(split_list,strus_path) - else: - shutil.rmtree(strus_path) - shutil.copytree(split_list,strus_path) - - os.chdir(cwd) - os.chdir(work_path) - f = open('record.calypso','a+') - f.write('3\n') - f.close() - os.chdir(cwd) - - -#def model_devi(iter_index,calypso_model_devi_path,all_models,jdata): -# -# # Model Devi -# -# -# dlog.info('$$$$$$$$$$$$$$$ Model Devi Start $$$$$$$$$$$$$$$$$$') -# -# iter_name = make_iter_name(iter_index) -# work_path = os.path.join(iter_name, model_devi_name) -# -# cwd = os.getcwd() -# Devis = [] -# pcount = 0 -# strus_lists = glob.glob(os.path.join(calypso_model_devi_path,'*.structures')) -# for num, strus_path in enumerate(strus_lists): -# os.chdir(strus_path) -# os.system('rename .vasp .poscar *') -# os.chdir(cwd) -# -# structure_list = glob.glob(os.path.join(strus_path,'*.poscar')) -# -# # every 500 confs in one task dir -# if len(structure_list) == 0: -# continue -# else: -# num_per_task = math.ceil(len(structure_list)/500) -# graphs = [DP(model) for model in all_models] -# for temp in range(num_per_task): -# task_name = os.path.join(calypso_model_devi_path,'task.%03d.%03d'%(num,temp)) -# put_poscar = os.path.join(task_name,'traj') -# if not os.path.exists(task_name): -# os.mkdir(task_name) -# os.mkdir(put_poscar) -# else: -# shutil.rmtree(task_name) -# os.mkdir(task_name) -# os.mkdir(put_poscar) -# devis = [] -# try: -# temp_sl = structure_list[temp*500:(temp+1)*500] -# except Exception as err: -# dlog.info('err %s'%str(err)) -# temp_sl = structure_list[temp*500:] -# -# new_index = 0 -# for index,sname in enumerate(temp_sl): -# shutil.copyfile(sname,os.path.join(put_poscar,'%s.poscar'%str(index))) -# os.chdir(put_poscar) -# pdata = dpdata.System('%s.poscar'%str(index),type_map = jdata['type_map']) -# nopbc = pdata.nopbc -# coord = pdata.data['coords'] -# cell = pdata.data['cells'] -# atom_types = pdata.data['atom_types'] -# devi = calc_model_devi(coord,cell,atom_types,graphs,nopbc=nopbc) -# # ------------------------------------------------------------------------------------ -# # append min-distance in devi list -# dis_temp = dpdata.System('%s.poscar'%str(index)) -# dis = dis_temp.to_ase_structure()[0].get_all_distances(mic=True) -# row,col = np.diag_indices_from(dis) -# dis[row,col] = 10000 -# min_dis = np.nanmin(dis) -# devi = np.append(devi[0],min_dis) -# t = [devi] -# devi = np.array(t) -# # ------------------------------------------------------------------------------------ -# temp_d = copy.deepcopy(devi) -# temp_D = copy.deepcopy(devi) -# devis.append(temp_d) -# Devis.append(temp_D) -# devis[index][0][0] = np.array(index) -# Devis[pcount][0][0] = np.array(pcount) -# pcount += 1 -# new_index += 1 -# os.chdir(cwd) -# os.chdir(task_name) -# devis = np.vstack(devis) -# write_model_devi_out(devis,'model_devi.out') -# os.chdir(cwd) -# -# os.chdir(calypso_model_devi_path) -# Devis = np.vstack(Devis) -# write_model_devi_out(Devis,'Model_Devi.out') -# os.chdir(cwd) -# -# os.chdir(work_path) -# f = open('record.calypso','a+') -# f.write('4\n') -# f.close() -# os.chdir(cwd) - From 68cd1c9f9d7139e77cf2b1d67fbd0a5fc307f500 Mon Sep 17 00:00:00 2001 From: zhenyu Date: Tue, 22 Mar 2022 15:52:48 +0800 Subject: [PATCH 24/46] remove some useless outputs --- dpgen/generator/lib/model_devi_calypso.py | 6 +++--- dpgen/generator/run.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dpgen/generator/lib/model_devi_calypso.py b/dpgen/generator/lib/model_devi_calypso.py index 21b6df4f4..be32454e3 100644 --- a/dpgen/generator/lib/model_devi_calypso.py +++ b/dpgen/generator/lib/model_devi_calypso.py @@ -94,7 +94,7 @@ def gen_structures(iter_index,jdata,mdata): for ii in range(int(PickUpStep)-1,maxstep+1): - dlog.info('$$$$$$$$$$$$$$$$$$$$ step %s $$$$$$$$$$$$$$$$$'%ii) + dlog.info('CALYPSO step %s'%ii) if ii == maxstep : while True: if len(glob.glob('OUTCAR_*')) == popsize: @@ -274,7 +274,7 @@ def analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): # Analysis - dlog.info('$$$$$$$$$$$$$$$ Analysis Started $$$$$$$$$$$$$$$$$$') + #dlog.info('$$$$$$$$$$$$$$$ Analysis Started $$$$$$$$$$$$$$$$$$') ms = dpdata.MultiSystems() @@ -287,7 +287,7 @@ def analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): traj_path = os.path.join(calypso_run_opt_path,'traj') traj_list = glob.glob(traj_path+'/*.traj') - dlog.info('len(traj_list) %s'%str(len(traj_list))) + #dlog.info('len(traj_list) %s'%str(len(traj_list))) # read confs from traj record_traj_num = 0 diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index 19280df92..9e71a12b1 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -1398,7 +1398,7 @@ def run_multi_model_devi (iter_index, jdata, mdata) : - dlog.info('############## run calypso model devi #######################') + dlog.info('start running CALYPSO') iter_name = make_iter_name(iter_index) @@ -1444,7 +1444,7 @@ def run_multi_model_devi (iter_index, os.chdir(cwd) elif lines[-1].strip().strip('\n') == '4': - dlog.info('Model Devi is done.') + #dlog.info('Model Devi is done.') break def post_model_devi (iter_index, From ff23f23d801c0d2039fc6e188c39e228693977ee Mon Sep 17 00:00:00 2001 From: zhenyu Date: Tue, 22 Mar 2022 16:08:20 +0800 Subject: [PATCH 25/46] restore dispatcher/PBS.py --- dpgen/dispatcher/PBS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpgen/dispatcher/PBS.py b/dpgen/dispatcher/PBS.py index d6362de2b..0fed5a888 100644 --- a/dpgen/dispatcher/PBS.py +++ b/dpgen/dispatcher/PBS.py @@ -94,7 +94,7 @@ def sub_script_head(self, res): if res['numb_gpu'] == 0: ret += '#PBS -l nodes=%d:ppn=%d\n' % (res['numb_node'], res['task_per_node']) else : - ret += '#PBS -l select=%d:ncpus=%d:ngpus=%d\n' % (res['numb_node'], res['task_per_node'], res['numb_gpu']) + ret += '#PBS -l nodes=%d:ppn=%d:gpus=%d\n' % (res['numb_node'], res['task_per_node'], res['numb_gpu']) ret += '#PBS -l walltime=%s\n' % (res['time_limit']) if res['mem_limit'] > 0 : ret += "#PBS -l mem=%dG \n" % res['mem_limit'] From 3796cd4b4d79e5a589f0521455371abd1b63f99e Mon Sep 17 00:00:00 2001 From: zhenyu Date: Wed, 23 Mar 2022 16:46:20 +0800 Subject: [PATCH 26/46] refactor: make some changes in calypso related code - rename model_devi_calypso.py run_calypso.py - rename calypso.py make_calypso.py - add new file calypso_check_outcar.py calypso_run_opt.py - add new file calypso_run_model_devi.py - delete write_modd.py - modify README.md and run.py --- README.md | 2 +- dpgen/generator/lib/calypso.py | 438 ------------------ dpgen/generator/lib/calypso_check_outcar.py | 107 +++++ dpgen/generator/lib/calypso_run_model_devi.py | 124 +++++ dpgen/generator/lib/calypso_run_opt.py | 176 +++++++ dpgen/generator/lib/make_calypso.py | 147 ++++++ .../{model_devi_calypso.py => run_calypso.py} | 96 +--- dpgen/generator/lib/write_modd.py | 127 ----- dpgen/generator/run.py | 24 +- 9 files changed, 571 insertions(+), 670 deletions(-) delete mode 100644 dpgen/generator/lib/calypso.py create mode 100644 dpgen/generator/lib/calypso_check_outcar.py create mode 100644 dpgen/generator/lib/calypso_run_model_devi.py create mode 100644 dpgen/generator/lib/calypso_run_opt.py create mode 100644 dpgen/generator/lib/make_calypso.py rename dpgen/generator/lib/{model_devi_calypso.py => run_calypso.py} (80%) delete mode 100644 dpgen/generator/lib/write_modd.py diff --git a/README.md b/README.md index 9bd5d056a..04e6aa004 100644 --- a/README.md +++ b/README.md @@ -378,7 +378,7 @@ In each iteration, there are three stages of work, namely, `00.train 01.model_d + 00.train: DP-GEN will train several (default 4) models based on initial and generated data. The only difference between these models is the random seed for neural network initialization. -+ 01.model_devi : represent for model-deviation. Model-deviation engine in `01.model_devi` can be chosen between Molecular Dynamics(default LAMMPS) or Structures Prediction(CALYPSO). DP-GEN will use models obtained from 00.train to run Molecular Dynamics or to run structure optimization with ASE in CALYPSO. Larger deviation for structure properties (default is force of atoms) means less accuracy of the models. Using this criterion, a few fructures will be selected and put into next stage `02.fp` for more accurate calculation based on First Principles. ++ 01.model_devi : represent for model-deviation. Model-deviation engine in `01.model_devi` can be chosen between Molecular Dynamics(LAMMPS and GROMACS) or Structures Prediction(CALYPSO). DP-GEN will use models obtained from 00.train to run Molecular Dynamics or to run structure optimization with ASE in CALYPSO. Larger deviation for structure properties (default is force of atoms) means less accuracy of the models. Using this criterion, a few structures will be selected and put into next stage `02.fp` for more accurate calculation based on First Principles. + 02.fp : Selected structures will be calculated by first principles methods(default VASP). DP-GEN will obtain some new data and put them together with initial data and data generated in previous iterations. After that a new training will be set up and DP-GEN will enter next iteration! diff --git a/dpgen/generator/lib/calypso.py b/dpgen/generator/lib/calypso.py deleted file mode 100644 index 56ece6bd7..000000000 --- a/dpgen/generator/lib/calypso.py +++ /dev/null @@ -1,438 +0,0 @@ -#!/usr/bin/env python3 -import os -import shutil -import json -import numpy as np - -def make_run_opt_script(fmax ) : - - ret ="import os,sys,glob,time \n" - ret+="import numpy as np \n" - ret+="from ase.io import read \n" - ret+="from ase.optimize import BFGS,QuasiNewton,LBFGS \n" - ret+="from ase.constraints import UnitCellFilter, ExpCellFilter\n" - ret+="from deepmd.calculator import DP \n" - ret+=" \n" - ret+="def Get_Element_Num(elements): \n" - ret+=" '''Using the Atoms.symples to Know Element&Num''' \n" - ret+=" element = [] \n" - ret+=" ele = {} \n" - ret+=" element.append(elements[0]) \n" - ret+=" for x in elements: \n" - ret+=" if x not in element : \n" - ret+=" element.append(x) \n" - ret+=" for x in element: \n" - ret+=" ele[x] = elements.count(x) \n" - ret+=" return element, ele \n" - ret+=" \n" - ret+="def Write_Contcar(element, ele, lat, pos): \n" - ret+=" '''Write CONTCAR''' \n" - ret+=" f = open('CONTCAR','w') \n" - ret+=" f.write('ASE-DPKit-Optimization\\n') \n" - ret+=" f.write('1.0\\n') \n" - ret+=" for i in range(3): \n" - ret+=" f.write('%15.10f %15.10f %15.10f\\n' % tuple(lat[i])) \n" - ret+=" for x in element: \n" - ret+=" f.write(x + ' ') \n" - ret+=" f.write('\\n') \n" - ret+=" for x in element: \n" - ret+=" f.write(str(ele[x]) + ' ') \n" - ret+=" f.write('\\n') \n" - ret+=" f.write('Direct\\n') \n" - ret+=" na = sum(ele.values()) \n" - ret+=" dpos = np.dot(pos,np.linalg.inv(lat)) \n" - ret+=" for i in range(na): \n" - ret+=" f.write('%15.10f %15.10f %15.10f\\n' % tuple(dpos[i])) \n" - ret+=" \n" - ret+="def Write_Outcar(element, ele, volume, lat, pos, ene, force, stress,pstress): \n" - ret+=" '''Write OUTCAR''' \n" - ret+=" f = open('OUTCAR','w') \n" - ret+=" for x in element: \n" - ret+=" f.write('VRHFIN =' + str(x) + '\\n') \n" - ret+=" f.write('ions per type =') \n" - ret+=" for x in element: \n" - ret+=" f.write('%5d' % ele[x]) \n" - ret+=" #f.write('\\nvolume of cell :\\n') \n" - ret+=" f.write('\\nDirection XX YY ZZ XY YZ ZX\\n') \n" - ret+=" f.write('in kB') \n" - ret+=" f.write('%15.6f' % stress[0]) \n" - ret+=" f.write('%15.6f' % stress[1]) \n" - ret+=" f.write('%15.6f' % stress[2]) \n" - ret+=" f.write('%15.6f' % stress[3]) \n" - ret+=" f.write('%15.6f' % stress[4]) \n" - ret+=" f.write('%15.6f' % stress[5]) \n" - ret+=" f.write('\\n') \n" - ret+=" ext_pressure = np.sum(stress[0] + stress[1] + stress[2])/3.0 - pstress \n" - ret+=" f.write('external pressure = %20.6f kB Pullay stress = %20.6f kB\\n'% (ext_pressure, pstress))\n" - ret+=" f.write('volume of cell : %20.6f\\n' % volume) \n" - ret+=" f.write('direct lattice vectors\\n') \n" - ret+=" for i in range(3): \n" - ret+=" f.write('%10.6f %10.6f %10.6f\\n' % tuple(lat[i])) \n" - ret+=" f.write('POSITION TOTAL-FORCE(eV/Angst)\\n') \n" - ret+=" f.write('-------------------------------------------------------------------\\n') \n" - ret+=" na = sum(ele.values()) \n" - ret+=" for i in range(na): \n" - ret+=" f.write('%15.6f %15.6f %15.6f' % tuple(pos[i])) \n" - ret+=" f.write('%15.6f %15.6f %15.6f\\n' % tuple(force[i])) \n" - ret+=" f.write('-------------------------------------------------------------------\\n') \n" - ret+=" f.write('energy without entropy= %20.6f %20.6f\\n' % (ene, ene/na)) \n" - ret+=" enthalpy = ene + pstress * volume / 1602.17733 \n" - ret+=" f.write('enthalpy is TOTEN = %20.6f %20.6f\\n' % (enthalpy, enthalpy/na)) \n" - ret+=" \n" - ret+="def Write_calylog(templog): \n" - ret+=" '''For Write Evolve Structures Log into caly.log''' \n" - ret+=" f = open('opt.log','a+') \n" - ret+=" f.write(templog+'\\n') \n" - ret+=" f.close() \n" - ret+="def read_stress():\n" - ret+=" pstress = 0\n" - ret+=" #assert os.path.exists('./input.dat'), 'input.dat does not exist!'\n" - ret+=" try:\n" - ret+=" f = open('input.dat','r')\n" - ret+=" except:\n" - ret+=" assert os.path.exists('../input.dat'),' now we are in %s, do not find ../input.dat'%(os.getcwd())\n" - ret+=" f = open('../input.dat','r')\n" - ret+=" lines = f.readlines()\n" - ret+=" f.close()\n" - ret+=" for line in lines:\n" - ret+=" if line[0] == '#':\n" - ret+=" continue\n" - ret+=" if 'PSTRESS' in line:\n" - ret+=" pstress = float(line.split('=')[1])\n" - ret+=" return pstress\n" - ret+=" \n" - ret+=" \n" - ret+="def run_opt(stress): \n" - ret+=" '''Using the ASE&DP to Optimize Configures''' \n" - ret+=" ''' > 600 Steps Called Failure Optimization''' \n" - ret+=" \n" - ret+=" os.system('mv OUTCAR OUTCAR-last')\n" - ret+=" is_dpgen = True\n" - ret+=" nn = 1\n" - ret+=" try:\n" - ret+=" model_path = sys.argv[1] \n" - ret+=" Model_List = glob.glob('%s/graph*pb'%model_path) \n" - ret+=" calc = DP(model=Model_List[0]) # init the model before iteration \n" - ret+=" except:\n" - ret+=" assert os.path.exists('graph.pb'), 'did not found graph.pb in this directory %s, or did you forget to add args?'%(os.getcwd())\n" - ret+=" calc = DP(model='graph.pb') # init the model before iteration \n" - ret+=" is_dpgen = False\n" - ret+=" nn = 3\n" - ret+=" \n" - ret+=" print('Start to Optimize Structures by DP----------') \n" - ret+=" \n" - ret+=" Opt_Step = 200 \n" - ret+=" start = time.time() \n" - ret+=" # pstress kbar\n" - ret+=" pstress = stress\n" - ret+=" # kBar to eV/A^3\n" - ret+=" # 1 eV/A^3 = 160.21766028 GPa\n" - ret+=" # 1 / 160.21766028 ~ 0.006242\n" - ret+=" aim_stress = 1.0 * pstress* 0.01 * 0.6242 / 10.0 \n" - ret+=" to_be_opti = read('POSCAR') \n" - ret+=" to_be_opti.calc = calc \n" - ret+=" ucf = UnitCellFilter(to_be_opti, scalar_pressure=aim_stress)\n" - ret+=" atoms_vol_2 = to_be_opti.get_volume() \n" - ret+=" for i in range(nn): \n" - ret+=" if is_dpgen:\n" - ret+=" opt = LBFGS(ucf,trajectory='traj.traj') \n" - ret+=" else:\n" - ret+=" opt = LBFGS(ucf) \n" - ret+=" #opt = QuasiNewton(to_be_opti) \n" - ret+=" #opt = BFGS(to_be_opti) \n" - ret+=" #opt = BFGS(to_be_opti,trajectory='traj.traj',logfile='opt.log') \n" - ret+=" opt.run(fmax=%s,steps=200) \n"%str(fmax) - ret+=" \n" - ret+=" atoms_lat = to_be_opti.cell \n" - ret+=" atoms_pos = to_be_opti.positions \n" - ret+=" atoms_force = to_be_opti.get_forces() \n" - ret+=" atoms_stress = to_be_opti.get_stress() \n" - ret+=" # eV/A^3 to GPa\n" - ret+=" atoms_stress = atoms_stress/(0.01*0.6242)\n" - ret+=" #atoms_num = to_be_opti.get_atomic_numbers() \n" - ret+=" atoms_symbols = to_be_opti.get_chemical_symbols() \n" - ret+=" #atoms_formula = to_be_opti.get_chemical_formula() \n" - ret+=" atoms_ene = to_be_opti.get_potential_energy() \n" - ret+=" atoms_vol = to_be_opti.get_volume() \n" - ret+=" \n" - ret+=" element, ele = Get_Element_Num(atoms_symbols) \n" - ret+=" \n" - ret+=" Write_Contcar(element, ele, atoms_lat, atoms_pos) \n" - ret+=" Write_Outcar(element, ele, atoms_vol, atoms_lat, atoms_pos,atoms_ene, atoms_force, atoms_stress * -10.0, pstress) \n" - ret+=" \n" - ret+=" \n" - ret+=" stop = time.time() \n" - ret+=" _cwd = os.getcwd() \n" - ret+=" _cwd = os.path.basename(_cwd) \n" - ret+=" print('%s is done, time: %s' % (_cwd,stop-start)) \n" - ret+="stress = read_stress()\n" - ret+="run_opt(stress) \n" - return ret - - -def make_check_outcar_script( ) : - - ret = "import numpy as np \n" - ret+= "import os,sys,glob,time \n" - ret+= "from deepmd.calculator import DP \n" - ret+= "from ase.io import read \n" - ret+= " \n" - ret+= "def Get_Element_Num(elements): \n" - ret+= " '''Using the Atoms.symples to Know Element&Num''' \n" - ret+= " element = [] \n" - ret+= " ele = {} \n" - ret+= " element.append(elements[0]) \n" - ret+= " for x in elements: \n" - ret+= " if x not in element : \n" - ret+= " element.append(x) \n" - ret+= " for x in element: \n" - ret+= " ele[x] = elements.count(x) \n" - ret+= " return element, ele \n" - ret+= " \n" - ret+= "def Write_Contcar(element, ele, lat, pos): \n" - ret+= " '''Write CONTCAR''' \n" - ret+= " f = open('CONTCAR','w') \n" - ret+= " f.write('ASE-DPKit-FAILED-nan\\n') \n" - ret+= " f.write('1.0\\n') \n" - ret+= " for i in range(3): \n" - ret+= " f.write('%15.10f %15.10f %15.10f\\n' % tuple(lat[i])) \n" - ret+= " for x in element: \n" - ret+= " f.write(x + ' ') \n" - ret+= " f.write('\\n') \n" - ret+= " for x in element: \n" - ret+= " f.write(str(ele[x]) + ' ') \n" - ret+= " f.write('\\n') \n" - ret+= " f.write('Direct\\n') \n" - ret+= " na = sum(ele.values()) \n" - ret+= " dpos = np.dot(pos,np.linalg.inv(lat)) \n" - ret+= " for i in range(na): \n" - ret+= " f.write('%15.10f %15.10f %15.10f\\n' % tuple(dpos[i])) \n" - ret+= " \n" - ret+= "def Write_Outcar(element, ele, volume, lat, pos, ene, force, stress,pstress): \n" - ret+= " '''Write OUTCAR''' \n" - ret+= " f = open('OUTCAR','w') \n" - ret+= " for x in element: \n" - ret+= " f.write('VRHFIN =' + str(x) + '\\n') \n" - ret+= " f.write('ions per type =') \n" - ret+= " for x in element: \n" - ret+= " f.write('%5d' % ele[x]) \n" - ret+= " #f.write('\\nvolume of cell :\\n') \n" - ret+= " f.write('\\nDirection XX YY ZZ XY YZ ZX\\n') \n" - ret+= " f.write('in kB') \n" - ret+= " f.write('%15.6f' % stress[0]) \n" - ret+= " f.write('%15.6f' % stress[1]) \n" - ret+= " f.write('%15.6f' % stress[2]) \n" - ret+= " f.write('%15.6f' % stress[3]) \n" - ret+= " f.write('%15.6f' % stress[4]) \n" - ret+= " f.write('%15.6f' % stress[5]) \n" - ret+= " f.write('\\n') \n" - ret+= " ext_pressure = np.sum(stress[0] + stress[1] + stress[2])/3.0 - pstress \n" - ret+= " f.write('external pressure = %20.6f kB Pullay stress = %20.6f kB\\n'% (ext_pressure, pstress))\n" - ret+= " f.write('volume of cell : %20.6f\\n' % volume) \n" - ret+= " f.write('direct lattice vectors\\n') \n" - ret+= " for i in range(3): \n" - ret+= " f.write('%10.6f %10.6f %10.6f\\n' % tuple(lat[i])) \n" - ret+= " f.write('POSITION TOTAL-FORCE(eV/Angst)\\n') \n" - ret+= " f.write('-------------------------------------------------------------------\\n') \n" - ret+= " na = sum(ele.values()) \n" - ret+= " for i in range(na): \n" - ret+= " f.write('%15.6f %15.6f %15.6f' % tuple(pos[i])) \n" - ret+= " f.write('%15.6f %15.6f %15.6f\\n' % tuple(force[i])) \n" - ret+= " f.write('-------------------------------------------------------------------\\n') \n" - ret+= " f.write('energy without entropy= %20.6f %20.6f\\n' % (ene, ene)) \n" - ret+= " enthalpy = ene + pstress * volume / 1602.17733 \n" - ret+= " f.write('enthalpy is TOTEN = %20.6f %20.6f\\n' % (enthalpy, enthalpy)) \n" - ret+= " \n" - ret+= "def check(): \n" - ret+= " \n" - ret+= " from deepmd.calculator import DP \n" - ret+= " from ase.io import read \n" - ret+= " model_path = sys.argv[1] \n" - ret+= " Model_List = glob.glob('%s/graph*pb'%model_path) \n" - ret+= " calc = DP(model='%s'%(Model_List[0])) # init the model before iteration \n" - ret+= " \n" - ret+= " to_be_opti = read('POSCAR') \n" - ret+= " to_be_opti.calc = calc \n" - ret+= " # --------------------------------- \n" - ret+= " # for failed outcar \n" - ret+= " atoms_symbols_f = to_be_opti.get_chemical_symbols() \n" - ret+= " element_f, ele_f = Get_Element_Num(atoms_symbols_f) \n" - ret+= " atoms_vol_f = to_be_opti.get_volume() \n" - ret+= " atoms_stress_f = to_be_opti.get_stress() \n" - ret+= " atoms_stress_f = atoms_stress_f/(0.01*0.6242) \n" - ret+= " atoms_lat_f = to_be_opti.cell \n" - ret+= " atoms_pos_f = to_be_opti.positions \n" - ret+= " atoms_force_f = to_be_opti.get_forces() \n" - ret+= " atoms_ene_f = 610612509 \n" - ret+= " # --------------------------------- \n" - ret+= " Write_Contcar(element_f, ele_f, atoms_lat_f, atoms_pos_f) \n" - ret+= " Write_Outcar(element_f, ele_f, atoms_vol_f, atoms_lat_f, atoms_pos_f,atoms_ene_f, atoms_force_f, atoms_stress_f * -10.0, 0) \n" - ret+= " \n" - ret+= "cwd = os.getcwd() \n" - ret+= "if not os.path.exists(os.path.join(cwd,'OUTCAR')): \n" - ret+= " check() \n" - return ret - - -def make_calypso_input(nameofatoms,numberofatoms, - numberofformula,volume, - distanceofion,psoratio,popsize, - maxstep,icode,split,vsc, - maxnumatom,ctrlrange,pstress,fmax): - ret = "################################ The Basic Parameters of CALYPSO ################################\n" - ret+= "# A string of one or several words contain a descriptive name of the system (max. 40 characters).\n" - assert (nameofatoms != None ) - ret+= "SystemName = %s\n"%(''.join(nameofatoms)) - ret+= "# Number of different atomic species in the simulation.\n" - ret+= "NumberOfSpecies = %d\n"%(len(nameofatoms)) - ret+= "# Element symbols of the different chemical species.\n" - ret+= "NameOfAtoms = %s\n"%(' '.join(nameofatoms)) - ret+= "# Number of atoms for each chemical species in one formula unit. \n" - assert numberofatoms != None or len(numberofatoms) == len(nameofatoms) - ret+= "NumberOfAtoms = %s\n"%(' '.join(list(map(str,numberofatoms)))) - ret+= "# The range of formula unit per cell in your simulation. \n" - assert numberofformula != None or len(numberofformula) == 2 or type(numberofformula) == type([1,2]) - ret+= "NumberOfFormula = %s\n"%(' '.join(list(map(str,numberofformula)))) - ret+= "# The volume per formula unit. Unit is in angstrom^3.\n" - ret+= "Volume = %s\n"%(volume[0]) - ret+= "# Minimal distance between atoms of each chemical species. Unit is in angstrom.\n" - assert len(distanceofion) == len(nameofatoms) #"check distance of ions and the number of atoms" - assert len(distanceofion[0]) == len(nameofatoms) - ret+= "@DistanceOfIon \n" - for temp in distanceofion: - ret+="%4s \n"%(' '.join(list(map(str,temp)))) - ret+= "@End\n" - ret+= "# It determines which algorithm should be adopted in the simulation.\n" - ret+= "Ialgo = 2\n" - ret+= "# Ialgo = 1 for Global PSO\n" - ret+= "# Ialgo = 2 for Local PSO (default value)\n" - ret+= "# The proportion of the structures generated by PSO.\n" - assert (0 <= psoratio[0] <= 1 ) - ret+= "PsoRatio = %s\n"%(psoratio[0]) - ret+= "# The population size. Normally, it has a larger number for larger systems.\n" - assert popsize[0] != None or popsize!=None or type(popsize) == type([0,1]) or type(popsize[0]) == type(0) - ret+= "PopSize = %d\n"%(popsize[0]) - assert maxstep[0] != None or maxstep!=None or type(maxstep) == type([0,1]) or type(maxstep[0]) == type(0) - ret+= "# The Max step for iteration\n" - ret+= "MaxStep = %d\n"%(maxstep[0]) - ret+= "#It determines which method should be adopted in generation the random structure. \n" - ret+= "GenType= 1 \n" - ret+= "# 1 under symmetric constraints\n" - ret+= "# 2 grid method for large system\n" - ret+= "# 3 and 4 core grow method \n" - ret+= "# 0 combination of all method\n" - ret+= "# If GenType=3 or 4, it determined the small unit to grow the whole structure\n" - ret+= "# It determines which local optimization method should be interfaced in the simulation.\n" - assert icode != None or type(icode) == type([0,1]) or type(icode[0]) == type(0) - ret+= "ICode= %d\n"%(icode[0]) - ret+= "# ICode= 1 interfaced with VASP\n" - ret+= "# ICode= 2 interfaced with SIESTA\n" - ret+= "# ICode= 3 interfaced with GULP\n" - ret+= "# The number of lbest for local PSO\n" - ret+= "NumberOfLbest=4\n" - ret+= "# The Number of local optimization for each structure.\n" - ret+= "NumberOfLocalOptim= 3\n" - ret+= "# The command to perform local optimiztion calculation (e.g., VASP, SIESTA) on your computer.\n" - ret+= "Command = sh submit.sh\n" - ret+= "MaxTime = 9000 \n" - ret+= "# If True, a previous calculation will be continued.\n" - ret+= "PickUp = F\n" - ret+= "# At which step will the previous calculation be picked up.\n" - ret+= "PickStep = 1\n" - ret+= "# If True, the local optimizations performed by parallel\n" - ret+= "Parallel = F\n" - ret+= "# The number node for parallel \n" - ret+= "NumberOfParallel = 4\n" - assert split != None - ret+= "Split = %s\n"%(split) - assert pstress != None or type(pstress) == type([200]) - ret+= "PSTRESS = %f\n"%(pstress[0]) - assert fmax != None or type(fmax) == type([200]) - ret+= "fmax = %f\n"%(fmax[0]) - ret+= "################################ End of The Basic Parameters of CALYPSO #######################\n" - if vsc == 'T': - assert len(ctrlrange) == len(nameofatoms) #'check distance of ions and the number of atoms' - ret+= "##### The Parameters For Variational Stoichiometry ##############\n" - ret+= "## If True, Variational Stoichiometry structure prediction is performed\n" - ret+= "VSC = %s\n"%(vsc) - ret+= "# The Max Number of Atoms in unit cell\n" - ret+= "MaxNumAtom = %s\n"%(maxnumatom[0]) - ret+= "# The Variation Range for each type atom \n" - ret+= "@CtrlRange\n" - for ttemp in ctrlrange: - ret+="%4s \n"%(' '.join(list(map(str,ttemp)))) - ret+= "@end\n" - ret+= "###################End Parameters for VSC ##########################\n" - return ret - -def _make_model_devi_buffet(jdata,calypso_run_opt_path): - calypso_input_path = jdata.get('calypso_input_path') - shutil.copyfile(os.path.join(calypso_input_path,'input.dat'),os.path.join(calypso_run_opt_path, 'input.dat')) - # run opt script - run_opt_script = os.path.join(calypso_run_opt_path,'run_opt.py') - with open(run_opt_script,'w') as ffff: - ffff.write(make_run_opt_script(jdata.get('fmax',0.01))) - -def _make_model_devi_native_calypso(iter_index,model_devi_jobs, calypso_run_opt_path): - - for iiidx, jobbs in enumerate(model_devi_jobs): - if iter_index in jobbs.get('times'): - cur_job = model_devi_jobs[iiidx] - - work_path = os.path.dirname(calypso_run_opt_path) - with open(os.path.join(work_path, 'cur_job.json'), 'w') as outfile: - json.dump(cur_job, outfile, indent = 4) - - run_opt_script = os.path.join(calypso_run_opt_path,'run_opt.py') - with open(run_opt_script,'w') as ffff: - ffff.write(make_run_opt_script(cur_job.get('fmax',0.01))) - - # Crystal Parameters - nameofatoms = cur_job.get('NameOfAtoms') - numberofatoms = cur_job.get('NumberOfAtoms') - numberofformula = cur_job.get('NumberOfFormula',[1,1]) - volume = cur_job.get('Volume') - distanceofion = cur_job.get('DistanceOfIon') - psoratio = cur_job.get('PsoRatio') - popsize = cur_job.get('PopSize') - maxstep = cur_job.get('MaxStep') - icode = cur_job.get('ICode',[1]) - split = cur_job.get('Split','T') - # VSC Control - maxnumatom = None - ctrlrange = None - vsc = cur_job.get('VSC','F') - if vsc == 'T': - maxnumatom = cur_job.get('MaxNumAtom') - ctrlrange = cur_job.get('CtrlRange') - - # Optimization - pstress = cur_job.get('PSTRESS',[0.001]) - fmax = cur_job.get('fmax',[0.01]) - - # Cluster - - # 2D - - file_c = make_calypso_input(nameofatoms,numberofatoms, - numberofformula,volume, - distanceofion,psoratio,popsize, - maxstep,icode,split,vsc, - maxnumatom,ctrlrange,pstress,fmax) - with open(os.path.join(calypso_run_opt_path, 'input.dat'), 'w') as cin : - cin.write(file_c) - -def write_model_devi_out(devi, fname): - assert devi.shape[1] == 8 - #assert devi.shape[1] == 7 - header = "%5s" % "step" - for item in 'vf': - header += "%16s%16s%16s" % (f"max_devi_{item}", f"min_devi_{item}",f"avg_devi_{item}") - header += "%16s"%str('min_dis') - np.savetxt(fname, - devi, - fmt=['%5d'] + ['%17.6e' for _ in range(7)], - delimiter='', - header=header) - return devi - diff --git a/dpgen/generator/lib/calypso_check_outcar.py b/dpgen/generator/lib/calypso_check_outcar.py new file mode 100644 index 000000000..c07a00590 --- /dev/null +++ b/dpgen/generator/lib/calypso_check_outcar.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import numpy as np +import os,sys,glob,time +from deepmd.calculator import DP +from ase.io import read + +''' +check if structure optimization worked well +if not, this script will generate a fake outcar +''' + +def Get_Element_Num(elements): + '''Using the Atoms.symples to Know Element&Num''' + element = [] + ele = {} + element.append(elements[0]) + for x in elements: + if x not in element : + element.append(x) + for x in element: + ele[x] = elements.count(x) + return element, ele + +def Write_Contcar(element, ele, lat, pos): + '''Write CONTCAR''' + f = open('CONTCAR','w') + f.write('ASE-DPKit-FAILED-nan\n') + f.write('1.0\n') + for i in range(3): + f.write('%15.10f %15.10f %15.10f\n' % tuple(lat[i])) + for x in element: + f.write(x + ' ') + f.write('\n') + for x in element: + f.write(str(ele[x]) + ' ') + f.write('\n') + f.write('Direct\n') + na = sum(ele.values()) + dpos = np.dot(pos,np.linalg.inv(lat)) + for i in range(na): + f.write('%15.10f %15.10f %15.10f\n' % tuple(dpos[i])) + +def Write_Outcar(element, ele, volume, lat, pos, ene, force, stress,pstress): + '''Write OUTCAR''' + f = open('OUTCAR','w') + for x in element: + f.write('VRHFIN =' + str(x) + '\n') + f.write('ions per type =') + for x in element: + f.write('%5d' % ele[x]) + #f.write('\nvolume of cell :\n') + f.write('\nDirection XX YY ZZ XY YZ ZX\n') + f.write('in kB') + f.write('%15.6f' % stress[0]) + f.write('%15.6f' % stress[1]) + f.write('%15.6f' % stress[2]) + f.write('%15.6f' % stress[3]) + f.write('%15.6f' % stress[4]) + f.write('%15.6f' % stress[5]) + f.write('\n') + ext_pressure = np.sum(stress[0] + stress[1] + stress[2])/3.0 - pstress + f.write('external pressure = %20.6f kB Pullay stress = %20.6f kB\n'% (ext_pressure, pstress)) + f.write('volume of cell : %20.6f\n' % volume) + f.write('direct lattice vectors\n') + for i in range(3): + f.write('%10.6f %10.6f %10.6f\n' % tuple(lat[i])) + f.write('POSITION TOTAL-FORCE(eV/Angst)\n') + f.write('-------------------------------------------------------------------\n') + na = sum(ele.values()) + for i in range(na): + f.write('%15.6f %15.6f %15.6f' % tuple(pos[i])) + f.write('%15.6f %15.6f %15.6f\n' % tuple(force[i])) + f.write('-------------------------------------------------------------------\n') + f.write('energy without entropy= %20.6f %20.6f\n' % (ene, ene)) + enthalpy = ene + pstress * volume / 1602.17733 + f.write('enthalpy is TOTEN = %20.6f %20.6f\n' % (enthalpy, enthalpy)) + +def check(): + + from deepmd.calculator import DP + from ase.io import read + model_path = sys.argv[1] + Model_List = glob.glob('%s/graph*pb'%model_path) + calc = DP(model='%s'%(Model_List[0])) # init the model before iteration + + to_be_opti = read('POSCAR') + to_be_opti.calc = calc + # --------------------------------- + # for failed outcar + atoms_symbols_f = to_be_opti.get_chemical_symbols() + element_f, ele_f = Get_Element_Num(atoms_symbols_f) + atoms_vol_f = to_be_opti.get_volume() + atoms_stress_f = to_be_opti.get_stress() + atoms_stress_f = atoms_stress_f/(0.01*0.6242) + atoms_lat_f = to_be_opti.cell + atoms_pos_f = to_be_opti.positions + atoms_force_f = to_be_opti.get_forces() + atoms_ene_f = 610612509 + # --------------------------------- + Write_Contcar(element_f, ele_f, atoms_lat_f, atoms_pos_f) + Write_Outcar(element_f, ele_f, atoms_vol_f, atoms_lat_f, atoms_pos_f,atoms_ene_f, atoms_force_f, atoms_stress_f * -10.0, 0) + +cwd = os.getcwd() +if not os.path.exists(os.path.join(cwd,'OUTCAR')): + check() diff --git a/dpgen/generator/lib/calypso_run_model_devi.py b/dpgen/generator/lib/calypso_run_model_devi.py new file mode 100644 index 000000000..c4d5b5f27 --- /dev/null +++ b/dpgen/generator/lib/calypso_run_model_devi.py @@ -0,0 +1,124 @@ +import argparse +import copy +import dpdata +import glob +import math +import numpy as np +import os +import sys +import shutil +from deepmd.infer import calc_model_devi +from deepmd.infer import DeepPot as DP + + +def write_model_devi_out(devi, fname): + assert devi.shape[1] == 8 + #assert devi.shape[1] == 7 + header = '%5s' % 'step' + for item in 'vf': + header += '%16s%16s%16s' % (f'max_devi_{item}', f'min_devi_{item}',f'avg_devi_{item}') + header += '%16s'%str('min_dis') + np.savetxt(fname, + devi, + fmt=['%5d'] + ['%17.6e' for _ in range(7)], + delimiter='', + header=header) + return devi + + +def Modd(all_models,type_map): + + # Model Devi + + cwd = os.getcwd() + graphs = [DP(model) for model in all_models] + + Devis = [] + pcount = 0 + strus_lists = glob.glob(os.path.join(cwd,'*.structures')) + for num, strus_path in enumerate(strus_lists): + + structures_data = dpdata.System(strus_path,'deepmd/npy',type_map=type_map) + + # every 500 confs in one task dir + nnum = structures_data.get_nframes() + if nnum == 0: + continue + else: + num_per_task = math.ceil(nnum/500) + + + for temp in range(num_per_task): + task_name = os.path.join(cwd,'task.%03d.%03d'%(num,temp)) + put_poscar = os.path.join(task_name,'traj') + if not os.path.exists(task_name): + os.mkdir(task_name) + os.mkdir(put_poscar) + else: + shutil.rmtree(task_name) + os.mkdir(task_name) + os.mkdir(put_poscar) + devis = [] + if (nnum - (temp+1)*500) >= 0: + temp_sl = range(temp*500,(temp+1)*500) + else: + temp_sl = range(temp*500,nnum) + + new_index = 0 + for index,frameid in enumerate(temp_sl): + pdata = structures_data[frameid] + pdata.to_vasp_poscar(os.path.join(put_poscar,'%s.poscar'%str(index))) + nopbc = pdata.nopbc + coord = pdata.data['coords'] + cell = pdata.data['cells'] + atom_types = pdata.data['atom_types'] + devi = calc_model_devi(coord,cell,atom_types,graphs,nopbc=nopbc) + # ------------------------------------------------------------------------------------ + # append min-distance in devi list + dis = pdata.to_ase_structure()[0].get_all_distances(mic=True) + row,col = np.diag_indices_from(dis) + dis[row,col] = 10000 + min_dis = np.nanmin(dis) + devi = np.append(devi[0],min_dis) + t = [devi] + devi = np.array(t) + # ------------------------------------------------------------------------------------ + temp_d = copy.deepcopy(devi) + temp_D = copy.deepcopy(devi) + devis.append(temp_d) + Devis.append(temp_D) + devis[index][0][0] = np.array(index) + Devis[pcount][0][0] = np.array(pcount) + pcount += 1 + new_index += 1 + devis = np.vstack(devis) + write_model_devi_out(devis,os.path.join(task_name, 'model_devi.out')) + + Devis = np.vstack(Devis) + write_model_devi_out(Devis,os.path.join(cwd,'Model_Devi.out')) + + f = open(os.path.join(os.path.abspath(os.path.join(cwd,os.pardir)),'record.calypso'),'a+') + f.write('4') + f.close() + +if __name__ == '__main__': + + cwd = os.getcwd() + model_path = os.path.join(os.path.abspath(os.path.join(cwd,os.pardir)),'gen_stru_analy') + parser = argparse.ArgumentParser(description='calc model-devi by `all_models` and `type_map`') + parser.add_argument( + '--all_models', + type=str, + nargs='+', + default=model_path, + help='the path of models which will be used to do model-deviation', + ) + parser.add_argument( + '--type_map', + nargs='+', + help='the type map of models which will be used to do model-deviation', + ) + args = parser.parse_args() + #print(vars(args)) + Modd(args.all_models,args.type_map) + #Modd(sys.argv[1],sys.argv[2]) diff --git a/dpgen/generator/lib/calypso_run_opt.py b/dpgen/generator/lib/calypso_run_opt.py new file mode 100644 index 000000000..7f6b6d58c --- /dev/null +++ b/dpgen/generator/lib/calypso_run_opt.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import os,sys,glob,time +import numpy as np +from ase.io import read +from ase.optimize import BFGS,QuasiNewton,LBFGS +from ase.constraints import UnitCellFilter, ExpCellFilter +from deepmd.calculator import DP +''' +structure optimization with DP model and ASE +PSTRESS and fmax should exist in input.dat +''' + +def Get_Element_Num(elements): + '''Using the Atoms.symples to Know Element&Num''' + element = [] + ele = {} + element.append(elements[0]) + for x in elements: + if x not in element : + element.append(x) + for x in element: + ele[x] = elements.count(x) + return element, ele + +def Write_Contcar(element, ele, lat, pos): + '''Write CONTCAR''' + f = open('CONTCAR','w') + f.write('ASE-DPKit-Optimization\n') + f.write('1.0\n') + for i in range(3): + f.write('%15.10f %15.10f %15.10f\n' % tuple(lat[i])) + for x in element: + f.write(x + ' ') + f.write('\n') + for x in element: + f.write(str(ele[x]) + ' ') + f.write('\n') + f.write('Direct\n') + na = sum(ele.values()) + dpos = np.dot(pos,np.linalg.inv(lat)) + for i in range(na): + f.write('%15.10f %15.10f %15.10f\n' % tuple(dpos[i])) + +def Write_Outcar(element, ele, volume, lat, pos, ene, force, stress,pstress): + '''Write OUTCAR''' + f = open('OUTCAR','w') + for x in element: + f.write('VRHFIN =' + str(x) + '\n') + f.write('ions per type =') + for x in element: + f.write('%5d' % ele[x]) + #f.write('\nvolume of cell :\n') + f.write('\nDirection XX YY ZZ XY YZ ZX\n') + f.write('in kB') + f.write('%15.6f' % stress[0]) + f.write('%15.6f' % stress[1]) + f.write('%15.6f' % stress[2]) + f.write('%15.6f' % stress[3]) + f.write('%15.6f' % stress[4]) + f.write('%15.6f' % stress[5]) + f.write('\n') + ext_pressure = np.sum(stress[0] + stress[1] + stress[2])/3.0 - pstress + f.write('external pressure = %20.6f kB Pullay stress = %20.6f kB\n'% (ext_pressure, pstress)) + f.write('volume of cell : %20.6f\n' % volume) + f.write('direct lattice vectors\n') + for i in range(3): + f.write('%10.6f %10.6f %10.6f\n' % tuple(lat[i])) + f.write('POSITION TOTAL-FORCE(eV/Angst)\n') + f.write('-------------------------------------------------------------------\n') + na = sum(ele.values()) + for i in range(na): + f.write('%15.6f %15.6f %15.6f' % tuple(pos[i])) + f.write('%15.6f %15.6f %15.6f\n' % tuple(force[i])) + f.write('-------------------------------------------------------------------\n') + f.write('energy without entropy= %20.6f %20.6f\n' % (ene, ene/na)) + enthalpy = ene + pstress * volume / 1602.17733 + f.write('enthalpy is TOTEN = %20.6f %20.6f\n' % (enthalpy, enthalpy/na)) + +def Write_calylog(templog): + '''For Write Evolve Structures Log into caly.log''' + f = open('opt.log','a+') + f.write(templog+'\n') + f.close() +def read_stress_fmax(): + pstress = 0 + fmax = 0.01 + #assert os.path.exists('./input.dat'), 'input.dat does not exist!' + try: + f = open('input.dat','r') + except: + assert os.path.exists('../input.dat'),' now we are in %s, do not find ../input.dat'%(os.getcwd()) + f = open('../input.dat','r') + lines = f.readlines() + f.close() + for line in lines: + if line[0] == '#': + continue + if 'PSTRESS' in line or 'pstress' in line: + pstress = float(line.split('=')[1]) + elif 'fmax' in line: + fmax = float(line.split('=')[1]) + return fmax,pstress + + +def run_opt(fmax,stress): + '''Using the ASE&DP to Optimize Configures''' + ''' > 600 Steps Called Failure Optimization''' + + os.system('mv OUTCAR OUTCAR-last') + is_dpgen = True + nn = 1 + try: + model_path = sys.argv[1] + Model_List = glob.glob('%s/graph*pb'%model_path) + calc = DP(model=Model_List[0]) # init the model before iteration + except: + assert os.path.exists('graph.pb'), 'did not found graph.pb in this directory %s, or did you forget to add args?'%(os.getcwd()) + calc = DP(model='graph.pb') # init the model before iteration + is_dpgen = False + nn = 3 + + print('Start to Optimize Structures by DP----------') + + Opt_Step = 200 + start = time.time() + # pstress kbar + pstress = stress + # kBar to eV/A^3 + # 1 eV/A^3 = 160.21766028 GPa + # 1 / 160.21766028 ~ 0.006242 + aim_stress = 1.0 * pstress* 0.01 * 0.6242 / 10.0 + to_be_opti = read('POSCAR') + to_be_opti.calc = calc + ucf = UnitCellFilter(to_be_opti, scalar_pressure=aim_stress) + atoms_vol_2 = to_be_opti.get_volume() + for i in range(nn): + if is_dpgen: + opt = LBFGS(ucf,trajectory='traj.traj') + else: + opt = LBFGS(ucf) + #opt = QuasiNewton(to_be_opti) + #opt = BFGS(to_be_opti) + #opt = BFGS(to_be_opti,trajectory='traj.traj',logfile='opt.log') + opt.run(fmax=fmax,steps=200) + + atoms_lat = to_be_opti.cell + atoms_pos = to_be_opti.positions + atoms_force = to_be_opti.get_forces() + atoms_stress = to_be_opti.get_stress() + # eV/A^3 to GPa + atoms_stress = atoms_stress/(0.01*0.6242) + #atoms_num = to_be_opti.get_atomic_numbers() + atoms_symbols = to_be_opti.get_chemical_symbols() + #atoms_formula = to_be_opti.get_chemical_formula() + atoms_ene = to_be_opti.get_potential_energy() + atoms_vol = to_be_opti.get_volume() + + element, ele = Get_Element_Num(atoms_symbols) + + Write_Contcar(element, ele, atoms_lat, atoms_pos) + Write_Outcar(element, ele, atoms_vol, atoms_lat, atoms_pos,atoms_ene, atoms_force, atoms_stress * -10.0, pstress) + + + stop = time.time() + _cwd = os.getcwd() + _cwd = os.path.basename(_cwd) + print('%s is done, time: %s' % (_cwd,stop-start)) + +def run(): + fmax, stress = read_stress_fmax() + run_opt(famx, stress) + +if __name__=='__main__': + run() diff --git a/dpgen/generator/lib/make_calypso.py b/dpgen/generator/lib/make_calypso.py new file mode 100644 index 000000000..3a4f570df --- /dev/null +++ b/dpgen/generator/lib/make_calypso.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +import os +import shutil +import json +import numpy as np + +def make_calypso_input(nameofatoms,numberofatoms, + numberofformula,volume, + distanceofion,psoratio,popsize, + maxstep,icode,split,vsc, + maxnumatom,ctrlrange,pstress,fmax): + ret = "################################ The Basic Parameters of CALYPSO ################################\n" + ret+= "# A string of one or several words contain a descriptive name of the system (max. 40 characters).\n" + assert (nameofatoms != None ) + ret+= "SystemName = %s\n"%(''.join(nameofatoms)) + ret+= "# Number of different atomic species in the simulation.\n" + ret+= "NumberOfSpecies = %d\n"%(len(nameofatoms)) + ret+= "# Element symbols of the different chemical species.\n" + ret+= "NameOfAtoms = %s\n"%(' '.join(nameofatoms)) + ret+= "# Number of atoms for each chemical species in one formula unit. \n" + assert numberofatoms != None or len(numberofatoms) == len(nameofatoms) + ret+= "NumberOfAtoms = %s\n"%(' '.join(list(map(str,numberofatoms)))) + ret+= "# The range of formula unit per cell in your simulation. \n" + assert numberofformula != None or len(numberofformula) == 2 or type(numberofformula) == type([1,2]) + ret+= "NumberOfFormula = %s\n"%(' '.join(list(map(str,numberofformula)))) + ret+= "# The volume per formula unit. Unit is in angstrom^3.\n" + ret+= "Volume = %s\n"%(volume[0]) + ret+= "# Minimal distance between atoms of each chemical species. Unit is in angstrom.\n" + assert len(distanceofion) == len(nameofatoms) #"check distance of ions and the number of atoms" + assert len(distanceofion[0]) == len(nameofatoms) + ret+= "@DistanceOfIon \n" + for temp in distanceofion: + ret+="%4s \n"%(' '.join(list(map(str,temp)))) + ret+= "@End\n" + ret+= "# It determines which algorithm should be adopted in the simulation.\n" + ret+= "Ialgo = 2\n" + ret+= "# Ialgo = 1 for Global PSO\n" + ret+= "# Ialgo = 2 for Local PSO (default value)\n" + ret+= "# The proportion of the structures generated by PSO.\n" + assert (0 <= psoratio[0] <= 1 ) + ret+= "PsoRatio = %s\n"%(psoratio[0]) + ret+= "# The population size. Normally, it has a larger number for larger systems.\n" + assert popsize[0] != None or popsize!=None or type(popsize) == type([0,1]) or type(popsize[0]) == type(0) + ret+= "PopSize = %d\n"%(popsize[0]) + assert maxstep[0] != None or maxstep!=None or type(maxstep) == type([0,1]) or type(maxstep[0]) == type(0) + ret+= "# The Max step for iteration\n" + ret+= "MaxStep = %d\n"%(maxstep[0]) + ret+= "#It determines which method should be adopted in generation the random structure. \n" + ret+= "GenType= 1 \n" + ret+= "# 1 under symmetric constraints\n" + ret+= "# 2 grid method for large system\n" + ret+= "# 3 and 4 core grow method \n" + ret+= "# 0 combination of all method\n" + ret+= "# If GenType=3 or 4, it determined the small unit to grow the whole structure\n" + ret+= "# It determines which local optimization method should be interfaced in the simulation.\n" + assert icode != None or type(icode) == type([0,1]) or type(icode[0]) == type(0) + ret+= "ICode= %d\n"%(icode[0]) + ret+= "# ICode= 1 interfaced with VASP\n" + ret+= "# ICode= 2 interfaced with SIESTA\n" + ret+= "# ICode= 3 interfaced with GULP\n" + ret+= "# The number of lbest for local PSO\n" + ret+= "NumberOfLbest=4\n" + ret+= "# The Number of local optimization for each structure.\n" + ret+= "NumberOfLocalOptim= 3\n" + ret+= "# The command to perform local optimiztion calculation (e.g., VASP, SIESTA) on your computer.\n" + ret+= "Command = sh submit.sh\n" + ret+= "MaxTime = 9000 \n" + ret+= "# If True, a previous calculation will be continued.\n" + ret+= "PickUp = F\n" + ret+= "# At which step will the previous calculation be picked up.\n" + ret+= "PickStep = 1\n" + ret+= "# If True, the local optimizations performed by parallel\n" + ret+= "Parallel = F\n" + ret+= "# The number node for parallel \n" + ret+= "NumberOfParallel = 4\n" + assert split != None + ret+= "Split = %s\n"%(split) + assert pstress != None or type(pstress) == type([200]) + ret+= "PSTRESS = %f\n"%(pstress[0]) + assert fmax != None or type(fmax) == type([200]) + ret+= "fmax = %f\n"%(fmax[0]) + ret+= "################################ End of The Basic Parameters of CALYPSO #######################\n" + if vsc == 'T': + assert len(ctrlrange) == len(nameofatoms) #'check distance of ions and the number of atoms' + ret+= "##### The Parameters For Variational Stoichiometry ##############\n" + ret+= "## If True, Variational Stoichiometry structure prediction is performed\n" + ret+= "VSC = %s\n"%(vsc) + ret+= "# The Max Number of Atoms in unit cell\n" + ret+= "MaxNumAtom = %s\n"%(maxnumatom[0]) + ret+= "# The Variation Range for each type atom \n" + ret+= "@CtrlRange\n" + for ttemp in ctrlrange: + ret+="%4s \n"%(' '.join(list(map(str,ttemp)))) + ret+= "@end\n" + ret+= "###################End Parameters for VSC ##########################\n" + return ret + +def _make_model_devi_buffet(jdata,calypso_run_opt_path): + + calypso_input_path = jdata.get('calypso_input_path') + shutil.copyfile(os.path.join(calypso_input_path,'input.dat'),os.path.join(calypso_run_opt_path, 'input.dat')) + +def _make_model_devi_native_calypso(iter_index,model_devi_jobs, calypso_run_opt_path): + + for iiidx, jobbs in enumerate(model_devi_jobs): + if iter_index in jobbs.get('times'): + cur_job = model_devi_jobs[iiidx] + + work_path = os.path.dirname(calypso_run_opt_path) + with open(os.path.join(work_path, 'cur_job.json'), 'w') as outfile: + json.dump(cur_job, outfile, indent = 4) + + # Crystal Parameters + nameofatoms = cur_job.get('NameOfAtoms') + numberofatoms = cur_job.get('NumberOfAtoms') + numberofformula = cur_job.get('NumberOfFormula',[1,1]) + volume = cur_job.get('Volume') + distanceofion = cur_job.get('DistanceOfIon') + psoratio = cur_job.get('PsoRatio') + popsize = cur_job.get('PopSize') + maxstep = cur_job.get('MaxStep') + icode = cur_job.get('ICode',[1]) + split = cur_job.get('Split','T') + # VSC Control + maxnumatom = None + ctrlrange = None + vsc = cur_job.get('VSC','F') + if vsc == 'T': + maxnumatom = cur_job.get('MaxNumAtom') + ctrlrange = cur_job.get('CtrlRange') + + # Optimization + pstress = cur_job.get('PSTRESS',[0.001]) + fmax = cur_job.get('fmax',[0.01]) + + # Cluster + + # 2D + + file_c = make_calypso_input(nameofatoms,numberofatoms, + numberofformula,volume, + distanceofion,psoratio,popsize, + maxstep,icode,split,vsc, + maxnumatom,ctrlrange,pstress,fmax) + with open(os.path.join(calypso_run_opt_path, 'input.dat'), 'w') as cin : + cin.write(file_c) + diff --git a/dpgen/generator/lib/model_devi_calypso.py b/dpgen/generator/lib/run_calypso.py similarity index 80% rename from dpgen/generator/lib/model_devi_calypso.py rename to dpgen/generator/lib/run_calypso.py index be32454e3..97ca7c51b 100644 --- a/dpgen/generator/lib/model_devi_calypso.py +++ b/dpgen/generator/lib/run_calypso.py @@ -1,5 +1,5 @@ """ -calypso model devi: +calypso as model devi engine: 1. gen_structures 2. analysis 3. model devi @@ -40,6 +40,7 @@ def gen_structures(iter_index,jdata,mdata): model_devi_group_size = mdata['model_devi_group_size'] model_devi_resources = mdata['model_devi_resources'] + api_version = mdata.get('api_version', '0.9') iter_name = make_iter_name(iter_index) @@ -125,7 +126,6 @@ def gen_structures(iter_index,jdata,mdata): run_tasks = [os.path.basename(ii) for ii in run_tasks_] - api_version = mdata.get('api_version', '0.9') if LooseVersion(api_version) < LooseVersion('1.0'): warnings.warn(f"the dpdispatcher will be updated to new version." f"And the interface may be changed. Please check the documents for more details") @@ -182,6 +182,7 @@ def gen_structures(iter_index,jdata,mdata): else: # -------------------------------------------------------------- + # TODO(zhenyu) make this code work for other situation component = ['Mg','Al','Cu','MgAl','MgCu','AlCu','MgAlCu'] for idx,com in enumerate(component): @@ -359,94 +360,3 @@ def analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): f.close() os.chdir(cwd) - -#def model_devi(iter_index,calypso_model_devi_path,all_models,jdata): -# -# # Model Devi -# -# -# dlog.info('$$$$$$$$$$$$$$$ Model Devi Start $$$$$$$$$$$$$$$$$$') -# -# iter_name = make_iter_name(iter_index) -# work_path = os.path.join(iter_name, model_devi_name) -# -# cwd = os.getcwd() -# Devis = [] -# pcount = 0 -# strus_lists = glob.glob(os.path.join(calypso_model_devi_path,'*.structures')) -# for num, strus_path in enumerate(strus_lists): -# os.chdir(strus_path) -# os.system('rename .vasp .poscar *') -# os.chdir(cwd) -# -# structure_list = glob.glob(os.path.join(strus_path,'*.poscar')) -# -# # every 500 confs in one task dir -# if len(structure_list) == 0: -# continue -# else: -# num_per_task = math.ceil(len(structure_list)/500) -# graphs = [DP(model) for model in all_models] -# for temp in range(num_per_task): -# task_name = os.path.join(calypso_model_devi_path,'task.%03d.%03d'%(num,temp)) -# put_poscar = os.path.join(task_name,'traj') -# if not os.path.exists(task_name): -# os.mkdir(task_name) -# os.mkdir(put_poscar) -# else: -# shutil.rmtree(task_name) -# os.mkdir(task_name) -# os.mkdir(put_poscar) -# devis = [] -# try: -# temp_sl = structure_list[temp*500:(temp+1)*500] -# except Exception as err: -# dlog.info('err %s'%str(err)) -# temp_sl = structure_list[temp*500:] -# -# new_index = 0 -# for index,sname in enumerate(temp_sl): -# shutil.copyfile(sname,os.path.join(put_poscar,'%s.poscar'%str(index))) -# os.chdir(put_poscar) -# pdata = dpdata.System('%s.poscar'%str(index),type_map = jdata['type_map']) -# nopbc = pdata.nopbc -# coord = pdata.data['coords'] -# cell = pdata.data['cells'] -# atom_types = pdata.data['atom_types'] -# devi = calc_model_devi(coord,cell,atom_types,graphs,nopbc=nopbc) -# # ------------------------------------------------------------------------------------ -# # append min-distance in devi list -# dis_temp = dpdata.System('%s.poscar'%str(index)) -# dis = dis_temp.to_ase_structure()[0].get_all_distances(mic=True) -# row,col = np.diag_indices_from(dis) -# dis[row,col] = 10000 -# min_dis = np.nanmin(dis) -# devi = np.append(devi[0],min_dis) -# t = [devi] -# devi = np.array(t) -# # ------------------------------------------------------------------------------------ -# temp_d = copy.deepcopy(devi) -# temp_D = copy.deepcopy(devi) -# devis.append(temp_d) -# Devis.append(temp_D) -# devis[index][0][0] = np.array(index) -# Devis[pcount][0][0] = np.array(pcount) -# pcount += 1 -# new_index += 1 -# os.chdir(cwd) -# os.chdir(task_name) -# devis = np.vstack(devis) -# write_model_devi_out(devis,'model_devi.out') -# os.chdir(cwd) -# -# os.chdir(calypso_model_devi_path) -# Devis = np.vstack(Devis) -# write_model_devi_out(Devis,'Model_Devi.out') -# os.chdir(cwd) -# -# os.chdir(work_path) -# f = open('record.calypso','a+') -# f.write('4\n') -# f.close() -# os.chdir(cwd) - diff --git a/dpgen/generator/lib/write_modd.py b/dpgen/generator/lib/write_modd.py deleted file mode 100644 index 44f38d88b..000000000 --- a/dpgen/generator/lib/write_modd.py +++ /dev/null @@ -1,127 +0,0 @@ -def write_modd(): - - ret ="import argparse\n" - ret+="import copy\n" - ret+="import dpdata\n" - ret+="import glob\n" - ret+="import math\n" - ret+="import numpy as np\n" - ret+="import os\n" - ret+="import sys\n" - ret+="import shutil\n" - ret+="\n" - ret+="\n" - ret+="def write_model_devi_out(devi, fname):\n" - ret+=" assert devi.shape[1] == 8\n" - ret+=" #assert devi.shape[1] == 7\n" - ret+=" header = '%5s' % 'step'\n" - ret+=" for item in 'vf':\n" - ret+=" header += '%16s%16s%16s' % (f'max_devi_{item}', f'min_devi_{item}',f'avg_devi_{item}')\n" - ret+=" header += '%16s'%str('min_dis')\n" - ret+=" np.savetxt(fname,\n" - ret+=" devi,\n" - ret+=" fmt=['%5d'] + ['%17.6e' for _ in range(7)],\n" - ret+=" delimiter='',\n" - ret+=" header=header)\n" - ret+=" return devi\n" - ret+="\n" - ret+="\n" - ret+="def Modd(all_models,type_map):\n" - ret+=" from deepmd.infer import calc_model_devi\n" - ret+=" from deepmd.infer import DeepPot as DP\n" - ret+="\n" - ret+=" # Model Devi \n" - ret+="\n" - ret+=" cwd = os.getcwd()\n" - ret+=" graphs = [DP(model) for model in all_models]\n" - ret+="\n" - ret+=" Devis = []\n" - ret+=" pcount = 0\n" - ret+=" strus_lists = glob.glob(os.path.join(cwd,'*.structures'))\n" - ret+=" for num, strus_path in enumerate(strus_lists):\n" - ret+="\n" - ret+=" structures_data = dpdata.System(strus_path,'deepmd/npy',type_map=type_map)\n" - ret+="\n" - ret+=" # every 500 confs in one task dir\n" - ret+=" nnum = structures_data.get_nframes()\n" - ret+=" if nnum == 0:\n" - ret+=" continue\n" - ret+=" else:\n" - ret+=" num_per_task = math.ceil(nnum/500)\n" - ret+=" \n" - ret+="\n" - ret+=" for temp in range(num_per_task):\n" - ret+=" task_name = os.path.join(cwd,'task.%03d.%03d'%(num,temp)) \n" - ret+=" put_poscar = os.path.join(task_name,'traj')\n" - ret+=" if not os.path.exists(task_name):\n" - ret+=" os.mkdir(task_name)\n" - ret+=" os.mkdir(put_poscar)\n" - ret+=" else:\n" - ret+=" shutil.rmtree(task_name)\n" - ret+=" os.mkdir(task_name)\n" - ret+=" os.mkdir(put_poscar)\n" - ret+=" devis = []\n" - ret+=" if (nnum - (temp+1)*500) >= 0:\n" - ret+=" temp_sl = range(temp*500,(temp+1)*500)\n" - ret+=" else:\n" - ret+=" temp_sl = range(temp*500,nnum)\n" - ret+=" \n" - ret+=" new_index = 0\n" - ret+=" for index,frameid in enumerate(temp_sl):\n" - ret+=" pdata = structures_data[frameid]\n" - ret+=" pdata.to_vasp_poscar(os.path.join(put_poscar,'%s.poscar'%str(index)))\n" - ret+=" nopbc = pdata.nopbc\n" - ret+=" coord = pdata.data['coords']\n" - ret+=" cell = pdata.data['cells']\n" - ret+=" atom_types = pdata.data['atom_types']\n" - ret+=" devi = calc_model_devi(coord,cell,atom_types,graphs,nopbc=nopbc)\n" - ret+=" # ------------------------------------------------------------------------------------\n" - ret+=" # append min-distance in devi list\n" - ret+=" dis = pdata.to_ase_structure()[0].get_all_distances(mic=True)\n" - ret+=" row,col = np.diag_indices_from(dis)\n" - ret+=" dis[row,col] = 10000\n" - ret+=" min_dis = np.nanmin(dis)\n" - ret+=" devi = np.append(devi[0],min_dis)\n" - ret+=" t = [devi]\n" - ret+=" devi = np.array(t)\n" - ret+=" # ------------------------------------------------------------------------------------\n" - ret+=" temp_d = copy.deepcopy(devi)\n" - ret+=" temp_D = copy.deepcopy(devi)\n" - ret+=" devis.append(temp_d)\n" - ret+=" Devis.append(temp_D)\n" - ret+=" devis[index][0][0] = np.array(index)\n" - ret+=" Devis[pcount][0][0] = np.array(pcount)\n" - ret+=" pcount += 1\n" - ret+=" new_index += 1\n" - ret+=" devis = np.vstack(devis)\n" - ret+=" write_model_devi_out(devis,os.path.join(task_name, 'model_devi.out'))\n" - ret+="\n" - ret+=" Devis = np.vstack(Devis)\n" - ret+=" write_model_devi_out(Devis,os.path.join(cwd,'Model_Devi.out'))\n" - ret+="\n" - ret+=" f = open(os.path.join(os.path.abspath(os.path.join(cwd,os.pardir)),'record.calypso'),'a+')\n" - ret+=" f.write('4')\n" - ret+=" f.close()\n" - ret+="\n" - ret+="if __name__ == '__main__':\n" - ret+="\n" - ret+=" cwd = os.getcwd()\n" - ret+=" model_path = os.path.join(os.path.abspath(os.path.join(cwd,os.pardir)),'gen_stru_analy')\n" - ret+=" parser = argparse.ArgumentParser(description='calc model-devi by `all_models` and `type_map`')\n" - ret+=" parser.add_argument(\n" - ret+=" '--all_models',\n" - ret+=" type=str,\n" - ret+=" nargs='+',\n" - ret+=" default=model_path,\n" - ret+=" help='the path of models which will be used to do model-deviation',\n" - ret+=" )\n" - ret+=" parser.add_argument(\n" - ret+=" '--type_map',\n" - ret+=" nargs='+',\n" - ret+=" help='the type map of models which will be used to do model-deviation',\n" - ret+=" )\n" - ret+=" args = parser.parse_args()\n" - ret+=" #print(vars(args))\n" - ret+=" Modd(args.all_models,args.type_map)\n" - ret+=" #Modd(sys.argv[1],sys.argv[2])\n" - return ret diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index 9e71a12b1..770705e3e 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -41,10 +41,8 @@ from dpgen.generator.lib.utils import log_task from dpgen.generator.lib.utils import symlink_user_forward_files from dpgen.generator.lib.lammps import make_lammps_input, get_dumped_forces -from dpgen.generator.lib.calypso import make_run_opt_script,make_check_outcar_script -from dpgen.generator.lib.calypso import _make_model_devi_native_calypso,write_model_devi_out,_make_model_devi_buffet -from dpgen.generator.lib.model_devi_calypso import gen_structures,analysis -from dpgen.generator.lib.write_modd import write_modd +from dpgen.generator.lib.make_calypso import _make_model_devi_native_calypso,_make_model_devi_buffet +from dpgen.generator.lib.run_calypso import gen_structures,analysis from dpgen.generator.lib.parse_calypso import _parse_calypso_input,_parse_calypso_dis_mtx from dpgen.generator.lib.vasp import write_incar_dict from dpgen.generator.lib.vasp import make_vasp_incar_user_dict @@ -87,6 +85,9 @@ # for calypso calypso_run_opt_name = 'gen_stru_analy' calypso_model_devi_name = 'model_devi_results' +calypso_run_model_devi_file = os.path.join(ROOT_PATH,'generator/lib/calypso_run_model_devi.py') +check_outcar_file = os.path.join(ROOT_PATH,'generator/lib/calypso_check_outcar.py') +run_opt_file = os.path.join(ROOT_PATH,'generator/lib/calypso_run_opt.py') def get_job_names(jdata) : jobkeys = [] @@ -848,14 +849,15 @@ def make_model_devi (iter_index, calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) create_path(calypso_run_opt_path) create_path(calypso_model_devi_path) - # modd script - modd_script = os.path.join(calypso_model_devi_path,'modd.py') - with open(modd_script,'w') as fm: - fm.write(write_modd()) + # run model devi script + calypso_run_model_devi_script = os.path.join(calypso_model_devi_path,'calypso_run_model_devi.py') + shutil.copyfile(calypso_run_model_devi_file,calypso_run_model_devi_script) + # run confs opt script + run_opt_script = os.path.join(calypso_run_opt_path,'run_opt.py') + shutil.copyfile(run_opt_file,run_opt_script) # check outcar script check_outcar_script = os.path.join(calypso_run_opt_path,'check_outcar.py') - with open(check_outcar_script,'w') as ffff: - ffff.write(make_check_outcar_script()) + shutil.copyfile(check_outcar_file,check_outcar_script) for mm in models : model_name = os.path.basename(mm) if model_devi_engine != 'calypso': @@ -1437,7 +1439,7 @@ def run_multi_model_devi (iter_index, all_models = glob.glob(os.path.join(_calypso_run_opt_path, 'graph*pb')) cwd = os.getcwd() os.chdir(calypso_model_devi_path) - args = ' '.join(['modd.py', '--all_models',' '.join(all_models),'--type_map',' '.join(jdata.get('type_map'))]) + args = ' '.join(['calypso_run_model_devi.py', '--all_models',' '.join(all_models),'--type_map',' '.join(jdata.get('type_map'))]) deepmdkit_python = mdata.get('deepmdkit_python') os.system(f'{deepmdkit_python} {args} ') #Modd(iter_index,calypso_model_devi_path,all_models,jdata) From 496fa074815a00862e5fa7188c9b5e58e65fdd90 Mon Sep 17 00:00:00 2001 From: zhenyu Date: Wed, 23 Mar 2022 20:46:51 +0800 Subject: [PATCH 27/46] refactor: move calypso main code(run.py) into run_calypso.py and modify test_calypso.py --- dpgen/generator/lib/run_calypso.py | 63 ++++++++++++++++++++++++--- dpgen/generator/run.py | 68 +++--------------------------- tests/generator/context.py | 4 +- tests/generator/test_calypso.py | 24 ----------- 4 files changed, 65 insertions(+), 94 deletions(-) diff --git a/dpgen/generator/lib/run_calypso.py b/dpgen/generator/lib/run_calypso.py index 97ca7c51b..1985f3b97 100644 --- a/dpgen/generator/lib/run_calypso.py +++ b/dpgen/generator/lib/run_calypso.py @@ -16,15 +16,12 @@ import shutil from ase.io.vasp import write_vasp from ase.io.trajectory import Trajectory -#from deepmd.infer import calc_model_devi -#from deepmd.infer import DeepPot as DP from pathlib import Path from distutils.version import LooseVersion from dpgen import dlog from dpgen.generator.lib.utils import make_iter_name -from dpgen.generator.lib.calypso import write_model_devi_out -from dpgen.generator.lib.parse_calypso import _parse_calypso_input,_parse_calypso_dis_mtx -from dpgen.dispatcher.Dispatcher import Dispatcher, _split_tasks, make_dispatcher, make_submission +from dpgen.generator.lib.parse_calypso import _parse_calypso_input +from dpgen.dispatcher.Dispatcher import make_dispatcher, make_submission train_name = '00.train' model_devi_name = '01.model_devi' @@ -51,7 +48,7 @@ def gen_structures(iter_index,jdata,mdata): calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) # - calypso_path = jdata.get('calypso_path') + calypso_path = mdata.get('model_devi_calypso_path') #calypso_input_path = jdata.get('calypso_input_path') popsize = int(_parse_calypso_input('PopSize',calypso_run_opt_path)) maxstep = int(_parse_calypso_input('MaxStep',calypso_run_opt_path)) @@ -59,7 +56,7 @@ def gen_structures(iter_index,jdata,mdata): all_models = glob.glob(os.path.join(calypso_run_opt_path, 'graph*pb')) model_names = [os.path.basename(ii) for ii in all_models] - deepmdkit_python = mdata.get('deepmdkit_python') + deepmdkit_python = mdata.get('model_devi_deepmdkit_python') command = "%s run_opt.py %s 1>> model_devi.log 2>> model_devi.log" % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) command += " || %s check_outcar.py %s " % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) commands = [command] @@ -360,3 +357,55 @@ def analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): f.close() os.chdir(cwd) + +def run_calypso_model_devi (iter_index, + jdata, + mdata) : + + dlog.info('start running CALYPSO') + + + iter_name = make_iter_name(iter_index) + work_path = os.path.join(iter_name, model_devi_name) + assert(os.path.isdir(work_path)) + + calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) + calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) + + cwd = os.getcwd() + + record_calypso_path = os.path.join(work_path,'record.calypso') + while True: + if not os.path.exists(record_calypso_path): + f = open(record_calypso_path,'w') + f.write('1\n') + lines = '1' + f.close() + else: + f = open(record_calypso_path,'r') + lines = f.readlines() + f.close() + + if lines[-1].strip().strip('\n') == '1': + # Gen Structures + gen_structures(iter_index,jdata,mdata) + + elif lines[-1].strip().strip('\n') == '2': + # Analysis & to deepmd/raw + analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path) + + elif lines[-1].strip().strip('\n') == '3': + # Model Devi + _calypso_run_opt_path = os.path.abspath(calypso_run_opt_path) + all_models = glob.glob(os.path.join(_calypso_run_opt_path, 'graph*pb')) + cwd = os.getcwd() + os.chdir(calypso_model_devi_path) + args = ' '.join(['calypso_run_model_devi.py', '--all_models',' '.join(all_models),'--type_map',' '.join(jdata.get('type_map'))]) + deepmdkit_python = mdata.get('model_devi_deepmdkit_python') + os.system(f'{deepmdkit_python} {args} ') + #Modd(iter_index,calypso_model_devi_path,all_models,jdata) + os.chdir(cwd) + + elif lines[-1].strip().strip('\n') == '4': + #dlog.info('Model Devi is done.') + break diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index 770705e3e..572f042cc 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -42,7 +42,7 @@ from dpgen.generator.lib.utils import symlink_user_forward_files from dpgen.generator.lib.lammps import make_lammps_input, get_dumped_forces from dpgen.generator.lib.make_calypso import _make_model_devi_native_calypso,_make_model_devi_buffet -from dpgen.generator.lib.run_calypso import gen_structures,analysis +from dpgen.generator.lib.run_calypso import gen_structures,analysis,run_calypso_model_devi from dpgen.generator.lib.parse_calypso import _parse_calypso_input,_parse_calypso_dis_mtx from dpgen.generator.lib.vasp import write_incar_dict from dpgen.generator.lib.vasp import make_vasp_incar_user_dict @@ -793,7 +793,7 @@ def make_model_devi (iter_index, if (iter_index >= len(model_devi_jobs)) : return False else: - # mode 1: generate structures according to the user-provided input.dat file, so calypso_input_path and model_devi_max_iter and fmax is needed + # mode 1: generate structures according to the user-provided input.dat file, so calypso_input_path and model_devi_max_iter are needed if "calypso_input_path" in jdata: try: maxiter = jdata.get('model_devi_max_iter') @@ -1267,9 +1267,9 @@ def _make_model_devi_native_gromacs(iter_index, jdata, mdata, conf_systems): -def run_single_model_devi (iter_index, - jdata, - mdata) : +def run_md_model_devi (iter_index, + jdata, + mdata) : #rmdlog.info("This module has been run !") model_devi_exec = mdata['model_devi_command'] @@ -1391,64 +1391,10 @@ def run_model_devi(iter_index,jdata,mdata): model_devi_engine = jdata.get("model_devi_engine", "lammps") if model_devi_engine != "calypso": - run_single_model_devi(iter_index,jdata,mdata) + run_md_model_devi(iter_index,jdata,mdata) else: - run_multi_model_devi(iter_index,jdata,mdata) + run_calypso_model_devi(iter_index,jdata,mdata) - -def run_multi_model_devi (iter_index, - jdata, - mdata) : - - dlog.info('start running CALYPSO') - - - iter_name = make_iter_name(iter_index) - work_path = os.path.join(iter_name, model_devi_name) - assert(os.path.isdir(work_path)) - - calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) - calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) - - - cwd = os.getcwd() - - record_calypso_path = os.path.join(work_path,'record.calypso') - while True: - if not os.path.exists(record_calypso_path): - f = open(record_calypso_path,'w') - f.write('1\n') - lines = '1' - f.close() - else: - f = open(record_calypso_path,'r') - lines = f.readlines() - f.close() - - if lines[-1].strip().strip('\n') == '1': - # Gen Structures - gen_structures(iter_index,jdata,mdata) - - elif lines[-1].strip().strip('\n') == '2': - # Analysis & to deepmd/raw - analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path) - - elif lines[-1].strip().strip('\n') == '3': - # Model Devi - _calypso_run_opt_path = os.path.abspath(calypso_run_opt_path) - all_models = glob.glob(os.path.join(_calypso_run_opt_path, 'graph*pb')) - cwd = os.getcwd() - os.chdir(calypso_model_devi_path) - args = ' '.join(['calypso_run_model_devi.py', '--all_models',' '.join(all_models),'--type_map',' '.join(jdata.get('type_map'))]) - deepmdkit_python = mdata.get('deepmdkit_python') - os.system(f'{deepmdkit_python} {args} ') - #Modd(iter_index,calypso_model_devi_path,all_models,jdata) - os.chdir(cwd) - - elif lines[-1].strip().strip('\n') == '4': - #dlog.info('Model Devi is done.') - break - def post_model_devi (iter_index, jdata, mdata) : diff --git a/tests/generator/context.py b/tests/generator/context.py index 2c804751a..ac95cd44d 100644 --- a/tests/generator/context.py +++ b/tests/generator/context.py @@ -5,8 +5,8 @@ from dpgen.generator.lib.gaussian import detect_multiplicity from dpgen.generator.lib.ele_temp import NBandsEsti from dpgen.generator.lib.lammps import get_dumped_forces -from dpgen.generator.lib.calypso import make_run_opt_script,make_check_outcar_script -from dpgen.generator.lib.calypso import write_model_devi_out,make_calypso_input +from dpgen.generator.lib.calypso_run_model_devi import write_model_devi_out +from dpgen.generator.lib.make_calypso import make_calypso_input from dpgen.generator.lib.parse_calypso import _parse_calypso_input,_parse_calypso_dis_mtx param_file = 'param-mg-vasp.json' diff --git a/tests/generator/test_calypso.py b/tests/generator/test_calypso.py index f997c463f..b302cfca6 100644 --- a/tests/generator/test_calypso.py +++ b/tests/generator/test_calypso.py @@ -7,22 +7,13 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) __package__ = 'generator' -from .context import make_run_opt_script -from .context import make_check_outcar_script from .context import make_calypso_input from .context import write_model_devi_out from .context import _parse_calypso_input from .context import _parse_calypso_dis_mtx -#calypso_data = Path('.').joinpath('calypso_data') - -#assert os.path.exists(calypso_data) -#assert os.path.exists(calypso_path) -#assert os.path.exists(graph_path) - # temp dir test_path = Path('.').joinpath('calypso_test_path') -test_poscar_path = Path('.').joinpath('calypso_test_path','POSCAR') test_path.mkdir(parents=True,exist_ok=True) os.system('rm calypso_test_path/*') fmax = 0.01 @@ -46,21 +37,6 @@ def setUp(self): def tearDown(self): pass - def test_write_run_opt_script(self): - ret = make_run_opt_script(fmax) - for temp in ret.split('\n'): - if 'opt.run' in temp: - self.assertEqual(temp.strip(),'opt.run(fmax=0.01,steps=200)') - - def test_write_check_outcar_script(self): - ret = make_check_outcar_script() - #with open('calypso_test_path/check_outcar.py','w') as fc: - with open('check_outcar.py','w') as fc: - fc.write(ret) - #self.assertTrue(os.path.exists('calypso_test_path/check_outcar.py')) - self.assertTrue(os.path.exists('check_outcar.py')) - os.remove('check_outcar.py') - def test_write_model_devi_out(self): #devi = write_model_devi_out(model_devi, 'calypso_test_path/model_devi.out') #ndevi = np.loadtxt('calypso_test_path/model_devi.out') From 36dd23ae0c8c92be667e32acb3b33bfc318d38b8 Mon Sep 17 00:00:00 2001 From: zhenyu Date: Wed, 23 Mar 2022 20:56:37 +0800 Subject: [PATCH 28/46] docs: modify README.md related to CALYPSO --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 04e6aa004..7b4ee04d8 100644 --- a/README.md +++ b/README.md @@ -625,10 +625,8 @@ The bold notation of key (such as **calypso_path**) means that it's a necessary | :---------------- | :--------------------- | :-------------------------------------- | :-------------------------------------------------------------| | *in param file* | **model_devi_engine** | string | "calypso" | CALYPSO as model-deviation engine.| -| **calypso_path** | string | "/home/zhenyu/workplace/debug" | The absolute path of calypso.x.| -| **calypso_input_path** | string | "/home/zhenyu/workplace/debug" | The absolute path of CALYPSO input file named input.dat, when this keys exists, all the iters will use the same CALYPSO input file until reach the number of max iter specified by **model_devi_max_iter** and **model_devi_jobs** key will not work.| +| **calypso_input_path** | string | "/home/zhenyu/workplace/debug" | The absolute path of CALYPSO input file named input.dat(PSTRESS and fmax should be included), when this keys exists, all the iters will use the same CALYPSO input file until reach the number of max iter specified by **model_devi_max_iter** and **model_devi_jobs** key will not work.| | **model_devi_max_iter** | int | 10 | The max iter number code can run, it works when **calypso_input_path** exists.| -| **fmax** | float | 0.01 | The convergence criterion is that the force on all individual atoms should be less than *fmax*, it works when **calypso_input_path** exists.| | **model_devi_jobs** | List of Dict | [{ "times":[3],"NameOfAtoms":["Al","Cu"],"NumberOfAtoms":[1,10],"NumberOfFormula":[1,2],"Volume":[300],"DistanceOfIon":[[ 1.48,1.44],[ 1.44,1.41]],"PsoRatio":[0.6],"PopSize":[5],"MaxStep":[3],"ICode":[1],"Split":"T"},...] | Settings for exploration in `01.model_devi`. Different number in `times` List means different iteration index and iterations mentioned in List wil use same CALYPSO parameters.| | **model_devi_jobs["times"]** | List of int | [0,1,2] | Different number in `times` List means different iteration index and iterations mentioned in List wil use same CALYPSO parameters.| | **model_devi_jobs["NameOfAtoms"]** | List of string |["Al","Cu"] | Parameter of CALYPSO input file, means the element species of structures to be generated. | @@ -642,9 +640,10 @@ The bold notation of key (such as **calypso_path**) means that it's a necessary | **model_devi_jobs["ICode"]** | List of int |[13] | Parameter of CALYPSO input file, means the chosen of local optimization, 1 is vasp and 13 is ASE with dp. | | **model_devi_jobs["Split"]** | String |"T" | Parameter of CALYPSO input file, means that generating structures and optimizing structures are split into two parts, in dpgen workflow, Split must be T. | | **model_devi_jobs["PSTRESS"]** | List of float |[0.001] | Same as PSTRESS in INCAR. | -| **model_devi_jobs["fmax"]** | List of float |[0.01] | The convergence criterion of local optimization. | +| **model_devi_jobs["fmax"]** | List of float |[0.01] | The convergence criterion is that the force on all individual atoms should be less than *fmax*. | | *in machine file* -| **deepmdkit_python** | String | "/home/zhenyu/soft/deepmd-kit/bin/python" | A python path with deepmd package. | +| **model_devi["deepmdkit_python"]** | String | "/home/zhenyu/soft/deepmd-kit/bin/python" | A python path with deepmd package. | +| **model_devi["calypso_path"]** | string | "/home/zhenyu/workplace/debug" | The absolute path of calypso.x.| #### Rules for cp2k input at dictionary form Converting cp2k input is very simple as dictionary used to dpgen input. You just need follow some simple rule: From 0e9669e93e876c1bcdae3c46a5a17e1200d5eeff Mon Sep 17 00:00:00 2001 From: zhenyu Date: Thu, 24 Mar 2022 17:05:05 +0800 Subject: [PATCH 29/46] delete some useless comments in run.py --- dpgen/generator/run.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index 572f042cc..075f02705 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -2708,13 +2708,8 @@ def post_fp_vasp (iter_index, if all_sys is None: all_sys = _sys else: - #try: all_sys.append(_sys) - #except RuntimeError: - # dlog.info('%s has different formula, so pass '%oo) - # pass # save ele_temp, if any - # need to check with open(oo.replace('OUTCAR', 'job.json')) as fp: job_data = json.load(fp) if 'ele_temp' in job_data: From 961f63faff2c36d736c5879c6f4f8b8eaf991a9e Mon Sep 17 00:00:00 2001 From: zhenyu Date: Thu, 24 Mar 2022 17:32:29 +0800 Subject: [PATCH 30/46] add write_model_devi_out function into make_calypso.py for passing unittest --- dpgen/generator/lib/make_calypso.py | 14 ++++++++++++++ tests/generator/context.py | 3 +-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/dpgen/generator/lib/make_calypso.py b/dpgen/generator/lib/make_calypso.py index 3a4f570df..4c5ce7d66 100644 --- a/dpgen/generator/lib/make_calypso.py +++ b/dpgen/generator/lib/make_calypso.py @@ -145,3 +145,17 @@ def _make_model_devi_native_calypso(iter_index,model_devi_jobs, calypso_run_opt_ with open(os.path.join(calypso_run_opt_path, 'input.dat'), 'w') as cin : cin.write(file_c) +def write_model_devi_out(devi, fname): + assert devi.shape[1] == 8 + #assert devi.shape[1] == 7 + header = '%5s' % 'step' + for item in 'vf': + header += '%16s%16s%16s' % (f'max_devi_{item}', f'min_devi_{item}',f'avg_devi_{item}') + header += '%16s'%str('min_dis') + np.savetxt(fname, + devi, + fmt=['%5d'] + ['%17.6e' for _ in range(7)], + delimiter='', + header=header) + return devi + diff --git a/tests/generator/context.py b/tests/generator/context.py index ac95cd44d..3ee10b9c8 100644 --- a/tests/generator/context.py +++ b/tests/generator/context.py @@ -5,8 +5,7 @@ from dpgen.generator.lib.gaussian import detect_multiplicity from dpgen.generator.lib.ele_temp import NBandsEsti from dpgen.generator.lib.lammps import get_dumped_forces -from dpgen.generator.lib.calypso_run_model_devi import write_model_devi_out -from dpgen.generator.lib.make_calypso import make_calypso_input +from dpgen.generator.lib.make_calypso import make_calypso_input,write_model_devi_out from dpgen.generator.lib.parse_calypso import _parse_calypso_input,_parse_calypso_dis_mtx param_file = 'param-mg-vasp.json' From df2bf1cc67805cd646b904c0791be1dc3ceb8a38 Mon Sep 17 00:00:00 2001 From: zhenyu Date: Sat, 26 Mar 2022 10:59:23 +0800 Subject: [PATCH 31/46] fix a spelling mistake in calypso_run_opt.py (famx -> fmax) --- dpgen/generator/lib/calypso_run_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpgen/generator/lib/calypso_run_opt.py b/dpgen/generator/lib/calypso_run_opt.py index 7f6b6d58c..00a863c0e 100644 --- a/dpgen/generator/lib/calypso_run_opt.py +++ b/dpgen/generator/lib/calypso_run_opt.py @@ -170,7 +170,7 @@ def run_opt(fmax,stress): def run(): fmax, stress = read_stress_fmax() - run_opt(famx, stress) + run_opt(fmax, stress) if __name__=='__main__': run() From 43ef866fdc4ae02df3c1855baede69d531927b91 Mon Sep 17 00:00:00 2001 From: zhenyu Date: Sat, 26 Mar 2022 12:12:09 +0800 Subject: [PATCH 32/46] change runopt script's name from run_opt.py to calypso_run_opt.py and fix bug in runopt script --- dpgen/generator/lib/calypso_run_opt.py | 6 +++--- dpgen/generator/lib/run_calypso.py | 8 ++++---- dpgen/generator/run.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dpgen/generator/lib/calypso_run_opt.py b/dpgen/generator/lib/calypso_run_opt.py index 00a863c0e..3344d0f37 100644 --- a/dpgen/generator/lib/calypso_run_opt.py +++ b/dpgen/generator/lib/calypso_run_opt.py @@ -99,7 +99,7 @@ def read_stress_fmax(): continue if 'PSTRESS' in line or 'pstress' in line: pstress = float(line.split('=')[1]) - elif 'fmax' in line: + if 'fmax' in line: fmax = float(line.split('=')[1]) return fmax,pstress @@ -123,7 +123,7 @@ def run_opt(fmax,stress): print('Start to Optimize Structures by DP----------') - Opt_Step = 200 + Opt_Step = 600 start = time.time() # pstress kbar pstress = stress @@ -143,7 +143,7 @@ def run_opt(fmax,stress): #opt = QuasiNewton(to_be_opti) #opt = BFGS(to_be_opti) #opt = BFGS(to_be_opti,trajectory='traj.traj',logfile='opt.log') - opt.run(fmax=fmax,steps=200) + opt.run(fmax=fmax,steps=Opt_Step) atoms_lat = to_be_opti.cell atoms_pos = to_be_opti.positions diff --git a/dpgen/generator/lib/run_calypso.py b/dpgen/generator/lib/run_calypso.py index 1985f3b97..c15989486 100644 --- a/dpgen/generator/lib/run_calypso.py +++ b/dpgen/generator/lib/run_calypso.py @@ -57,14 +57,14 @@ def gen_structures(iter_index,jdata,mdata): model_names = [os.path.basename(ii) for ii in all_models] deepmdkit_python = mdata.get('model_devi_deepmdkit_python') - command = "%s run_opt.py %s 1>> model_devi.log 2>> model_devi.log" % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) + command = "%s calypso_run_opt.py %s 1>> model_devi.log 2>> model_devi.log" % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) command += " || %s check_outcar.py %s " % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) commands = [command] cwd = os.getcwd() os.chdir(calypso_run_opt_path) - forward_files = ['POSCAR', 'run_opt.py','check_outcar.py','input.dat'] + forward_files = ['POSCAR', 'calypso_run_opt.py','check_outcar.py','input.dat'] backward_files = ['OUTCAR','CONTCAR','traj.traj','model_devi.log'] run_calypso = calypso_path+'/calypso.x | tee log' @@ -109,7 +109,7 @@ def gen_structures(iter_index,jdata,mdata): except: shutil.rmtree('task.%03d'%pop) os.mkdir('task.%03d'%pop) - shutil.copyfile('run_opt.py',os.path.join('task.%03d'%pop,'run_opt.py')) + shutil.copyfile('calypso_run_opt.py',os.path.join('task.%03d'%pop,'calypso_run_opt.py')) shutil.copyfile('check_outcar.py',os.path.join('task.%03d'%pop,'check_outcar.py')) shutil.copyfile('POSCAR_%s'%str(pop-ii*int(popsize)+1),os.path.join('task.%03d'%(pop),'POSCAR')) shutil.copyfile('input.dat',os.path.join('task.%03d'%pop,'input.dat')) @@ -198,7 +198,7 @@ def gen_structures(iter_index,jdata,mdata): except: shutil.rmtree('task.%04d'%(idx+1)) os.mkdir('task.%04d'%(idx+1)) - shutil.copyfile('run_opt.py',os.path.join('task.%04d'%(idx+1),'run_opt.py')) + shutil.copyfile('calypso_run_opt.py',os.path.join('task.%04d'%(idx+1),'calypso_run_opt.py')) shutil.copyfile('check_outcar.py',os.path.join('task.%04d'%(idx+1),'check_outcar.py')) shutil.copyfile('POSCAR_%s'%str(idx+1),os.path.join('task.%04d'%(idx+1),'POSCAR')) shutil.copyfile('input.dat',os.path.join('task.%04d'%(idx+1),'input.dat')) diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index 075f02705..14c8b0709 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -853,7 +853,7 @@ def make_model_devi (iter_index, calypso_run_model_devi_script = os.path.join(calypso_model_devi_path,'calypso_run_model_devi.py') shutil.copyfile(calypso_run_model_devi_file,calypso_run_model_devi_script) # run confs opt script - run_opt_script = os.path.join(calypso_run_opt_path,'run_opt.py') + run_opt_script = os.path.join(calypso_run_opt_path,'calypso_run_opt.py') shutil.copyfile(run_opt_file,run_opt_script) # check outcar script check_outcar_script = os.path.join(calypso_run_opt_path,'check_outcar.py') From 33271e9b1329415cd42a5c443b5d73f4cabc604f Mon Sep 17 00:00:00 2001 From: zhenyu Date: Thu, 31 Mar 2022 15:11:41 +0800 Subject: [PATCH 33/46] fix unittest --- tests/generator/test_make_fp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/generator/test_make_fp.py b/tests/generator/test_make_fp.py index 57789fee9..8158888f8 100644 --- a/tests/generator/test_make_fp.py +++ b/tests/generator/test_make_fp.py @@ -913,7 +913,8 @@ def test_make_fp_pwmat(self): _check_poscars(self, 0, jdata['fp_task_max'], jdata['type_map']) _check_pwmat_input(self, 0) _check_potcar(self, 0, jdata['fp_pp_path'], jdata['fp_pp_files']) - shutil.rmtree('iter.000000') + os.system('rm -r iter.000000') + #shutil.rmtree('iter.000000') if __name__ == '__main__': unittest.main() From 56b1e0502413d519f723545166245aeeeb60ae7d Mon Sep 17 00:00:00 2001 From: zhenyu Date: Mon, 18 Jul 2022 19:38:23 +0800 Subject: [PATCH 34/46] make vsc mode work for any situation --- dpgen/generator/lib/run_calypso.py | 35 +++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/dpgen/generator/lib/run_calypso.py b/dpgen/generator/lib/run_calypso.py index c15989486..389d4153e 100644 --- a/dpgen/generator/lib/run_calypso.py +++ b/dpgen/generator/lib/run_calypso.py @@ -14,9 +14,11 @@ import re import glob import shutil +import sys from ase.io.vasp import write_vasp from ase.io.trajectory import Trajectory from pathlib import Path +from itertools import combinations from distutils.version import LooseVersion from dpgen import dlog from dpgen.generator.lib.utils import make_iter_name @@ -94,9 +96,9 @@ def gen_structures(iter_index,jdata,mdata): for ii in range(int(PickUpStep)-1,maxstep+1): dlog.info('CALYPSO step %s'%ii) if ii == maxstep : - while True: - if len(glob.glob('OUTCAR_*')) == popsize: - break + #while True: + # if len(glob.glob('OUTCAR_*')) == popsize: + # break os.system('%s'%run_calypso) break # run calypso @@ -180,12 +182,29 @@ def gen_structures(iter_index,jdata,mdata): else: # -------------------------------------------------------------- # TODO(zhenyu) make this code work for other situation - component = ['Mg','Al','Cu','MgAl','MgCu','AlCu','MgAlCu'] + type_map = jdata['type_map'] + how_many_spec = len(type_map) + if how_many_spec == 1: + dlog.info('vsc mode can not work in one-element situation' ) + sys.exit() + + comp_temp = list(map(list,list(combinations(type_map,1)))) + for hms in range(2,how_many_spec+1): + comp_temp.extend(list(map(list,list(combinations(type_map,hms))))) # comp_temp = [['Mg'],['Al'],['Cu'],['Mg','Al'],['Mg','Cu'],['Al','Cu'],['Mg','Al','Cu']] + + component = [] + for comp_temp_ in comp_temp: + component.append(''.join(comp_temp_)) # component = ['Mg','Al','Cu','MgAl','MgCu','AlCu','MgAlCu'] + + print(component) + #sys.exit() + calypso_input_path = jdata.get('calypso_input_path') for idx,com in enumerate(component): pwd = os.getcwd() os.mkdir(str(idx)) - shutil.copyfile(os.path.join(cwd,'calypso_input','input.dat.%s'%com),os.path.join(str(idx),'input.dat')) + #shutil.copyfile(os.path.join(cwd,'calypso_input','input.dat.%s'%com),os.path.join(str(idx),'input.dat')) + shutil.copyfile(os.path.join(calypso_input_path,'input.dat.%s'%com),os.path.join(str(idx),'input.dat')) os.chdir(str(idx)) os.system(run_calypso) os.chdir(pwd) @@ -330,7 +349,7 @@ def analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): if len(ms) == 0: print('too little confs, ') - return + sys.exit() if os.path.exists(os.path.join(result_path,'deepmd')): shutil.rmtree(os.path.join(result_path,'deepmd')) @@ -407,5 +426,5 @@ def run_calypso_model_devi (iter_index, os.chdir(cwd) elif lines[-1].strip().strip('\n') == '4': - #dlog.info('Model Devi is done.') - break + dlog.info('Model Devi is done.') + return From 2cfb895731de8401047496d16e9b638c486f6d3f Mon Sep 17 00:00:00 2001 From: zhenyu Date: Thu, 21 Jul 2022 19:51:58 +0800 Subject: [PATCH 35/46] feat: one iter can run different calypso loop in different pressure --- dpgen/generator/lib/make_calypso.py | 77 +++++++++++---------- dpgen/generator/lib/run_calypso.py | 100 ++++++++++++++++------------ dpgen/generator/run.py | 52 ++++++++++----- tests/generator/test_calypso.py | 8 +-- 4 files changed, 141 insertions(+), 96 deletions(-) diff --git a/dpgen/generator/lib/make_calypso.py b/dpgen/generator/lib/make_calypso.py index 4c5ce7d66..50b03dc73 100644 --- a/dpgen/generator/lib/make_calypso.py +++ b/dpgen/generator/lib/make_calypso.py @@ -3,6 +3,7 @@ import shutil import json import numpy as np +from dpgen.generator.lib.utils import create_path def make_calypso_input(nameofatoms,numberofatoms, numberofformula,volume, @@ -24,7 +25,7 @@ def make_calypso_input(nameofatoms,numberofatoms, assert numberofformula != None or len(numberofformula) == 2 or type(numberofformula) == type([1,2]) ret+= "NumberOfFormula = %s\n"%(' '.join(list(map(str,numberofformula)))) ret+= "# The volume per formula unit. Unit is in angstrom^3.\n" - ret+= "Volume = %s\n"%(volume[0]) + ret+= "Volume = %s\n"%(volume) ret+= "# Minimal distance between atoms of each chemical species. Unit is in angstrom.\n" assert len(distanceofion) == len(nameofatoms) #"check distance of ions and the number of atoms" assert len(distanceofion[0]) == len(nameofatoms) @@ -37,14 +38,14 @@ def make_calypso_input(nameofatoms,numberofatoms, ret+= "# Ialgo = 1 for Global PSO\n" ret+= "# Ialgo = 2 for Local PSO (default value)\n" ret+= "# The proportion of the structures generated by PSO.\n" - assert (0 <= psoratio[0] <= 1 ) - ret+= "PsoRatio = %s\n"%(psoratio[0]) + assert (0 <= psoratio <= 1 ) + ret+= "PsoRatio = %s\n"%(psoratio) ret+= "# The population size. Normally, it has a larger number for larger systems.\n" - assert popsize[0] != None or popsize!=None or type(popsize) == type([0,1]) or type(popsize[0]) == type(0) - ret+= "PopSize = %d\n"%(popsize[0]) - assert maxstep[0] != None or maxstep!=None or type(maxstep) == type([0,1]) or type(maxstep[0]) == type(0) + assert popsize != None or type(popsize) == type(0) + ret+= "PopSize = %d\n"%(popsize) + assert maxstep != None or type(maxstep) == type(0) ret+= "# The Max step for iteration\n" - ret+= "MaxStep = %d\n"%(maxstep[0]) + ret+= "MaxStep = %d\n"%(maxstep) ret+= "#It determines which method should be adopted in generation the random structure. \n" ret+= "GenType= 1 \n" ret+= "# 1 under symmetric constraints\n" @@ -53,8 +54,8 @@ def make_calypso_input(nameofatoms,numberofatoms, ret+= "# 0 combination of all method\n" ret+= "# If GenType=3 or 4, it determined the small unit to grow the whole structure\n" ret+= "# It determines which local optimization method should be interfaced in the simulation.\n" - assert icode != None or type(icode) == type([0,1]) or type(icode[0]) == type(0) - ret+= "ICode= %d\n"%(icode[0]) + assert icode != None or type(icode) == type(0) + ret+= "ICode= %d\n"%(icode) ret+= "# ICode= 1 interfaced with VASP\n" ret+= "# ICode= 2 interfaced with SIESTA\n" ret+= "# ICode= 3 interfaced with GULP\n" @@ -75,10 +76,10 @@ def make_calypso_input(nameofatoms,numberofatoms, ret+= "NumberOfParallel = 4\n" assert split != None ret+= "Split = %s\n"%(split) - assert pstress != None or type(pstress) == type([200]) - ret+= "PSTRESS = %f\n"%(pstress[0]) - assert fmax != None or type(fmax) == type([200]) - ret+= "fmax = %f\n"%(fmax[0]) + assert pstress != None or type(pstress) == type(200) + ret+= "PSTRESS = %f\n"%(pstress) + assert fmax != None or type(fmax) == type(0.01) + ret+= "fmax = %f\n"%(fmax) ret+= "################################ End of The Basic Parameters of CALYPSO #######################\n" if vsc == 'T': assert len(ctrlrange) == len(nameofatoms) #'check distance of ions and the number of atoms' @@ -86,7 +87,7 @@ def make_calypso_input(nameofatoms,numberofatoms, ret+= "## If True, Variational Stoichiometry structure prediction is performed\n" ret+= "VSC = %s\n"%(vsc) ret+= "# The Max Number of Atoms in unit cell\n" - ret+= "MaxNumAtom = %s\n"%(maxnumatom[0]) + ret+= "MaxNumAtom = %s\n"%(maxnumatom) ret+= "# The Variation Range for each type atom \n" ret+= "@CtrlRange\n" for ttemp in ctrlrange: @@ -95,32 +96,32 @@ def make_calypso_input(nameofatoms,numberofatoms, ret+= "###################End Parameters for VSC ##########################\n" return ret -def _make_model_devi_buffet(jdata,calypso_run_opt_path): +def _make_model_devi_buffet(jdata,caly_run_opt_path): calypso_input_path = jdata.get('calypso_input_path') - shutil.copyfile(os.path.join(calypso_input_path,'input.dat'),os.path.join(calypso_run_opt_path, 'input.dat')) + shutil.copyfile(os.path.join(calypso_input_path,'input.dat'),os.path.join(caly_run_opt_path[0], 'input.dat')) -def _make_model_devi_native_calypso(iter_index,model_devi_jobs, calypso_run_opt_path): +def _make_model_devi_native_calypso(iter_index,model_devi_jobs, caly_run_opt_path): for iiidx, jobbs in enumerate(model_devi_jobs): if iter_index in jobbs.get('times'): cur_job = model_devi_jobs[iiidx] - work_path = os.path.dirname(calypso_run_opt_path) - with open(os.path.join(work_path, 'cur_job.json'), 'w') as outfile: - json.dump(cur_job, outfile, indent = 4) - # Crystal Parameters nameofatoms = cur_job.get('NameOfAtoms') numberofatoms = cur_job.get('NumberOfAtoms') numberofformula = cur_job.get('NumberOfFormula',[1,1]) volume = cur_job.get('Volume') distanceofion = cur_job.get('DistanceOfIon') - psoratio = cur_job.get('PsoRatio') - popsize = cur_job.get('PopSize') - maxstep = cur_job.get('MaxStep') - icode = cur_job.get('ICode',[1]) + psoratio = cur_job.get('PsoRatio', 0.6) + popsize = cur_job.get('PopSize', 30) + maxstep = cur_job.get('MaxStep', 5) + icode = cur_job.get('ICode',1) split = cur_job.get('Split','T') + # Cluster + + # 2D + # VSC Control maxnumatom = None ctrlrange = None @@ -128,22 +129,26 @@ def _make_model_devi_native_calypso(iter_index,model_devi_jobs, calypso_run_opt_ if vsc == 'T': maxnumatom = cur_job.get('MaxNumAtom') ctrlrange = cur_job.get('CtrlRange') - # Optimization + fmax = cur_job.get('fmax',0.01) + # pstress is a List which contains the target stress pstress = cur_job.get('PSTRESS',[0.001]) - fmax = cur_job.get('fmax',[0.01]) - - # Cluster - - # 2D - - file_c = make_calypso_input(nameofatoms,numberofatoms, + # pressures + for press_idx, temp_caly_run_opt_path in enumerate(caly_run_opt_path): + # cur_press + cur_press = pstress[press_idx] + # cur_dir + work_path = os.path.dirname(temp_caly_run_opt_path) + # cur_job.json + with open(os.path.join(work_path, 'cur_job.json'), 'w') as outfile: + json.dump(cur_job, outfile, indent = 4) + file_c = make_calypso_input(nameofatoms,numberofatoms, numberofformula,volume, distanceofion,psoratio,popsize, maxstep,icode,split,vsc, - maxnumatom,ctrlrange,pstress,fmax) - with open(os.path.join(calypso_run_opt_path, 'input.dat'), 'w') as cin : - cin.write(file_c) + maxnumatom,ctrlrange,cur_press,fmax) + with open(os.path.join(temp_caly_run_opt_path, 'input.dat'), 'w') as cin : + cin.write(file_c) def write_model_devi_out(devi, fname): assert devi.shape[1] == 8 diff --git a/dpgen/generator/lib/run_calypso.py b/dpgen/generator/lib/run_calypso.py index 389d4153e..306ead2de 100644 --- a/dpgen/generator/lib/run_calypso.py +++ b/dpgen/generator/lib/run_calypso.py @@ -21,6 +21,7 @@ from itertools import combinations from distutils.version import LooseVersion from dpgen import dlog +from dpgen.generator.lib.utils import create_path from dpgen.generator.lib.utils import make_iter_name from dpgen.generator.lib.parse_calypso import _parse_calypso_input from dpgen.dispatcher.Dispatcher import make_dispatcher, make_submission @@ -31,7 +32,7 @@ calypso_run_opt_name = 'gen_stru_analy' calypso_model_devi_name = 'model_devi_results' -def gen_structures(iter_index,jdata,mdata): +def gen_structures(iter_index, jdata, mdata, caly_run_path, current_idx, length_of_caly_runopt_list): # run calypso # vsc means generate elemental, binary and ternary at the same time @@ -45,10 +46,9 @@ def gen_structures(iter_index,jdata,mdata): iter_name = make_iter_name(iter_index) work_path = os.path.join(iter_name, model_devi_name) assert(os.path.isdir(work_path)) - # - calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) + + calypso_run_opt_path = caly_run_path calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) - # calypso_path = mdata.get('model_devi_calypso_path') #calypso_input_path = jdata.get('calypso_input_path') @@ -181,7 +181,6 @@ def gen_structures(iter_index,jdata,mdata): else: # -------------------------------------------------------------- - # TODO(zhenyu) make this code work for other situation type_map = jdata['type_map'] how_many_spec = len(type_map) if how_many_spec == 1: @@ -196,7 +195,7 @@ def gen_structures(iter_index,jdata,mdata): for comp_temp_ in comp_temp: component.append(''.join(comp_temp_)) # component = ['Mg','Al','Cu','MgAl','MgCu','AlCu','MgAlCu'] - print(component) + dlog.info(component) #sys.exit() calypso_input_path = jdata.get('calypso_input_path') @@ -279,34 +278,58 @@ def gen_structures(iter_index,jdata,mdata): shutil.rmtree(t) # -------------------------------------------------------------- + if current_idx < length_of_caly_runopt_list - 1: + tobewrite = '1 %s\n'%(str(current_idx + 1)) + elif current_idx == length_of_caly_runopt_list - 1 : + tobewrite = '2\n' + os.chdir(cwd) os.chdir(work_path) f = open('record.calypso','a+') - f.write('2\n') + f.write(tobewrite) f.close() os.chdir(cwd) -def analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): +def gen_main(iter_index, jdata, mdata, caly_run_opt_list, gen_idx): - # Analysis + iter_name = make_iter_name(iter_index) + work_path = os.path.join(iter_name, model_devi_name) + caly_run_opt_list = glob.glob(os.path.join(work_path,'%s.*'%((calypso_run_opt_name)))) + caly_run_opt_list.sort() - #dlog.info('$$$$$$$$$$$$$$$ Analysis Started $$$$$$$$$$$$$$$$$$') + current_gen_path = os.path.join(work_path, '%s.%03d'%(calypso_run_opt_name, int(gen_idx))) + if current_gen_path not in caly_run_opt_list: + dlog.info(f"current gen path {current_gen_path} not in caly run opt list {caly_run_opt_list}") + sys.exit() + + indice = caly_run_opt_list.index(current_gen_path) + for iidx, temp_path in enumerate(caly_run_opt_list): + if iidx >= indice: + gen_structures(iter_index, jdata, mdata, temp_path, iidx, len(caly_run_opt_list)) + + +def analysis(iter_index, jdata, calypso_model_devi_path): + # Analysis ms = dpdata.MultiSystems() cwd = os.getcwd() iter_name = make_iter_name(iter_index) work_path = os.path.join(iter_name, model_devi_name) - result_path = os.path.join(calypso_run_opt_path,'results') - # trajs to be model devi - traj_path = os.path.join(calypso_run_opt_path,'traj') - traj_list = glob.glob(traj_path+'/*.traj') + deepmd_data_path = os.path.join(work_path,'confs', 'deepmd') + traj_pos_path = os.path.join(work_path,'confs', 'traj_confs') + create_path(deepmd_data_path) + create_path(traj_pos_path) - #dlog.info('len(traj_list) %s'%str(len(traj_list))) + # trajs to be model devi + # traj_path = os.path.join(calypso_run_opt_path,'traj') + # traj_list = glob.glob(traj_path+'/*.traj') + # 'gen_struc_analy.000/traj/*.traj' 'gen_struc_analy.001/traj/*.traj' 'gen_struc_analy.002/traj/*.traj' + traj_list = glob.glob(f'{work_path}/*/traj/*.traj') - # read confs from traj + # read poscar from traj file in confs/traj/*.traj record_traj_num = 0 for traj_name in traj_list: traj_num = os.path.basename(traj_name).split('.')[0] @@ -330,16 +353,13 @@ def analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): for idx, traj in enumerate(trajs): write_vasp(os.path.join( - traj_path,'%03d.%03d.poscar' % ( + traj_pos_path,'%03d.%03d.poscar' % ( int(traj_num), int(idx) ) ), traj) - traj_pos_list = glob.glob(traj_path+'/*.poscar') - - #dlog.info('traj_num %s'%str(len(traj_pos_list))) - #dlog.info('total_traj_num %s'%str(record_traj_num)) + traj_pos_list = glob.glob(traj_pos_path+'/*.poscar') for npos in traj_pos_list: try: @@ -348,20 +368,16 @@ def analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path): dlog.info(npos,'failed : ',e) if len(ms) == 0: - print('too little confs, ') + dlog.info('too little confs, ') sys.exit() - if os.path.exists(os.path.join(result_path,'deepmd')): - shutil.rmtree(os.path.join(result_path,'deepmd')) - ms.to_deepmd_raw(os.path.join(result_path,'deepmd')) - ms.to_deepmd_npy(os.path.join(result_path,'deepmd')) - + if os.path.exists(deepmd_data_path): + shutil.rmtree(deepmd_data_path) + ms.to_deepmd_raw(deepmd_data_path) + ms.to_deepmd_npy(deepmd_data_path) - split_lists = glob.glob(os.path.join(result_path,'deepmd','*')) + split_lists = glob.glob(os.path.join(deepmd_data_path,'*')) for i,split_list in enumerate(split_lists): - #ss = dpdata.System(split_list,fmt='deepmd') - #for j in range(ss.get_nframes()): - # ss.to('vasp/poscar',os.path.join(split_list,'%03d.%03d.poscar'%(i,j)),frame_idx=j) strus_path = os.path.join(calypso_model_devi_path,'%03d.structures'%i) if not os.path.exists(strus_path): shutil.copytree(split_list,strus_path) @@ -383,39 +399,40 @@ def run_calypso_model_devi (iter_index, dlog.info('start running CALYPSO') - iter_name = make_iter_name(iter_index) work_path = os.path.join(iter_name, model_devi_name) assert(os.path.isdir(work_path)) - calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) - cwd = os.getcwd() + caly_run_opt_list = glob.glob(os.path.join(work_path,'%s.*'%(str(calypso_run_opt_name)))) + caly_run_opt_list.sort() + cwd = os.getcwd() record_calypso_path = os.path.join(work_path,'record.calypso') while True: if not os.path.exists(record_calypso_path): f = open(record_calypso_path,'w') - f.write('1\n') - lines = '1' + f.write('1 0\n') + lines = ['1 0\n'] f.close() else: f = open(record_calypso_path,'r') lines = f.readlines() f.close() - if lines[-1].strip().strip('\n') == '1': + if lines[-1].strip().strip('\n').split()[0] == '1': # Gen Structures - gen_structures(iter_index,jdata,mdata) + gen_index = lines[-1].strip().strip('\n').split()[1] + gen_main(iter_index, jdata, mdata, caly_run_opt_list, gen_index) elif lines[-1].strip().strip('\n') == '2': # Analysis & to deepmd/raw - analysis(iter_index,jdata,calypso_run_opt_path,calypso_model_devi_path) + analysis(iter_index, jdata, calypso_model_devi_path) elif lines[-1].strip().strip('\n') == '3': # Model Devi - _calypso_run_opt_path = os.path.abspath(calypso_run_opt_path) + _calypso_run_opt_path = os.path.abspath(caly_run_opt_list[0]) all_models = glob.glob(os.path.join(_calypso_run_opt_path, 'graph*pb')) cwd = os.getcwd() os.chdir(calypso_model_devi_path) @@ -427,4 +444,5 @@ def run_calypso_model_devi (iter_index, elif lines[-1].strip().strip('\n') == '4': dlog.info('Model Devi is done.') - return + # return + break diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index 14c8b0709..072ca112d 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -794,6 +794,7 @@ def make_model_devi (iter_index, return False else: # mode 1: generate structures according to the user-provided input.dat file, so calypso_input_path and model_devi_max_iter are needed + run_mode = 1 if "calypso_input_path" in jdata: try: maxiter = jdata.get('model_devi_max_iter') @@ -803,10 +804,11 @@ def make_model_devi (iter_index, else: try: maxiter = max(model_devi_jobs[-1].get('times')) + run_mode = 2 except KeyError: raise KeyError('did not find model_devi_jobs["times"] key') if (iter_index > maxiter) : - print(f'iter_index is {iter_index} and maxiter is {maxiter}') + dlog.info(f'iter_index is {iter_index} and maxiter is {maxiter}') return False if "sys_configs_prefix" in jdata: @@ -823,7 +825,7 @@ def make_model_devi (iter_index, cur_job = model_devi_jobs[iter_index] sys_idx = expand_idx(cur_job['sys_idx']) else: - cur_job = [] + cur_job = [{'model_devi_engine':'calypso','input.dat':'user_provided'}] sys_idx = [] if (len(sys_idx) != len(list(set(sys_idx)))) : @@ -845,25 +847,43 @@ def make_model_devi (iter_index, work_path = os.path.join(iter_name, model_devi_name) create_path(work_path) if model_devi_engine == 'calypso': - calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) + _calypso_run_opt_path = os.path.join(work_path,calypso_run_opt_name) calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) - create_path(calypso_run_opt_path) create_path(calypso_model_devi_path) # run model devi script calypso_run_model_devi_script = os.path.join(calypso_model_devi_path,'calypso_run_model_devi.py') shutil.copyfile(calypso_run_model_devi_file,calypso_run_model_devi_script) - # run confs opt script - run_opt_script = os.path.join(calypso_run_opt_path,'calypso_run_opt.py') - shutil.copyfile(run_opt_file,run_opt_script) - # check outcar script - check_outcar_script = os.path.join(calypso_run_opt_path,'check_outcar.py') - shutil.copyfile(check_outcar_file,check_outcar_script) + # Create work path of calypso and dpdispatcher depending on the number of PRESSURE + caly_run_opt_path = [] + if run_mode == 1: + caly_run_opt_path.append('%s.%03d'%(_calypso_run_opt_path, 0)) + elif run_mode == 2: + for iiidx, jobbs in enumerate(model_devi_jobs): + if iter_index in jobbs.get('times'): + cur_job = model_devi_jobs[iiidx] + + pressures_list = cur_job.get('PSTRESS', [0.0001]) + for temp_idx in range(len(pressures_list)): + caly_run_opt_path.append('%s.%03d'%(_calypso_run_opt_path, temp_idx)) + # to different directory + # caly_run_opt_path = ['gen_struc_analy.000','gen_struc_analy.001','gen_struc_analy.002',] + for temp_caly_run_opt_path in caly_run_opt_path: + create_path(temp_caly_run_opt_path) + # run confs opt script + run_opt_script = os.path.join(temp_caly_run_opt_path,'calypso_run_opt.py') + shutil.copyfile(run_opt_file,run_opt_script) + # check outcar script + check_outcar_script = os.path.join(temp_caly_run_opt_path,'check_outcar.py') + shutil.copyfile(check_outcar_file,check_outcar_script) + + for mm in models : model_name = os.path.basename(mm) if model_devi_engine != 'calypso': os.symlink(mm, os.path.join(work_path, model_name)) else: - os.symlink(mm, os.path.join(calypso_run_opt_path, model_name)) + for temp_caly_run_opt_path in caly_run_opt_path: + os.symlink(mm, os.path.join(temp_caly_run_opt_path, model_name)) with open(os.path.join(work_path, 'cur_job.json'), 'w') as outfile: json.dump(cur_job, outfile, indent = 4) @@ -910,13 +930,13 @@ def make_model_devi (iter_index, elif model_devi_engine == "gromacs": _make_model_devi_native_gromacs(iter_index, jdata, mdata, conf_systems) elif model_devi_engine == "calypso": - _make_model_devi_native_calypso(iter_index,model_devi_jobs, calypso_run_opt_path) # generate input.dat automatic in each iter + _make_model_devi_native_calypso(iter_index,model_devi_jobs, caly_run_opt_path) # generate input.dat automatic in each iter else: raise RuntimeError("unknown model_devi engine", model_devi_engine) elif input_mode == "revise_template": _make_model_devi_revmat(iter_index, jdata, mdata, conf_systems) elif input_mode == "buffet": - _make_model_devi_buffet(jdata,calypso_run_opt_path) # generate confs according to the input.dat provided + _make_model_devi_buffet(jdata,caly_run_opt_path) # generate confs according to the input.dat provided else: raise RuntimeError('unknown model_devi input mode', input_mode) #Copy user defined forward_files @@ -1498,7 +1518,8 @@ def _select_by_model_devi_standard( if model_devi_engine == 'calypso': iter_name = modd_system_task[0].split('/')[0] _work_path = os.path.join(iter_name, model_devi_name) - calypso_run_opt_path = os.path.join(_work_path,calypso_run_opt_name) + # calypso_run_opt_path = os.path.join(_work_path,calypso_run_opt_name) + calypso_run_opt_path = glob.glob('%s/%s.*'%(_work_path, calypso_run_opt_name))[0] numofspecies = _parse_calypso_input('NumberOfSpecies',calypso_run_opt_path) min_dis = _parse_calypso_dis_mtx(numofspecies,calypso_run_opt_path) fp_candidate = [] @@ -1689,7 +1710,8 @@ def _make_fp_vasp_inner (modd_path, if model_devi_engine == 'calypso': iter_name = work_path.split('/')[0] _work_path = os.path.join(iter_name, model_devi_name) - calypso_run_opt_path = os.path.join(_work_path,calypso_run_opt_name) + # calypso_run_opt_path = os.path.join(_work_path,calypso_run_opt_name) + calypso_run_opt_path = glob.glob('%s/%s.*'%(_work_path, calypso_run_opt_name))[0] numofspecies = _parse_calypso_input('NumberOfSpecies',calypso_run_opt_path) min_dis = _parse_calypso_dis_mtx(numofspecies,calypso_run_opt_path) diff --git a/tests/generator/test_calypso.py b/tests/generator/test_calypso.py index b302cfca6..958b9e40e 100644 --- a/tests/generator/test_calypso.py +++ b/tests/generator/test_calypso.py @@ -46,9 +46,9 @@ def test_write_model_devi_out(self): os.remove('model_devi.out') def test_make_calypso_input(self): - ret = make_calypso_input(["Mg","Al","Cu"],[1,1,1],[1,4],[30],[ + ret = make_calypso_input(["Mg","Al","Cu"],[1,1,1],[1,4],30,[ [1.48,1.44,1.59],[1.44,1.41,1.56],[1.59,1.56,1.70] - ],[0.6],[5],[3],[13],"T","T",[31],[[1,10],[1,10],[1,10]],[0],[0.01]) + ],0.6,5,3,13,"T","T",31,[[1,10],[1,10],[1,10]],0,0.01) #with open('calypso_test_path/input.dat','w') as fin: with open('input.dat','w') as fin: fin.write(ret) @@ -69,9 +69,9 @@ def test_make_calypso_input(self): break def test_parse_calypso_input(self): - ret = make_calypso_input(["Mg","Al","Cu"],[1,1,1],[1,4],[30],[ + ret = make_calypso_input(["Mg","Al","Cu"],[1,1,1],[1,4],30,[ [1.48,1.44,1.59],[1.44,1.41,1.56],[1.59,1.56,1.70] - ],[0.6],[5],[3],[13],"T","T",[31],[[1,10],[1,10],[1,10]],[0],[0.01]) + ],0.6,5,3,13,"T","T",31,[[1,10],[1,10],[1,10]],0,0.01) #with open('calypso_test_path/input.dat','w') as fin: with open('input.dat','w') as fin: fin.write(ret) From e1a603aecac6d81ff2ac6b9fa3a2f0600ac5d23a Mon Sep 17 00:00:00 2001 From: zhenyu Date: Fri, 22 Jul 2022 09:13:07 +0800 Subject: [PATCH 36/46] write model_devi_jobs[current_iter] into cur_jobs.json and modify a comment --- dpgen/generator/lib/make_calypso.py | 10 +++++----- dpgen/generator/run.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dpgen/generator/lib/make_calypso.py b/dpgen/generator/lib/make_calypso.py index 50b03dc73..19dd153b8 100644 --- a/dpgen/generator/lib/make_calypso.py +++ b/dpgen/generator/lib/make_calypso.py @@ -107,6 +107,11 @@ def _make_model_devi_native_calypso(iter_index,model_devi_jobs, caly_run_opt_pat if iter_index in jobbs.get('times'): cur_job = model_devi_jobs[iiidx] + work_path = os.path.dirname(caly_run_opt_path[0]) + # cur_job.json + with open(os.path.join(work_path, 'cur_job.json'), 'w') as outfile: + json.dump(cur_job, outfile, indent = 4) + # Crystal Parameters nameofatoms = cur_job.get('NameOfAtoms') numberofatoms = cur_job.get('NumberOfAtoms') @@ -137,11 +142,6 @@ def _make_model_devi_native_calypso(iter_index,model_devi_jobs, caly_run_opt_pat for press_idx, temp_caly_run_opt_path in enumerate(caly_run_opt_path): # cur_press cur_press = pstress[press_idx] - # cur_dir - work_path = os.path.dirname(temp_caly_run_opt_path) - # cur_job.json - with open(os.path.join(work_path, 'cur_job.json'), 'w') as outfile: - json.dump(cur_job, outfile, indent = 4) file_c = make_calypso_input(nameofatoms,numberofatoms, numberofformula,volume, distanceofion,psoratio,popsize, diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index d7d7ba5f7..facded36b 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -839,7 +839,7 @@ def make_model_devi (iter_index, # run model devi script calypso_run_model_devi_script = os.path.join(calypso_model_devi_path,'calypso_run_model_devi.py') shutil.copyfile(calypso_run_model_devi_file,calypso_run_model_devi_script) - # Create work path of calypso and dpdispatcher depending on the number of PRESSURE + # Create work path list caly_run_opt_path = [] if run_mode == 1: caly_run_opt_path.append('%s.%03d'%(_calypso_run_opt_path, 0)) From 94509794d32f3cd413f0318e6d74032988eddb30 Mon Sep 17 00:00:00 2001 From: zhenyu Date: Fri, 22 Jul 2022 09:34:48 +0800 Subject: [PATCH 37/46] modify README.md and make_calypso.py --- README.md | 14 +++++++------- dpgen/generator/lib/make_calypso.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e721759ef..417645c90 100644 --- a/README.md +++ b/README.md @@ -591,15 +591,15 @@ The bold notation of key (such as **calypso_path**) means that it's a necessary | **model_devi_jobs["NameOfAtoms"]** | List of string |["Al","Cu"] | Parameter of CALYPSO input file, means the element species of structures to be generated. | | **model_devi_jobs["NumberOfAtoms"]** | List of int |[1,10] | Parameter of CALYPSO input file, means the number of atoms for each chemical species in one formula unit. | | **model_devi_jobs["NumberOfFormula"]** | List of int |[1,2] | Parameter of CALYPSO input file, means the range of formula unit per cell. | -| **model_devi_jobs["Volume"]** | List of int |[300] | Parameter of CALYPSO input file, means the colume per formula unit(angstrom^3). | +| **model_devi_jobs["Volume"]** | int | 300 | Parameter of CALYPSO input file, means the colume per formula unit(angstrom^3). | | **model_devi_jobs["DistanceOfIon"]** | List of float |[[ 1.48,1.44],[ 1.44,1.41]] | Parameter of CALYPSO input file, means minimal distance between atoms of each chemical species. Unit is in angstrom. | -| **model_devi_jobs["PsoRatio"]** | List of float |[0.6] | Parameter of CALYPSO input file, means the proportion of the structures generated by PSO. | -| **model_devi_jobs["PopSize"]** | List of int |[5] | Parameter of CALYPSO input file, means the number of structures to be generated in one step in CALYPSO. | -| **model_devi_jobs["MaxStep"]** | List of int |[3] | Parameter of CALYPSO input file, means the number of max step in CALYPSO.| -| **model_devi_jobs["ICode"]** | List of int |[13] | Parameter of CALYPSO input file, means the chosen of local optimization, 1 is vasp and 13 is ASE with dp. | +| **model_devi_jobs["PsoRatio"]** | float or int | 0.6 | Parameter of CALYPSO input file, means the proportion of the structures generated by PSO. | +| **model_devi_jobs["PopSize"]** | int | 5 | Parameter of CALYPSO input file, means the number of structures to be generated in one step in CALYPSO. | +| **model_devi_jobs["MaxStep"]** | int | 3 | Parameter of CALYPSO input file, means the number of max step in CALYPSO.| +| **model_devi_jobs["ICode"]** | int | 13 | Parameter of CALYPSO input file, means the chosen of local optimization, 1 is vasp and 13 is ASE with dp. | | **model_devi_jobs["Split"]** | String |"T" | Parameter of CALYPSO input file, means that generating structures and optimizing structures are split into two parts, in dpgen workflow, Split must be T. | -| **model_devi_jobs["PSTRESS"]** | List of float |[0.001] | Same as PSTRESS in INCAR. | -| **model_devi_jobs["fmax"]** | List of float |[0.01] | The convergence criterion is that the force on all individual atoms should be less than *fmax*. | +| **model_devi_jobs["PSTRESS"]** | List of float or int |[0.001] | Same as PSTRESS in INCAR. | +| **model_devi_jobs["fmax"]** | float | 0.01 | The convergence criterion is that the force on all individual atoms should be less than *fmax*. | | *in machine file* | **model_devi["deepmdkit_python"]** | String | "/home/zhenyu/soft/deepmd-kit/bin/python" | A python path with deepmd package. | | **model_devi["calypso_path"]** | string | "/home/zhenyu/workplace/debug" | The absolute path of calypso.x.| diff --git a/dpgen/generator/lib/make_calypso.py b/dpgen/generator/lib/make_calypso.py index 19dd153b8..63d123af8 100644 --- a/dpgen/generator/lib/make_calypso.py +++ b/dpgen/generator/lib/make_calypso.py @@ -76,7 +76,7 @@ def make_calypso_input(nameofatoms,numberofatoms, ret+= "NumberOfParallel = 4\n" assert split != None ret+= "Split = %s\n"%(split) - assert pstress != None or type(pstress) == type(200) + assert pstress != None or type(pstress) == type(200) or type(pstress) == type(0.001) ret+= "PSTRESS = %f\n"%(pstress) assert fmax != None or type(fmax) == type(0.01) ret+= "fmax = %f\n"%(fmax) From b478712bfc1099c84d78cefe9ce41dee4fa6c7ab Mon Sep 17 00:00:00 2001 From: zhenyu Date: Mon, 1 Aug 2022 14:25:28 +0800 Subject: [PATCH 38/46] fix calypso_run_opt script and check outcar script to support Bohrium --- dpgen/generator/lib/calypso_check_outcar.py | 7 ++++--- dpgen/generator/lib/calypso_run_opt.py | 7 ++++--- dpgen/generator/lib/run_calypso.py | 6 ++++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/dpgen/generator/lib/calypso_check_outcar.py b/dpgen/generator/lib/calypso_check_outcar.py index c07a00590..421fd623b 100644 --- a/dpgen/generator/lib/calypso_check_outcar.py +++ b/dpgen/generator/lib/calypso_check_outcar.py @@ -81,9 +81,10 @@ def check(): from deepmd.calculator import DP from ase.io import read - model_path = sys.argv[1] - Model_List = glob.glob('%s/graph*pb'%model_path) - calc = DP(model='%s'%(Model_List[0])) # init the model before iteration + # model_path = sys.argv[1] + # Model_List = glob.glob('%s/graph*pb'%model_path) + # calc = DP(model='%s'%(Model_List[0])) # init the model before iteration + calc = DP(model='../graph.000.pb') # init the model before iteration to_be_opti = read('POSCAR') to_be_opti.calc = calc diff --git a/dpgen/generator/lib/calypso_run_opt.py b/dpgen/generator/lib/calypso_run_opt.py index 3344d0f37..7605ea02e 100644 --- a/dpgen/generator/lib/calypso_run_opt.py +++ b/dpgen/generator/lib/calypso_run_opt.py @@ -112,9 +112,10 @@ def run_opt(fmax,stress): is_dpgen = True nn = 1 try: - model_path = sys.argv[1] - Model_List = glob.glob('%s/graph*pb'%model_path) - calc = DP(model=Model_List[0]) # init the model before iteration + # model_path = sys.argv[1] + # Model_List = glob.glob('%s/graph*pb'%model_path) + # calc = DP(model=Model_List[0]) # init the model before iteration + calc = DP(model='../graph.000.pb') # init the model before iteration except: assert os.path.exists('graph.pb'), 'did not found graph.pb in this directory %s, or did you forget to add args?'%(os.getcwd()) calc = DP(model='graph.pb') # init the model before iteration diff --git a/dpgen/generator/lib/run_calypso.py b/dpgen/generator/lib/run_calypso.py index 92bae1bcd..b0a6c5a7a 100644 --- a/dpgen/generator/lib/run_calypso.py +++ b/dpgen/generator/lib/run_calypso.py @@ -59,8 +59,10 @@ def gen_structures(iter_index, jdata, mdata, caly_run_path, current_idx, length_ model_names = [os.path.basename(ii) for ii in all_models] deepmdkit_python = mdata.get('model_devi_deepmdkit_python') - command = "%s calypso_run_opt.py %s 1>> model_devi.log 2>> model_devi.log" % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) - command += " || %s check_outcar.py %s " % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) + command = "%s calypso_run_opt.py 1>> model_devi.log 2>> model_devi.log" % (deepmdkit_python) + # command = "%s calypso_run_opt.py %s 1>> model_devi.log 2>> model_devi.log" % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) + # command += " || %s check_outcar.py %s " % (deepmdkit_python,os.path.abspath(calypso_run_opt_path)) + command += " || %s check_outcar.py " % (deepmdkit_python) commands = [command] cwd = os.getcwd() From cbae76924900e29fcc7cb4d0c90e47dd1375499e Mon Sep 17 00:00:00 2001 From: zhenyu Date: Mon, 1 Aug 2022 20:04:01 +0800 Subject: [PATCH 39/46] Support different pressure in vsc mode --- .gitignore | 3 +- dpgen/generator/lib/make_calypso.py | 25 ++++- dpgen/generator/lib/parse_calypso.py | 11 +- dpgen/generator/lib/run_calypso.py | 44 +++++--- dpgen/generator/run.py | 23 +++- examples/run/dp-calypso-vasp/machine.json | 70 ++++++++++++ examples/run/dp-calypso-vasp/param.json | 127 ++++++++++++++++++++++ 7 files changed, 276 insertions(+), 27 deletions(-) create mode 100644 examples/run/dp-calypso-vasp/machine.json create mode 100644 examples/run/dp-calypso-vasp/param.json diff --git a/.gitignore b/.gitignore index b2e21dd28..ecfb6d73e 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,5 @@ dbconfig.json .idea/* _build tests/generator/calypso_test_path -doc/api/ \ No newline at end of file +doc/api/ + diff --git a/dpgen/generator/lib/make_calypso.py b/dpgen/generator/lib/make_calypso.py index 63d123af8..2cd58d32e 100644 --- a/dpgen/generator/lib/make_calypso.py +++ b/dpgen/generator/lib/make_calypso.py @@ -2,6 +2,7 @@ import os import shutil import json +import glob import numpy as np from dpgen.generator.lib.utils import create_path @@ -99,7 +100,29 @@ def make_calypso_input(nameofatoms,numberofatoms, def _make_model_devi_buffet(jdata,caly_run_opt_path): calypso_input_path = jdata.get('calypso_input_path') - shutil.copyfile(os.path.join(calypso_input_path,'input.dat'),os.path.join(caly_run_opt_path[0], 'input.dat')) + if jdata.get('vsc', False): + # [input.dat.Li.250, input.dat.Li.300] + one_ele_inputdat_list = list( + set(glob.glob( + f"{jdata.get('calypso_input_path')}/input.dat.{jdata.get('type_map')[0]}.*" + )) + ) + # [input.dat.La, input.dat.H, input.dat.LaH,] only one pressure + if len(one_ele_inputdat_list) == 0: + os.system(f"cp {calypso_input_path}/input.dat.* {caly_run_opt_path[0]}") + # different pressure, 250GPa and 300GPa + # [input.dat.La.250, input.dat.H.250, input.dat.LaH.250, input.dat.La.300, input.dat.H.300, input.dat.LaH.300,] + else: + pressures_list = [temp.split('.')[-1] for temp in one_ele_inputdat_list] + pressures_list = list(map(int, pressures_list)) + # caly_run_opt_path = ['gen_struc_analy.000','gen_struc_analy.001'] + for press_idx, temp_caly_run_opt_path in enumerate(caly_run_opt_path): + cur_press = pressures_list[press_idx] + os.system(f"cp {calypso_input_path}/input.dat.*.{cur_press} {temp_caly_run_opt_path}") + elif not jdata.get('vsc', False): + shutil.copyfile(os.path.join(calypso_input_path,'input.dat'),os.path.join(caly_run_opt_path[0], 'input.dat')) + if not os.path.exists(os.path.join(caly_run_opt_path[0], 'input.dat')): + raise FileNotFoundError('input.dat') def _make_model_devi_native_calypso(iter_index,model_devi_jobs, caly_run_opt_path): diff --git a/dpgen/generator/lib/parse_calypso.py b/dpgen/generator/lib/parse_calypso.py index 123a77e94..df842c789 100644 --- a/dpgen/generator/lib/parse_calypso.py +++ b/dpgen/generator/lib/parse_calypso.py @@ -2,10 +2,13 @@ import os def _parse_calypso_input(var,input_path): - try: - f = open(input_path,'r') - except: - f = open(os.path.join(input_path,'input.dat'),'r') + + if os.path.basename(input_path) != 'input.dat': + input_path = os.path.join(input_path,'input.dat') + if not os.path.exists(input_path): + raise FileNotFoundError(input_path) + + f = open(input_path,'r') lines = f.readlines() f.close() diff --git a/dpgen/generator/lib/run_calypso.py b/dpgen/generator/lib/run_calypso.py index b0a6c5a7a..91d72b2f2 100644 --- a/dpgen/generator/lib/run_calypso.py +++ b/dpgen/generator/lib/run_calypso.py @@ -52,8 +52,6 @@ def gen_structures(iter_index, jdata, mdata, caly_run_path, current_idx, length_ calypso_path = mdata.get('model_devi_calypso_path') #calypso_input_path = jdata.get('calypso_input_path') - popsize = int(_parse_calypso_input('PopSize',calypso_run_opt_path)) - maxstep = int(_parse_calypso_input('MaxStep',calypso_run_opt_path)) all_models = glob.glob(os.path.join(calypso_run_opt_path, 'graph*pb')) model_names = [os.path.basename(ii) for ii in all_models] @@ -91,9 +89,9 @@ def gen_structures(iter_index, jdata, mdata, caly_run_path, current_idx, length_ os.mkdir('opt') except: pass - #shutil.rmtree('opt') - #os.mkdir('opt') + popsize = int(_parse_calypso_input('PopSize', '.')) + maxstep = int(_parse_calypso_input('MaxStep', '.')) for ii in range(int(PickUpStep)-1,maxstep+1): dlog.info('CALYPSO step %s'%ii) @@ -188,24 +186,30 @@ def gen_structures(iter_index, jdata, mdata, caly_run_path, current_idx, length_ comp_temp = list(map(list,list(combinations(type_map,1)))) for hms in range(2,how_many_spec+1): - comp_temp.extend(list(map(list,list(combinations(type_map,hms))))) # comp_temp = [['Mg'],['Al'],['Cu'],['Mg','Al'],['Mg','Cu'],['Al','Cu'],['Mg','Al','Cu']] + # comp_temp = [['Mg'],['Al'],['Cu'],['Mg','Al'],['Mg','Cu'],['Al','Cu'],['Mg','Al','Cu']] + comp_temp.extend(list(map(list,list(combinations(type_map,hms))))) component = [] for comp_temp_ in comp_temp: component.append(''.join(comp_temp_)) # component = ['Mg','Al','Cu','MgAl','MgCu','AlCu','MgAlCu'] dlog.info(component) - #sys.exit() - calypso_input_path = jdata.get('calypso_input_path') + # calypso_input_path = jdata.get('calypso_input_path') - for idx,com in enumerate(component): - pwd = os.getcwd() - os.mkdir(str(idx)) - shutil.copyfile(os.path.join(calypso_input_path,'input.dat.%s'%com),os.path.join(str(idx),'input.dat')) - os.chdir(str(idx)) + pwd = os.getcwd() + if len(glob.glob(f'input.dat.{component[0]}.*')) != 0: + os.system('for i in input.dat.*;do mv $i ${i%.*};done') + for idx, com in enumerate(component): + if not os.path.exists(com): + os.mkdir(com) + #shutil.copyfile(os.path.join(calypso_input_path,'input.dat.%s'%com),os.path.join(com,'input.dat')) + shutil.copyfile('input.dat.%s'%com ,os.path.join(com,'input.dat')) + os.chdir(com) os.system(run_calypso) os.chdir(pwd) - + + shutil.copyfile('input.dat.%s'%component[-1], 'input.dat') + name_list = Path('.').glob('*/POSCAR_*') for idx,name in enumerate(name_list): shutil.copyfile(name,'POSCAR_%s'%(idx+1)) @@ -219,7 +223,9 @@ def gen_structures(iter_index, jdata, mdata, caly_run_path, current_idx, length_ shutil.copyfile('POSCAR_%s'%str(idx+1),os.path.join('task.%04d'%(idx+1),'POSCAR')) shutil.copyfile('input.dat',os.path.join('task.%04d'%(idx+1),'input.dat')) - all_task = glob.glob( "task.*") + # sys.exit() + + all_task = glob.glob("task.*") all_task.sort() run_tasks_ = all_task @@ -293,9 +299,6 @@ def gen_main(iter_index, jdata, mdata, caly_run_opt_list, gen_idx): iter_name = make_iter_name(iter_index) work_path = os.path.join(iter_name, model_devi_name) - caly_run_opt_list = glob.glob(os.path.join(work_path,'%s.*'%((calypso_run_opt_name)))) - caly_run_opt_list.sort() - current_gen_path = os.path.join(work_path, '%s.%03d'%(calypso_run_opt_name, int(gen_idx))) if current_gen_path not in caly_run_opt_list: dlog.info(f"current gen path {current_gen_path} not in caly run opt list {caly_run_opt_list}") @@ -403,7 +406,12 @@ def run_calypso_model_devi (iter_index, calypso_model_devi_path = os.path.join(work_path,calypso_model_devi_name) - caly_run_opt_list = glob.glob(os.path.join(work_path,'%s.*'%(str(calypso_run_opt_name)))) + _caly_run_opt_list = glob.glob(os.path.join(work_path,'%s.*'%(str(calypso_run_opt_name)))) + caly_run_opt_list = _caly_run_opt_list.copy() + # check if gen_struc_analy.000.bk000 in caly_run_opt_list + for temp_value in _caly_run_opt_list: + if 'bk' in temp_value: + caly_run_opt_list.remove(temp_value) caly_run_opt_list.sort() cwd = os.getcwd() diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index facded36b..6013ed779 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -842,7 +842,22 @@ def make_model_devi (iter_index, # Create work path list caly_run_opt_path = [] if run_mode == 1: - caly_run_opt_path.append('%s.%03d'%(_calypso_run_opt_path, 0)) + if jdata.get('vsc', False) and len(jdata.get('type_map')) > 1: + # [input.dat.Li.250, input.dat.Li.300] + one_ele_inputdat_list = glob.glob( + f"{jdata.get('calypso_input_path')}/input.dat.{jdata.get('type_map')[0]}.*" + ) + if len(one_ele_inputdat_list) == 0: + number_of_pressure = 1 + else: + number_of_pressure = len(list(set(one_ele_inputdat_list))) + + # caly_run_opt_path = ['gen_struc_analy.000','gen_struc_analy.001'] + for temp_idx in range(number_of_pressure): + caly_run_opt_path.append('%s.%03d'%(_calypso_run_opt_path, temp_idx)) + elif not jdata.get('vsc', False): + caly_run_opt_path.append('%s.%03d'%(_calypso_run_opt_path, 0)) + elif run_mode == 2: for iiidx, jobbs in enumerate(model_devi_jobs): if iter_index in jobbs.get('times'): @@ -862,14 +877,16 @@ def make_model_devi (iter_index, check_outcar_script = os.path.join(temp_caly_run_opt_path,'check_outcar.py') shutil.copyfile(check_outcar_file,check_outcar_script) - for mm in models : model_name = os.path.basename(mm) if model_devi_engine != 'calypso': os.symlink(mm, os.path.join(work_path, model_name)) else: for temp_caly_run_opt_path in caly_run_opt_path: - os.symlink(mm, os.path.join(temp_caly_run_opt_path, model_name)) + models_path = os.path.join(temp_caly_run_opt_path, model_name) + if not os.path.exists(models_path): + os.symlink(mm, models_path) + with open(os.path.join(work_path, 'cur_job.json'), 'w') as outfile: json.dump(cur_job, outfile, indent = 4) diff --git a/examples/run/dp-calypso-vasp/machine.json b/examples/run/dp-calypso-vasp/machine.json new file mode 100644 index 000000000..f07f2dd87 --- /dev/null +++ b/examples/run/dp-calypso-vasp/machine.json @@ -0,0 +1,70 @@ +{ + "api_version": "1.0", + "deepmd_version": "2.0.1", + "train" :[ + { + "command": "dp", + "machine": { + "batch_type": "pbs", + "context_type": "lazylocal", + "local_root" : "./" + }, + "resources": { + "envs": {"LD_LIBRARY_PATH": "/opt/software/cuda/10.1/lib64:$LD_LIBRARY_PATH"}, + "custom_flags":["#PBS -l walltime=48:0:0"], + "number_node": 1, + "local_root":"./", + "cpu_per_node": 16, + "gpu_per_node": 1, + "queue_name": "T4_16_62", + "group_size": 1 + } + }], + "model_devi": + [{ + "calypso_path":"/home/zhenyu/workplace/dpgen/AA-2400/debug/calypso_input", + "deepmdkit_python":"/home/zhenyu/soft/deepmd-kit/envs/debug/bin/python", + "_deepmdkit_python":"this python will be used to optimize structures generated by CALYPSO with ase", + "_deepmdkit_python":"and also will be used to calculate model deviation in iter.000001/01.model_devi/model_devi_results", + "_deepmdkit_python":"with calypso_run_model_devi.py script", + "command": "", + "machine": { + "_batch_type": "pbs", + "batch_type": "shell", + "context_type": "lazylocal", + "remote_root": "/home/zhenyu/workplace/dpgen/AA-2400/debug/temp", + "local_root" : "./" + }, + "resources": { + "envs": {"LD_LIBRARY_PATH": "/opt/software/cuda/10.1/lib64:$LD_LIBRARY_PATH"}, + "custom_flags":["#PBS -l walltime=48:0:0"], + "number_node": 1, + "local_root":"./", + "cpu_per_node": 16, + "gpu_per_node": 1, + "queue_name": "T4_16_62", + "group_size": 50 + } + }], + "fp": + [{ + "command": "mpirun -n 32 /opt/vasp/vasp_std", + "machine": { + "batch_type": "shell", + "context_type": "lazylocal", + "local_root" : "./", + "remote_root": "/home/zhenyu/workplace/dpgen/AA-2400/debug/temp" + }, + "resources": { + "number_node": 1, + "cpu_per_node": 12, + "gpu_per_node": 0, + "queue_name": "C_32_64", + "group_size": 10, + "local_root":"./", + "custom_flags":["#PBS -l walltime=200:0:0"], + "source_list": ["/opt/intel/oneapi/setvars.sh"] + } + } + ] +} diff --git a/examples/run/dp-calypso-vasp/param.json b/examples/run/dp-calypso-vasp/param.json new file mode 100644 index 000000000..1818a12fb --- /dev/null +++ b/examples/run/dp-calypso-vasp/param.json @@ -0,0 +1,127 @@ +{ + "model_devi_engine":"calypso", + "_calypso_path":"/home/zhenyu/workplace/dpgen/AA-2400/debug", + "_calypso_input_path":"/home/zhenyu/workplace/dpgen/AA-2400/debug/calypso_input", + "_model_devi_max_iter": 50, + "vsc":false, + + "type_map": [ + "Mg", + "Al", + "Cu" + ], + "mass_map": [ + 24, + 27, + 64 + ], + "init_data_prefix":"/home/zhenyu/workplace/dpgen/AA-2400/data/", + "init_data_sys": [ + "data.init/data.init/AlCuMg/init.000" + ], + "_comment": " that's all ", + + "training_init_model": false, + "training_iter0_model_path": "/home/zhenyu/workplace/dpgen/AA-2400/pb/00[0-3]", + "training_reuse_iter": 0, + "training_reuse_old_ratio": 0.9, + "training_reuse_start_lr": 1e-4, + "training_reuse_stop_batch": 1000000, + "training_reuse_start_pref_e": 0.2, + "training_reuse_start_pref_f": 100, + + "numb_models": 4, + "default_training_param": { + "model": { + "descriptor": { + "type": "se_e2_a", + "sel": [ + 500, + 700, + 1100 + ], + "rcut_smth": 2.0, + "rcut": 12.0, + "neuron": [ + 25, + 50, + 100 + ], + "resnet_dt": false, + "axis_neuron": 12, + "type_one_side": true, + "seed": 1801819940, + "_activation_function": "tanh" + }, + "fitting_net": { + "neuron": [ + 240, + 240, + 240 + ], + "resnet_dt": true, + "seed": 2375417769 + }, + "type_map": [ + "Mg", + "Al", + "Cu" + ] + }, + "learning_rate": { + "type": "exp", + "start_lr": 0.001, + "decay_steps": 5000 + }, + "loss": { + "start_pref_e": 0.2, + "limit_pref_e": 2, + "start_pref_f": 100, + "limit_pref_f": 1, + "start_pref_v": 0, + "limit_pref_v": 0 + }, + "training": { + "stop_batch": 6000, + "seed": 3982377700, + "_comment": "that's all", + "disp_file": "lcurve.out", + "disp_freq": 2000, + "numb_test": 4, + "save_freq": 2000, + "save_ckpt": "model.ckpt", + "disp_training": true, + "time_training": true, + "profiling": false, + "profiling_file": "timeline.json", + "set_prefix": "set" + } + }, + "sys_configs":"", + "model_devi_dt": 0.002, + "model_devi_skip": 0, + "model_devi_f_trust_lo": 0.05, + "model_devi_f_trust_hi": 0.15, + "model_devi_e_trust_lo": 10000000000.0, + "model_devi_e_trust_hi": 10000000000.0, + "model_devi_clean_traj": true, + "model_devi_jobs": [ +{"times":[0],"NameOfAtoms":["Mg","Al","Cu"],"NumberOfAtoms":[1,1,1],"NumberOfFormula":[1,1],"Volume":[30],"DistanceOfIon":[[1.48,1.44,1.59],[1.44,1.41,1.56],[1.59,1.56,1.70]],"PsoRatio":[0.6],"PopSize":[500],"MaxStep":[3],"ICode":[1],"Split":"T","VSC":"T","MaxNumAtom":[31],"CtrlRange":[[1,10],[1,10],[1,10]]}, +{"times":[1],"NameOfAtoms":["Mg","Al"],"NumberOfAtoms":[1,1],"NumberOfFormula":[1,1],"Volume":[30],"DistanceOfIon":[[1.48,1.44],[1.44,1.41]],"PsoRatio":[0.6],"PopSize":[500],"MaxStep":[3],"ICode":[1],"Split":"T"}, +{"times":[3],"NameOfAtoms":["Al","Cu"],"NumberOfAtoms":[1,10],"NumberOfFormula":[1,2],"Volume":[300],"DistanceOfIon":[[1.48,1.44],[1.44,1.41]],"PsoRatio":[0.6],"PopSize":[5],"MaxStep":[3],"ICode":[1],"Split":"T"}, +{"times":[2],"NameOfAtoms":["Mg","Al","Cu"],"NumberOfAtoms":[1,1,1],"NumberOfFormula":[1,1],"Volume":[30],"DistanceOfIon":[[1.48,1.44,1.59],[1.44,1.41,1.56],[1.59,1.56,1.70]],"PsoRatio":[0.6],"PopSize":[5],"MaxStep":[1],"ICode":[1],"Split":"T","VSC":"T","MaxNumAtom":[31],"CtrlRange":[[1,10],[1,10],[1,10]],"PSTRESS":[0],"fmax":[0.01]}, +{"times":[4],"NameOfAtoms":["Mg","Al","Cu"],"NumberOfAtoms":[1,1,1],"NumberOfFormula":[1,1],"Volume":30,"DistanceOfIon":[[1.48,1.44,1.59],[1.44,1.41,1.56],[1.59,1.56,1.70]],"PsoRatio":0.6,"PopSize":5,"MaxStep":1,"ICode":1,"Split":"T","VSC":"T","MaxNumAtom":31,"CtrlRange":[[1,10],[1,10],[1,10]],"PSTRESS":[0, 400 ],"fmax":0.01} + ], + + "fp_style": "vasp", + "shuffle_poscar": false, + "fp_task_max": 10, + "fp_task_min": 1, + "fp_pp_path": "/home/zhenyu/workplace/dpgen/AA-2400/vasp_input", + "fp_pp_files": [ + "POTCAR.Mg", + "POTCAR.Al", + "POTCAR.Cu" + ], + "fp_incar":"/home/zhenyu/workplace/dpgen/AA-2400/vasp_input/INCAR" +} From 0d5b7fa743b69afb5a11934af1b5cb0d47bce920 Mon Sep 17 00:00:00 2001 From: zhenyu Date: Mon, 1 Aug 2022 21:02:01 +0800 Subject: [PATCH 40/46] delete some useless comments and modify the assert code in calypso_run_opt.py --- dpgen/generator/lib/calypso_check_outcar.py | 186 ++++++++------- dpgen/generator/lib/calypso_run_opt.py | 239 +++++++++----------- dpgen/generator/lib/make_calypso.py | 40 ++-- 3 files changed, 216 insertions(+), 249 deletions(-) diff --git a/dpgen/generator/lib/calypso_check_outcar.py b/dpgen/generator/lib/calypso_check_outcar.py index 421fd623b..5e9b8490b 100644 --- a/dpgen/generator/lib/calypso_check_outcar.py +++ b/dpgen/generator/lib/calypso_check_outcar.py @@ -1,108 +1,104 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import numpy as np -import os,sys,glob,time -from deepmd.calculator import DP -from ase.io import read +import numpy as np +import os,sys,glob,time +from deepmd.calculator import DP +from ase.io import read ''' check if structure optimization worked well if not, this script will generate a fake outcar ''' - -def Get_Element_Num(elements): + +def Get_Element_Num(elements): '''Using the Atoms.symples to Know Element&Num''' - element = [] - ele = {} - element.append(elements[0]) - for x in elements: - if x not in element : - element.append(x) - for x in element: - ele[x] = elements.count(x) - return element, ele - -def Write_Contcar(element, ele, lat, pos): - '''Write CONTCAR''' - f = open('CONTCAR','w') - f.write('ASE-DPKit-FAILED-nan\n') - f.write('1.0\n') - for i in range(3): - f.write('%15.10f %15.10f %15.10f\n' % tuple(lat[i])) - for x in element: - f.write(x + ' ') - f.write('\n') - for x in element: - f.write(str(ele[x]) + ' ') - f.write('\n') - f.write('Direct\n') - na = sum(ele.values()) - dpos = np.dot(pos,np.linalg.inv(lat)) - for i in range(na): - f.write('%15.10f %15.10f %15.10f\n' % tuple(dpos[i])) - -def Write_Outcar(element, ele, volume, lat, pos, ene, force, stress,pstress): - '''Write OUTCAR''' - f = open('OUTCAR','w') - for x in element: - f.write('VRHFIN =' + str(x) + '\n') - f.write('ions per type =') - for x in element: - f.write('%5d' % ele[x]) - #f.write('\nvolume of cell :\n') + element = [] + ele = {} + element.append(elements[0]) + for x in elements: + if x not in element : + element.append(x) + for x in element: + ele[x] = elements.count(x) + return element, ele + +def Write_Contcar(element, ele, lat, pos): + '''Write CONTCAR''' + f = open('CONTCAR','w') + f.write('ASE-DPKit-FAILED-nan\n') + f.write('1.0\n') + for i in range(3): + f.write('%15.10f %15.10f %15.10f\n' % tuple(lat[i])) + for x in element: + f.write(x + ' ') + f.write('\n') + for x in element: + f.write(str(ele[x]) + ' ') + f.write('\n') + f.write('Direct\n') + na = sum(ele.values()) + dpos = np.dot(pos,np.linalg.inv(lat)) + for i in range(na): + f.write('%15.10f %15.10f %15.10f\n' % tuple(dpos[i])) + +def Write_Outcar(element, ele, volume, lat, pos, ene, force, stress,pstress): + '''Write OUTCAR''' + f = open('OUTCAR','w') + for x in element: + f.write('VRHFIN =' + str(x) + '\n') + f.write('ions per type =') + for x in element: + f.write('%5d' % ele[x]) f.write('\nDirection XX YY ZZ XY YZ ZX\n') - f.write('in kB') - f.write('%15.6f' % stress[0]) - f.write('%15.6f' % stress[1]) - f.write('%15.6f' % stress[2]) - f.write('%15.6f' % stress[3]) - f.write('%15.6f' % stress[4]) - f.write('%15.6f' % stress[5]) - f.write('\n') - ext_pressure = np.sum(stress[0] + stress[1] + stress[2])/3.0 - pstress + f.write('in kB') + f.write('%15.6f' % stress[0]) + f.write('%15.6f' % stress[1]) + f.write('%15.6f' % stress[2]) + f.write('%15.6f' % stress[3]) + f.write('%15.6f' % stress[4]) + f.write('%15.6f' % stress[5]) + f.write('\n') + ext_pressure = np.sum(stress[0] + stress[1] + stress[2])/3.0 - pstress f.write('external pressure = %20.6f kB Pullay stress = %20.6f kB\n'% (ext_pressure, pstress)) - f.write('volume of cell : %20.6f\n' % volume) - f.write('direct lattice vectors\n') - for i in range(3): - f.write('%10.6f %10.6f %10.6f\n' % tuple(lat[i])) - f.write('POSITION TOTAL-FORCE(eV/Angst)\n') - f.write('-------------------------------------------------------------------\n') - na = sum(ele.values()) - for i in range(na): - f.write('%15.6f %15.6f %15.6f' % tuple(pos[i])) - f.write('%15.6f %15.6f %15.6f\n' % tuple(force[i])) - f.write('-------------------------------------------------------------------\n') - f.write('energy without entropy= %20.6f %20.6f\n' % (ene, ene)) - enthalpy = ene + pstress * volume / 1602.17733 - f.write('enthalpy is TOTEN = %20.6f %20.6f\n' % (enthalpy, enthalpy)) - -def check(): - - from deepmd.calculator import DP - from ase.io import read - # model_path = sys.argv[1] - # Model_List = glob.glob('%s/graph*pb'%model_path) - # calc = DP(model='%s'%(Model_List[0])) # init the model before iteration - calc = DP(model='../graph.000.pb') # init the model before iteration - - to_be_opti = read('POSCAR') - to_be_opti.calc = calc - # --------------------------------- - # for failed outcar - atoms_symbols_f = to_be_opti.get_chemical_symbols() - element_f, ele_f = Get_Element_Num(atoms_symbols_f) - atoms_vol_f = to_be_opti.get_volume() - atoms_stress_f = to_be_opti.get_stress() - atoms_stress_f = atoms_stress_f/(0.01*0.6242) - atoms_lat_f = to_be_opti.cell - atoms_pos_f = to_be_opti.positions - atoms_force_f = to_be_opti.get_forces() - atoms_ene_f = 610612509 - # --------------------------------- - Write_Contcar(element_f, ele_f, atoms_lat_f, atoms_pos_f) + f.write('volume of cell : %20.6f\n' % volume) + f.write('direct lattice vectors\n') + for i in range(3): + f.write('%10.6f %10.6f %10.6f\n' % tuple(lat[i])) + f.write('POSITION TOTAL-FORCE(eV/Angst)\n') + f.write('-------------------------------------------------------------------\n') + na = sum(ele.values()) + for i in range(na): + f.write('%15.6f %15.6f %15.6f' % tuple(pos[i])) + f.write('%15.6f %15.6f %15.6f\n' % tuple(force[i])) + f.write('-------------------------------------------------------------------\n') + f.write('energy without entropy= %20.6f %20.6f\n' % (ene, ene)) + enthalpy = ene + pstress * volume / 1602.17733 + f.write('enthalpy is TOTEN = %20.6f %20.6f\n' % (enthalpy, enthalpy)) + +def check(): + + from deepmd.calculator import DP + from ase.io import read + calc = DP(model='../graph.000.pb') # init the model before iteration + + to_be_opti = read('POSCAR') + to_be_opti.calc = calc + # --------------------------------- + # for failed outcar + atoms_symbols_f = to_be_opti.get_chemical_symbols() + element_f, ele_f = Get_Element_Num(atoms_symbols_f) + atoms_vol_f = to_be_opti.get_volume() + atoms_stress_f = to_be_opti.get_stress() + atoms_stress_f = atoms_stress_f/(0.01*0.6242) + atoms_lat_f = to_be_opti.cell + atoms_pos_f = to_be_opti.positions + atoms_force_f = to_be_opti.get_forces() + atoms_ene_f = 610612509 + # --------------------------------- + Write_Contcar(element_f, ele_f, atoms_lat_f, atoms_pos_f) Write_Outcar(element_f, ele_f, atoms_vol_f, atoms_lat_f, atoms_pos_f,atoms_ene_f, atoms_force_f, atoms_stress_f * -10.0, 0) -cwd = os.getcwd() -if not os.path.exists(os.path.join(cwd,'OUTCAR')): - check() +cwd = os.getcwd() +if not os.path.exists(os.path.join(cwd,'OUTCAR')): + check() diff --git a/dpgen/generator/lib/calypso_run_opt.py b/dpgen/generator/lib/calypso_run_opt.py index 7605ea02e..dc84864fd 100644 --- a/dpgen/generator/lib/calypso_run_opt.py +++ b/dpgen/generator/lib/calypso_run_opt.py @@ -1,88 +1,82 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import os,sys,glob,time -import numpy as np -from ase.io import read -from ase.optimize import BFGS,QuasiNewton,LBFGS +import os,sys,glob,time +import numpy as np +from ase.io import read +from ase.optimize import BFGS,QuasiNewton,LBFGS from ase.constraints import UnitCellFilter, ExpCellFilter -from deepmd.calculator import DP +from deepmd.calculator import DP ''' structure optimization with DP model and ASE PSTRESS and fmax should exist in input.dat ''' - -def Get_Element_Num(elements): - '''Using the Atoms.symples to Know Element&Num''' - element = [] - ele = {} - element.append(elements[0]) - for x in elements: - if x not in element : - element.append(x) - for x in element: - ele[x] = elements.count(x) - return element, ele - -def Write_Contcar(element, ele, lat, pos): - '''Write CONTCAR''' - f = open('CONTCAR','w') - f.write('ASE-DPKit-Optimization\n') - f.write('1.0\n') - for i in range(3): - f.write('%15.10f %15.10f %15.10f\n' % tuple(lat[i])) - for x in element: - f.write(x + ' ') - f.write('\n') - for x in element: - f.write(str(ele[x]) + ' ') - f.write('\n') - f.write('Direct\n') - na = sum(ele.values()) - dpos = np.dot(pos,np.linalg.inv(lat)) - for i in range(na): - f.write('%15.10f %15.10f %15.10f\n' % tuple(dpos[i])) - -def Write_Outcar(element, ele, volume, lat, pos, ene, force, stress,pstress): - '''Write OUTCAR''' - f = open('OUTCAR','w') - for x in element: - f.write('VRHFIN =' + str(x) + '\n') - f.write('ions per type =') - for x in element: - f.write('%5d' % ele[x]) - #f.write('\nvolume of cell :\n') + +def Get_Element_Num(elements): + '''Using the Atoms.symples to Know Element&Num''' + element = [] + ele = {} + element.append(elements[0]) + for x in elements: + if x not in element : + element.append(x) + for x in element: + ele[x] = elements.count(x) + return element, ele + +def Write_Contcar(element, ele, lat, pos): + '''Write CONTCAR''' + f = open('CONTCAR','w') + f.write('ASE-DPKit-Optimization\n') + f.write('1.0\n') + for i in range(3): + f.write('%15.10f %15.10f %15.10f\n' % tuple(lat[i])) + for x in element: + f.write(x + ' ') + f.write('\n') + for x in element: + f.write(str(ele[x]) + ' ') + f.write('\n') + f.write('Direct\n') + na = sum(ele.values()) + dpos = np.dot(pos,np.linalg.inv(lat)) + for i in range(na): + f.write('%15.10f %15.10f %15.10f\n' % tuple(dpos[i])) + +def Write_Outcar(element, ele, volume, lat, pos, ene, force, stress,pstress): + '''Write OUTCAR''' + f = open('OUTCAR','w') + for x in element: + f.write('VRHFIN =' + str(x) + '\n') + f.write('ions per type =') + for x in element: + f.write('%5d' % ele[x]) f.write('\nDirection XX YY ZZ XY YZ ZX\n') - f.write('in kB') - f.write('%15.6f' % stress[0]) - f.write('%15.6f' % stress[1]) - f.write('%15.6f' % stress[2]) - f.write('%15.6f' % stress[3]) - f.write('%15.6f' % stress[4]) - f.write('%15.6f' % stress[5]) - f.write('\n') - ext_pressure = np.sum(stress[0] + stress[1] + stress[2])/3.0 - pstress + f.write('in kB') + f.write('%15.6f' % stress[0]) + f.write('%15.6f' % stress[1]) + f.write('%15.6f' % stress[2]) + f.write('%15.6f' % stress[3]) + f.write('%15.6f' % stress[4]) + f.write('%15.6f' % stress[5]) + f.write('\n') + ext_pressure = np.sum(stress[0] + stress[1] + stress[2])/3.0 - pstress f.write('external pressure = %20.6f kB Pullay stress = %20.6f kB\n'% (ext_pressure, pstress)) - f.write('volume of cell : %20.6f\n' % volume) - f.write('direct lattice vectors\n') - for i in range(3): - f.write('%10.6f %10.6f %10.6f\n' % tuple(lat[i])) + f.write('volume of cell : %20.6f\n' % volume) + f.write('direct lattice vectors\n') + for i in range(3): + f.write('%10.6f %10.6f %10.6f\n' % tuple(lat[i])) f.write('POSITION TOTAL-FORCE(eV/Angst)\n') - f.write('-------------------------------------------------------------------\n') - na = sum(ele.values()) - for i in range(na): - f.write('%15.6f %15.6f %15.6f' % tuple(pos[i])) - f.write('%15.6f %15.6f %15.6f\n' % tuple(force[i])) - f.write('-------------------------------------------------------------------\n') - f.write('energy without entropy= %20.6f %20.6f\n' % (ene, ene/na)) - enthalpy = ene + pstress * volume / 1602.17733 - f.write('enthalpy is TOTEN = %20.6f %20.6f\n' % (enthalpy, enthalpy/na)) - -def Write_calylog(templog): - '''For Write Evolve Structures Log into caly.log''' - f = open('opt.log','a+') - f.write(templog+'\n') - f.close() + f.write('-------------------------------------------------------------------\n') + na = sum(ele.values()) + for i in range(na): + f.write('%15.6f %15.6f %15.6f' % tuple(pos[i])) + f.write('%15.6f %15.6f %15.6f\n' % tuple(force[i])) + f.write('-------------------------------------------------------------------\n') + f.write('energy without entropy= %20.6f %20.6f\n' % (ene, ene/na)) + enthalpy = ene + pstress * volume / 1602.17733 + f.write('enthalpy is TOTEN = %20.6f %20.6f\n' % (enthalpy, enthalpy/na)) + def read_stress_fmax(): pstress = 0 fmax = 0.01 @@ -102,76 +96,53 @@ def read_stress_fmax(): if 'fmax' in line: fmax = float(line.split('=')[1]) return fmax,pstress - - -def run_opt(fmax,stress): - '''Using the ASE&DP to Optimize Configures''' - ''' > 600 Steps Called Failure Optimization''' + +def run_opt(fmax,stress): + '''Using the ASE&DP to Optimize Configures''' + calc = DP(model='../graph.000.pb') # init the model before iteration os.system('mv OUTCAR OUTCAR-last') - is_dpgen = True - nn = 1 - try: - # model_path = sys.argv[1] - # Model_List = glob.glob('%s/graph*pb'%model_path) - # calc = DP(model=Model_List[0]) # init the model before iteration - calc = DP(model='../graph.000.pb') # init the model before iteration - except: - assert os.path.exists('graph.pb'), 'did not found graph.pb in this directory %s, or did you forget to add args?'%(os.getcwd()) - calc = DP(model='graph.pb') # init the model before iteration - is_dpgen = False - nn = 3 - - print('Start to Optimize Structures by DP----------') - - Opt_Step = 600 - start = time.time() + + print('Start to Optimize Structures by DP----------') + + Opt_Step = 1000 + start = time.time() # pstress kbar pstress = stress # kBar to eV/A^3 # 1 eV/A^3 = 160.21766028 GPa # 1 / 160.21766028 ~ 0.006242 - aim_stress = 1.0 * pstress* 0.01 * 0.6242 / 10.0 - to_be_opti = read('POSCAR') - to_be_opti.calc = calc + aim_stress = 1.0 * pstress* 0.01 * 0.6242 / 10.0 + to_be_opti = read('POSCAR') + to_be_opti.calc = calc ucf = UnitCellFilter(to_be_opti, scalar_pressure=aim_stress) - atoms_vol_2 = to_be_opti.get_volume() - for i in range(nn): - if is_dpgen: - opt = LBFGS(ucf,trajectory='traj.traj') - else: - opt = LBFGS(ucf) - #opt = QuasiNewton(to_be_opti) - #opt = BFGS(to_be_opti) - #opt = BFGS(to_be_opti,trajectory='traj.traj',logfile='opt.log') - opt.run(fmax=fmax,steps=Opt_Step) - - atoms_lat = to_be_opti.cell - atoms_pos = to_be_opti.positions - atoms_force = to_be_opti.get_forces() - atoms_stress = to_be_opti.get_stress() - # eV/A^3 to GPa - atoms_stress = atoms_stress/(0.01*0.6242) - #atoms_num = to_be_opti.get_atomic_numbers() - atoms_symbols = to_be_opti.get_chemical_symbols() - #atoms_formula = to_be_opti.get_chemical_formula() - atoms_ene = to_be_opti.get_potential_energy() - atoms_vol = to_be_opti.get_volume() - - element, ele = Get_Element_Num(atoms_symbols) - - Write_Contcar(element, ele, atoms_lat, atoms_pos) - Write_Outcar(element, ele, atoms_vol, atoms_lat, atoms_pos,atoms_ene, atoms_force, atoms_stress * -10.0, pstress) - - - stop = time.time() - _cwd = os.getcwd() - _cwd = os.path.basename(_cwd) - print('%s is done, time: %s' % (_cwd,stop-start)) + atoms_vol_2 = to_be_opti.get_volume() + # opt + opt = LBFGS(ucf,trajectory='traj.traj') + opt.run(fmax=fmax,steps=Opt_Step) + + atoms_lat = to_be_opti.cell + atoms_pos = to_be_opti.positions + atoms_force = to_be_opti.get_forces() + atoms_stress = to_be_opti.get_stress() + # eV/A^3 to GPa + atoms_stress = atoms_stress/(0.01*0.6242) + atoms_symbols = to_be_opti.get_chemical_symbols() + atoms_ene = to_be_opti.get_potential_energy() + atoms_vol = to_be_opti.get_volume() + element, ele = Get_Element_Num(atoms_symbols) + + Write_Contcar(element, ele, atoms_lat, atoms_pos) + Write_Outcar(element, ele, atoms_vol, atoms_lat, atoms_pos,atoms_ene, atoms_force, atoms_stress * -10.0, pstress) + + stop = time.time() + _cwd = os.getcwd() + _cwd = os.path.basename(_cwd) + print('%s is done, time: %s' % (_cwd,stop-start)) def run(): fmax, stress = read_stress_fmax() - run_opt(fmax, stress) + run_opt(fmax, stress) if __name__=='__main__': - run() + run() diff --git a/dpgen/generator/lib/make_calypso.py b/dpgen/generator/lib/make_calypso.py index 2cd58d32e..363a33fd6 100644 --- a/dpgen/generator/lib/make_calypso.py +++ b/dpgen/generator/lib/make_calypso.py @@ -13,17 +13,17 @@ def make_calypso_input(nameofatoms,numberofatoms, maxnumatom,ctrlrange,pstress,fmax): ret = "################################ The Basic Parameters of CALYPSO ################################\n" ret+= "# A string of one or several words contain a descriptive name of the system (max. 40 characters).\n" - assert (nameofatoms != None ) + assert nameofatoms is not None ret+= "SystemName = %s\n"%(''.join(nameofatoms)) ret+= "# Number of different atomic species in the simulation.\n" ret+= "NumberOfSpecies = %d\n"%(len(nameofatoms)) ret+= "# Element symbols of the different chemical species.\n" ret+= "NameOfAtoms = %s\n"%(' '.join(nameofatoms)) ret+= "# Number of atoms for each chemical species in one formula unit. \n" - assert numberofatoms != None or len(numberofatoms) == len(nameofatoms) + assert numberofatoms is not None and len(numberofatoms) == len(nameofatoms) ret+= "NumberOfAtoms = %s\n"%(' '.join(list(map(str,numberofatoms)))) ret+= "# The range of formula unit per cell in your simulation. \n" - assert numberofformula != None or len(numberofformula) == 2 or type(numberofformula) == type([1,2]) + assert numberofformula is not None and len(numberofformula) == 2 and type(numberofformula) is list ret+= "NumberOfFormula = %s\n"%(' '.join(list(map(str,numberofformula)))) ret+= "# The volume per formula unit. Unit is in angstrom^3.\n" ret+= "Volume = %s\n"%(volume) @@ -42,9 +42,9 @@ def make_calypso_input(nameofatoms,numberofatoms, assert (0 <= psoratio <= 1 ) ret+= "PsoRatio = %s\n"%(psoratio) ret+= "# The population size. Normally, it has a larger number for larger systems.\n" - assert popsize != None or type(popsize) == type(0) + assert popsize is not None and type(popsize) is int ret+= "PopSize = %d\n"%(popsize) - assert maxstep != None or type(maxstep) == type(0) + assert maxstep is not None and type(maxstep) is int ret+= "# The Max step for iteration\n" ret+= "MaxStep = %d\n"%(maxstep) ret+= "#It determines which method should be adopted in generation the random structure. \n" @@ -55,7 +55,7 @@ def make_calypso_input(nameofatoms,numberofatoms, ret+= "# 0 combination of all method\n" ret+= "# If GenType=3 or 4, it determined the small unit to grow the whole structure\n" ret+= "# It determines which local optimization method should be interfaced in the simulation.\n" - assert icode != None or type(icode) == type(0) + assert icode is not None and type(icode) is int ret+= "ICode= %d\n"%(icode) ret+= "# ICode= 1 interfaced with VASP\n" ret+= "# ICode= 2 interfaced with SIESTA\n" @@ -75,11 +75,11 @@ def make_calypso_input(nameofatoms,numberofatoms, ret+= "Parallel = F\n" ret+= "# The number node for parallel \n" ret+= "NumberOfParallel = 4\n" - assert split != None + assert split is not None ret+= "Split = %s\n"%(split) - assert pstress != None or type(pstress) == type(200) or type(pstress) == type(0.001) + assert pstress is not None and (type(pstress) is int or type(pstress) is float) ret+= "PSTRESS = %f\n"%(pstress) - assert fmax != None or type(fmax) == type(0.01) + assert fmax is not None or type(fmax) is float ret+= "fmax = %f\n"%(fmax) ret+= "################################ End of The Basic Parameters of CALYPSO #######################\n" if vsc == 'T': @@ -97,7 +97,7 @@ def make_calypso_input(nameofatoms,numberofatoms, ret+= "###################End Parameters for VSC ##########################\n" return ret -def _make_model_devi_buffet(jdata,caly_run_opt_path): +def _make_model_devi_buffet(jdata,calypso_run_opt_path): calypso_input_path = jdata.get('calypso_input_path') if jdata.get('vsc', False): @@ -109,28 +109,28 @@ def _make_model_devi_buffet(jdata,caly_run_opt_path): ) # [input.dat.La, input.dat.H, input.dat.LaH,] only one pressure if len(one_ele_inputdat_list) == 0: - os.system(f"cp {calypso_input_path}/input.dat.* {caly_run_opt_path[0]}") + os.system(f"cp {calypso_input_path}/input.dat.* {calypso_run_opt_path[0]}") # different pressure, 250GPa and 300GPa # [input.dat.La.250, input.dat.H.250, input.dat.LaH.250, input.dat.La.300, input.dat.H.300, input.dat.LaH.300,] else: pressures_list = [temp.split('.')[-1] for temp in one_ele_inputdat_list] pressures_list = list(map(int, pressures_list)) - # caly_run_opt_path = ['gen_struc_analy.000','gen_struc_analy.001'] - for press_idx, temp_caly_run_opt_path in enumerate(caly_run_opt_path): + # calypso_run_opt_path = ['gen_struc_analy.000','gen_struc_analy.001'] + for press_idx, temp_calypso_run_opt_path in enumerate(calypso_run_opt_path): cur_press = pressures_list[press_idx] - os.system(f"cp {calypso_input_path}/input.dat.*.{cur_press} {temp_caly_run_opt_path}") + os.system(f"cp {calypso_input_path}/input.dat.*.{cur_press} {temp_calypso_run_opt_path}") elif not jdata.get('vsc', False): - shutil.copyfile(os.path.join(calypso_input_path,'input.dat'),os.path.join(caly_run_opt_path[0], 'input.dat')) - if not os.path.exists(os.path.join(caly_run_opt_path[0], 'input.dat')): + shutil.copyfile(os.path.join(calypso_input_path,'input.dat'),os.path.join(calypso_run_opt_path[0], 'input.dat')) + if not os.path.exists(os.path.join(calypso_run_opt_path[0], 'input.dat')): raise FileNotFoundError('input.dat') -def _make_model_devi_native_calypso(iter_index,model_devi_jobs, caly_run_opt_path): +def _make_model_devi_native_calypso(iter_index,model_devi_jobs, calypso_run_opt_path): for iiidx, jobbs in enumerate(model_devi_jobs): if iter_index in jobbs.get('times'): cur_job = model_devi_jobs[iiidx] - work_path = os.path.dirname(caly_run_opt_path[0]) + work_path = os.path.dirname(calypso_run_opt_path[0]) # cur_job.json with open(os.path.join(work_path, 'cur_job.json'), 'w') as outfile: json.dump(cur_job, outfile, indent = 4) @@ -162,7 +162,7 @@ def _make_model_devi_native_calypso(iter_index,model_devi_jobs, caly_run_opt_pat # pstress is a List which contains the target stress pstress = cur_job.get('PSTRESS',[0.001]) # pressures - for press_idx, temp_caly_run_opt_path in enumerate(caly_run_opt_path): + for press_idx, temp_calypso_run_opt_path in enumerate(calypso_run_opt_path): # cur_press cur_press = pstress[press_idx] file_c = make_calypso_input(nameofatoms,numberofatoms, @@ -170,7 +170,7 @@ def _make_model_devi_native_calypso(iter_index,model_devi_jobs, caly_run_opt_pat distanceofion,psoratio,popsize, maxstep,icode,split,vsc, maxnumatom,ctrlrange,cur_press,fmax) - with open(os.path.join(temp_caly_run_opt_path, 'input.dat'), 'w') as cin : + with open(os.path.join(temp_calypso_run_opt_path, 'input.dat'), 'w') as cin : cin.write(file_c) def write_model_devi_out(devi, fname): From d70594857f50e2c68884a50dca6a03996aaa58cf Mon Sep 17 00:00:00 2001 From: zhenyu Date: Mon, 1 Aug 2022 21:07:22 +0800 Subject: [PATCH 41/46] change sys.exit() into RuntimeError in run_calypso.py --- dpgen/generator/lib/run_calypso.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpgen/generator/lib/run_calypso.py b/dpgen/generator/lib/run_calypso.py index 91d72b2f2..5aa8cda76 100644 --- a/dpgen/generator/lib/run_calypso.py +++ b/dpgen/generator/lib/run_calypso.py @@ -370,7 +370,7 @@ def analysis(iter_index, jdata, calypso_model_devi_path): if len(ms) == 0: dlog.info('too little confs, ') - sys.exit() + raise RuntimeError('no confs found in Analysis part and this should not happen!') if os.path.exists(deepmd_data_path): shutil.rmtree(deepmd_data_path) From 2901d23920235a99c7ce86ac370f3c7bbab3e014 Mon Sep 17 00:00:00 2001 From: zhenyu Date: Tue, 2 Aug 2022 14:04:41 +0800 Subject: [PATCH 42/46] compatible with new python interface of calc_model_devi func --- dpgen/generator/lib/calypso_run_model_devi.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dpgen/generator/lib/calypso_run_model_devi.py b/dpgen/generator/lib/calypso_run_model_devi.py index c4d5b5f27..95f7e7d37 100644 --- a/dpgen/generator/lib/calypso_run_model_devi.py +++ b/dpgen/generator/lib/calypso_run_model_devi.py @@ -70,9 +70,12 @@ def Modd(all_models,type_map): pdata.to_vasp_poscar(os.path.join(put_poscar,'%s.poscar'%str(index))) nopbc = pdata.nopbc coord = pdata.data['coords'] - cell = pdata.data['cells'] + cell = pdata.data['cells'] if not nopbc else None atom_types = pdata.data['atom_types'] - devi = calc_model_devi(coord,cell,atom_types,graphs,nopbc=nopbc) + try: + devi = calc_model_devi(coord,cell,atom_types,graphs,nopbc=nopbc) + except TypeError: + devi = calc_model_devi(coord,cell,atom_types,graphs) # ------------------------------------------------------------------------------------ # append min-distance in devi list dis = pdata.to_ase_structure()[0].get_all_distances(mic=True) From 2dca9e68ac3e445e4fa3d5c6b5ebef274de530d3 Mon Sep 17 00:00:00 2001 From: zhenyu Date: Tue, 2 Aug 2022 20:05:40 +0800 Subject: [PATCH 43/46] add comments to explain the meaning of run_mode --- dpgen/generator/run.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index 6013ed779..f3ee34f8b 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -841,6 +841,9 @@ def make_model_devi (iter_index, shutil.copyfile(calypso_run_model_devi_file,calypso_run_model_devi_script) # Create work path list caly_run_opt_path = [] + + # mode 1: generate structures according to the user-provided input.dat file, + # so calypso_input_path and model_devi_max_iter are needed if run_mode == 1: if jdata.get('vsc', False) and len(jdata.get('type_map')) > 1: # [input.dat.Li.250, input.dat.Li.300] @@ -858,6 +861,8 @@ def make_model_devi (iter_index, elif not jdata.get('vsc', False): caly_run_opt_path.append('%s.%03d'%(_calypso_run_opt_path, 0)) + # mode 2: control each iteration to generate structures in specific way + # by providing model_devi_jobs key elif run_mode == 2: for iiidx, jobbs in enumerate(model_devi_jobs): if iter_index in jobbs.get('times'): From 1f091e1df9d4ca997be7498231d5d553f4530dde Mon Sep 17 00:00:00 2001 From: zhenyu Date: Wed, 3 Aug 2022 19:36:44 +0800 Subject: [PATCH 44/46] change caly_run_opt_path into calypso_run_opt_path --- dpgen/generator/run.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/dpgen/generator/run.py b/dpgen/generator/run.py index f3ee34f8b..24baf1ed4 100644 --- a/dpgen/generator/run.py +++ b/dpgen/generator/run.py @@ -840,7 +840,7 @@ def make_model_devi (iter_index, calypso_run_model_devi_script = os.path.join(calypso_model_devi_path,'calypso_run_model_devi.py') shutil.copyfile(calypso_run_model_devi_file,calypso_run_model_devi_script) # Create work path list - caly_run_opt_path = [] + calypso_run_opt_path = [] # mode 1: generate structures according to the user-provided input.dat file, # so calypso_input_path and model_devi_max_iter are needed @@ -855,11 +855,11 @@ def make_model_devi (iter_index, else: number_of_pressure = len(list(set(one_ele_inputdat_list))) - # caly_run_opt_path = ['gen_struc_analy.000','gen_struc_analy.001'] + # calypso_run_opt_path = ['gen_struc_analy.000','gen_struc_analy.001'] for temp_idx in range(number_of_pressure): - caly_run_opt_path.append('%s.%03d'%(_calypso_run_opt_path, temp_idx)) + calypso_run_opt_path.append('%s.%03d'%(_calypso_run_opt_path, temp_idx)) elif not jdata.get('vsc', False): - caly_run_opt_path.append('%s.%03d'%(_calypso_run_opt_path, 0)) + calypso_run_opt_path.append('%s.%03d'%(_calypso_run_opt_path, 0)) # mode 2: control each iteration to generate structures in specific way # by providing model_devi_jobs key @@ -870,16 +870,16 @@ def make_model_devi (iter_index, pressures_list = cur_job.get('PSTRESS', [0.0001]) for temp_idx in range(len(pressures_list)): - caly_run_opt_path.append('%s.%03d'%(_calypso_run_opt_path, temp_idx)) + calypso_run_opt_path.append('%s.%03d'%(_calypso_run_opt_path, temp_idx)) # to different directory - # caly_run_opt_path = ['gen_struc_analy.000','gen_struc_analy.001','gen_struc_analy.002',] - for temp_caly_run_opt_path in caly_run_opt_path: - create_path(temp_caly_run_opt_path) + # calypso_run_opt_path = ['gen_struc_analy.000','gen_struc_analy.001','gen_struc_analy.002',] + for temp_calypso_run_opt_path in calypso_run_opt_path: + create_path(temp_calypso_run_opt_path) # run confs opt script - run_opt_script = os.path.join(temp_caly_run_opt_path,'calypso_run_opt.py') + run_opt_script = os.path.join(temp_calypso_run_opt_path,'calypso_run_opt.py') shutil.copyfile(run_opt_file,run_opt_script) # check outcar script - check_outcar_script = os.path.join(temp_caly_run_opt_path,'check_outcar.py') + check_outcar_script = os.path.join(temp_calypso_run_opt_path,'check_outcar.py') shutil.copyfile(check_outcar_file,check_outcar_script) for mm in models : @@ -887,8 +887,8 @@ def make_model_devi (iter_index, if model_devi_engine != 'calypso': os.symlink(mm, os.path.join(work_path, model_name)) else: - for temp_caly_run_opt_path in caly_run_opt_path: - models_path = os.path.join(temp_caly_run_opt_path, model_name) + for temp_calypso_run_opt_path in calypso_run_opt_path: + models_path = os.path.join(temp_calypso_run_opt_path, model_name) if not os.path.exists(models_path): os.symlink(mm, models_path) @@ -946,13 +946,13 @@ def make_model_devi (iter_index, elif model_devi_engine == "amber": _make_model_devi_amber(iter_index, jdata, mdata, conf_systems) elif model_devi_engine == "calypso": - _make_model_devi_native_calypso(iter_index,model_devi_jobs, caly_run_opt_path) # generate input.dat automatic in each iter + _make_model_devi_native_calypso(iter_index,model_devi_jobs, calypso_run_opt_path) # generate input.dat automatic in each iter else: raise RuntimeError("unknown model_devi engine", model_devi_engine) elif input_mode == "revise_template": _make_model_devi_revmat(iter_index, jdata, mdata, conf_systems) elif input_mode == "buffet": - _make_model_devi_buffet(jdata,caly_run_opt_path) # generate confs according to the input.dat provided + _make_model_devi_buffet(jdata,calypso_run_opt_path) # generate confs according to the input.dat provided else: raise RuntimeError('unknown model_devi input mode', input_mode) #Copy user defined forward_files From fbca550bb4087b002f2a473146e84a7d2cb607d2 Mon Sep 17 00:00:00 2001 From: zhenyu Date: Thu, 4 Aug 2022 14:00:26 +0800 Subject: [PATCH 45/46] fix bug when key Volume is not given CALYPSO will not run --- dpgen/generator/lib/make_calypso.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dpgen/generator/lib/make_calypso.py b/dpgen/generator/lib/make_calypso.py index 363a33fd6..bb2db0af8 100644 --- a/dpgen/generator/lib/make_calypso.py +++ b/dpgen/generator/lib/make_calypso.py @@ -26,7 +26,10 @@ def make_calypso_input(nameofatoms,numberofatoms, assert numberofformula is not None and len(numberofformula) == 2 and type(numberofformula) is list ret+= "NumberOfFormula = %s\n"%(' '.join(list(map(str,numberofformula)))) ret+= "# The volume per formula unit. Unit is in angstrom^3.\n" - ret+= "Volume = %s\n"%(volume) + if volume is None: + ret+= "# volume not found, CALYPSO will set one!\n" + else: + ret+= "Volume = %s\n"%(volume) ret+= "# Minimal distance between atoms of each chemical species. Unit is in angstrom.\n" assert len(distanceofion) == len(nameofatoms) #"check distance of ions and the number of atoms" assert len(distanceofion[0]) == len(nameofatoms) From 7631283b7b2ad0ea7fb6663db3db63bd109dc6ce Mon Sep 17 00:00:00 2001 From: zhenyu Date: Thu, 4 Aug 2022 22:52:59 +0800 Subject: [PATCH 46/46] fix bug of the number of confs is not correct when different pressure are in use --- dpgen/generator/lib/run_calypso.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dpgen/generator/lib/run_calypso.py b/dpgen/generator/lib/run_calypso.py index 5aa8cda76..5fc0ad934 100644 --- a/dpgen/generator/lib/run_calypso.py +++ b/dpgen/generator/lib/run_calypso.py @@ -334,6 +334,7 @@ def analysis(iter_index, jdata, calypso_model_devi_path): record_traj_num = 0 for traj_name in traj_list: traj_num = os.path.basename(traj_name).split('.')[0] + press_num = traj_name.split('/')[-3].split('.')[-1] trajs_origin = Trajectory(traj_name) record_traj_num += len(trajs_origin) @@ -354,8 +355,8 @@ def analysis(iter_index, jdata, calypso_model_devi_path): for idx, traj in enumerate(trajs): write_vasp(os.path.join( - traj_pos_path,'%03d.%03d.poscar' % ( - int(traj_num), int(idx) + traj_pos_path,'%d.%03d.%03d.poscar' % ( + int(press_num), int(traj_num), int(idx) ) ), traj)