I have seen a lot of nans during training，Do you have any good suggestions for modification? Training with sel_atten_v2

[XuFanffei]

The input.json is as follows:
{
"model": {
"descriptor": {
"type": "se_atten_v2",
"sel": 120,
"rcut_smth": 2.0,
"rcut": 10.0,
"neuron": [
25,
50,
100
],
"resnet_dt": false,
"axis_neuron": 12,
"seed": 1801819940,
"attn": 128,
"attn_layer": 2,
"attn_mask": false,
"attn_dotr": true,
"_activation_function": "tanh"
},
"fitting_net": {
"neuron": [
240,
240,
240
],
"resnet_dt": true,
"_coord_norm": true,
"_type_fitting_net": false,
"seed": 2375417769,
"_activation_function": "tanh"
},
"type_map": [
"W",
"N",
"Ge",
"Si",
"C",
"Pb",
"P",
"Bi",
"Se",
"Mo",
"Sn",
"As",
"Te",
"S",
"H",
"O",
"K",
"F",
"Cr",
"Zr",
"Sb",
"Ce",
"Ga",
"Cs",
"Cl",
"Rb",
"I",
"Hf",
"B",
"Tb",
"Li",
"Br",
"Al",
"Na",
"Sc",
"Pr",
"Ti",
"In",
"Mg",
"Cd",
"H.75",
"Ca",
"Pt",
"Cu",
"Hg",
"H1.25",
"Pd",
"Be",
"H.5",
"Po"
]
},
"learning_rate": {
"type": "exp",
"start_lr": 0.001,
"decay_steps": 10000,
"stop_lr": 5e-08,
"_decay_rate": 0.95
},
"loss": {
"start_pref_e": 0.02,
"limit_pref_e": 2,
"start_pref_f": 1000,
"limit_pref_f": 1,
"start_pref_v": 0,
"limit_pref_v": 0
},
"training": {
"training_data": {
"systems": [
"../dp_merge/110",
"../dp_merge/84",
"../dp_merge/60",
"../dp_merge/17",
"../dp_merge/7",
"../dp_merge/36",
"../dp_merge/232",
"../dp_merge/335",
"../dp_merge/74",
"../dp_merge/31",
"../dp_merge/104",
"../dp_merge/152",
"../dp_merge/10",
"../dp_merge/67",
"../dp_merge/28",
"../dp_merge/527",
"../dp_merge/55",
"../dp_merge/22",
"../dp_merge/214",
"../dp_merge/14",
"../dp_merge/63",
"../dp_merge/87",
"../dp_merge/35",
"../dp_merge/4",
"../dp_merge/246",
"../dp_merge/93",
"../dp_merge/148",
"../dp_merge/9",
"../dp_merge/3",
"../dp_merge/170",
"../dp_merge/107",
"../dp_merge/32",
"../dp_merge/13",
"../dp_merge/80",
"../dp_merge/19",
"../dp_merge/126",
"../dp_merge/383",
"../dp_merge/21",
"../dp_merge/88",
"../dp_merge/23",
"../dp_merge/116",
"../dp_merge/29",
"../dp_merge/215",
"../dp_merge/30",
"../dp_merge/1",
"../dp_merge/6",
"../dp_merge/102",
"../dp_merge/108",
"../dp_merge/40",
"../dp_merge/37",
"../dp_merge/239",
"../dp_merge/78",
"../dp_merge/24",
"../dp_merge/59",
"../dp_merge/85",
"../dp_merge/154",
"../dp_merge/123",
"../dp_merge/150",
"../dp_merge/18",
"../dp_merge/12",
"../dp_merge/216",
"../dp_merge/20",
"../dp_merge/149",
"../dp_merge/98",
"../dp_merge/143",
"../dp_merge/134",
"../dp_merge/2",
"../dp_merge/8",
"../dp_merge/431"
],
"batch_size": "auto"
},
"validation_data": {
"systems": [
"../dp_merge/192",
"../dp_merge/5",
"../dp_merge/95",
"../dp_merge/144",
"../dp_merge/112",
"../dp_merge/118",
"../dp_merge/27",
"../dp_merge/288",
"../dp_merge/120",
"../dp_merge/68",
"../dp_merge/15",
"../dp_merge/359"
],
"batch_size": 1,
"numb_btch": 4,
"_comment": "that's all"
},
"numb_steps": 10000000,
"seed": 3982377700,
"_comment": "that's all",
"disp_file": "lcurve.out",
"disp_freq": 2000,
"numb_test": 1,
"save_freq": 2000,
"save_ckpt": "model.ckpt",
"disp_training": true,
"time_training": true,
"profiling": false,
"profiling_file": "timeline.json"
}
}

[njzjz]

The learning rate (1e-2) is too large, which is likely to cause NaN.

Your configuration is 1e-3 instead of 1e-2, though. You might make a mistake here.

[th-nic]

Was this issue resolved? Can you please share the solution. I am facing similar issues.