运行时报错信息如下：
Traceback (most recent call last):
File "run_classifier.py", line 287, in
main(args)
File "run_classifier.py", line 187, in main
main_program=train_program)
File "/home/slurm/job/tmp/job-168618/ERNIE/python/lib/python2.7/site-packages/paddle/fluid/parallel_executor.py", line 122, in init
self.compiled_program.compile(place=self.place, scope=self.scope)
File "/home/slurm/job/tmp/job-168618/ERNIE/python/lib/python2.7/site-packages/paddle/fluid/compiler.py", line 282, in compile
scope=self.scope)
File "/home/slurm/job/tmp/job-168618/ERNIE/python/lib/python2.7/site-packages/paddle/fluid/compiler.py", line 253, in compile_data_parallel
self.exec_strategy, self.build_strategy, self.graph)
paddle.fluid.core.EnforceNotMet: internal error at [/ssd2/liyukun01/paddle-env/repos/Paddle/paddle/fluid/platform/nccl_helper.h:113]
PaddlePaddle Call Stacks:
0 0x7f3274456992p void paddle::platform::EnforceNotMet::Init<char const*>(char const*, char const*, int) + 354
1 0x7f3274456d16p paddle::platform::EnforceNotMet::EnforceNotMet(std::exception_ptr::exception_ptr, char const*, int) + 134
2 0x7f327459dfb5p paddle::platform::NCCLContextMap::NCCLContextMap(std::vector<boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, std::allocator<boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> > > const&, ncclUniqueId*, unsigned long, unsigned long) + 3317
3 0x7f327459a2a2p paddle::framework::ParallelExecutor::ParallelExecutor(std::vector<boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, std::allocator<boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> > > const&, std::vector<std::string, std::allocatorstd::string > const&, std::string const&, paddle::framework::Scope*, std::vector<paddle::framework::Scope*, std::allocatorpaddle::framework::Scope* > const&, paddle::framework::details::ExecutionStrategy const&, paddle::framework::details::BuildStrategy const&, paddle::framework::ir::Graph*) + 2962
4 0x7f32744b4bfbp
5 0x7f327447b3c9p
6 0x7f32de9b6f13p PyObject_Call + 67
7 0x7f32de9c5a4dp
8 0x7f32de9b6f13p PyObject_Call + 67
9 0x7f32dea2520fp
10 0x7f32dea2197fp
11 0x7f32de9b6f13p PyObject_Call + 67
12 0x7f32dea6cb66p PyEval_EvalFrameEx + 15014
13 0x7f32dea7286dp PyEval_EvalCodeEx + 2061
14 0x7f32dea6f9fcp PyEval_EvalFrameEx + 26940
15 0x7f32dea7286dp PyEval_EvalCodeEx + 2061
16 0x7f32dea6f9fcp PyEval_EvalFrameEx + 26940
17 0x7f32dea7286dp PyEval_EvalCodeEx + 2061
18 0x7f32de9e9015p
19 0x7f32de9b6f13p PyObject_Call + 67
20 0x7f32de9c5a4dp
21 0x7f32de9b6f13p PyObject_Call + 67
22 0x7f32dea2520fp
23 0x7f32dea2197fp
24 0x7f32de9b6f13p PyObject_Call + 67
25 0x7f32dea6cb66p PyEval_EvalFrameEx + 15014
26 0x7f32dea7286dp PyEval_EvalCodeEx + 2061
27 0x7f32dea6f9fcp PyEval_EvalFrameEx + 26940
28 0x7f32dea7286dp PyEval_EvalCodeEx + 2061
29 0x7f32dea729a2p PyEval_EvalCode + 50
30 0x7f32dea9b782p PyRun_FileExFlags + 146
31 0x7f32dea9caf9p PyRun_SimpleFileExFlags + 217
32 0x7f32deab282dp Py_Main + 3149
33 0x7f32ddcafbd5p __libc_start_main + 245
34 0x4007a1p
*** Aborted at 1581944112 (unix time) try "date -d @1581944112" if you are using GNU date ***
PC: @ 0x0 (unknown)
*** SIGSEGV (@0x5b22120) received by PID 28165 (TID 0x7f32def9f700) from PID 95559968; stack trace: ***
@ 0x7f32de755160 (unknown)
@ 0x5b22120 (unknown)
script/run_ChnSentiCorp.sh: line 39: 28165 Segmentation fault python -u run_classifier.py --use_cuda true --verbose true --do_train true --do_val false --do_test true --batch_size 1024 --init_pretraining_params ${MODEL_PATH} --train_set ${TASK_DATA_PATH}/train_filelist --test_set ${TASK_DATA_PATH}/test_filelist.without_punc_stopword --vocab_path config/vocab_word.txt --checkpoints ./checkpoints --save_steps 100000000 --weight_decay 0.01 --warmup_proportion 0.0 --validation_steps 50 --epoch 1 --max_seq_len 70 --ernie_config_path config/ernie_config_distillation.json --learning_rate 1e-3 --skip_steps 50 --num_iteration_per_drop_scope 1 --num_labels 1 --random_seed 1
=============================================
代码：
运行时报错信息如下：
Traceback (most recent call last):
File "run_classifier.py", line 287, in
main(args)
File "run_classifier.py", line 187, in main
main_program=train_program)
File "/home/slurm/job/tmp/job-168618/ERNIE/python/lib/python2.7/site-packages/paddle/fluid/parallel_executor.py", line 122, in init
self.compiled_program.compile(place=self.place, scope=self.scope)
File "/home/slurm/job/tmp/job-168618/ERNIE/python/lib/python2.7/site-packages/paddle/fluid/compiler.py", line 282, in compile
scope=self.scope)
File "/home/slurm/job/tmp/job-168618/ERNIE/python/lib/python2.7/site-packages/paddle/fluid/compiler.py", line 253, in compile_data_parallel
self.exec_strategy, self.build_strategy, self.graph)
paddle.fluid.core.EnforceNotMet: internal error at [/ssd2/liyukun01/paddle-env/repos/Paddle/paddle/fluid/platform/nccl_helper.h:113]
PaddlePaddle Call Stacks:
0 0x7f3274456992p void paddle::platform::EnforceNotMet::Init<char const*>(char const*, char const*, int) + 354
1 0x7f3274456d16p paddle::platform::EnforceNotMet::EnforceNotMet(std::exception_ptr::exception_ptr, char const*, int) + 134
2 0x7f327459dfb5p paddle::platform::NCCLContextMap::NCCLContextMap(std::vector<boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, std::allocator<boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> > > const&, ncclUniqueId*, unsigned long, unsigned long) + 3317
3 0x7f327459a2a2p paddle::framework::ParallelExecutor::ParallelExecutor(std::vector<boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, std::allocator<boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> > > const&, std::vector<std::string, std::allocatorstd::string > const&, std::string const&, paddle::framework::Scope*, std::vector<paddle::framework::Scope*, std::allocatorpaddle::framework::Scope* > const&, paddle::framework::details::ExecutionStrategy const&, paddle::framework::details::BuildStrategy const&, paddle::framework::ir::Graph*) + 2962
4 0x7f32744b4bfbp
5 0x7f327447b3c9p
6 0x7f32de9b6f13p PyObject_Call + 67
7 0x7f32de9c5a4dp
8 0x7f32de9b6f13p PyObject_Call + 67
9 0x7f32dea2520fp
10 0x7f32dea2197fp
11 0x7f32de9b6f13p PyObject_Call + 67
12 0x7f32dea6cb66p PyEval_EvalFrameEx + 15014
13 0x7f32dea7286dp PyEval_EvalCodeEx + 2061
14 0x7f32dea6f9fcp PyEval_EvalFrameEx + 26940
15 0x7f32dea7286dp PyEval_EvalCodeEx + 2061
16 0x7f32dea6f9fcp PyEval_EvalFrameEx + 26940
17 0x7f32dea7286dp PyEval_EvalCodeEx + 2061
18 0x7f32de9e9015p
19 0x7f32de9b6f13p PyObject_Call + 67
20 0x7f32de9c5a4dp
21 0x7f32de9b6f13p PyObject_Call + 67
22 0x7f32dea2520fp
23 0x7f32dea2197fp
24 0x7f32de9b6f13p PyObject_Call + 67
25 0x7f32dea6cb66p PyEval_EvalFrameEx + 15014
26 0x7f32dea7286dp PyEval_EvalCodeEx + 2061
27 0x7f32dea6f9fcp PyEval_EvalFrameEx + 26940
28 0x7f32dea7286dp PyEval_EvalCodeEx + 2061
29 0x7f32dea729a2p PyEval_EvalCode + 50
30 0x7f32dea9b782p PyRun_FileExFlags + 146
31 0x7f32dea9caf9p PyRun_SimpleFileExFlags + 217
32 0x7f32deab282dp Py_Main + 3149
33 0x7f32ddcafbd5p __libc_start_main + 245
34 0x4007a1p
*** Aborted at 1581944112 (unix time) try "date -d @1581944112" if you are using GNU date ***
PC: @ 0x0 (unknown)
*** SIGSEGV (@0x5b22120) received by PID 28165 (TID 0x7f32def9f700) from PID 95559968; stack trace: ***
@ 0x7f32de755160 (unknown)
@ 0x5b22120 (unknown)
script/run_ChnSentiCorp.sh: line 39: 28165 Segmentation fault python -u run_classifier.py --use_cuda true --verbose true --do_train true --do_val false --do_test true --batch_size 1024 --init_pretraining_params ${MODEL_PATH} --train_set ${TASK_DATA_PATH}/train_filelist --test_set ${TASK_DATA_PATH}/test_filelist.without_punc_stopword --vocab_path config/vocab_word.txt --checkpoints ./checkpoints --save_steps 100000000 --weight_decay 0.01 --warmup_proportion 0.0 --validation_steps 50 --epoch 1 --max_seq_len 70 --ernie_config_path config/ernie_config_distillation.json --learning_rate 1e-3 --skip_steps 50 --num_iteration_per_drop_scope 1 --num_labels 1 --random_seed 1
=============================================
代码：