2021-08-09 21:15:51 [INFO] Start to evaluate(total_samples=198, total_steps=198)...

---------------------------------------------------------------------------OSError Traceback (most recent call last) in
39 warmup_start_lr=0.0,
40 save_dir='output/mask_rcnn_r50_fpn',
---> 41 use_vdl=True)
42
/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/models/detector.py in train(self, num_epochs, train_dataset, train_batch_size, eval_dataset, optimizer, save_interval_epochs, log_interval_steps, save_dir, pretrain_weights, learning_rate, warmup_steps, warmup_start_lr, lr_decay_epochs, lr_decay_gamma, metric, use_ema, early_stop, early_stop_patience, use_vdl, resume_checkpoint)
284 early_stop=early_stop,
285 early_stop_patience=early_stop_patience,
--> 286 use_vdl=use_vdl)
287
288 def quant_aware_train(self,
/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/models/base.py in train_loop(self, num_epochs, train_dataset, train_batch_size, eval_dataset, save_interval_epochs, log_interval_steps, save_dir, ema, early_stop, early_stop_patience, use_vdl)
393 eval_dataset,
394 batch_size=eval_batch_size,
--> 395 return_details=True)
396 # 保存最优模型
397 if local_rank == 0:
/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/models/detector.py in evaluate(self, eval_dataset, batch_size, metric, return_details)
452 with paddle.no_grad():
453 for step, data in enumerate(self.eval_data_loader):
--> 454 outputs = self.run(self.net, data, 'eval')
455 eval_metric.update(data, outputs)
456 eval_metric.accumulate()
/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/models/detector.py in run(self, net, inputs, mode)
98
99 def run(self, net, inputs, mode):
--> 100 net_out = net(inputs)
101 if mode in ['train', 'eval']:
102 outputs = net_out
/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/dygraph/layers.py in __call__(self, *inputs, **kwargs)
896 self._built = True
897
--> 898 outputs = self.forward(*inputs, **kwargs)
899
900 for forward_post_hook in self._forward_post_hooks.values():
/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/ppdet/modeling/architectures/meta_arch.py in forward(self, inputs)
27 out = self.get_loss()
28 else:
---> 29 out = self.get_pred()
30 return out
31
/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/ppdet/modeling/architectures/mask_rcnn.py in get_pred(self)
131
132 def get_pred(self):
--> 133 bbox_pred, bbox_num, mask_pred = self._forward()
134 output = {'bbox': bbox_pred, 'bbox_num': bbox_num, 'mask': mask_pred}
135 return output
/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/ppdet/modeling/architectures/mask_rcnn.py in _forward(self)
101 return rpn_loss, bbox_loss, mask_loss
102 else:
--> 103 rois, rois_num, _ = self.rpn_head(body_feats, self.inputs)
104 preds, feat_func = self.bbox_head(body_feats, rois, rois_num, None)
105
/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/dygraph/layers.py in __call__(self, *inputs, **kwargs)
896 self._built = True
897
--> 898 outputs = self.forward(*inputs, **kwargs)
899
900 for forward_post_hook in self._forward_post_hooks.values():
/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/ppdet/modeling/proposal_generator/rpn_head.py in forward(self, feats, inputs)
141
142 rois, rois_num = self._gen_proposal(scores, deltas, anchors, inputs,
--> 143 batch_size)
144 if self.training:
145 loss = self.get_loss(scores, deltas, anchors, inputs)
/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/ppdet/modeling/proposal_generator/rpn_head.py in _gen_proposal(self, scores, bbox_deltas, anchors, inputs, batch_size)
176 bbox_deltas=rpn_delta[i:i + 1],
177 anchors=anchor,
--> 178 im_shape=im_shape[i:i + 1])
179 if rpn_rois.shape[0] > 0:
180 rpn_rois_list.append(rpn_rois)
/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/ppdet/modeling/proposal_generator/proposal_generator.py in __call__(self, scores, bbox_deltas, anchors, im_shape)
78 min_size=self.min_size,
79 eta=self.eta,
---> 80 return_rois_num=True)
81 return rpn_rois, rpn_rois_prob, rpn_rois_num, self.post_nms_top_n
/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/ppdet/modeling/ops.py in generate_proposals(scores, bbox_deltas, im_shape, anchors, variances, pre_nms_top_n, post_nms_top_n, nms_thresh, min_size, eta, pixel_offset, return_rois_num, name)
1516 min_size, 'eta', eta, 'pixel_offset', pixel_offset)
1517 rpn_rois, rpn_roi_probs, rpn_rois_num = core.ops.generate_proposals_v2(
-> 1518 scores, bbox_deltas, im_shape, anchors, variances, *attrs)
1519 return rpn_rois, rpn_roi_probs, rpn_rois_num
1520
OSError: (External) Cuda error(700), an illegal memory access was encountered.
[Advise: Please search for the error code(700) on website( https://docs.nvidia.com/cuda/archive/10.0/cuda-runtime-api/group__CUDART__TYPES.html#group__CUDART__TYPES_1g3f51e3575c2178246db0a94a430e0038 ) to get Nvidia's official solution about CUDA Error.] (at /paddle/paddle/fluid/platform/gpu_info.cc:382)
[operator < generate_proposals_v2 > error]