Out of memory error on GPU

paddle version: develop

batch_size=1
train_batch_reader = paddle.batch(paddle.reader.shuffle(reader, buf_size=BATCH_SIZE*2), BATCH_SIZE, drop_last=True)
def reader_flyingchairs(dataset):
    n = len(dataset)

    def reader():
        for i in range(n):
            a, b = dataset[i]
            yield a[0][:,0,:,:].transpose(1,2,0), a[0][:,1,:,:].transpose(1,2,0), b[0].transpose(1, 2, 0)# a single entry of data is created each time
    return reader

some code

    with fluid.dygraph.guard(place=fluid.CUDAPlace(0)):
        for epoch in range(epoch_num):
            for batch_id, data in enumerate(train_batch_reader()):
                start = time.time()
                im1_data = np.array(
                    [x[0] for x in data]).astype('float32')
                im2_data = np.array(
                    [x[1] for x in data]).astype('float32')
                flo_data = np.array(
                    [x[2] for x in data]).astype('float32')
                im_all = np.concatenate((im1_data, im2_data), axis=3).astype(np.float32)
                im_all = im_all/255.0
                im_all = np.swapaxes(np.swapaxes(im_all, 1, 2), 1, 3)
                label = flo_data/20.0
                label = np.swapaxes(np.swapaxes(label, 1, 2), 1, 3)
                print(im_all.shape)
                im_all = fluid.dygraph.to_variable(im_all)
                end = time.time()
                print('read data {} s ----'.format(end - start))
                start = time.time()
                res = model(im_all)
                end = time.time()
                model.clear_gradients()
                print('batch {} time: {} s'.format(batch_id, end - start))

when running after some batch, memory error occurs

read data 0.010515689849853516 s ----
batch 55 time: 0.06910538673400879 s
(1, 6, 384, 512)
read data 0.01302027702331543 s ----
batch 56 time: 0.0730290412902832 s
(1, 6, 384, 512)
read data 0.011269807815551758 s ----
batch 57 time: 0.08451676368713379 s
(1, 6, 384, 512)
read data 0.009588003158569336 s ----
Traceback (most recent call last):
  File "train.py", line 134, in <module>
    main()
  File "train.py", line 119, in main
    flo = model(im_all)
  File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/layers.py", line 181, in __call__
    outputs = self.forward(*inputs, **kwargs)
  File "/models/dy_model.py", line 196, in forward
    x = fluid.layers.concat(input=[fluid.layers.leaky_relu(self.conv6_4(x), 0.1), x], axis=1)
  File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/layers.py", line 181, in __call__
    outputs = self.forward(*inputs, **kwargs)
  File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/nn.py", line 251, in forward
    'use_mkldnn': False,
  File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/layer_object_helper.py", line 52, in append_op
    stop_gradient=stop_gradient)
  File "/usr/local/lib/python3.7/site-packages/paddle/fluid/framework.py", line 2499, in append_op
    kwargs.get("stop_gradient", False))
  File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/tracer.py", line 47, in trace_op
    not stop_gradient)
RuntimeError:

--------------------------------------------
C++ Call Stacks (More useful to developers):
--------------------------------------------
0   std::string paddle::platform::GetTraceBackString<std::string>(std::string&&, char const*, int)
1   paddle::memory::detail::GPUAllocator::Alloc(unsigned long*, unsigned long)
2   paddle::memory::detail::BuddyAllocator::RefillPool(unsigned long)
3   paddle::memory::detail::BuddyAllocator::Alloc(unsigned long)
4   void* paddle::memory::legacy::Alloc<paddle::platform::CUDAPlace>(paddle::platform::CUDAPlace const&, unsigned long)
5   paddle::memory::allocation::NaiveBestFitAllocator::AllocateImpl(unsigned long)
6   paddle::memory::allocation::Allocator::Allocate(unsigned long)
7   paddle::memory::allocation::RetryAllocator::AllocateImpl(unsigned long)
8   paddle::memory::allocation::AllocatorFacade::Alloc(paddle::platform::Place const&, unsigned long)
9   paddle::memory::Alloc(paddle::platform::Place const&, unsigned long)
10  paddle::memory::Alloc(paddle::platform::DeviceContext const&, unsigned long)
11  paddle::platform::CudnnWorkspaceHandle::ReallocWorkspace(unsigned long)
12  paddle::operators::CUDNNConvOpKernel<float>::Compute(paddle::framework::ExecutionContext const&) const
13  std::_Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CUDAPlace, false, 0ul, paddle::operators::CUDNNConvOpKernel<float>, paddle::operators::CUDNNConvOpKernel<double>, paddle::operators::CUDNNConvOpKernel<paddle::platform::float16> >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&)
14  paddle::imperative::PreparedOp::Run(std::map<std::string, std::vector<std::shared_ptr<paddle::imperative::VarBase>, std::allocator<std::shared_ptr<paddle::imperative::VarBase> > >, std::less<std::string>, std::allocator<std::pair<std::string const, std::vector<std::shared_ptr<paddle::imperative::VarBase>, std::allocator<std::shared_ptr<paddle::imperative::VarBase> > > > > > const*, std::map<std::string, std::vector<std::shared_ptr<paddle::imperative::VarBase>, std::allocator<std::shared_ptr<paddle::imperative::VarBase> > >, std::less<std::string>, std::allocator<std::pair<std::string const, std::vector<std::shared_ptr<paddle::imperative::VarBase>, std::allocator<std::shared_ptr<paddle::imperative::VarBase> > > > > > const*, std::unordered_map<std::string, boost::variant<boost::blank, int, float, std::string, std::vector<int, std::allocator<int> >, std::vector<float, std::allocator<float> >, std::vector<std::string, std::allocator<std::string> >, bool, std::vector<bool, std::allocator<bool> >, paddle::framework::BlockDesc*, long, std::vector<paddle::framework::BlockDesc*, std::allocator<paddle::framework::BlockDesc*> >, std::vector<long, std::allocator<long> >, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, boost::variant<boost::blank, int, float, std::string, std::vector<int, std::allocator<int> >, std::vector<float, std::allocator<float> >, std::vector<std::string, std::allocator<std::string> >, bool, std::vector<bool, std::allocator<bool> >, paddle::framework::BlockDesc*, long, std::vector<paddle::framework::BlockDesc*, std::allocator<paddle::framework::BlockDesc*> >, std::vector<long, std::allocator<long> >, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> > > > const*)
15  paddle::imperative::OpBase::Run(std::map<std::string, std::vector<std::shared_ptr<paddle::imperative::VarBase>, std::allocator<std::shared_ptr<paddle::imperative::VarBase> > >, std::less<std::string>, std::allocator<std::pair<std::string const, std::vector<std::shared_ptr<paddle::imperative::VarBase>, std::allocator<std::shared_ptr<paddle::imperative::VarBase> > > > > > const&, std::map<std::string, std::vector<std::shared_ptr<paddle::imperative::VarBase>, std::allocator<std::shared_ptr<paddle::imperative::VarBase> > >, std::less<std::string>, std::allocator<std::pair<std::string const, std::vector<std::shared_ptr<paddle::imperative::VarBase>, std::allocator<std::shared_ptr<paddle::imperative::VarBase> > > > > > const&)
16  paddle::imperative::Tracer::TraceOp(std::string const&, std::map<std::string, std::vector<std::shared_ptr<paddle::imperative::VarBase>, std::allocator<std::shared_ptr<paddle::imperative::VarBase> > >, std::less<std::string>, std::allocator<std::pair<std::string const, std::vector<std::shared_ptr<paddle::imperative::VarBase>, std::allocator<std::shared_ptr<paddle::imperative::VarBase> > > > > > const&, std::map<std::string, std::vector<std::shared_ptr<paddle::imperative::VarBase>, std::allocator<std::shared_ptr<paddle::imperative::VarBase> > >, std::less<std::string>, std::allocator<std::pair<std::string const, std::vector<std::shared_ptr<paddle::imperative::VarBase>, std::allocator<std::shared_ptr<paddle::imperative::VarBase> > > > > > const&, std::unordered_map<std::string, boost::variant<boost::blank, int, float, std::string, std::vector<int, std::allocator<int> >, std::vector<float, std::allocator<float> >, std::vector<std::string, std::allocator<std::string> >, bool, std::vector<bool, std::allocator<bool> >, paddle::framework::BlockDesc*, long, std::vector<paddle::framework::BlockDesc*, std::allocator<paddle::framework::BlockDesc*> >, std::vector<long, std::allocator<long> >, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, boost::variant<boost::blank, int, float, std::string, std::vector<int, std::allocator<int> >, std::vector<float, std::allocator<float> >, std::vector<std::string, std::allocator<std::string> >, bool, std::vector<bool, std::allocator<bool> >, paddle::framework::BlockDesc*, long, std::vector<paddle::framework::BlockDesc*, std::allocator<paddle::framework::BlockDesc*> >, std::vector<long, std::allocator<long> >, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> > > >, paddle::platform::Place const&, bool)

----------------------
Error Message Summary:
----------------------
ResourceExhaustedError:

Out of memory error on GPU 0. Cannot allocate 36.583496MB memory on GPU 0, available memory is only 25.937500MB.

Please check whether there is any other process using GPU 0.
1. If yes, please stop them, or start PaddlePaddle on another GPU.
2. If no, please try one of the following suggestions:
   1) Decrease the batch size of your model.
   2) FLAGS_fraction_of_gpu_memory_to_use is 0.92 now, please set it to a higher value but less than 1.0.
      The command is `export FLAGS_fraction_of_gpu_memory_to_use=xxx`.