Pass 800, Batch 0, Cost 0.081233
Pass 900, Batch 0, Cost 0.033355

---------------------------------------------------------------------------EnforceNotMet Traceback (most recent call last) in
8 # event_handler_plot(train_prompt, step, metrics[0])
9 step += 1
---> 10 avg_loss_val, acc_val = train_test(train_p=test_program,train_r=test_reader,train_f=feeder)
11 print("Test with Epoch %d, avg_cost: %s, acc: %s" %(epoch_id, avg_loss_val, acc_val))
12 # event_handler_plot(test_prompt, step, metrics[0])
in train_test(train_p, train_f, train_r)
6 avg_loss_set = []
7 for test_data in train_r():
----> 8 acc_np,avg_loss_np=exe.run(program=train_p,feed=train_f.feed(test_data),fetch_list=[acc,avg_loss])
9 acc_set.append(float(acc_np))
10 avg_loss_set.append(float(avg_loss_np))
/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/paddle/fluid/executor.py in run(self, program, feed, fetch_list, feed_var_name, fetch_var_name, scope, return_numpy, use_program_cache)
648 scope=scope,
649 return_numpy=return_numpy,
--> 650 use_program_cache=use_program_cache)
651 else:
652 if fetch_list and program._is_data_parallel and program._program and (
/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/paddle/fluid/executor.py in _run(self, program, exe, feed, fetch_list, feed_var_name, fetch_var_name, scope, return_numpy, use_program_cache)
746 self._feed_data(program, feed, feed_var_name, scope)
747 if not use_program_cache:
--> 748 exe.run(program.desc, scope, 0, True, True, fetch_var_name)
749 else:
750 exe.run_cached_prepared_ctx(ctx, scope, False, False, False)
EnforceNotMet: Invoke operator batch_norm_grad error.
Python Callstacks:
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/paddle/fluid/framework.py", line 1748, in append_op
attrs=kwargs.get("attrs", None))
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op
return self.main_program.current_block().append_op(*args, **kwargs)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/paddle/fluid/layers/nn.py", line 3219, in batch_norm
"use_global_stats": use_global_stats
File "", line 4, in cnn
conv1 = fluid.layers.batch_norm(conv1)
File "", line 3, in train
predict = cnn()
File "", line 2, in
predict,[avg_loss,acc]=train()
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 3265, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 3183, in run_ast_nodes
if (yield from self.run_code(code, result)):
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 3018, in run_cell_async
interactivity=interactivity, compiler=compiler, result=result)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/IPython/core/async_helpers.py", line 67, in _pseudo_sync_runner
coro.send(None)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2843, in _run_cell
return runner(coro)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2817, in run_cell
raw_cell, store_history, silent, shell_futures)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/ipykernel/zmqshell.py", line 536, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/ipykernel/ipkernel.py", line 294, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/tornado/gen.py", line 326, in wrapper
yielded = next(result)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/ipykernel/kernelbase.py", line 534, in execute_request
user_expressions, allow_stdin,
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/tornado/gen.py", line 326, in wrapper
yielded = next(result)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/ipykernel/kernelbase.py", line 267, in dispatch_shell
yield gen.maybe_future(handler(stream, idents, msg))
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/tornado/gen.py", line 326, in wrapper
yielded = next(result)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/ipykernel/kernelbase.py", line 357, in process_one
yield gen.maybe_future(dispatch(*args))
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/tornado/gen.py", line 1147, in run
yielded = self.gen.send(value)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/tornado/gen.py", line 1080, in __init__
self.run()
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/tornado/gen.py", line 346, in wrapper
runner = Runner(result, future, yielded)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/ipykernel/kernelbase.py", line 370, in dispatch_queue
yield self.process_one()
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/tornado/gen.py", line 1147, in run
yielded = self.gen.send(value)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/tornado/gen.py", line 1233, in inner
self.run()
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/tornado/stack_context.py", line 300, in null_wrapper
return fn(*args, **kwargs)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/tornado/ioloop.py", line 758, in _run_callback
ret = callback()
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/asyncio/events.py", line 127, in _run
self._callback(*self._args)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/asyncio/base_events.py", line 1425, in _run_once
handle._run()
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/asyncio/base_events.py", line 421, in run_forever
self._run_once()
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/tornado/platform/asyncio.py", line 132, in start
self.asyncio_loop.run_forever()
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/ipykernel/kernelapp.py", line 505, in start
self.io_loop.start()
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/site-packages/ipykernel_launcher.py", line 16, in
app.launch_new_instance()
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/opt/conda/envs/python35-paddle120-env/lib/python3.5/runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
C++ Callstacks:
holder_ should not be null
Tensor holds no memory. Call Tensor::mutable_data first. at [/paddle/paddle/fluid/framework/tensor.cc:23]
PaddlePaddle Call Stacks:
0 0x7f521937b808p void paddle::platform::EnforceNotMet::Init(std::string, char const*, int) + 360
1 0x7f521937bb57p paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) + 87
2 0x7f521b32c769p paddle::framework::Tensor::check_memory_size() const + 185
3 0x7f52193826e9p float const* paddle::framework::Tensor::data() const + 25
4 0x7f5219c7606ap paddle::operators::BatchNormGradKernel::Compute(paddle::framework::ExecutionContext const&) const + 1882
5 0x7f5219c77be3p std::_Function_handler, paddle::operators::BatchNormGradKernel, paddle::operators::BatchNormGradKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) + 35
6 0x7f521b2d6657p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&, paddle::framework::RuntimeContext*) const + 375
7 0x7f521b2d6a31p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant const&) const + 529
8 0x7f521b2d402cp paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant const&) + 332
9 0x7f521950547ep paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool, bool) + 382
10 0x7f521950851fp paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, std::vector > const&, bool) + 143
11 0x7f521936c96dp
12 0x7f52193adca6p
13 0x7f529e812199p PyCFunction_Call + 233
14 0x7f529e8ad3f9p PyEval_EvalFrameEx + 33545
15 0x7f529e8af4b6p
16 0x7f529e8ac5b5p PyEval_EvalFrameEx + 29893
17 0x7f529e8af4b6p
18 0x7f529e8ac5b5p PyEval_EvalFrameEx + 29893
19 0x7f529e8af4b6p
20 0x7f529e8ac5b5p PyEval_EvalFrameEx + 29893
21 0x7f529e8af4b6p
22 0x7f529e8af5a8p PyEval_EvalCodeEx + 72
23 0x7f529e8af5ebp PyEval_EvalCode + 59
24 0x7f529e8a2c5dp
25 0x7f529e812179p PyCFunction_Call + 201
26 0x7f529e8acdbep PyEval_EvalFrameEx + 31950
27 0x7f529e7e6410p _PyGen_Send + 128
28 0x7f529e8ab953p PyEval_EvalFrameEx + 26723