本人新人小白，尝试超分辨任务，但是一旦使用残差块MSE会高达 1e11,不使用可以收敛但是速度极慢

class ResidualBlock(fluid.dygraph.Layer):
    def __init__(self,kernel_size=3,n_channels=64):
        super(ResidualBlock,self).__init__()
        self.convblock1 = Conv2D (n_channels,n_channels,kernel_size,padding = kernel_size//2,act='relu')
        self.convblock2 = Conv2D (n_channels,n_channels,kernel_size,padding = kernel_size//2)

    def forward(self,inputs):
        residual = inputs 
        outputs = self.convblock1(inputs) 
        outputs = self.convblock2(outputs)
        outputs = outputs + residual 
        return outputs
        
class SRCNN(fluid.dygraph.Layer):                                                                                                                                                                                                                    
    def __init__(self):
        super(SRCNN, self).__init__()
        self._simple_net_1 = Conv2D(3, 64, 9,padding=4,act='relu')
        self._simple_net_2 = Conv2D(64, 64, 1,act='relu')
        self.residual_blocks = fluid.dygraph.Sequential(*[ResidualBlock() for i in range(16)])
        self.sub = Conv2D(64, 64*4, 3,padding =1)
        self._simple_net_3 = Conv2D(64, 3, 5,padding=2)
    def forward(self, img_input):
        x = self._simple_net_1(img_input)
        x = self._simple_net_2(x)
        #x = self.residual_blocks(x)
        x =self.sub(x)
        x =fluid.layers.pixel_shuffle(x,upscale_factor=2)
        x = self._simple_net_3(x)
        return x

with fluid.dygraph.guard():
    epoch_num = 20
    srcnn = SRCNN()
    train_reader = paddle.batch(paddle.reader.shuffle(read_data3(),buf_size=320), batch_size=32)
    adam = fluid.optimizer.AdamOptimizer(learning_rate=0.0001, parameter_list=srcnn.parameters())
    step = 0
    for epoch in range(epoch_num):
        avg_loss = 0
        for batch_id, data in enumerate(train_reader()):
            step+=1
            LR = np.array([x[0]for x in data]).astype('float32')
            HR = np.array([x[1]for x in data]).astype('float32')
            LR = fluid.dygraph.to_variable(LR)
            HR = fluid.dygraph.to_variable(HR)
            SR = srcnn(LR)
            
            if batch_id%100 == 0:
                display(LR)
                display(SR)
                display(HR)
                
            loss = fluid.layers.mse_loss(SR,HR)
            #loss = fluid.layers.mean(fluid.layers.abs(SR,HR))
            avg_loss += loss.numpy()
            loss.backward()
            adam.minimize(loss)
            srcnn.clear_gradients()
            if batch_id%100 == 0:
                print("step{}, epoch: {}, batch_id: {}, loss is: {}, psnr is {}".format(step, epoch, batch_id, loss.numpy(),test() ))