# Anchors
            if not opt.noautoanchor:
                check_anchors(dataset, model=model, thr=hyp['anchor_t'], imgsz=imgsz)
            # model.half().float()
            model.float()  # pre-reduce anchor precision

# Save model
            if (not nosave) or (final_epoch and not evolve):  # if save
                ckpt = {'epoch': epoch,
                        'best_fitness': best_fitness,
                        # 'model': deepcopy(de_parallel(model)).half(),
                        'model': deepcopy(de_parallel(model)),
                        # 'ema': deepcopy(ema.ema).half(),
                        'ema': deepcopy(ema.ema),
                        'updates': ema.updates,
                        'optimizer': optimizer.state_dict(),
                        'wandb_id': loggers.wandb.wandb_run.id if loggers.wandb else None,
                        'date': datetime.now().isoformat()}

 if RANK in [-1, 0]:
        LOGGER.info(f'\n{epoch - start_epoch + 1} epochs completed in {(time.time() - t0) / 3600:.3f} hours.')
        for f in last, best:
            if f.exists():
                strip_optimizer(f)  # strip optimizers
                if f is best:
                    LOGGER.info(f'\nValidating {f}...')
                    results, _, _ = val.run(data_dict,
                                            batch_size=batch_size // WORLD_SIZE * 2,
                                            imgsz=imgsz,
                                            # model=attempt_load(f, device).half(),
                                            model=attempt_load(f, device),
                                            iou_thres=0.65 if is_coco else 0.60,  # best pycocotools results at 0.65
                                            single_cls=single_cls,
                                            dataloader=val_loader,
                                            save_dir=save_dir,
                                            save_json=is_coco,
                                            verbose=True,
                                            plots=True,
                                            callbacks=callbacks,
                                            compute_loss=compute_loss)  # val best model with plots

以上就将对BN层添加L1正则化的程序加好了，核心思想就是修改反向传播的梯度。

1.2 查看稀疏训练效果

如果想查看系数训练的效果，可加入下方程序:

  # =============== show bn weights ===================== #
        module_list = []
        module_bias_list = []
        for i, layer in model.named_modules():
            if isinstance(layer, nn.BatchNorm2d) and i not in ignore_bn_list:
                bnw = layer.state_dict()['weight']
                bnb = layer.state_dict()['bias']
                module_list.append(bnw)
                module_bias_list.append(bnb)
                # bnw = bnw.sort()
                # print(f"{i} : {bnw} : ")
        size_list = [idx.data.shape[0] for idx in module_list]

        bn_weights = torch.zeros(sum(size_list))
        bnb_weights = torch.zeros(sum(size_list))
        index = 0
        for idx, size in enumerate(size_list):
            bn_weights[index:(index + size)] = module_list[idx].data.abs().clone()
            bnb_weights[index:(index + size)] = module_bias_list[idx].data.abs().clone()
            index += size

        # print("bn_weights:", torch.sort(bn_weights))
        # print("bn_bias:", torch.sort(bnb_weights))
        # tb_writer.add_histogram('bn_weights/hist', bn_weights.numpy(), epoch, bins='doane')
        # tb_writer.add_histogram('bn_bias/hist', bnb_weights.numpy(), epoch, bins='doane')
        loggers.tb.add_histogram('bn_weights/hist', bn_weights.numpy(), epoch, bins='doane')
        loggers.tb.add_histogram('bn_bias/hist', bnb_weights.numpy(), epoch, bins='doane')

将其加到：一个batch训练结束之后的程序后边就好

我在tensorboard中的图：

 纵轴是epoch,横轴是权重，可以看到我一共进行了100轮稀疏训练，我的项目中非bottoltneck中的bn层加起来参数大概有8000多个，那可以看到在49个epoch的时候，0附近的权重已经有5000多个了。那接下来我可以设置60%的剪枝率，把它们都剪掉