# InternLM3 压缩版## 压缩信息压缩自上游 [internlm/internlm3-8b-instruct](https://huggingface.co/internlm/internlm3-8b-instruct).### 压缩配置- **原始模型**: internlm3-8b-instruct (48 layers, 4096 hidden, 8.80B params)- **压缩模型**: - Layers: 6 - Hidden Size: 512 - Attention Heads: 4 - KV Heads: 1 - Intermediate Size: 1280 - Tie Embeddings: False - Parameters: 147.33M - Size (bfloat16): 281.01 MB## 压缩方法```python#!/usr/bin/env python3"""InternLM3 模型压缩执行脚本 - HuggingFace 格式使用 transformers 原生 API 创建压缩配置的模型并保存为 HuggingFace transformers 兼容格式"""import osimport jsonimport shutildef compress_internlm3_hf_format( original_config_path: str, output_dir: str, num_layers: int = 6, # 48/8=6, 保持1/8的统一压缩比例 hidden_size: int = 512, # 4096/8=512 tie_embeddings: bool = False,): """ 压缩 InternLM3 模型 - 保存为 HuggingFace 格式（使用 transformers 原生 API） Args: original_config_path: 原始模型配置文件路径 output_dir: 输出目录 num_layers: 压缩后的层数 hidden_size: 压缩后的隐藏层维度 tie_embeddings: 是否共享 embedding """ # 使用 transformers 原生 API，不依赖 Paddle from transformers import AutoModelForCausalLM # 读取原始配置 with open(original_config_path, "r") as f: original_config = json.load(f) # 创建压缩配置（使用 HF 兼容的配置格式） num_attention_heads = max(1, hidden_size // 128) num_key_value_heads = max(1, num_attention_heads // 4) intermediate_size = int(hidden_size * 2.5) head_dim = hidden_size // num_attention_heads compressed_config_dict = { "architectures": ["InternLM3ForCausalLM"], "attention_dropout": 0.0, "auto_map": { "AutoConfig": "configuration_internlm3.InternLM3Config", "AutoModel": "modeling_internlm3.InternLM3Model", "AutoModelForCausalLM": "modeling_internlm3.InternLM3ForCausalLM" }, "bias": False, "bos_token_id": original_config.get("bos_token_id", 1), "eos_token_id": original_config.get("eos_token_id", 2), "head_dim": head_dim, "hidden_act": "silu", "hidden_size": hidden_size, "initializer_range": original_config.get("initializer_range", 0.02), "intermediate_size": intermediate_size, "max_position_embeddings": original_config.get("max_position_embeddings", 32768), "model_type": "internlm3", "num_attention_heads": num_attention_heads, "num_hidden_layers": num_layers, "num_key_value_heads": num_key_value_heads, "pad_token_id": original_config.get("pad_token_id", 2), "qkv_bias": False, "rms_norm_eps": original_config.get("rms_norm_eps", 1e-05), "rope_scaling": original_config.get("rope_scaling", {"factor": 6.0, "rope_type": "dynamic"}), "rope_theta": original_config.get("rope_theta", 50000000), "tie_word_embeddings": tie_embeddings, "torch_dtype": "bfloat16", "transformers_version": "4.47.1", "use_cache": True, "vocab_size": original_config["vocab_size"], "_compression_info": { "compressed": True, "original_model": "internlm3-8b-instruct" } } # 从原始目录加载 modeling_internlm3.py（如果存在） import sys original_dir_full = os.path.dirname(os.path.dirname(original_config_path)) modeling_file = os.path.join(original_dir_full, "modeling_internlm3.py") if os.path.exists(modeling_file): sys.path.insert(0, original_dir_full) print(f" 添加 {original_dir_full} 到 Python 路径以加载 modeling_internlm3") # 先保存配置文件，然后从配置文件加载（使用 trust_remote_code） os.makedirs(output_dir, exist_ok=True) temp_config_path = os.path.join(output_dir, "config.json") with open(temp_config_path, "w") as f: json.dump(compressed_config_dict, f, indent=2) print(f" 配置文件已保存到: {temp_config_path}") # 复制 modeling_internlm3.py 和 configuration_internlm3.py for fname in ["modeling_internlm3.py", "configuration_internlm3.py"]: src = os.path.join(original_dir_full, fname) if os.path.exists(src): shutil.copy2(src, os.path.join(output_dir, fname)) print(f" {fname} 已复制") # 使用 AutoConfig 和 AutoModelForCausalLM 创建模型 from transformers import AutoConfig config = AutoConfig.from_pretrained(output_dir, trust_remote_code=True) model = AutoModelForCausalLM.from_config(config, trust_remote_code=True) # 计算参数量 total_params = sum(p.numel() for p in model.parameters()) os.makedirs(output_dir, exist_ok=True) # 1. 保存 config.json config_path = os.path.join(output_dir, "config.json") with open(config_path, "w") as f: json.dump(compressed_config_dict, f, indent=2) print(f" config.json 已保存") # 2. 保存为 safetensors 格式（使用 safetensors.torch 正确处理 bfloat16） import torch from safetensors.torch import save_file state_dict = model.state_dict() # 确保所有权重都在 CPU 上且不需要梯度 state_dict = {k: v.detach().cpu() for k, v in state_dict.items()} safetensors_path = os.path.join(output_dir, "model.safetensors") save_file(state_dict, safetensors_path) print(f" model.safetensors 已保存") # 3. 生成 model.safetensors.index.json index_dict = { "metadata": {"total_size": total_params * 2}, "weight_map": {name: "model.safetensors" for name in state_dict.keys()} } index_path = os.path.join(output_dir, "model.safetensors.index.json") with open(index_path, "w") as f: json.dump(index_dict, f, indent=2) print(f" model.safetensors.index.json 已保存") # 4. 保存 generation_config.json generation_config = { "bos_token_id": compressed_config_dict["bos_token_id"], "eos_token_id": compressed_config_dict["eos_token_id"], "pad_token_id": compressed_config_dict["pad_token_id"], "max_length": 2048, "temperature": 0.7, "top_p": 0.9, "do_sample": True, } with open(os.path.join(output_dir, "generation_config.json"), "w") as f: json.dump(generation_config, f, indent=2) print(f" generation_config.json 已保存") # 5. 复制 Python 代码文件（用于 remote code） code_files = [ ("modeling_internlm3.py", "modeling_internlm3.py"), ("configuration_internlm3.py", "configuration_internlm3.py"), ("tokenization_internlm3.py", "tokenization_internlm3.py"), ] for src_fname, dst_fname in code_files: src = os.path.join(original_dir_full, src_fname) if os.path.exists(src): dst = os.path.join(output_dir, dst_fname) shutil.copy2(src, dst) print(f" {dst_fname} 已复制") # 6. 复制 tokenizer 相关文件 tokenizer_files = [ "tokenizer.model", "tokenizer_config.json", "special_tokens_map.json", ] for fname in tokenizer_files: src = os.path.join(original_dir_full, fname) if os.path.exists(src): dst = os.path.join(output_dir, fname) shutil.copy2(src, dst) print(f" {fname} 已复制") readme_content = f"""TODO""" with open(os.path.join(output_dir, "README.md"), "w") as f: f.write(readme_content)def main(): original_config_path = "/mnt/learncat/llm/internlm/internlm3-8b-instruct/internlm3-8b-instruct/config.json" output_dir = "/mnt/learncat/llm/internlm/internlm3-8b-instruct-mini" # 压缩配置: 6层 x 512维 (1/8统一比例压缩) # 原始: 48层 x 4096维 -> 压缩: 6层 x 512维 compress_internlm3_hf_format( original_config_path=original_config_path, output_dir=output_dir, num_layers=6, hidden_size=512, tie_embeddings=False, )if __name__ == "__main__": main()```## 加载对齐方法```pythonclass InternLM3CompatibilityTest(unittest.TestCase): """测试 Paddle 模型推理与 Transformers 参考值对齐""" MINI_MODEL_PATH = "learncat/internlm3-8b-instruct-mini-raw" # 以下参考值由 tf4.53 环境的 transformers 生成，固定随机种子42和固定输入 # 参考代码（不在当前环境执行）: # ------------------------------------------------------------ # import torch # import numpy as np # from transformers import AutoModelForCausalLM # np.random.seed(42) # torch.manual_seed(42) # input_ids = np.array([[100, 200, 300, 400, 500, 600, 700, 800, 900, 950]]) # model = AutoModelForCausalLM.from_pretrained( # "/mnt/caoyuanye/llm/internlm/internlm3-8b-instruct-mini", # trust_remote_code=True, torch_dtype=torch.float32 # ).cpu().eval() # torch_input = torch.from_numpy(input_ids).long() # with torch.no_grad(): # logits = model(torch_input).logits[0, -1, :].cpu().numpy() # print(logits[:20]) # print(np.argsort(logits)[-10:][::-1]) # ------------------------------------------------------------ REF_LOGITS_FIRST_20 = [ -0.144880, -0.648041, 0.386456, 0.213346, 0.771256, 0.620751, 0.073640, -0.021458, -0.764580, 0.317350, -0.025523, -0.056741, -0.671094, -0.187901, 0.286278, -0.182251, 0.849138, 0.340502, 0.324327, 0.609751, ] REF_TOP10_TOKEN_IDS = [72267, 94359, 95067, 121546, 19719, 125351, 74467, 115313, 87550, 24000] # @classmethod # def setUpClass(cls) -> None: # if not os.path.exists(cls.MINI_MODEL_PATH): # cls.skipTest(f"Mini model not found at {cls.MINI_MODEL_PATH}") def test_torch_paddle_model_alignment(self): """验证 Paddle 输出与 Transformers 参考值对齐""" np.random.seed(42) paddle.seed(42) input_ids = np.array([[100, 200, 300, 400, 500, 600, 700, 800, 900, 950]]) paddle_input = paddle.to_tensor(input_ids, dtype="int64") paddle_model = InternLM3ForCausalLM.from_pretrained( self.MINI_MODEL_PATH, dtype=paddle.float32 ) paddle_model.eval() with paddle.no_grad(): paddle_logits = paddle_model(paddle_input)[0][0, -1, :].cpu().numpy() # 指标1: 前20个logits值对齐 < 1e-2 paddle_first20 = paddle_logits[:20] max_diff = np.max(np.abs(paddle_first20 - np.array(self.REF_LOGITS_FIRST_20))) print(f"paddle and transformer models differ: {max_diff}") self.assertLess(max_diff, 1e-2, f"First 20 logits diff={max_diff}") # 指标2: Top-10 token 相同 paddle_top10 = set(np.argsort(paddle_logits)[-10:]) ref_top10 = set(self.REF_TOP10_TOKEN_IDS) self.assertEqual(paddle_top10, ref_top10, f"Top-10 tokens mismatch") print("paddle and transformer has same 10 tokens id")```