<aside> 🧐 RoadMap to Computer Science
</aside>
from transformers import AutoTokenizer, AutoModelForCausalLM
import torch
# 定义模型名称或路径
model_name_or_path = "/mnt/alluxio/alluxio-fuse/user/tc_agi/minicpm/564721/job_564721_ckpt_263500"
# 加载tokenizer和模型
tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
model = AutoModelForCausalLM.from_pretrained(model_name_or_path)
# 获取所有token
all_tokens = list(tokenizer.get_vocab().keys())
# 将所有token转换为输入张量
inputs = tokenizer(all_tokens, return_tensors="pt", padding=True, truncation=True)
# 通过模型获取所有token的embedding
with torch.no_grad():
outputs = model(**inputs)
# 获取所有token的embedding
all_token_embeddings = outputs.last_hidden_state
ValueError: The checkpoint you are trying to load has model type `cpm_dragonfly` but Transformers does not recognize this architecture. This could be because of an issue with the checkpoint, or because your version of Transformers is out of date.