<aside> 🧐 RoadMap to Computer Science

</aside>

Roadmap

from transformers import AutoTokenizer, AutoModelForCausalLM
import torch

# 定义模型名称或路径
model_name_or_path = "/mnt/alluxio/alluxio-fuse/user/tc_agi/minicpm/564721/job_564721_ckpt_263500"

# 加载tokenizer和模型
tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
model = AutoModelForCausalLM.from_pretrained(model_name_or_path)

# 获取所有token
all_tokens = list(tokenizer.get_vocab().keys())

# 将所有token转换为输入张量
inputs = tokenizer(all_tokens, return_tensors="pt", padding=True, truncation=True)

# 通过模型获取所有token的embedding
with torch.no_grad():
    outputs = model(**inputs)

# 获取所有token的embedding
all_token_embeddings = outputs.last_hidden_state

ValueError: The checkpoint you are trying to load has model type `cpm_dragonfly` but Transformers does not recognize this architecture. This could be because of an issue with the checkpoint, or because your version of Transformers is out of date.