03-llama-3.1-3b-accelerate-deepspeed.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Package                  Version\n",
      "------------------------ ----------------\n",
      "accelerate               1.0.1\n",
      "aiohappyeyeballs         2.4.3\n",
      "aiohttp                  3.10.10\n",
      "aiosignal                1.3.1\n",
      "annotated-types          0.7.0\n",
      "apturl                   0.5.2\n",
      "asttokens                2.4.1\n",
      "async-timeout            4.0.3\n",
      "attrs                    21.2.0\n",
      "bcrypt                   3.2.0\n",
      "blinker                  1.4\n",
      "Brlapi                   0.8.3\n",
      "certifi                  2020.6.20\n",
      "chardet                  4.0.0\n",
      "charset-normalizer       3.4.0\n",
      "click                    8.0.3\n",
      "colorama                 0.4.4\n",
      "comm                     0.2.2\n",
      "cryptography             3.4.8\n",
      "cupshelpers              1.0\n",
      "datasets                 3.0.2\n",
      "dbus-python              1.2.18\n",
      "debugpy                  1.8.7\n",
      "decorator                5.1.1\n",
      "deepspeed                0.15.3\n",
      "defer                    1.0.6\n",
      "dill                     0.3.8\n",
      "distro                   1.7.0\n",
      "distro-info              1.1+ubuntu0.2\n",
      "docker                   5.0.3\n",
      "dockerpty                0.4.1\n",
      "docopt                   0.6.2\n",
      "duplicity                0.8.21\n",
      "evaluate                 0.4.3\n",
      "exceptiongroup           1.2.2\n",
      "executing                2.1.0\n",
      "fasteners                0.14.1\n",
      "filelock                 3.16.1\n",
      "frozenlist               1.5.0\n",
      "fsspec                   2024.9.0\n",
      "future                   0.18.2\n",
      "gyp                      0.1\n",
      "hjson                    3.1.0\n",
      "httplib2                 0.20.2\n",
      "huggingface-hub          0.26.1\n",
      "idna                     3.3\n",
      "importlib-metadata       4.6.4\n",
      "ipykernel                6.29.5\n",
      "ipython                  8.29.0\n",
      "jedi                     0.19.1\n",
      "jeepney                  0.7.1\n",
      "Jinja2                   3.1.4\n",
      "jsonschema               3.2.0\n",
      "jupyter_client           8.6.3\n",
      "jupyter_core             5.7.2\n",
      "keyring                  23.5.0\n",
      "language-selector        0.1\n",
      "launchpadlib             1.10.16\n",
      "lazr.restfulclient       0.14.4\n",
      "lazr.uri                 1.0.6\n",
      "lockfile                 0.12.2\n",
      "louis                    3.20.0\n",
      "macaroonbakery           1.3.1\n",
      "Mako                     1.1.3\n",
      "MarkupSafe               2.0.1\n",
      "matplotlib-inline        0.1.7\n",
      "modelscope               1.19.2\n",
      "monotonic                1.6\n",
      "more-itertools           8.10.0\n",
      "mpmath                   1.3.0\n",
      "msgpack                  1.1.0\n",
      "multidict                6.1.0\n",
      "multiprocess             0.70.16\n",
      "nest-asyncio             1.6.0\n",
      "netifaces                0.11.0\n",
      "networkx                 3.4.2\n",
      "ninja                    1.11.1.1\n",
      "numpy                    2.1.2\n",
      "nvidia-cublas-cu12       12.4.5.8\n",
      "nvidia-cuda-cupti-cu12   12.4.127\n",
      "nvidia-cuda-nvrtc-cu12   12.4.127\n",
      "nvidia-cuda-runtime-cu12 12.4.127\n",
      "nvidia-cudnn-cu12        9.1.0.70\n",
      "nvidia-cufft-cu12        11.2.1.3\n",
      "nvidia-curand-cu12       10.3.5.147\n",
      "nvidia-cusolver-cu12     11.6.1.9\n",
      "nvidia-cusparse-cu12     12.3.1.170\n",
      "nvidia-ml-py             12.560.30\n",
      "nvidia-nccl-cu12         2.21.5\n",
      "nvidia-nvjitlink-cu12    12.4.127\n",
      "nvidia-nvtx-cu12         12.4.127\n",
      "oauthlib                 3.2.0\n",
      "olefile                  0.46\n",
      "packaging                24.1\n",
      "pandas                   2.2.3\n",
      "paramiko                 2.9.3\n",
      "parso                    0.8.4\n",
      "peft                     0.13.2\n",
      "pexpect                  4.8.0\n",
      "Pillow                   9.0.1\n",
      "pip                      22.0.2\n",
      "platformdirs             4.3.6\n",
      "prompt_toolkit           3.0.48\n",
      "propcache                0.2.0\n",
      "protobuf                 5.28.3\n",
      "psutil                   6.1.0\n",
      "ptyprocess               0.7.0\n",
      "pure_eval                0.2.3\n",
      "py-cpuinfo               9.0.0\n",
      "pyarrow                  17.0.0\n",
      "pycairo                  1.20.1\n",
      "pycups                   2.0.1\n",
      "pydantic                 2.9.2\n",
      "pydantic_core            2.23.4\n",
      "Pygments                 2.18.0\n",
      "PyGObject                3.42.1\n",
      "PyJWT                    2.3.0\n",
      "pymacaroons              0.13.0\n",
      "PyNaCl                   1.5.0\n",
      "pyparsing                2.4.7\n",
      "pyRFC3339                1.1\n",
      "pyrsistent               0.18.1\n",
      "python-apt               2.4.0+ubuntu4\n",
      "python-dateutil          2.9.0.post0\n",
      "python-debian            0.1.43+ubuntu1.1\n",
      "python-dotenv            0.19.2\n",
      "pytz                     2022.1\n",
      "pyxdg                    0.27\n",
      "PyYAML                   5.4.1\n",
      "pyzmq                    26.2.0\n",
      "ray                      2.36.1\n",
      "regex                    2024.9.11\n",
      "reportlab                3.6.8\n",
      "requests                 2.32.3\n",
      "safetensors              0.4.5\n",
      "screen-resolution-extra  0.0.0\n",
      "SecretStorage            3.3.1\n",
      "setuptools               59.6.0\n",
      "six                      1.16.0\n",
      "ssh-import-id            5.11\n",
      "stack-data               0.6.3\n",
      "sympy                    1.13.1\n",
      "systemd-python           234\n",
      "texttable                1.6.4\n",
      "tokenizers               0.20.1\n",
      "torch                    2.5.0\n",
      "tornado                  6.4.1\n",
      "tqdm                     4.66.5\n",
      "traitlets                5.14.3\n",
      "transformers             4.45.2\n",
      "triton                   3.1.0\n",
      "typing_extensions        4.12.2\n",
      "tzdata                   2024.2\n",
      "ubuntu-drivers-common    0.0.0\n",
      "ubuntu-pro-client        8001\n",
      "ufw                      0.36.1\n",
      "unattended-upgrades      0.1\n",
      "urllib3                  1.26.5\n",
      "usb-creator              0.3.7\n",
      "vboxapi                  1.0\n",
      "wadllib                  1.3.6\n",
      "wcwidth                  0.2.13\n",
      "webcolors                24.6.0\n",
      "webencodings             0.5.1\n",
      "websocket-client         1.8.0\n",
      "wheel                    0.37.1\n",
      "widgetsnbextension       4.0.11\n",
      "xdg                      5\n",
      "xkit                     0.0.0\n",
      "xxhash                   3.5.0\n",
      "yarl                     1.15.4\n",
      "zipp                     1.0.0\n"
     ]
    }
   ],
   "source": [
    "# 在jupyter 中用pip list 查看安装的包\n",
    "!pip list\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 设置环境变量 HF_ENDPOINT=https://hf.neolink-ai.com\n",
    "import os\n",
    "os.environ[\"HF_ENDPOINT\"] = \"https://hf.neolink-ai.com\"\n",
    "\n",
    "# 设置 https proxy为http://10.161.0.82:7899/\n",
    "os.environ[\"HTTPS_PROXY\"] = \"http://10.161.0.82:7899/\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DatasetDict({\n",
      "    train: Dataset({\n",
      "        features: ['question', 'answer'],\n",
      "        num_rows: 7473\n",
      "    })\n",
      "    test: Dataset({\n",
      "        features: ['question', 'answer'],\n",
      "        num_rows: 1319\n",
      "    })\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "# 使用load_dataset 加载数据 (\"gsm8k\", \"main\")\n",
    "from datasets import load_dataset\n",
    "\n",
    "dataset = load_dataset(\"gsm8k\", \"main\")\n",
    "print(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PreTrainedTokenizerFast(name_or_path='/home/chendan/Desktop/workspace/workspace/checkpoints/Llama-3___2-1B', vocab_size=128000, model_max_length=131072, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'bos_token': '<|begin_of_text|>', 'eos_token': '<|end_of_text|>'}, clean_up_tokenization_spaces=True),  added_tokens_decoder={\n",
      "\t128000: AddedToken(\"<|begin_of_text|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128001: AddedToken(\"<|end_of_text|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128002: AddedToken(\"<|reserved_special_token_0|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128003: AddedToken(\"<|reserved_special_token_1|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128004: AddedToken(\"<|finetune_right_pad_id|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128005: AddedToken(\"<|reserved_special_token_2|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128006: AddedToken(\"<|start_header_id|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128007: AddedToken(\"<|end_header_id|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128008: AddedToken(\"<|eom_id|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128009: AddedToken(\"<|eot_id|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128010: AddedToken(\"<|python_tag|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128011: AddedToken(\"<|reserved_special_token_3|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128012: AddedToken(\"<|reserved_special_token_4|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128013: AddedToken(\"<|reserved_special_token_5|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128014: AddedToken(\"<|reserved_special_token_6|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128015: AddedToken(\"<|reserved_special_token_7|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128016: AddedToken(\"<|reserved_special_token_8|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128017: AddedToken(\"<|reserved_special_token_9|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128018: AddedToken(\"<|reserved_special_token_10|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128019: AddedToken(\"<|reserved_special_token_11|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128020: AddedToken(\"<|reserved_special_token_12|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128021: AddedToken(\"<|reserved_special_token_13|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128022: AddedToken(\"<|reserved_special_token_14|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128023: AddedToken(\"<|reserved_special_token_15|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128024: AddedToken(\"<|reserved_special_token_16|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128025: AddedToken(\"<|reserved_special_token_17|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128026: AddedToken(\"<|reserved_special_token_18|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128027: AddedToken(\"<|reserved_special_token_19|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128028: AddedToken(\"<|reserved_special_token_20|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128029: AddedToken(\"<|reserved_special_token_21|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128030: AddedToken(\"<|reserved_special_token_22|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128031: AddedToken(\"<|reserved_special_token_23|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128032: AddedToken(\"<|reserved_special_token_24|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128033: AddedToken(\"<|reserved_special_token_25|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128034: AddedToken(\"<|reserved_special_token_26|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128035: AddedToken(\"<|reserved_special_token_27|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128036: AddedToken(\"<|reserved_special_token_28|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128037: AddedToken(\"<|reserved_special_token_29|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128038: AddedToken(\"<|reserved_special_token_30|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128039: AddedToken(\"<|reserved_special_token_31|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128040: AddedToken(\"<|reserved_special_token_32|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128041: AddedToken(\"<|reserved_special_token_33|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128042: AddedToken(\"<|reserved_special_token_34|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128043: AddedToken(\"<|reserved_special_token_35|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128044: AddedToken(\"<|reserved_special_token_36|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128045: AddedToken(\"<|reserved_special_token_37|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128046: AddedToken(\"<|reserved_special_token_38|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128047: AddedToken(\"<|reserved_special_token_39|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128048: AddedToken(\"<|reserved_special_token_40|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128049: AddedToken(\"<|reserved_special_token_41|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128050: AddedToken(\"<|reserved_special_token_42|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128051: AddedToken(\"<|reserved_special_token_43|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128052: AddedToken(\"<|reserved_special_token_44|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128053: AddedToken(\"<|reserved_special_token_45|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128054: AddedToken(\"<|reserved_special_token_46|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128055: AddedToken(\"<|reserved_special_token_47|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128056: AddedToken(\"<|reserved_special_token_48|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128057: AddedToken(\"<|reserved_special_token_49|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128058: AddedToken(\"<|reserved_special_token_50|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128059: AddedToken(\"<|reserved_special_token_51|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128060: AddedToken(\"<|reserved_special_token_52|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128061: AddedToken(\"<|reserved_special_token_53|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128062: AddedToken(\"<|reserved_special_token_54|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128063: AddedToken(\"<|reserved_special_token_55|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128064: AddedToken(\"<|reserved_special_token_56|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128065: AddedToken(\"<|reserved_special_token_57|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128066: AddedToken(\"<|reserved_special_token_58|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128067: AddedToken(\"<|reserved_special_token_59|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128068: AddedToken(\"<|reserved_special_token_60|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128069: AddedToken(\"<|reserved_special_token_61|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128070: AddedToken(\"<|reserved_special_token_62|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128071: AddedToken(\"<|reserved_special_token_63|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128072: AddedToken(\"<|reserved_special_token_64|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128073: AddedToken(\"<|reserved_special_token_65|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128074: AddedToken(\"<|reserved_special_token_66|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128075: AddedToken(\"<|reserved_special_token_67|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128076: AddedToken(\"<|reserved_special_token_68|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128077: AddedToken(\"<|reserved_special_token_69|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128078: AddedToken(\"<|reserved_special_token_70|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128079: AddedToken(\"<|reserved_special_token_71|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128080: AddedToken(\"<|reserved_special_token_72|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128081: AddedToken(\"<|reserved_special_token_73|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128082: AddedToken(\"<|reserved_special_token_74|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128083: AddedToken(\"<|reserved_special_token_75|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128084: AddedToken(\"<|reserved_special_token_76|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128085: AddedToken(\"<|reserved_special_token_77|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128086: AddedToken(\"<|reserved_special_token_78|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128087: AddedToken(\"<|reserved_special_token_79|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128088: AddedToken(\"<|reserved_special_token_80|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128089: AddedToken(\"<|reserved_special_token_81|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128090: AddedToken(\"<|reserved_special_token_82|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128091: AddedToken(\"<|reserved_special_token_83|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128092: AddedToken(\"<|reserved_special_token_84|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128093: AddedToken(\"<|reserved_special_token_85|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128094: AddedToken(\"<|reserved_special_token_86|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128095: AddedToken(\"<|reserved_special_token_87|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128096: AddedToken(\"<|reserved_special_token_88|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128097: AddedToken(\"<|reserved_special_token_89|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128098: AddedToken(\"<|reserved_special_token_90|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128099: AddedToken(\"<|reserved_special_token_91|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128100: AddedToken(\"<|reserved_special_token_92|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128101: AddedToken(\"<|reserved_special_token_93|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128102: AddedToken(\"<|reserved_special_token_94|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128103: AddedToken(\"<|reserved_special_token_95|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128104: AddedToken(\"<|reserved_special_token_96|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128105: AddedToken(\"<|reserved_special_token_97|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128106: AddedToken(\"<|reserved_special_token_98|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128107: AddedToken(\"<|reserved_special_token_99|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128108: AddedToken(\"<|reserved_special_token_100|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128109: AddedToken(\"<|reserved_special_token_101|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128110: AddedToken(\"<|reserved_special_token_102|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128111: AddedToken(\"<|reserved_special_token_103|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128112: AddedToken(\"<|reserved_special_token_104|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128113: AddedToken(\"<|reserved_special_token_105|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128114: AddedToken(\"<|reserved_special_token_106|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128115: AddedToken(\"<|reserved_special_token_107|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128116: AddedToken(\"<|reserved_special_token_108|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128117: AddedToken(\"<|reserved_special_token_109|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128118: AddedToken(\"<|reserved_special_token_110|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128119: AddedToken(\"<|reserved_special_token_111|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128120: AddedToken(\"<|reserved_special_token_112|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128121: AddedToken(\"<|reserved_special_token_113|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128122: AddedToken(\"<|reserved_special_token_114|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128123: AddedToken(\"<|reserved_special_token_115|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128124: AddedToken(\"<|reserved_special_token_116|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128125: AddedToken(\"<|reserved_special_token_117|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128126: AddedToken(\"<|reserved_special_token_118|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128127: AddedToken(\"<|reserved_special_token_119|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128128: AddedToken(\"<|reserved_special_token_120|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128129: AddedToken(\"<|reserved_special_token_121|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128130: AddedToken(\"<|reserved_special_token_122|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128131: AddedToken(\"<|reserved_special_token_123|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128132: AddedToken(\"<|reserved_special_token_124|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128133: AddedToken(\"<|reserved_special_token_125|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128134: AddedToken(\"<|reserved_special_token_126|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128135: AddedToken(\"<|reserved_special_token_127|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128136: AddedToken(\"<|reserved_special_token_128|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128137: AddedToken(\"<|reserved_special_token_129|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128138: AddedToken(\"<|reserved_special_token_130|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128139: AddedToken(\"<|reserved_special_token_131|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128140: AddedToken(\"<|reserved_special_token_132|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128141: AddedToken(\"<|reserved_special_token_133|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128142: AddedToken(\"<|reserved_special_token_134|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128143: AddedToken(\"<|reserved_special_token_135|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128144: AddedToken(\"<|reserved_special_token_136|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128145: AddedToken(\"<|reserved_special_token_137|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128146: AddedToken(\"<|reserved_special_token_138|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128147: AddedToken(\"<|reserved_special_token_139|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128148: AddedToken(\"<|reserved_special_token_140|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128149: AddedToken(\"<|reserved_special_token_141|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128150: AddedToken(\"<|reserved_special_token_142|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128151: AddedToken(\"<|reserved_special_token_143|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128152: AddedToken(\"<|reserved_special_token_144|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128153: AddedToken(\"<|reserved_special_token_145|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128154: AddedToken(\"<|reserved_special_token_146|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128155: AddedToken(\"<|reserved_special_token_147|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128156: AddedToken(\"<|reserved_special_token_148|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128157: AddedToken(\"<|reserved_special_token_149|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128158: AddedToken(\"<|reserved_special_token_150|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128159: AddedToken(\"<|reserved_special_token_151|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128160: AddedToken(\"<|reserved_special_token_152|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128161: AddedToken(\"<|reserved_special_token_153|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128162: AddedToken(\"<|reserved_special_token_154|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128163: AddedToken(\"<|reserved_special_token_155|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128164: AddedToken(\"<|reserved_special_token_156|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128165: AddedToken(\"<|reserved_special_token_157|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128166: AddedToken(\"<|reserved_special_token_158|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128167: AddedToken(\"<|reserved_special_token_159|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128168: AddedToken(\"<|reserved_special_token_160|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128169: AddedToken(\"<|reserved_special_token_161|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128170: AddedToken(\"<|reserved_special_token_162|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128171: AddedToken(\"<|reserved_special_token_163|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128172: AddedToken(\"<|reserved_special_token_164|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128173: AddedToken(\"<|reserved_special_token_165|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128174: AddedToken(\"<|reserved_special_token_166|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128175: AddedToken(\"<|reserved_special_token_167|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128176: AddedToken(\"<|reserved_special_token_168|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128177: AddedToken(\"<|reserved_special_token_169|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128178: AddedToken(\"<|reserved_special_token_170|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128179: AddedToken(\"<|reserved_special_token_171|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128180: AddedToken(\"<|reserved_special_token_172|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128181: AddedToken(\"<|reserved_special_token_173|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128182: AddedToken(\"<|reserved_special_token_174|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128183: AddedToken(\"<|reserved_special_token_175|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128184: AddedToken(\"<|reserved_special_token_176|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128185: AddedToken(\"<|reserved_special_token_177|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128186: AddedToken(\"<|reserved_special_token_178|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128187: AddedToken(\"<|reserved_special_token_179|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128188: AddedToken(\"<|reserved_special_token_180|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128189: AddedToken(\"<|reserved_special_token_181|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128190: AddedToken(\"<|reserved_special_token_182|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128191: AddedToken(\"<|reserved_special_token_183|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128192: AddedToken(\"<|reserved_special_token_184|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128193: AddedToken(\"<|reserved_special_token_185|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128194: AddedToken(\"<|reserved_special_token_186|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128195: AddedToken(\"<|reserved_special_token_187|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128196: AddedToken(\"<|reserved_special_token_188|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128197: AddedToken(\"<|reserved_special_token_189|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128198: AddedToken(\"<|reserved_special_token_190|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128199: AddedToken(\"<|reserved_special_token_191|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128200: AddedToken(\"<|reserved_special_token_192|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128201: AddedToken(\"<|reserved_special_token_193|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128202: AddedToken(\"<|reserved_special_token_194|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128203: AddedToken(\"<|reserved_special_token_195|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128204: AddedToken(\"<|reserved_special_token_196|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128205: AddedToken(\"<|reserved_special_token_197|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128206: AddedToken(\"<|reserved_special_token_198|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128207: AddedToken(\"<|reserved_special_token_199|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128208: AddedToken(\"<|reserved_special_token_200|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128209: AddedToken(\"<|reserved_special_token_201|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128210: AddedToken(\"<|reserved_special_token_202|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128211: AddedToken(\"<|reserved_special_token_203|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128212: AddedToken(\"<|reserved_special_token_204|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128213: AddedToken(\"<|reserved_special_token_205|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128214: AddedToken(\"<|reserved_special_token_206|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128215: AddedToken(\"<|reserved_special_token_207|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128216: AddedToken(\"<|reserved_special_token_208|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128217: AddedToken(\"<|reserved_special_token_209|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128218: AddedToken(\"<|reserved_special_token_210|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128219: AddedToken(\"<|reserved_special_token_211|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128220: AddedToken(\"<|reserved_special_token_212|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128221: AddedToken(\"<|reserved_special_token_213|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128222: AddedToken(\"<|reserved_special_token_214|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128223: AddedToken(\"<|reserved_special_token_215|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128224: AddedToken(\"<|reserved_special_token_216|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128225: AddedToken(\"<|reserved_special_token_217|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128226: AddedToken(\"<|reserved_special_token_218|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128227: AddedToken(\"<|reserved_special_token_219|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128228: AddedToken(\"<|reserved_special_token_220|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128229: AddedToken(\"<|reserved_special_token_221|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128230: AddedToken(\"<|reserved_special_token_222|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128231: AddedToken(\"<|reserved_special_token_223|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128232: AddedToken(\"<|reserved_special_token_224|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128233: AddedToken(\"<|reserved_special_token_225|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128234: AddedToken(\"<|reserved_special_token_226|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128235: AddedToken(\"<|reserved_special_token_227|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128236: AddedToken(\"<|reserved_special_token_228|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128237: AddedToken(\"<|reserved_special_token_229|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128238: AddedToken(\"<|reserved_special_token_230|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128239: AddedToken(\"<|reserved_special_token_231|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128240: AddedToken(\"<|reserved_special_token_232|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128241: AddedToken(\"<|reserved_special_token_233|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128242: AddedToken(\"<|reserved_special_token_234|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128243: AddedToken(\"<|reserved_special_token_235|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128244: AddedToken(\"<|reserved_special_token_236|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128245: AddedToken(\"<|reserved_special_token_237|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128246: AddedToken(\"<|reserved_special_token_238|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128247: AddedToken(\"<|reserved_special_token_239|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128248: AddedToken(\"<|reserved_special_token_240|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128249: AddedToken(\"<|reserved_special_token_241|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128250: AddedToken(\"<|reserved_special_token_242|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128251: AddedToken(\"<|reserved_special_token_243|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128252: AddedToken(\"<|reserved_special_token_244|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128253: AddedToken(\"<|reserved_special_token_245|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128254: AddedToken(\"<|reserved_special_token_246|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t128255: AddedToken(\"<|reserved_special_token_247|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# 使用AutoTokenizer 加载tokenizer\n",
    "from transformers import AutoTokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"/home/chendan/Desktop/workspace/workspace/checkpoints/Llama-3___2-1B\")\n",
    "print(tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DatasetDict({\n",
      "    train: Dataset({\n",
      "        features: ['input_ids', 'attention_mask'],\n",
      "        num_rows: 7473\n",
      "    })\n",
      "    test: Dataset({\n",
      "        features: ['input_ids', 'attention_mask'],\n",
      "        num_rows: 1319\n",
      "    })\n",
      "})\n",
      "[128000, 45, 4306, 689, 6216, 27203, 311, 220, 2166, 315, 1077, 4885, 304, 5936, 11, 323, 1243, 1364, 6216, 4376, 439, 1690, 27203, 304, 3297, 13, 2650, 1690, 27203, 1550, 42701, 689, 4662, 31155, 304, 5936, 323, 3297, 30, 128000, 45, 4306, 689, 6216, 220, 2166, 14, 17, 284, 1134, 2166, 14, 17, 28, 1187, 2511, 1187, 27203, 304, 3297, 627, 45, 4306, 689, 6216, 220, 2166, 10, 1187, 284, 1134, 2166, 10, 1187, 28, 5332, 2511, 5332, 27203, 31155, 304, 5936, 323, 3297, 627, 827, 220, 5332]\n",
      "[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n"
     ]
    }
   ],
   "source": [
    "# 定义一个tokenize_function\n",
    "def tokenize_function(examples):\n",
    "    return tokenizer(examples[\"question\"], examples[\"answer\"], truncation=True, max_length=None)\n",
    "\n",
    "# 使用map方法应用tokenize_function到dataset的train和test,并且移除question和answer列\n",
    "dataset_tokenized = dataset.map(tokenize_function, batched=True, remove_columns=[\"question\", \"answer\"])\n",
    "# 打印dataset_tokenized的结构\n",
    "print(dataset_tokenized)\n",
    "# 打印dataset_tokenized的train的第一个元素的input_ids和attention_mask\n",
    "print(dataset_tokenized[\"train\"][0][\"input_ids\"])\n",
    "print(dataset_tokenized[\"train\"][0][\"attention_mask\"])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}