modernvbert / special_tokens_map.json
paultltc's picture
Upload folder using huggingface_hub
21e384c verified
{
"additional_special_tokens": [
"<global-img>",
"<row_1_col_1>",
"<row_1_col_2>",
"<row_1_col_3>",
"<row_1_col_4>",
"<row_1_col_5>",
"<row_1_col_6>",
"<row_2_col_1>",
"<row_2_col_2>",
"<row_2_col_3>",
"<row_2_col_4>",
"<row_2_col_5>",
"<row_2_col_6>",
"<row_3_col_1>",
"<row_3_col_2>",
"<row_3_col_3>",
"<row_3_col_4>",
"<row_3_col_5>",
"<row_3_col_6>",
"<row_4_col_1>",
"<row_4_col_2>",
"<row_4_col_3>",
"<row_4_col_4>",
"<row_4_col_5>",
"<row_4_col_6>",
"<row_5_col_1>",
"<row_5_col_2>",
"<row_5_col_3>",
"<row_5_col_4>",
"<row_5_col_5>",
"<row_5_col_6>",
"<row_6_col_1>",
"<row_6_col_2>",
"<row_6_col_3>",
"<row_6_col_4>",
"<row_6_col_5>",
"<row_6_col_6>",
"<end_of_utterance>",
"<fake_token_around_image>",
"<image>"
],
"cls_token": {
"content": "[CLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"mask_token": {
"content": "[MASK]",
"lstrip": true,
"normalized": false,
"rstrip": false,
"single_word": false
},
"pad_token": {
"content": "[PAD]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"sep_token": {
"content": "[SEP]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"unk_token": {
"content": "[UNK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
}
}