{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [], "normalizer": { "type": "Replace", "pattern": { "Regex": "[^ !\"$',.:;?A-Za-z\u00a1\u00ab\u00bb\u00bf\u00e6\u00e7\u00f0\u00f8\u0127\u014b\u0153\u01c0-\u01c3\u0250-\u0268\u026a-\u0276\u0278-\u027b\u027d\u027e\u0280-\u0284\u0288-\u0292\u0294\u0295\u0298\u0299\u029b-\u029d\u029f\u02a1\u02a2\u02a4\u02a7\u02b0-\u02b2\u02b4\u02b7\u02bc\u02c8\u02cc\u02d0\u02d1\u02de\u02e0\u02e4\u0329\u03b2\u03b8\u03c7\u1d7b\u2014\u201c\u201d\u2026\u2191-\u2193\u2197\u2198\u2c71]" }, "content": "" }, "pre_tokenizer": { "type": "Split", "pattern": { "Regex": "" }, "behavior": "Isolated", "invert": false }, "post_processor": { "type": "TemplateProcessing", "single": [ { "SpecialToken": { "id": "$", "type_id": 0 } }, { "Sequence": { "id": "A", "type_id": 0 } }, { "SpecialToken": { "id": "$", "type_id": 0 } } ], "special_tokens": { "$": { "id": "$", "ids": [ 0 ], "tokens": [ "$" ] } } }, "decoder": null, "model": { "vocab": { "$": 0, ";": 1, ":": 2, ",": 3, ".": 4, "!": 5, "?": 6, "\u00a1": 7, "\u00bf": 8, "\u2014": 9, "\u2026": 10, "\"": 11, "\u00ab": 12, "\u00bb": 13, "\u201c": 14, "\u201d": 15, " ": 16, "A": 17, "B": 18, "C": 19, "D": 20, "E": 21, "F": 22, "G": 23, "H": 24, "I": 25, "J": 26, "K": 27, "L": 28, "M": 29, "N": 30, "O": 31, "P": 32, "Q": 33, "R": 34, "S": 35, "T": 36, "U": 37, "V": 38, "W": 39, "X": 40, "Y": 41, "Z": 42, "a": 43, "b": 44, "c": 45, "d": 46, "e": 47, "f": 48, "g": 49, "h": 50, "i": 51, "j": 52, "k": 53, "l": 54, "m": 55, "n": 56, "o": 57, "p": 58, "q": 59, "r": 60, "s": 61, "t": 62, "u": 63, "v": 64, "w": 65, "x": 66, "y": 67, "z": 68, "\u0251": 69, "\u0250": 70, "\u0252": 71, "\u00e6": 72, "\u0253": 73, "\u0299": 74, "\u03b2": 75, "\u0254": 76, "\u0255": 77, "\u00e7": 78, "\u0257": 79, "\u0256": 80, "\u00f0": 81, "\u02a4": 82, "\u0259": 83, "\u0258": 84, "\u025a": 85, "\u025b": 86, "\u025c": 87, "\u025d": 88, "\u025e": 89, "\u025f": 90, "\u0284": 91, "\u0261": 92, "\u0260": 93, "\u0262": 94, "\u029b": 95, "\u0266": 96, "\u0267": 97, "\u0127": 98, "\u0265": 99, "\u029c": 100, "\u0268": 101, "\u026a": 102, "\u029d": 103, "\u026d": 104, "\u026c": 105, "\u026b": 106, "\u026e": 107, "\u029f": 108, "\u0271": 109, "\u026f": 110, "\u0270": 111, "\u014b": 112, "\u0273": 113, "\u0272": 114, "\u0274": 115, "\u00f8": 116, "\u0275": 117, "\u0278": 118, "\u03b8": 119, "\u0153": 120, "\u0276": 121, "\u0298": 122, "\u0279": 123, "\u027a": 124, "\u027e": 125, "\u027b": 126, "\u0280": 127, "\u0281": 128, "\u027d": 129, "\u0282": 130, "\u0283": 131, "\u0288": 132, "\u02a7": 133, "\u0289": 134, "\u028a": 135, "\u028b": 136, "\u2c71": 137, "\u028c": 138, "\u0263": 139, "\u0264": 140, "\u028d": 141, "\u03c7": 142, "\u028e": 143, "\u028f": 144, "\u0291": 145, "\u0290": 146, "\u0292": 147, "\u0294": 148, "\u02a1": 149, "\u0295": 150, "\u02a2": 151, "\u01c0": 152, "\u01c1": 153, "\u01c2": 154, "\u01c3": 155, "\u02c8": 156, "\u02cc": 157, "\u02d0": 158, "\u02d1": 159, "\u02bc": 160, "\u02b4": 161, "\u02b0": 162, "\u02b1": 163, "\u02b2": 164, "\u02b7": 165, "\u02e0": 166, "\u02e4": 167, "\u02de": 168, "\u2193": 169, "\u2191": 170, "\u2192": 171, "\u2197": 172, "\u2198": 173, "\u0329": 175, "'": 176, "\u1d7b": 177 } } }