564 lines
13 KiB
JSON
564 lines
13 KiB
JSON
|
{
|
|||
|
|
"version":"1.0",
|
|||
|
|
"truncation":null,
|
|||
|
|
"padding":null,
|
|||
|
|
"normalizer":null,
|
|||
|
|
"pre_tokenizer":{
|
|||
|
|
"type":"Whitespace"
|
|||
|
|
},
|
|||
|
|
"post_processor":null,
|
|||
|
|
"decoder":null,
|
|||
|
|
"added_tokens":[
|
|||
|
|
{
|
|||
|
|
"id":0,
|
|||
|
|
"special":true,
|
|||
|
|
"content":"[STOP]",
|
|||
|
|
"single_word":false,
|
|||
|
|
"lstrip":false,
|
|||
|
|
"rstrip":false,
|
|||
|
|
"normalized":false
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id":1,
|
|||
|
|
"special":true,
|
|||
|
|
"content":"[UNK]",
|
|||
|
|
"single_word":false,
|
|||
|
|
"lstrip":false,
|
|||
|
|
"rstrip":false,
|
|||
|
|
"normalized":false
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id":2,
|
|||
|
|
"special":true,
|
|||
|
|
"content":"[SPACE]",
|
|||
|
|
"single_word":false,
|
|||
|
|
"lstrip":false,
|
|||
|
|
"rstrip":false,
|
|||
|
|
"normalized":false
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"model":{
|
|||
|
|
"type":"BPE",
|
|||
|
|
"language": "ja",
|
|||
|
|
"dropout":null,
|
|||
|
|
"unk_token":"[UNK]",
|
|||
|
|
"continuing_subword_prefix":null,
|
|||
|
|
"end_of_word_suffix":null,
|
|||
|
|
"fuse_unk":false,
|
|||
|
|
"vocab":{
|
|||
|
|
"[STOP]": 0,
|
|||
|
|
"[UNK]": 1,
|
|||
|
|
"[SPACE]": 2,
|
|||
|
|
|
|||
|
|
"ー":255,
|
|||
|
|
|
|||
|
|
"ア": 14,
|
|||
|
|
"イ": 22,
|
|||
|
|
"ウ": 34,
|
|||
|
|
"エ": 18,
|
|||
|
|
"オ": 28,
|
|||
|
|
"カ": 254,
|
|||
|
|
"ガ": 230,
|
|||
|
|
"キ": 253,
|
|||
|
|
"キャ": 215,
|
|||
|
|
"キュ": 214,
|
|||
|
|
"キョ": 213,
|
|||
|
|
"ギ": 229,
|
|||
|
|
"ギャ": 194,
|
|||
|
|
"ギュ": 193,
|
|||
|
|
"ギョ": 192,
|
|||
|
|
"ク": 252,
|
|||
|
|
"グ": 228,
|
|||
|
|
"ケ": 93,
|
|||
|
|
"ゲ": 112,
|
|||
|
|
"コ": 251,
|
|||
|
|
"ゴ": 119,
|
|||
|
|
"サ": 150,
|
|||
|
|
"ザ": 227,
|
|||
|
|
"シ": 250,
|
|||
|
|
"シャ": 212,
|
|||
|
|
"シュ": 211,
|
|||
|
|
"ショ": 210,
|
|||
|
|
"ジ": 226,
|
|||
|
|
"ジャ": 191,
|
|||
|
|
"ジュ": 190,
|
|||
|
|
"ジョ": 189,
|
|||
|
|
"ス": 133,
|
|||
|
|
"ズ": 225,
|
|||
|
|
"セ": 66,
|
|||
|
|
"ゼ": 224,
|
|||
|
|
"ソ": 123,
|
|||
|
|
"ゾ": 223,
|
|||
|
|
"タ": 249,
|
|||
|
|
"ダ": 222,
|
|||
|
|
"チ": 248,
|
|||
|
|
"チャ": 209,
|
|||
|
|
"チュ": 208,
|
|||
|
|
"チョ": 207,
|
|||
|
|
"ヂ": 226,
|
|||
|
|
"ツ": 247,
|
|||
|
|
"ヅ": 225,
|
|||
|
|
"テ": 136,
|
|||
|
|
"デ": 126,
|
|||
|
|
"ト": 51,
|
|||
|
|
"ド": 134,
|
|||
|
|
"ナ": 246,
|
|||
|
|
"ニ": 245,
|
|||
|
|
"ニャ": 206,
|
|||
|
|
"ニュ": 205,
|
|||
|
|
"ニョ": 204,
|
|||
|
|
"ヌ": 244,
|
|||
|
|
"ネ": 111,
|
|||
|
|
"ノ": 95,
|
|||
|
|
"ハ": 102,
|
|||
|
|
"バ": 221,
|
|||
|
|
"パ": 218,
|
|||
|
|
"ヒ": 243,
|
|||
|
|
"ヒャ": 203,
|
|||
|
|
"ヒュ": 202,
|
|||
|
|
"ヒョ": 201,
|
|||
|
|
"ビ": 220,
|
|||
|
|
"ビャ": 188,
|
|||
|
|
"ビュ": 187,
|
|||
|
|
"ビョ": 186,
|
|||
|
|
"ピ": 217,
|
|||
|
|
"ピャ": 185,
|
|||
|
|
"ピュ": 184,
|
|||
|
|
"ピョ": 183,
|
|||
|
|
"フ": 242,
|
|||
|
|
"ブ": 219,
|
|||
|
|
"プ": 216,
|
|||
|
|
"ヘ": 62,
|
|||
|
|
"ベ": 67,
|
|||
|
|
"ペ": 124,
|
|||
|
|
"ホ": 241,
|
|||
|
|
"ボ": 165,
|
|||
|
|
"ポ": 139,
|
|||
|
|
"マ": 240,
|
|||
|
|
"ミ": 239,
|
|||
|
|
"ミャ": 200,
|
|||
|
|
"ミュ": 199,
|
|||
|
|
"ミョ": 198,
|
|||
|
|
"ム": 238,
|
|||
|
|
"メ": 80,
|
|||
|
|
"モ": 115,
|
|||
|
|
"ヤ": 237,
|
|||
|
|
"ユ": 236,
|
|||
|
|
"ヨ": 235,
|
|||
|
|
"ラ": 234,
|
|||
|
|
"リ": 101,
|
|||
|
|
"リャ": 197,
|
|||
|
|
"リュ": 196,
|
|||
|
|
"リョ": 195,
|
|||
|
|
"ル": 233,
|
|||
|
|
"レ": 46,
|
|||
|
|
"ロ": 89,
|
|||
|
|
"ワ": 232,
|
|||
|
|
"ヲ": 231,
|
|||
|
|
"ン": 27,
|
|||
|
|
"ッ":182,
|
|||
|
|
"ャ":181,
|
|||
|
|
"ョ":180,
|
|||
|
|
"ュ":179,
|
|||
|
|
|
|||
|
|
"あ": 14,
|
|||
|
|
"い": 22,
|
|||
|
|
"う": 34,
|
|||
|
|
"え": 18,
|
|||
|
|
"お": 28,
|
|||
|
|
"か": 254,
|
|||
|
|
"が": 230,
|
|||
|
|
"き": 253,
|
|||
|
|
"きゃ": 215,
|
|||
|
|
"きゅ": 214,
|
|||
|
|
"きょ": 213,
|
|||
|
|
"ぎ": 229,
|
|||
|
|
"ぎゃ": 194,
|
|||
|
|
"ぎゅ": 193,
|
|||
|
|
"ぎょ": 192,
|
|||
|
|
"く": 252,
|
|||
|
|
"ぐ": 228,
|
|||
|
|
"け": 93,
|
|||
|
|
"げ": 112,
|
|||
|
|
"こ": 251,
|
|||
|
|
"ご": 119,
|
|||
|
|
"さ": 150,
|
|||
|
|
"ざ": 227,
|
|||
|
|
"し": 250,
|
|||
|
|
"しゃ": 212,
|
|||
|
|
"しゅ": 211,
|
|||
|
|
"しょ": 210,
|
|||
|
|
"じ": 226,
|
|||
|
|
"じゃ": 191,
|
|||
|
|
"じゅ": 190,
|
|||
|
|
"じょ": 189,
|
|||
|
|
"す": 133,
|
|||
|
|
"ず": 225,
|
|||
|
|
"せ": 66,
|
|||
|
|
"ぜ": 224,
|
|||
|
|
"そ": 123,
|
|||
|
|
"ぞ": 223,
|
|||
|
|
"た": 249,
|
|||
|
|
"だ": 222,
|
|||
|
|
"ち": 248,
|
|||
|
|
"ちゃ": 209,
|
|||
|
|
"ちゅ": 208,
|
|||
|
|
"ちょ": 207,
|
|||
|
|
"ぢ": 226,
|
|||
|
|
"つ": 247,
|
|||
|
|
"づ": 225,
|
|||
|
|
"て": 136,
|
|||
|
|
"で": 126,
|
|||
|
|
"と": 51,
|
|||
|
|
"ど": 134,
|
|||
|
|
"な": 246,
|
|||
|
|
"に": 245,
|
|||
|
|
"にゃ": 206,
|
|||
|
|
"にゅ": 205,
|
|||
|
|
"にょ": 204,
|
|||
|
|
"ぬ": 244,
|
|||
|
|
"ね": 111,
|
|||
|
|
"の": 95,
|
|||
|
|
"は": 102,
|
|||
|
|
"ば": 221,
|
|||
|
|
"ぱ": 218,
|
|||
|
|
"ひ": 243,
|
|||
|
|
"ひゃ": 203,
|
|||
|
|
"ひゅ": 202,
|
|||
|
|
"ひょ": 201,
|
|||
|
|
"び": 220,
|
|||
|
|
"びゃ": 188,
|
|||
|
|
"びゅ": 187,
|
|||
|
|
"びょ": 186,
|
|||
|
|
"ぴ": 217,
|
|||
|
|
"ぴゃ": 185,
|
|||
|
|
"ぴゅ": 184,
|
|||
|
|
"ぴょ": 183,
|
|||
|
|
"ふ": 242,
|
|||
|
|
"ぶ": 219,
|
|||
|
|
"ぷ": 216,
|
|||
|
|
"へ": 62,
|
|||
|
|
"べ": 67,
|
|||
|
|
"ぺ": 124,
|
|||
|
|
"ほ": 241,
|
|||
|
|
"ぼ": 165,
|
|||
|
|
"ぽ": 139,
|
|||
|
|
"ま": 240,
|
|||
|
|
"み": 239,
|
|||
|
|
"みゃ": 200,
|
|||
|
|
"みゅ": 199,
|
|||
|
|
"みょ": 198,
|
|||
|
|
"む": 238,
|
|||
|
|
"め": 80,
|
|||
|
|
"も": 115,
|
|||
|
|
"や": 237,
|
|||
|
|
"ゆ": 236,
|
|||
|
|
"よ": 235,
|
|||
|
|
"ら": 234,
|
|||
|
|
"り": 101,
|
|||
|
|
"りゃ": 197,
|
|||
|
|
"りゅ": 196,
|
|||
|
|
"りょ": 195,
|
|||
|
|
"る": 233,
|
|||
|
|
"れ": 46,
|
|||
|
|
"ろ": 89,
|
|||
|
|
"わ": 232,
|
|||
|
|
"を": 231,
|
|||
|
|
"ん": 27,
|
|||
|
|
"っ":182,
|
|||
|
|
"ゃ":181,
|
|||
|
|
"ょ":180,
|
|||
|
|
"ゅ":179,
|
|||
|
|
|
|||
|
|
"ba": 221,
|
|||
|
|
"bi": 220,
|
|||
|
|
"bu": 219,
|
|||
|
|
"bya": 188,
|
|||
|
|
"byo": 186,
|
|||
|
|
"byu": 187,
|
|||
|
|
"cha": 209,
|
|||
|
|
"chi": 248,
|
|||
|
|
"cho": 207,
|
|||
|
|
"chu": 208,
|
|||
|
|
"da": 222,
|
|||
|
|
"fu": 242,
|
|||
|
|
"ga": 230,
|
|||
|
|
"gi": 229,
|
|||
|
|
"gu": 228,
|
|||
|
|
"gya": 194,
|
|||
|
|
"gyo": 192,
|
|||
|
|
"gyu": 193,
|
|||
|
|
"hi": 243,
|
|||
|
|
"ho": 241,
|
|||
|
|
"hya": 203,
|
|||
|
|
"hyo": 201,
|
|||
|
|
"hyu": 202,
|
|||
|
|
"ja": 191,
|
|||
|
|
"ji": 226,
|
|||
|
|
"jo": 189,
|
|||
|
|
"ju": 190,
|
|||
|
|
"ka": 254,
|
|||
|
|
"ki": 253,
|
|||
|
|
"ko": 251,
|
|||
|
|
"ku": 252,
|
|||
|
|
"kya": 215,
|
|||
|
|
"kyo": 213,
|
|||
|
|
"kyu": 214,
|
|||
|
|
"ma": 240,
|
|||
|
|
"mi": 239,
|
|||
|
|
"mu": 238,
|
|||
|
|
"mya": 200,
|
|||
|
|
"myo": 198,
|
|||
|
|
"myu": 199,
|
|||
|
|
"na": 246,
|
|||
|
|
"ni": 245,
|
|||
|
|
"nu": 244,
|
|||
|
|
"nya": 206,
|
|||
|
|
"nyo": 204,
|
|||
|
|
"nyu": 205,
|
|||
|
|
"pa": 218,
|
|||
|
|
"pi": 217,
|
|||
|
|
"pu": 216,
|
|||
|
|
"py": 183,
|
|||
|
|
"pya": 185,
|
|||
|
|
"pyu": 184,
|
|||
|
|
"ra": 234,
|
|||
|
|
"ru": 233,
|
|||
|
|
"rya": 197,
|
|||
|
|
"ryo": 195,
|
|||
|
|
"ryu": 196,
|
|||
|
|
"sha": 212,
|
|||
|
|
"shi": 250,
|
|||
|
|
"sho": 210,
|
|||
|
|
"shu": 211,
|
|||
|
|
"ta": 249,
|
|||
|
|
"tsu": 247,
|
|||
|
|
"wa": 232,
|
|||
|
|
"wo": 231,
|
|||
|
|
"ya": 237,
|
|||
|
|
"yo": 235,
|
|||
|
|
"yu": 236,
|
|||
|
|
"za": 227,
|
|||
|
|
"ze": 224,
|
|||
|
|
"zo": 223,
|
|||
|
|
"zu": 225,
|
|||
|
|
|
|||
|
|
"!": 3,
|
|||
|
|
"'": 4,
|
|||
|
|
"(": 5,
|
|||
|
|
")": 6,
|
|||
|
|
",": 7,
|
|||
|
|
"、": 7,
|
|||
|
|
"-": 8,
|
|||
|
|
".": 9,
|
|||
|
|
"。": 9,
|
|||
|
|
"…": 9,
|
|||
|
|
"/": 10,
|
|||
|
|
":": 11,
|
|||
|
|
";": 12,
|
|||
|
|
"?": 13,
|
|||
|
|
|
|||
|
|
"a": 14,
|
|||
|
|
"b": 15,
|
|||
|
|
"c": 16,
|
|||
|
|
"d": 17,
|
|||
|
|
"e": 18,
|
|||
|
|
"f": 19,
|
|||
|
|
"g": 20,
|
|||
|
|
"h": 21,
|
|||
|
|
"i": 22,
|
|||
|
|
"j": 23,
|
|||
|
|
"k": 24,
|
|||
|
|
"l": 25,
|
|||
|
|
"m": 26,
|
|||
|
|
"n": 27,
|
|||
|
|
"o": 28,
|
|||
|
|
"p": 29,
|
|||
|
|
"q": 30,
|
|||
|
|
"r": 31,
|
|||
|
|
"s": 32,
|
|||
|
|
"t": 33,
|
|||
|
|
"u": 34,
|
|||
|
|
"v": 35,
|
|||
|
|
"w": 36,
|
|||
|
|
"x": 37,
|
|||
|
|
"y": 38,
|
|||
|
|
"z": 39,
|
|||
|
|
|
|||
|
|
"th": 40,
|
|||
|
|
"in": 41,
|
|||
|
|
"the": 42,
|
|||
|
|
"an": 43,
|
|||
|
|
"er": 44,
|
|||
|
|
"ou": 45,
|
|||
|
|
"re": 46,
|
|||
|
|
"on": 47,
|
|||
|
|
"at": 48,
|
|||
|
|
"ed": 49,
|
|||
|
|
"en": 50,
|
|||
|
|
"to": 51,
|
|||
|
|
"ing": 52,
|
|||
|
|
"and": 53,
|
|||
|
|
"is": 54,
|
|||
|
|
"as": 55,
|
|||
|
|
"al": 56,
|
|||
|
|
"or": 57,
|
|||
|
|
"of": 58,
|
|||
|
|
"ar": 59,
|
|||
|
|
"it": 60,
|
|||
|
|
"es": 61,
|
|||
|
|
"he": 62,
|
|||
|
|
"st": 63,
|
|||
|
|
"le": 64,
|
|||
|
|
"om": 65,
|
|||
|
|
"se": 66,
|
|||
|
|
"be": 67,
|
|||
|
|
"ad": 68,
|
|||
|
|
"ow": 69,
|
|||
|
|
"ly": 70,
|
|||
|
|
"ch": 71,
|
|||
|
|
"wh": 72,
|
|||
|
|
"that": 73,
|
|||
|
|
"you": 74,
|
|||
|
|
"li": 75,
|
|||
|
|
"ve": 76,
|
|||
|
|
"ac": 77,
|
|||
|
|
"ti": 78,
|
|||
|
|
"ld": 79,
|
|||
|
|
"me": 80,
|
|||
|
|
"was": 81,
|
|||
|
|
"gh": 82,
|
|||
|
|
"id": 83,
|
|||
|
|
"ll": 84,
|
|||
|
|
"wi": 85,
|
|||
|
|
"ent": 86,
|
|||
|
|
"for": 87,
|
|||
|
|
"ay": 88,
|
|||
|
|
"ro": 89,
|
|||
|
|
"ver": 90,
|
|||
|
|
"ic": 91,
|
|||
|
|
"her": 92,
|
|||
|
|
"ke": 93,
|
|||
|
|
"his": 94,
|
|||
|
|
"no": 95,
|
|||
|
|
"ut": 96,
|
|||
|
|
"un": 97,
|
|||
|
|
"ir": 98,
|
|||
|
|
"lo": 99,
|
|||
|
|
"we": 100,
|
|||
|
|
"ri": 101,
|
|||
|
|
"ha": 102,
|
|||
|
|
"with": 103,
|
|||
|
|
"ght": 104,
|
|||
|
|
"out": 105,
|
|||
|
|
"im": 106,
|
|||
|
|
"ion": 107,
|
|||
|
|
"all": 108,
|
|||
|
|
"ab": 109,
|
|||
|
|
"one": 110,
|
|||
|
|
"ne": 111,
|
|||
|
|
"ge": 112,
|
|||
|
|
"ould": 113,
|
|||
|
|
"ter": 114,
|
|||
|
|
"mo": 115,
|
|||
|
|
"had": 116,
|
|||
|
|
"ce": 117,
|
|||
|
|
"she": 118,
|
|||
|
|
"go": 119,
|
|||
|
|
"sh": 120,
|
|||
|
|
"ur": 121,
|
|||
|
|
"am": 122,
|
|||
|
|
"so": 123,
|
|||
|
|
"pe": 124,
|
|||
|
|
"my": 125,
|
|||
|
|
"de": 126,
|
|||
|
|
"are": 127,
|
|||
|
|
"but": 128,
|
|||
|
|
"ome": 129,
|
|||
|
|
"fr": 130,
|
|||
|
|
"ther": 131,
|
|||
|
|
"fe": 132,
|
|||
|
|
"su": 133,
|
|||
|
|
"do": 134,
|
|||
|
|
"con": 135,
|
|||
|
|
"te": 136,
|
|||
|
|
"ain": 137,
|
|||
|
|
"ere": 138,
|
|||
|
|
"po": 139,
|
|||
|
|
"if": 140,
|
|||
|
|
"they": 141,
|
|||
|
|
"us": 142,
|
|||
|
|
"ag": 143,
|
|||
|
|
"tr": 144,
|
|||
|
|
"now": 145,
|
|||
|
|
"oun": 146,
|
|||
|
|
"this": 147,
|
|||
|
|
"have": 148,
|
|||
|
|
"not": 149,
|
|||
|
|
"sa": 150,
|
|||
|
|
"il": 151,
|
|||
|
|
"up": 152,
|
|||
|
|
"thing": 153,
|
|||
|
|
"from": 154,
|
|||
|
|
"ap": 155,
|
|||
|
|
"him": 156,
|
|||
|
|
"ack": 157,
|
|||
|
|
"ation": 158,
|
|||
|
|
"ant": 159,
|
|||
|
|
"our": 160,
|
|||
|
|
"op": 161,
|
|||
|
|
"like": 162,
|
|||
|
|
"ust": 163,
|
|||
|
|
"ess": 164,
|
|||
|
|
"bo": 165,
|
|||
|
|
"ok": 166,
|
|||
|
|
"ul": 167,
|
|||
|
|
"ind": 168,
|
|||
|
|
"ex": 169,
|
|||
|
|
"com": 170,
|
|||
|
|
"some": 171,
|
|||
|
|
"there": 172,
|
|||
|
|
"ers": 173,
|
|||
|
|
"co": 174,
|
|||
|
|
"res": 175,
|
|||
|
|
"man": 176,
|
|||
|
|
"ard": 177,
|
|||
|
|
"pl": 178
|
|||
|
|
},
|
|||
|
|
"merges":[
|
|||
|
|
"キ ャ",
|
|||
|
|
"キ ュ",
|
|||
|
|
"キ ョ",
|
|||
|
|
"シ ャ",
|
|||
|
|
"シ ュ",
|
|||
|
|
"シ ョ",
|
|||
|
|
"チ ャ",
|
|||
|
|
"チ ュ",
|
|||
|
|
"チ ョ",
|
|||
|
|
"ニ ャ",
|
|||
|
|
"ニ ュ",
|
|||
|
|
"ニ ョ",
|
|||
|
|
"ヒ ャ",
|
|||
|
|
"ヒ ュ",
|
|||
|
|
"ヒ ョ",
|
|||
|
|
"ミ ャ",
|
|||
|
|
"ミ ュ",
|
|||
|
|
"ミ ョ",
|
|||
|
|
"リ ャ",
|
|||
|
|
"リ ュ",
|
|||
|
|
"リ ョ",
|
|||
|
|
"ギ ャ",
|
|||
|
|
"ギ ュ",
|
|||
|
|
"ギ ョ",
|
|||
|
|
"ジ ャ",
|
|||
|
|
"ジ ュ",
|
|||
|
|
"ジ ョ",
|
|||
|
|
"ビ ャ",
|
|||
|
|
"ビ ュ",
|
|||
|
|
"ビ ョ",
|
|||
|
|
"ピ ャ",
|
|||
|
|
"ピ ュ",
|
|||
|
|
"ピ ョ"
|
|||
|
|
]
|
|||
|
|
}
|
|||
|
|
}
|