Upload tokenizer

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<pad>",
+  "unk_token": "<pad>"
+}

tokenizer.json

@@ -0,0 +1,214 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "<s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 77,
+      "content": "<|endoftext|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Strip",
+    "strip_left": true,
+    "strip_right": true
+  },
+  "pre_tokenizer": {
+    "type": "Sequence",
+    "pretokenizers": [
+      {
+        "type": "Whitespace"
+      },
+      {
+        "type": "Split",
+        "pattern": {
+          "Regex": "([PQRKNB]|[a-h][1-8]|[bnqkr]|\" \")"
+        },
+        "behavior": "Isolated",
+        "invert": false
+      }
+    ]
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "<s>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "<s>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "<pad>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "<pad>",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "<pad>": {
+        "id": "<pad>",
+        "ids": [
+          0
+        ],
+        "tokens": [
+          "<pad>"
+        ]
+      },
+      "<s>": {
+        "id": "<s>",
+        "ids": [
+          1
+        ],
+        "tokens": [
+          "<s>"
+        ]
+      }
+    }
+  },
+  "decoder": {
+    "type": "Replace",
+    "pattern": {
+      "String": "<pad>"
+    },
+    "content": ""
+  },
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "<pad>": 0,
+      "<s>": 1,
+      "</s>": 2,
+      "P": 3,
+      "N": 4,
+      "R": 5,
+      "B": 6,
+      "Q": 7,
+      "K": 8,
+      "c2": 9,
+      "c4": 10,
+      "e7": 11,
+      "e5": 12,
+      "g2": 13,
+      "g3": 14,
+      "b8": 15,
+      "c6": 16,
+      "f1": 17,
+      "g8": 18,
+      "f6": 19,
+      "b1": 20,
+      "c3": 21,
+      "f8": 22,
+      "b4": 23,
+      "d5": 24,
+      "e8": 25,
+      "e2": 26,
+      "e3": 27,
+      "e4": 28,
+      "a2": 29,
+      "a3": 30,
+      "d6": 31,
+      "d8": 32,
+      "a1": 33,
+      "e6": 34,
+      "b2": 35,
+      "b3": 36,
+      "c7": 37,
+      "d2": 38,
+      "d3": 39,
+      "f7": 40,
+      "f5": 41,
+      "a7": 42,
+      "a5": 43,
+      "g1": 44,
+      "d4": 45,
+      "g4": 46,
+      "c5": 47,
+      "e1": 48,
+      "d7": 49,
+      "d1": 50,
+      "h8": 51,
+      "c1": 52,
+      "a4": 53,
+      "h2": 54,
+      "h3": 55,
+      "a8": 56,
+      "f4": 57,
+      "g7": 58,
+      "g5": 59,
+      "b6": 60,
+      "b7": 61,
+      "c8": 62,
+      "h7": 63,
+      "h4": 64,
+      "f3": 65,
+      "h5": 66,
+      "h1": 67,
+      "f2": 68,
+      "h6": 69,
+      "g6": 70,
+      "b5": 71,
+      "a6": 72,
+      "q": 73,
+      "r": 74,
+      "b": 75,
+      "n": 76
+    },
+    "unk_token": "<pad>"
+  }
+}

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_bos_token": true,
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "max_length": 512,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_to_multiple_of": null,
+  "pad_token": "<pad>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "stride": 0,
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "<pad>"
+}