Update gptx_tokenizer.py
Browse files- gptx_tokenizer.py +1 -1
gptx_tokenizer.py
CHANGED
@@ -17,7 +17,7 @@ PAD_TOKEN = "<pad>"
|
|
17 |
BOS_TOKEN = "<s>"
|
18 |
EOS_TOKEN = "</s>"
|
19 |
UNK_TOKEN = "<unk>"
|
20 |
-
REPO_ID = "openGPT-X/Teuken-7B-
|
21 |
|
22 |
class HFGPTXTokenizer(PreTrainedTokenizer):
|
23 |
"""
|
|
|
17 |
BOS_TOKEN = "<s>"
|
18 |
EOS_TOKEN = "</s>"
|
19 |
UNK_TOKEN = "<unk>"
|
20 |
+
REPO_ID = "openGPT-X/Teuken-7B-chat-v0.4"
|
21 |
|
22 |
class HFGPTXTokenizer(PreTrainedTokenizer):
|
23 |
"""
|