solve "cannot load pytorch checkpoint issue" and fix tokenizer

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+SimSun.ttf filter=lfs diff=lfs merge=lfs -text

SimSun.ttf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca4da082cd970f0c8abaa79f213ddcbc475f7b5afabcb81b385998f9ebfbb53f
+size 10499104

config.json

@@ -35,7 +35,7 @@
   "visual": {
     "heads": 16,
     "image_size": 448,
-    "image_start_id": 50470,
+    "image_start_id": 50508,
     "layers": 48,
     "mlp_ratio": 4.9231,
     "output_dim": 4096,
@@ -43,4 +43,4 @@
     "width": 1664
   },
   "vocab_size": 51200
-}
+}

modeling_vitphi.py

@@ -45,8 +45,8 @@ from einops import rearrange
 from transformers.activations import ACT2FN
 from transformers import PretrainedConfig, PreTrainedModel
 from transformers.modeling_outputs import CausalLMOutputWithPast
-from .configuration_vitphi import MixFormerVLSequentialConfig
-from .visual import VisionTransformer
+from configuration_vitphi import MixFormerVLSequentialConfig
+from visual import VisionTransformer
 # from configuration_vitphi import MixFormerVLSequentialConfig
 # from visual import VisionTransformer
 

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:45305856b9096f2473b8822eae24ba326387a7d0be0a1ec4f2f862f9af0f1011
-size 6724937785
+oid sha256:196df56fad9a8cda2cfd334a4c75dd21a5fc4522f2ed28ddb02e8ff50c31de4d
+size 6726979333

tokenization_vitphi.py

@@ -44,12 +44,26 @@ IMEND = "<|im_end|>"
 # as different as possible to minimize the impact
 EXTRAS = tuple((f"<|extra_{i}|>" for i in range(205)))
 SPECIAL_TOKENS = (
-                     # ENDOFTEXT,
+                    ENDOFTEXT,
                      IMSTART,
                      IMEND,
                  ) + EXTRAS
 IMG_TOKEN_SPAN = 256
 
+def bytes_to_unicode():
+    bs = (
+        list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
+    )
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8 + n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
 
 def _load_tiktoken_bpe(tiktoken_bpe_file: str) -> Dict[bytes, int]:
     with open(tiktoken_bpe_file, "rb") as f:
@@ -119,9 +133,22 @@ class VitPhiTokenizer(PreTrainedTokenizer):
             box_end_tag='</box>',
             quad_start_tag='<quad>',
             quad_end_tag='</quad>',
+            unk_token="<|endoftext|>",
+            bos_token="<|endoftext|>",
+            eos_token="<|endoftext|>",
+            pad_token=None,
             **kwargs,
     ):
-        super().__init__(**kwargs)
+        bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token
+        eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
+        unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token
+        pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
+        super().__init__(errors=errors, 
+                unk_token=unk_token, 
+                bos_token=bos_token, 
+                eos_token=eos_token,
+                pad_token=pad_token,
+                **kwargs)
         self.image_start_tag = image_start_tag
         self.image_end_tag = image_end_tag
         self.image_pad_tag = image_pad_tag
@@ -140,14 +167,17 @@ class VitPhiTokenizer(PreTrainedTokenizer):
         )
 
         self.errors = errors  # how to handle errors in decoding
+        self.byte_encoder = bytes_to_unicode()
 
         self.mergeable_ranks = _load_tiktoken_bpe(vocab_file)  # type: dict[bytes, int]
         self.special_tokens = {
             token: index
             for index, token in enumerate(
-                SPECIAL_TOKENS + self.IMAGE_ST, start=len(self.mergeable_ranks)
+                SPECIAL_TOKENS + self.IMAGE_ST, start=len(self.mergeable_ranks)-1
             )
         }
+        self.special_tokens[ENDOFTEXT] = 50256
+       # print(self.special_tokens)
         self.img_start_id = self.special_tokens[self.image_start_tag]
         self.img_end_id = self.special_tokens[self.image_end_tag]
         self.img_pad_id = self.special_tokens[self.image_pad_tag]
@@ -165,8 +195,8 @@ class VitPhiTokenizer(PreTrainedTokenizer):
             special_tokens=self.special_tokens,
         )
         assert (
-                len(self.mergeable_ranks) + len(self.special_tokens) == enc.n_vocab
-        ), f"{len(self.mergeable_ranks) + len(self.special_tokens)} != {enc.n_vocab} in encoding"
+                len(self.mergeable_ranks) + len(self.special_tokens) - 1 == enc.n_vocab # has a common word
+        ), f"{len(self.mergeable_ranks) + len(self.special_tokens)} - 1 != {enc.n_vocab} in encoding"
 
         self.decoder = {
             v: k for k, v in self.mergeable_ranks.items()
@@ -174,9 +204,9 @@ class VitPhiTokenizer(PreTrainedTokenizer):
         self.decoder.update({v: k for k, v in self.special_tokens.items()})
 
         self.tokenizer = enc  # type: tiktoken.Encoding
-
-        self.eod_id = self.tokenizer.eot_token
+ 
         self.im_start_id = self.special_tokens[IMSTART]
+        self.eod_id = self.im_start_id - 1
         self.im_end_id = self.special_tokens[IMEND]
 
     def __len__(self) -> int:
@@ -251,12 +281,14 @@ class VitPhiTokenizer(PreTrainedTokenizer):
             `List[bytes|str]`: The list of tokens.
         """
         tokens = []
+        text = "".join([self.byte_encoder[b] for b in text.encode("utf-8")])
+        #text = text.replace(" ", self.byte_encoder[" ".encode("utf-8")[0]])
         text = unicodedata.normalize("NFC", text)
+        #print("----after nfc------:", text)
 
         # this implementation takes a detour: text -> token id -> token surface forms
         for t in self.tokenizer.encode(
-                text, allowed_special=allowed_special, disallowed_special=disallowed_special
-        ):
+                    text, allowed_special=allowed_special, disallowed_special=disallowed_special ):
             tokens.append(self.decoder[t])
 
         def _encode_imgurl(img_tokens):