config.json

{
  "model_type": "new",
  "vocab_size": 250002,
  "hidden_size": 1024,
  "num_hidden_layers": 24,
  "num_attention_heads": 16,
  "intermediate_size": 4096,
  "hidden_act": "gelu",
  "max_position_embeddings": 8194,
  "type_vocab_size": 1,
  "layer_norm_type": "layer_norm",
  "layer_norm_eps": 1e-5,
  "position_embedding_type": "rope",
  "rope_theta": 10000.0,
  "rope_scaling": null,
  "logn_attention_scale": false,
  "logn_attention_clip1": false,
  "architectures": ["GTEModel"],
  "model_max_length": 8194
}