slicing and feature engineering

config.json

@@ -1,53 +1,26 @@
 {
-  "_name_or_path": "microsoft/graphcodebert-base",
   "architectures": [
-    "RobertaForSequenceClassification"
+    "CodeBERTForVulnerabilityDetectionWithFeatures"
   ],
   "attention_probs_dropout_prob": 0.1,
   "bos_token_id": 0,
   "classifier_dropout": null,
   "eos_token_id": 2,
-  "gradient_checkpointing": false,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
   "hidden_size": 768,
-  "id2label": {
-    "0": "LABEL_0",
-    "1": "LABEL_1",
-    "2": "LABEL_2",
-    "3": "LABEL_3",
-    "4": "LABEL_4",
-    "5": "LABEL_5",
-    "6": "LABEL_6",
-    "7": "LABEL_7",
-    "8": "LABEL_8",
-    "9": "LABEL_9"
-  },
   "initializer_range": 0.02,
   "intermediate_size": 3072,
-  "label2id": {
-    "LABEL_0": 0,
-    "LABEL_1": 1,
-    "LABEL_2": 2,
-    "LABEL_3": 3,
-    "LABEL_4": 4,
-    "LABEL_5": 5,
-    "LABEL_6": 6,
-    "LABEL_7": 7,
-    "LABEL_8": 8,
-    "LABEL_9": 9
-  },
-  "layer_norm_eps": 1e-05,
+  "layer_norm_eps": 1e-12,
   "max_position_embeddings": 514,
-  "model_type": "roberta",
+  "model_type": "codebert",
   "num_attention_heads": 12,
   "num_hidden_layers": 12,
-  "output_past": true,
+  "output_attentions": true,
   "pad_token_id": 1,
   "position_embedding_type": "absolute",
-  "problem_type": "single_label_classification",
   "torch_dtype": "float32",
-  "transformers_version": "4.40.1",
+  "transformers_version": "4.41.1",
   "type_vocab_size": 1,
   "use_cache": true,
   "vocab_size": 50265

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6e9fbc6c8e352b7fb9cdd20e15a359a3c1b1869da5086b212ed38025d1968db9
-size 498637432
+oid sha256:b9bcae3293b5d29ff9f520b875765460e825e12ec2ecccdf5fb2635ab3d058ec
+size 498692860

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eeda6c76b7102eb22693937d10b880ae3af91f76df0a3a54359358853fc95e33
+size 997506746

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfc880f662532368769bf627f8612e2c4b90986f8b4c7b89b3f45762dc6d424d
+size 14244

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1b227ad2a10700168d2b014fe9a1d55c46d4ec5e2f18bd52971da8f64d4b44b
+size 1064

trainer_state.json

@@ -0,0 +1,236 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.685408299866131,
+  "eval_steps": 500,
+  "global_step": 3500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.6693440428380187,
+      "grad_norm": 14.38033676147461,
+      "learning_rate": 4.97e-05,
+      "loss": 1.6741,
+      "step": 500
+    },
+    {
+      "epoch": 0.6693440428380187,
+      "eval_accuracy": 0.6358458961474037,
+      "eval_f1_class_0": 0.9606299212598425,
+      "eval_f1_class_1": 0.4562647754137116,
+      "eval_f1_class_2": 0.5818673883626523,
+      "eval_f1_class_3": 0.6752,
+      "eval_f1_class_4": 0.4695259593679458,
+      "eval_f1_class_5": 0.6699186991869919,
+      "eval_f1_class_6": 0.6111908177905309,
+      "eval_f1_class_7": 0.6285714285714286,
+      "eval_f1_class_8": 0.7056603773584904,
+      "eval_f1_macro": 0.6398699297012882,
+      "eval_f1_micro": 0.6358458961474037,
+      "eval_f1_weighted": 0.6376816033602213,
+      "eval_loss": 1.0920162200927734,
+      "eval_recall_weighted": 0.6358458961474037,
+      "eval_runtime": 52.0874,
+      "eval_samples_per_second": 57.307,
+      "eval_steps_per_second": 1.805,
+      "step": 500
+    },
+    {
+      "epoch": 1.3386880856760375,
+      "grad_norm": 13.991486549377441,
+      "learning_rate": 4.643472022955524e-05,
+      "loss": 0.77,
+      "step": 1000
+    },
+    {
+      "epoch": 1.3386880856760375,
+      "eval_accuracy": 0.8395309882747068,
+      "eval_f1_class_0": 0.9654088050314467,
+      "eval_f1_class_1": 0.7083333333333334,
+      "eval_f1_class_2": 0.8022598870056498,
+      "eval_f1_class_3": 0.8431952662721893,
+      "eval_f1_class_4": 0.7962674961119751,
+      "eval_f1_class_5": 0.8715447154471545,
+      "eval_f1_class_6": 0.8243430152143845,
+      "eval_f1_class_7": 0.8963210702341138,
+      "eval_f1_class_8": 0.8755760368663594,
+      "eval_f1_macro": 0.8425832917240673,
+      "eval_f1_micro": 0.8395309882747068,
+      "eval_f1_weighted": 0.8406783887287406,
+      "eval_loss": 0.5749590992927551,
+      "eval_recall_weighted": 0.8395309882747068,
+      "eval_runtime": 52.1038,
+      "eval_samples_per_second": 57.289,
+      "eval_steps_per_second": 1.804,
+      "step": 1000
+    },
+    {
+      "epoch": 2.0080321285140563,
+      "grad_norm": 2.8914544582366943,
+      "learning_rate": 4.284791965566714e-05,
+      "loss": 0.4007,
+      "step": 1500
+    },
+    {
+      "epoch": 2.0080321285140563,
+      "eval_accuracy": 0.8931323283082077,
+      "eval_f1_class_0": 0.96875,
+      "eval_f1_class_1": 0.7915492957746477,
+      "eval_f1_class_2": 0.8837209302325582,
+      "eval_f1_class_3": 0.838235294117647,
+      "eval_f1_class_4": 0.844574780058651,
+      "eval_f1_class_5": 0.9504,
+      "eval_f1_class_6": 0.9249617151607963,
+      "eval_f1_class_7": 0.9345794392523364,
+      "eval_f1_class_8": 0.9199999999999999,
+      "eval_f1_macro": 0.895196828288515,
+      "eval_f1_micro": 0.8931323283082077,
+      "eval_f1_weighted": 0.8935187314222036,
+      "eval_loss": 0.3943232595920563,
+      "eval_recall_weighted": 0.8931323283082077,
+      "eval_runtime": 52.1055,
+      "eval_samples_per_second": 57.288,
+      "eval_steps_per_second": 1.804,
+      "step": 1500
+    },
+    {
+      "epoch": 2.677376171352075,
+      "grad_norm": 2.264118194580078,
+      "learning_rate": 3.926111908177906e-05,
+      "loss": 0.2025,
+      "step": 2000
+    },
+    {
+      "epoch": 2.677376171352075,
+      "eval_accuracy": 0.9165829145728643,
+      "eval_f1_class_0": 0.9734789391575663,
+      "eval_f1_class_1": 0.8170563961485556,
+      "eval_f1_class_2": 0.9121338912133892,
+      "eval_f1_class_3": 0.910384068278805,
+      "eval_f1_class_4": 0.8715447154471545,
+      "eval_f1_class_5": 0.9562499999999999,
+      "eval_f1_class_6": 0.9147058823529411,
+      "eval_f1_class_7": 0.9559748427672956,
+      "eval_f1_class_8": 0.9525368248772503,
+      "eval_f1_macro": 0.918229506693662,
+      "eval_f1_micro": 0.9165829145728643,
+      "eval_f1_weighted": 0.9169458307013622,
+      "eval_loss": 0.392281711101532,
+      "eval_recall_weighted": 0.9165829145728643,
+      "eval_runtime": 52.1029,
+      "eval_samples_per_second": 57.29,
+      "eval_steps_per_second": 1.804,
+      "step": 2000
+    },
+    {
+      "epoch": 3.3467202141900936,
+      "grad_norm": 9.838397979736328,
+      "learning_rate": 3.5674318507890965e-05,
+      "loss": 0.1351,
+      "step": 2500
+    },
+    {
+      "epoch": 3.3467202141900936,
+      "eval_accuracy": 0.9252931323283082,
+      "eval_f1_class_0": 0.9750778816199377,
+      "eval_f1_class_1": 0.8636363636363636,
+      "eval_f1_class_2": 0.909814323607427,
+      "eval_f1_class_3": 0.8973607038123167,
+      "eval_f1_class_4": 0.8852459016393442,
+      "eval_f1_class_5": 0.9549839228295819,
+      "eval_f1_class_6": 0.9319526627218935,
+      "eval_f1_class_7": 0.9635499207606973,
+      "eval_f1_class_8": 0.9556962025316457,
+      "eval_f1_macro": 0.9263686536843565,
+      "eval_f1_micro": 0.9252931323283082,
+      "eval_f1_weighted": 0.9251223151765453,
+      "eval_loss": 0.38727396726608276,
+      "eval_recall_weighted": 0.9252931323283082,
+      "eval_runtime": 52.0796,
+      "eval_samples_per_second": 57.316,
+      "eval_steps_per_second": 1.805,
+      "step": 2500
+    },
+    {
+      "epoch": 4.016064257028113,
+      "grad_norm": 3.1932365894317627,
+      "learning_rate": 3.208751793400287e-05,
+      "loss": 0.1193,
+      "step": 3000
+    },
+    {
+      "epoch": 4.016064257028113,
+      "eval_accuracy": 0.9366834170854271,
+      "eval_f1_class_0": 0.9753086419753085,
+      "eval_f1_class_1": 0.8781954887218045,
+      "eval_f1_class_2": 0.9465020576131686,
+      "eval_f1_class_3": 0.8873626373626373,
+      "eval_f1_class_4": 0.9245283018867925,
+      "eval_f1_class_5": 0.9478672985781991,
+      "eval_f1_class_6": 0.948301329394387,
+      "eval_f1_class_7": 0.9695999999999999,
+      "eval_f1_class_8": 0.9602543720190778,
+      "eval_f1_macro": 0.9375466808390418,
+      "eval_f1_micro": 0.9366834170854271,
+      "eval_f1_weighted": 0.9367077203287543,
+      "eval_loss": 0.36861082911491394,
+      "eval_recall_weighted": 0.9366834170854271,
+      "eval_runtime": 52.0777,
+      "eval_samples_per_second": 57.318,
+      "eval_steps_per_second": 1.805,
+      "step": 3000
+    },
+    {
+      "epoch": 4.685408299866131,
+      "grad_norm": 16.49643325805664,
+      "learning_rate": 2.850071736011478e-05,
+      "loss": 0.063,
+      "step": 3500
+    },
+    {
+      "epoch": 4.685408299866131,
+      "eval_accuracy": 0.9396984924623115,
+      "eval_f1_class_0": 0.9751552795031054,
+      "eval_f1_class_1": 0.8555240793201134,
+      "eval_f1_class_2": 0.9222222222222223,
+      "eval_f1_class_3": 0.9184549356223176,
+      "eval_f1_class_4": 0.9200603318250377,
+      "eval_f1_class_5": 0.9730586370839936,
+      "eval_f1_class_6": 0.9650986342943855,
+      "eval_f1_class_7": 0.964968152866242,
+      "eval_f1_class_8": 0.9774193548387097,
+      "eval_f1_macro": 0.9413290697306808,
+      "eval_f1_micro": 0.9396984924623115,
+      "eval_f1_weighted": 0.9399337568761867,
+      "eval_loss": 0.3695576786994934,
+      "eval_recall_weighted": 0.9396984924623115,
+      "eval_runtime": 52.0778,
+      "eval_samples_per_second": 57.318,
+      "eval_steps_per_second": 1.805,
+      "step": 3500
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 7470,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.47218650942464e+16,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1ae381d8a4e60b845958278ae33592715df0c591a7af3c62d64bd55e46c5dd18
-size 4920
+oid sha256:41dd377079db2bb5319531916f5985e2f6d42d7d70265ab3bb800398e5955fc2
+size 5048