Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+capybara_sharegpt.json filter=lfs diff=lfs merge=lfs -text
+merged_all.json filter=lfs diff=lfs merge=lfs -text
+synthia-v1.3_sharegpt_12500.json filter=lfs diff=lfs merge=lfs -text

capybara_sharegpt.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1947d28999416a2f468d1e09654cfdfc9bab8ccd03aa184598d20f0000dd6e4
+size 76361785

merged_all.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f915219dc691de207f9b8b448f005386330035f3c7f6cc6379dd183ac7662d2f
+size 306656943

remove_empty_output.py

@@ -0,0 +1,13 @@
+import json
+
+with open('merged_all.json', 'r') as file:
+    data = json.load(file)
+
+print(f"Normal len: {len(data)}")
+
+data = [row for row in data if row["output"] != ""]
+
+print(f"After len: {len(data)}")
+
+with open('merged_all.json', 'w') as file:
+    json.dump(data, file, indent=1)

synthia-v1.3_sharegpt_12500.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbdbc7413a3c7fc65a900518f0db8627bb5ced53e1e8ee82613d09856c1b3b70
+size 30638009