Spaces:
Runtime error
Runtime error
File size: 2,353 Bytes
55d70df b5a963d f445716 55d70df f445716 55d70df f445716 55d70df f445716 55d70df f445716 55d70df f445716 55d70df f445716 55d70df f445716 55d70df f445716 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 |
import json
import spacy
from spacy.tokens import DocBin
def read_in_chunks(file_path, chunk_size=1024):
with open(file_path, 'r', encoding='utf-8') as file:
while True:
data = file.read(chunk_size)
if not data:
break
yield data
def convert_json_to_spacy(json_file_path, spacy_file_path):
# Read the file in chunks and combine the chunks
file_content = ""
for chunk in read_in_chunks(json_file_path):
file_content += chunk
# Parse the JSON data
data = json.loads(file_content)
# Prepare the data for spaCy
spacy_format = []
for item in data:
text = item[0] # The first element in the list is the text
entities = item[1]['entities'] # The second element contains the dictionary with 'entities'
spacy_entities = [(start, end, label) for start, end, label in entities]
spacy_format.append({"text": text, "entities": spacy_entities})
# Create a blank English model
nlp = spacy.blank("en")
# Initialize a DocBin object
doc_bin = DocBin()
# Convert the data to spaCy Doc objects and add to DocBin
for entry in spacy_format:
doc = nlp.make_doc(entry["text"])
# Convert entities
entities = []
seen_positions = set() # To track positions and avoid overlap
for start, end, label in entry["entities"]:
# Ensure span is within the document's length
if start < 0 or end > len(doc.text) or start >= end:
print(f"Invalid span: start={start}, end={end}, label={label}")
continue
# Check for overlaps and prioritize entities
if not any(start < e_end and end > e_start for e_start, e_end, _ in seen_positions):
span = doc.char_span(start, end, label=label)
if span is not None:
entities.append(span)
seen_positions.add((start, end, label))
else:
print(f"Overlapping span: start={start}, end={end}, label={label}")
# Set entities
doc.ents = entities
# Add to DocBin
doc_bin.add(doc)
# Save the DocBin to a .spacy file
doc_bin.to_disk(spacy_file_path)
print(f"Data has been successfully saved to {spacy_file_path}!")
|