Added nltk punkt download.
Browse files
worker.py
CHANGED
@@ -3,6 +3,9 @@ import tokenizers
|
|
3 |
from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification
|
4 |
import numpy as np
|
5 |
import torch
|
|
|
|
|
|
|
6 |
from nltk.tokenize import sent_tokenize
|
7 |
|
8 |
|
|
|
3 |
from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification
|
4 |
import numpy as np
|
5 |
import torch
|
6 |
+
import nltk
|
7 |
+
|
8 |
+
nltk.download("punkt")
|
9 |
from nltk.tokenize import sent_tokenize
|
10 |
|
11 |
|