specific tokenizer defined

analyzer.py

@@ -3,6 +3,7 @@ from typing import Dict, Optional, Union
 from transformers import (
     AutoModelForSequenceClassification,
     AutoModelForTokenClassification,
+    BertTokenizer,
     AutoTokenizer,
     TokenClassificationPipeline,
 )
@@ -34,19 +35,19 @@ class NewsAnalyzer:
             model=AutoModelForSequenceClassification.from_pretrained(
                 category_model_name
             ),
-            tokenizer=AutoTokenizer.from_pretrained(category_model_name),
+            tokenizer=BertTokenizer.from_pretrained(category_model_name),
             emojis=CATEGORY_EMOJIS,
         )
         self.fake_pipe = NewsPipeline(
             model=AutoModelForSequenceClassification.from_pretrained(fake_model_name),
-            tokenizer=AutoTokenizer.from_pretrained(fake_model_name),
+            tokenizer=BertTokenizer.from_pretrained(fake_model_name),
             emojis=FAKE_EMOJIS,
         )
         self.clickbait_pipe = NewsPipeline(
             model=AutoModelForSequenceClassification.from_pretrained(
                 clickbait_model_name
             ),
-            tokenizer=AutoTokenizer.from_pretrained(clickbait_model_name),
+            tokenizer=BertTokenizer.from_pretrained(clickbait_model_name),
             emojis=CLICKBAIT_EMOJIS,
         )
         self.ner_pipe = TokenClassificationPipeline(
@@ -67,14 +68,3 @@ class NewsAnalyzer:
                 "content": self.ner_pipe(content) if content else None,
             },
         }
-
-
-if __name__ == "__main__":
-    analyzer = NewsAnalyzer(
-        category_model_name="elozano/news-category",
-        fake_model_name="elozano/news-fake",
-        clickbait_model_name="elozano/news-clickbait",
-        ner_model_name="dslim/bert-base-NER",
-    )
-    prediction = analyzer(headline="Lakers Won!")
-    print(prediction)