app.py

import gradio as gr

# Assuming we have a function to load a tokenizer by name (you would need to replace this with actual implementation)
def load_tokenizer(tokenizer_name):
    if tokenizer_name == "aranizer_sp32k":
        tokenizer = aranizer_sp32k.get_tokenizer()
    # Add conditions for other tokenizers

    return tokenizer

def tokenize_and_encode(text, tokenizer_choice):
    tokenizer = load_tokenizer(tokenizer_choice)
    tokens = tokenizer.tokenize(text)
    encoded_output = tokenizer.encode(text, add_special_tokens=True)
    decoded_text = tokenizer.decode(encoded_output)
    return tokens, encoded_output, decoded_text

iface = gr.Interface(
    fn=tokenize_and_encode,
    inputs=[gr.inputs.Textbox(lines=5, label="النص العربي"), gr.inputs.Dropdown(choices=["aranizer_bpe32k", "aranizer_bpe50k", "aranizer_bpe64k", "aranizer_bpe86k", "aranizer_sp32k", "aranizer_sp50k", "aranizer_sp64k", "aranizer_sp86k"], label="اختر المحلل اللفظي")],
    outputs=[gr.outputs.Textbox(label="Tokens"), gr.outputs.Textbox(label="Encoded Output"), gr.outputs.Textbox(label="Decoded Text")],
    title="مقارنة المحللات اللفظية للنص العربي",
    description="حدد نوع المحلل اللفظي وأدخل نصًا لرؤية النتائج.",
    language="ar",
)

iface.launch()