{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "gpuType": "T4" }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" }, "accelerator": "GPU" }, "cells": [ { "cell_type": "markdown", "source": [ "# **Gliese-OCR-7B-Post2.0-final**" ], "metadata": { "id": "zeCQura_Ri5N" } }, { "cell_type": "markdown", "source": [ "\n", "The [Gliese-OCR-7B-Post2.0-final](https://huggingface.co/prithivMLmods/Gliese-OCR-7B-Post2.0-final) model is a refined and optimized version of Gliese-OCR-7B-Post1.0, built upon the Qwen2.5-VL architecture. It represents the final iteration in the Gliese-OCR series, offering enhanced efficiency, precision, and visualization capabilities for document OCR, visual analysis, and information extraction.\n", "\n", "Fine-tuned with extended document visualization data and OCR-focused objectives, this model delivers superior accuracy across a wide range of document types, including scanned PDFs, handwritten pages, structured forms, and analytical reports." ], "metadata": { "id": "pTrk5nv-HAMV" } }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "oXHcxUMZGah0" }, "outputs": [], "source": [ "%%capture\n", "!pip install git+https://github.com/huggingface/accelerate.git \\\n", " git+https://github.com/huggingface/peft.git \\\n", " transformers-stream-generator huggingface_hub albumentations \\\n", " pyvips-binary qwen-vl-utils sentencepiece opencv-python docling-core \\\n", " python-docx torchvision safetensors matplotlib num2words \\\n", "\n", "!pip install transformers requests pymupdf hf_xet spaces pyvips pillow gradio \\\n", " einops torch fpdf timm av decord bitsandbytes\n", "#Hold tight, this will take around 1-2 minutes." ] }, { "cell_type": "markdown", "source": [ "### **Load Gliese-OCR-7B-Post2.0 with 4-bit quantization**" ], "metadata": { "id": "5vGwuV-4HaJv" } }, { "cell_type": "code", "source": [ "import os\n", "import sys\n", "import random\n", "import uuid\n", "import json\n", "import time\n", "from threading import Thread\n", "from typing import Iterable\n", "from huggingface_hub import snapshot_download\n", "\n", "import gradio as gr\n", "import spaces\n", "import torch\n", "import numpy as np\n", "from PIL import Image\n", "import cv2\n", "\n", "from transformers import (\n", " Qwen2_5_VLForConditionalGeneration,\n", " AutoProcessor,\n", " TextIteratorStreamer,\n", " BitsAndBytesConfig,\n", ")\n", "\n", "from transformers.image_utils import load_image\n", "from gradio.themes import Soft\n", "from gradio.themes.utils import colors, fonts, sizes\n", "\n", "# --- Theme and CSS Setup ---\n", "colors.steel_blue = colors.Color(\n", " name=\"steel_blue\",\n", " c50=\"#EBF3F8\",\n", " c100=\"#D3E5F0\",\n", " c200=\"#A8CCE1\",\n", " c300=\"#7DB3D2\",\n", " c400=\"#529AC3\",\n", " c500=\"#4682B4\",\n", " c600=\"#3E72A0\",\n", " c700=\"#36638C\",\n", " c800=\"#2E5378\",\n", " c900=\"#264364\",\n", " c950=\"#1E3450\",\n", ")\n", "\n", "class SteelBlueTheme(Soft):\n", " def __init__(\n", " self,\n", " *,\n", " primary_hue: colors.Color | str = colors.gray,\n", " secondary_hue: colors.Color | str = colors.steel_blue,\n", " neutral_hue: colors.Color | str = colors.slate,\n", " text_size: sizes.Size | str = sizes.text_lg,\n", " font: fonts.Font | str | Iterable[fonts.Font | str] = (\n", " fonts.GoogleFont(\"Outfit\"), \"Arial\", \"sans-serif\",\n", " ),\n", " font_mono: fonts.Font | str | Iterable[fonts.Font | str] = (\n", " fonts.GoogleFont(\"IBM Plex Mono\"), \"ui-monospace\", \"monospace\",\n", " ),\n", " ):\n", " super().__init__(\n", " primary_hue=primary_hue,\n", " secondary_hue=secondary_hue,\n", " neutral_hue=neutral_hue,\n", " text_size=text_size,\n", " font=font,\n", " font_mono=font_mono,\n", " )\n", " super().set(\n", " background_fill_primary=\"*primary_50\",\n", " background_fill_primary_dark=\"*primary_900\",\n", " body_background_fill=\"linear-gradient(135deg, *primary_200, *primary_100)\",\n", " body_background_fill_dark=\"linear-gradient(135deg, *primary_900, *primary_800)\",\n", " button_primary_text_color=\"white\",\n", " button_primary_text_color_hover=\"white\",\n", " button_primary_background_fill=\"linear-gradient(90deg, *secondary_500, *secondary_600)\",\n", " button_primary_background_fill_hover=\"linear-gradient(90deg, *secondary_600, *secondary_700)\",\n", " button_primary_background_fill_dark=\"linear-gradient(90deg, *secondary_600, *secondary_800)\",\n", " button_primary_background_fill_hover_dark=\"linear-gradient(90deg, *secondary_500, *secondary_500)\",\n", " button_secondary_text_color=\"black\",\n", " button_secondary_text_color_hover=\"white\",\n", " button_secondary_background_fill=\"linear-gradient(90deg, *primary_300, *primary_300)\",\n", " button_secondary_background_fill_hover=\"linear-gradient(90deg, *primary_400, *primary_400)\",\n", " button_secondary_background_fill_dark=\"linear-gradient(90deg, *primary_500, *primary_600)\",\n", " button_secondary_background_fill_hover_dark=\"linear-gradient(90deg, *primary_500, *primary_500)\",\n", " slider_color=\"*secondary_500\",\n", " slider_color_dark=\"*secondary_600\",\n", " block_title_text_weight=\"600\",\n", " block_border_width=\"3px\",\n", " block_shadow=\"*shadow_drop_lg\",\n", " button_primary_shadow=\"*shadow_drop_lg\",\n", " button_large_padding=\"11px\",\n", " color_accent_soft=\"*primary_100\",\n", " block_label_background_fill=\"*primary_200\",\n", " )\n", "\n", "steel_blue_theme = SteelBlueTheme()\n", "\n", "css = \"\"\"\n", "#main-title h1 {\n", " font-size: 2.3em !important;\n", "}\n", "#output-title h2 {\n", " font-size: 2.1em !important;\n", "}\n", "\"\"\"\n", "\n", "# --- Model Configuration ---\n", "MAX_MAX_NEW_TOKENS = 4096\n", "DEFAULT_MAX_NEW_TOKENS = 1024\n", "MAX_INPUT_TOKEN_LENGTH = int(os.getenv(\"MAX_INPUT_TOKEN_LENGTH\", \"4096\"))\n", "\n", "print(\"CUDA_VISIBLE_DEVICES=\", os.environ.get(\"CUDA_VISIBLE_DEVICES\"))\n", "print(\"torch.__version__ =\", torch.__version__)\n", "print(\"torch.version.cuda =\", torch.version.cuda)\n", "print(\"cuda available:\", torch.cuda.is_available())\n", "print(\"cuda device count:\", torch.cuda.device_count())\n", "if torch.cuda.is_available():\n", " print(\"current device:\", torch.cuda.current_device())\n", " print(\"device name:\", torch.cuda.get_device_name(torch.cuda.current_device()))\n", "\n", "# Define 4-bit quantization configuration\n", "# This config will load the model in 4-bit to save VRAM.\n", "quantization_config = BitsAndBytesConfig(\n", " load_in_4bit=True,\n", " bnb_4bit_compute_dtype=torch.float16,\n", " bnb_4bit_quant_type=\"nf4\",\n", " bnb_4bit_use_double_quant=True,\n", ")\n", "\n", "# Load Gliese-OCR-7B-Post2.0 with 4-bit quantization\n", "MODEL_ID_X = \"prithivMLmods/Gliese-OCR-7B-Post2.0-final\"\n", "print(f\"Loading {MODEL_ID_X}🤗. This will use 4-bit quantization to save VRAM.\")\n", "\n", "processor = AutoProcessor.from_pretrained(MODEL_ID_X, trust_remote_code=True)\n", "model = Qwen2_5_VLForConditionalGeneration.from_pretrained(\n", " MODEL_ID_X,\n", " trust_remote_code=True,\n", " quantization_config=quantization_config,\n", " device_map=\"auto\"\n", ").eval()\n", "\n", "\n", "@spaces.GPU\n", "def generate_image(text: str, image: Image.Image,\n", " max_new_tokens: int, temperature: float, top_p: float,\n", " top_k: int, repetition_penalty: float):\n", " \"\"\"\n", " Generates responses using the Nanonets model for image input.\n", " Yields raw text and Markdown-formatted text.\n", " \"\"\"\n", " if image is None:\n", " yield \"Please upload an image.\", \"Please upload an image.\"\n", " return\n", "\n", " messages = [{\n", " \"role\": \"user\",\n", " \"content\": [\n", " {\"type\": \"image\"},\n", " {\"type\": \"text\", \"text\": text},\n", " ]\n", " }]\n", " prompt_full = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n", "\n", " inputs = processor(\n", " text=[prompt_full],\n", " images=[image],\n", " return_tensors=\"pt\",\n", " padding=True).to(model.device)\n", "\n", " streamer = TextIteratorStreamer(processor, skip_prompt=True, skip_special_tokens=True)\n", " generation_kwargs = {\n", " **inputs,\n", " \"streamer\": streamer,\n", " \"max_new_tokens\": max_new_tokens,\n", " \"do_sample\": True,\n", " \"temperature\": temperature,\n", " \"top_p\": top_p,\n", " \"top_k\": top_k,\n", " \"repetition_penalty\": repetition_penalty,\n", " }\n", " thread = Thread(target=model.generate, kwargs=generation_kwargs)\n", " thread.start()\n", " buffer = \"\"\n", " for new_text in streamer:\n", " buffer += new_text\n", " buffer = buffer.replace(\"<|im_end|>\", \"\")\n", " time.sleep(0.01)\n", " yield buffer, buffer\n", "\n", "with gr.Blocks(css=css, theme=steel_blue_theme) as demo:\n", " gr.Markdown(\"# **Gliese-OCR-7B-Post2.0 (4-bit)**\", elem_id=\"main-title\")\n", " with gr.Row():\n", " with gr.Column(scale=2):\n", " image_query = gr.Textbox(label=\"Query Input\", placeholder=\"Enter your query here...\")\n", " image_upload = gr.Image(type=\"pil\", label=\"Upload Image\", height=290)\n", "\n", " image_submit = gr.Button(\"Submit\", variant=\"primary\")\n", "\n", " with gr.Accordion(\"Advanced options\", open=False):\n", " max_new_tokens = gr.Slider(label=\"Max new tokens\", minimum=1, maximum=MAX_MAX_NEW_TOKENS, step=1, value=DEFAULT_MAX_NEW_TOKENS)\n", " temperature = gr.Slider(label=\"Temperature\", minimum=0.1, maximum=4.0, step=0.1, value=0.7)\n", " top_p = gr.Slider(label=\"Top-p (nucleus sampling)\", minimum=0.05, maximum=1.0, step=0.05, value=0.9)\n", " top_k = gr.Slider(label=\"Top-k\", minimum=1, maximum=1000, step=1, value=50)\n", " repetition_penalty = gr.Slider(label=\"Repetition penalty\", minimum=1.0, maximum=2.0, step=0.05, value=1.1)\n", "\n", " with gr.Column(scale=3):\n", " gr.Markdown(\"## Output\", elem_id=\"output-title\")\n", " output = gr.Textbox(label=\"Raw Output Stream\", interactive=False, lines=11, show_copy_button=True)\n", " with gr.Accordion(\"(Result.md)\", open=False):\n", " markdown_output = gr.Markdown(label=\"(Result.Md)\")\n", "\n", " image_submit.click(\n", " fn=generate_image,\n", " inputs=[image_query, image_upload, max_new_tokens, temperature, top_p, top_k, repetition_penalty],\n", " outputs=[output, markdown_output]\n", " )\n", "\n", "if __name__ == \"__main__\":\n", " demo.queue(max_size=50).launch(show_error=True)" ], "metadata": { "id": "cbREUHUJGrzT" }, "execution_count": null, "outputs": [] } ] }