Upload 19 files

app.py

@@ -0,0 +1,53 @@
+import gradio as gr
+import pandas as pd
+from models import Reciept_Analyzer
+from utils import find_product
+import os
+model = Reciept_Analyzer()
+
+sample_images = []
+for img_file in os.listdir("samples/"):
+    sample_images.append(os.path.join("samples", img_file))
+
+def predict(image):
+    results = model.forward(image)
+    return results
+
+def handle_input(image, item_name):
+    df = predict(image)  # Phân tích hóa đơn và trả về dataframe
+    print(df)
+    result = find_product(item_name, df)
+    return result
+
+# Thiết kế giao diện với Gradio
+def create_interface():
+    with gr.Blocks() as app:
+        gr.Markdown("# Ứng dụng phân tích hóa đơn siêu thị")
+
+        with gr.Row():
+            # Cột bên trái
+            with gr.Column():
+                gr.Markdown("### Tải lên hóa đơn hoặc chọn ảnh mẫu")
+                image_input = gr.Image(label="Ảnh hóa đơn", type="filepath")
+
+                gr.Markdown("### Ảnh mẫu")
+                example = gr.Examples(
+                    inputs=image_input,
+                    examples=sample_images
+                )
+
+            # Cột bên phải
+            with gr.Column():
+                gr.Markdown("### Tìm kiếm thông tin item")
+                item_input = gr.Textbox(label="Tên item cần tìm")
+                output = gr.Textbox(label="Kết quả")
+
+                search_button = gr.Button("Tìm kiếm")
+                search_button.click(fn=handle_input, inputs=[image_input, item_input], outputs=output)
+
+    return app
+
+
+# Chạy ứng dụng
+app = create_interface()
+app.launch()

models.py

@@ -0,0 +1,164 @@
+import os
+import torch
+import numpy as np
+from ultralytics import YOLO
+from transformers import AutoProcessor
+from transformers import AutoModelForTokenClassification
+from utils import normalize_box, unnormalize_box, draw_output, create_df
+from PIL import Image, ImageDraw
+from vietocr.tool.predictor import Predictor
+from vietocr.tool.config import Cfg
+
+class Reciept_Analyzer:
+    def __init__(self,
+                 processor_pretrained='microsoft/layoutlmv3-base',
+                 layoutlm_pretrained=os.path.join(
+                      'models', 'checkpoint'),
+                 yolo_pretrained=os.path.join(
+                      'models', 'best.pt'),
+                 vietocr_pretrained=os.path.join(
+                      'models', 'vietocr', 'vgg_seq2seq.pth')
+                 ):
+        
+        print("Initializing processor")
+        self.processor = AutoProcessor.from_pretrained(
+            processor_pretrained, apply_ocr=False)
+        print("Finished initializing processor")
+
+        print("Initializing LayoutLM model")
+        self.lalm_model = AutoModelForTokenClassification.from_pretrained(
+            layoutlm_pretrained)
+        print("Finished initializing LayoutLM model")
+
+        if yolo_pretrained is not None:
+            print("Initializing YOLO model")
+            self.yolo_model = YOLO(yolo_pretrained)
+            print("Finished initializing YOLO model")
+
+        print("Initializing VietOCR model")
+        config = Cfg.load_config_from_name('vgg_seq2seq')
+        config['weights'] = vietocr_pretrained
+        config['cnn']['pretrained']= False
+        config['device'] = 'cuda:0' if torch.cuda.is_available() else 'cpu'
+        self.vietocr = Predictor(config)
+        print("Finished initializing VietOCR model")
+
+    def forward(self, img, output_path="output", is_save_cropped_img=False):
+        input_image = Image.open(img)
+
+        # detection with YOLOv8
+        bboxes = self.yolov8_det(input_image)
+        
+        # sort
+        sorted_bboxes = self.sort_bboxes(bboxes)
+        
+        # draw bbox
+        image_draw = input_image.copy()
+        self.draw_bbox(image_draw, sorted_bboxes, output_path)
+
+        # crop images
+        cropped_images, normalized_boxes = self.get_cropped_images(input_image, sorted_bboxes, is_save_cropped_img, output_path)
+        
+        # recognition with VietOCR
+        texts, mapping_bbox_texts = self.ocr(cropped_images, normalized_boxes)
+
+        # KIE with LayoutLMv3
+        pred_texts, pred_label, boxes  = self.kie(input_image, texts, normalized_boxes, mapping_bbox_texts, output_path)
+        
+        # create dataframe
+        return create_df(pred_texts, pred_label)
+        
+
+    def yolov8_det(self, img):
+        return self.yolo_model.predict(source=img, conf=0.3, iou=0.1)[0].boxes.xyxy.int()
+
+    def sort_bboxes(self, bboxes):
+        bbox_list = []
+        for box in bboxes:
+            tlx, tly, brx, bry = map(int, box)
+            bbox_list.append([tlx, tly, brx, bry])
+            bbox_list.sort(key=lambda x: (x[1], x[2]))
+        return bbox_list
+    
+    def draw_bbox(self, image_draw, bboxes, output_path):
+        # draw bbox
+        draw = ImageDraw.Draw(image_draw)
+        for box in bboxes:
+            draw.rectangle(box, outline='red', width=2)
+        image_draw.save(os.path.join(output_path, 'bbox.jpg'))
+        print(f"Exported image with bounding boxes to {os.path.join(output_path, 'bbox.jpg')}")
+
+    def get_cropped_images(self, input_image, bboxes, is_save_cropped=False, output_path="output"):
+        normalized_boxes = []
+        cropped_images = []
+
+        # OCR
+        if is_save_cropped:
+            cropped_folder = os.path.join(output_path, "cropped")
+            if not os.path.exists(cropped_folder):
+                os.makedirs(cropped_folder)
+            i = 0
+        for box in bboxes:
+            tlx, tly, brx, bry = map(int, box)
+            normalized_box = normalize_box(box, input_image.width, input_image.height)
+            normalized_boxes.append(normalized_box)
+            cropped_ = input_image.crop((tlx, tly, brx, bry))
+            if is_save_cropped:
+                cropped_.save(os.path.join(cropped_folder, f'cropped_{i}.jpg'))
+                i += 1
+            cropped_images.append(cropped_)
+
+        return cropped_images, normalized_boxes
+
+    def ocr(self, cropped_images, normalized_boxes):
+        mapping_bbox_texts = {}
+        texts = []
+        for img, normalized_box in zip(cropped_images, normalized_boxes):
+            result = self.vietocr.predict(img)
+            text = result.strip().replace('\n', ' ')
+            texts.append(text)
+            mapping_bbox_texts[','.join(map(str, normalized_box))] = text
+        
+        return texts, mapping_bbox_texts
+    
+    def kie(self, img, texts, boxes, mapping_bbox_texts, output_path):
+        encoding = self.processor(img, texts,
+                                  boxes=boxes,
+                                  return_offsets_mapping=True,
+                                  return_tensors='pt',
+                                  max_length=512,
+                                  padding='max_length')
+        offset_mapping = encoding.pop('offset_mapping')
+
+        with torch.no_grad():
+            outputs = self.lalm_model(**encoding)
+
+        id2label = self.lalm_model.config.id2label
+        logits = outputs.logits
+        token_boxes = encoding.bbox.squeeze().tolist()
+        offset_mapping = offset_mapping.squeeze().tolist()
+
+        predictions = logits.argmax(-1).squeeze().tolist()
+        is_subword = np.array(offset_mapping)[:, 0] != 0
+
+        true_predictions = []
+        true_boxes = []
+        true_texts = []
+        for idx in range(len(predictions)):
+            if not is_subword[idx] and token_boxes[idx] != [0, 0, 0, 0]:
+                true_predictions.append(id2label[predictions[idx]])
+                true_boxes.append(unnormalize_box(
+                    token_boxes[idx], img.width, img.height))
+                true_texts.append(mapping_bbox_texts.get(
+                    ','.join(map(str, token_boxes[idx])), ''))
+
+        if isinstance(output_path, str):
+            os.makedirs(output_path, exist_ok=True)
+            img_output = draw_output(
+                image=img,
+                true_predictions=true_predictions,
+                true_boxes=true_boxes
+            )
+            img_output.save(os.path.join(output_path, 'result.jpg'))
+            print(f"Exported result to {os.path.join(output_path, 'result.jpg')}")
+        return true_texts, true_predictions, true_boxes

models/best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a76aef7a2e063cfba3fc466a864ed10d80c460d3e87c5667cfeb5b4d67c4df0
+size 207624591

models/checkpoint/config.json

@@ -0,0 +1,118 @@
+{
+  "_name_or_path": "microsoft/layoutlmv3-base",
+  "architectures": [
+    "LayoutLMv3ForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "coordinate_size": 128,
+  "eos_token_id": 2,
+  "has_relative_attention_bias": true,
+  "has_spatial_attention_bias": true,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "ADDR",
+    "1": "TITLE",
+    "2": "PRODUCT_NAME_PREFIX",
+    "3": "PRODUCT_NAME",
+    "4": "AMOUNT_PREFIX",
+    "5": "AMOUNT",
+    "6": "UNIT_PREFIX",
+    "7": "UNIT",
+    "8": "UPRICE_PREFIX",
+    "9": "UPRICE",
+    "10": "SUB_TPRICE_PREFIX",
+    "11": "SUB_TPRICE",
+    "12": "TAMOUNT_PREFIX",
+    "13": "TAMOUNT",
+    "14": "TPRICE_PREFIX",
+    "15": "TPRICE",
+    "16": "RECEMONEY_PREFIX",
+    "17": "RECEMONEY",
+    "18": "OTHER",
+    "19": "BILLID_PREFIX",
+    "20": "BILLID",
+    "21": "DATETIME_PREFIX",
+    "22": "DATETIME",
+    "23": "CASHIER_PREFIX",
+    "24": "CASHIER",
+    "25": "SHOP_NAME",
+    "26": "PHONE_PREFIX",
+    "27": "FPRICE_PREFIX",
+    "28": "FPRICE",
+    "29": "REMAMONEY_PREFIX",
+    "30": "REMAMONEY",
+    "31": "PHONE",
+    "32": "TDISCOUNT_PREFIX",
+    "33": "TDISCOUNT",
+    "34": "ADDR_PREFIX",
+    "35": "UDISCOUNT_PREFIX",
+    "36": "UDISCOUNT"
+  },
+  "initializer_range": 0.02,
+  "input_size": 224,
+  "intermediate_size": 3072,
+  "label2id": {
+    "ADDR": 0,
+    "ADDR_PREFIX": 34,
+    "AMOUNT": 5,
+    "AMOUNT_PREFIX": 4,
+    "BILLID": 20,
+    "BILLID_PREFIX": 19,
+    "CASHIER": 24,
+    "CASHIER_PREFIX": 23,
+    "DATETIME": 22,
+    "DATETIME_PREFIX": 21,
+    "FPRICE": 28,
+    "FPRICE_PREFIX": 27,
+    "OTHER": 18,
+    "PHONE": 31,
+    "PHONE_PREFIX": 26,
+    "PRODUCT_NAME": 3,
+    "PRODUCT_NAME_PREFIX": 2,
+    "RECEMONEY": 17,
+    "RECEMONEY_PREFIX": 16,
+    "REMAMONEY": 30,
+    "REMAMONEY_PREFIX": 29,
+    "SHOP_NAME": 25,
+    "SUB_TPRICE": 11,
+    "SUB_TPRICE_PREFIX": 10,
+    "TAMOUNT": 13,
+    "TAMOUNT_PREFIX": 12,
+    "TDISCOUNT": 33,
+    "TDISCOUNT_PREFIX": 32,
+    "TITLE": 1,
+    "TPRICE": 15,
+    "TPRICE_PREFIX": 14,
+    "UDISCOUNT": 36,
+    "UDISCOUNT_PREFIX": 35,
+    "UNIT": 7,
+    "UNIT_PREFIX": 6,
+    "UPRICE": 9,
+    "UPRICE_PREFIX": 8
+  },
+  "layer_norm_eps": 1e-05,
+  "max_2d_position_embeddings": 1024,
+  "max_position_embeddings": 514,
+  "max_rel_2d_pos": 256,
+  "max_rel_pos": 128,
+  "model_type": "layoutlmv3",
+  "num_attention_heads": 12,
+  "num_channels": 3,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "patch_size": 16,
+  "rel_2d_pos_bins": 64,
+  "rel_pos_bins": 32,
+  "second_input_size": 112,
+  "shape_size": 128,
+  "text_embed": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.27.2",
+  "type_vocab_size": 1,
+  "visual_embed": true,
+  "vocab_size": 50265
+}

models/checkpoint/preprocessor_config.json

@@ -0,0 +1,27 @@
+{
+  "apply_ocr": false,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "feature_extractor_type": "LayoutLMv3FeatureExtractor",
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "LayoutLMv3ImageProcessor",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "ocr_lang": null,
+  "processor_class": "LayoutLMv3Processor",
+  "resample": 2,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 224,
+    "width": 224
+  },
+  "tesseract_config": ""
+}

models/checkpoint/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32d465ffd4cbb9b024db495cf54841f73cb717de6dba135fc8258fe9c253296f
+size 503863089

models/checkpoint/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd747db282a4c24e2cf6e4bc50b907e598e871654c02eb66d5fe56403e68a3d8
+size 14575

models/checkpoint/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d13d935c351b9a072808b2f35968a51ccc83f30be9b4047e7a9be1cf7fb0a97
+size 627

models/checkpoint/special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

models/checkpoint/tokenizer_config.json

@@ -0,0 +1,86 @@
+{
+  "add_prefix_space": true,
+  "apply_ocr": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token_box": [
+    0,
+    0,
+    0,
+    0
+  ],
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 512,
+  "only_label_first_subword": true,
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token_box": [
+    0,
+    0,
+    0,
+    0
+  ],
+  "pad_token_label": -100,
+  "processor_class": "LayoutLMv3Processor",
+  "sep_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token_box": [
+    0,
+    0,
+    0,
+    0
+  ],
+  "special_tokens_map_file": null,
+  "tokenizer_class": "LayoutLMv3Tokenizer",
+  "trim_offsets": true,
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

models/checkpoint/trainer_state.json

@@ -0,0 +1,676 @@
+{
+  "best_metric": 0.9322203066872333,
+  "best_model_checkpoint": "/content/drive/MyDrive/checkpoint_layoutlm2/checkpoint-3500",
+  "epoch": 26.31578947368421,
+  "global_step": 5000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.53,
+      "eval_accuracy": 0.5655867006431259,
+      "eval_f1": 0.49600000000000005,
+      "eval_loss": 1.9622313976287842,
+      "eval_precision": 0.6104310803618946,
+      "eval_recall": 0.41769847050254916,
+      "eval_runtime": 31.7963,
+      "eval_samples_per_second": 5.976,
+      "eval_steps_per_second": 1.51,
+      "step": 100
+    },
+    {
+      "epoch": 1.05,
+      "eval_accuracy": 0.7380172309185778,
+      "eval_f1": 0.6739791685869665,
+      "eval_loss": 1.2630114555358887,
+      "eval_precision": 0.682471532574202,
+      "eval_recall": 0.6656955571740714,
+      "eval_runtime": 32.3716,
+      "eval_samples_per_second": 5.869,
+      "eval_steps_per_second": 1.483,
+      "step": 200
+    },
+    {
+      "epoch": 1.58,
+      "eval_accuracy": 0.7893459531610242,
+      "eval_f1": 0.7480542986425339,
+      "eval_loss": 0.9176977872848511,
+      "eval_precision": 0.7436128103634401,
+      "eval_recall": 0.7525491624180627,
+      "eval_runtime": 32.1494,
+      "eval_samples_per_second": 5.91,
+      "eval_steps_per_second": 1.493,
+      "step": 300
+    },
+    {
+      "epoch": 2.11,
+      "eval_accuracy": 0.8295109816769809,
+      "eval_f1": 0.8007957319830002,
+      "eval_loss": 0.7080672979354858,
+      "eval_precision": 0.7954014729656906,
+      "eval_recall": 0.8062636562272396,
+      "eval_runtime": 32.0545,
+      "eval_samples_per_second": 5.927,
+      "eval_steps_per_second": 1.497,
+      "step": 400
+    },
+    {
+      "epoch": 2.63,
+      "learning_rate": 9.666666666666667e-06,
+      "loss": 1.4294,
+      "step": 500
+    },
+    {
+      "epoch": 2.63,
+      "eval_accuracy": 0.8411600533915787,
+      "eval_f1": 0.8213092550790069,
+      "eval_loss": 0.6136476397514343,
+      "eval_precision": 0.814615797958087,
+      "eval_recall": 0.8281136198106337,
+      "eval_runtime": 32.0473,
+      "eval_samples_per_second": 5.929,
+      "eval_steps_per_second": 1.498,
+      "step": 500
+    },
+    {
+      "epoch": 3.16,
+      "eval_accuracy": 0.8741657565829389,
+      "eval_f1": 0.8656448866182666,
+      "eval_loss": 0.5390823483467102,
+      "eval_precision": 0.8524271844660194,
+      "eval_recall": 0.879278951201748,
+      "eval_runtime": 33.0229,
+      "eval_samples_per_second": 5.754,
+      "eval_steps_per_second": 1.454,
+      "step": 600
+    },
+    {
+      "epoch": 3.68,
+      "eval_accuracy": 0.8869069287707803,
+      "eval_f1": 0.878114599262391,
+      "eval_loss": 0.47398096323013306,
+      "eval_precision": 0.8677333333333334,
+      "eval_recall": 0.8887472687545521,
+      "eval_runtime": 32.5116,
+      "eval_samples_per_second": 5.844,
+      "eval_steps_per_second": 1.476,
+      "step": 700
+    },
+    {
+      "epoch": 4.21,
+      "eval_accuracy": 0.8989200339764591,
+      "eval_f1": 0.893278463648834,
+      "eval_loss": 0.4286675751209259,
+      "eval_precision": 0.8972992834833732,
+      "eval_recall": 0.8892935178441369,
+      "eval_runtime": 32.15,
+      "eval_samples_per_second": 5.91,
+      "eval_steps_per_second": 1.493,
+      "step": 800
+    },
+    {
+      "epoch": 4.74,
+      "eval_accuracy": 0.9040165028515956,
+      "eval_f1": 0.8944705457173835,
+      "eval_loss": 0.40272650122642517,
+      "eval_precision": 0.8877331420373027,
+      "eval_recall": 0.9013109978150037,
+      "eval_runtime": 32.2339,
+      "eval_samples_per_second": 5.894,
+      "eval_steps_per_second": 1.489,
+      "step": 900
+    },
+    {
+      "epoch": 5.26,
+      "learning_rate": 9.333333333333334e-06,
+      "loss": 0.455,
+      "step": 1000
+    },
+    {
+      "epoch": 5.26,
+      "eval_accuracy": 0.9131173401286252,
+      "eval_f1": 0.9035698147311342,
+      "eval_loss": 0.369215190410614,
+      "eval_precision": 0.8970034092948143,
+      "eval_recall": 0.9102330662782229,
+      "eval_runtime": 32.0091,
+      "eval_samples_per_second": 5.936,
+      "eval_steps_per_second": 1.5,
+      "step": 1000
+    },
+    {
+      "epoch": 5.79,
+      "eval_accuracy": 0.9172430530275452,
+      "eval_f1": 0.912167606768735,
+      "eval_loss": 0.36320310831069946,
+      "eval_precision": 0.8973049145675532,
+      "eval_recall": 0.9275309541150765,
+      "eval_runtime": 31.8391,
+      "eval_samples_per_second": 5.968,
+      "eval_steps_per_second": 1.508,
+      "step": 1100
+    },
+    {
+      "epoch": 6.32,
+      "eval_accuracy": 0.9166363305424099,
+      "eval_f1": 0.9132502018480307,
+      "eval_loss": 0.34437164664268494,
+      "eval_precision": 0.9000884173297966,
+      "eval_recall": 0.92680262199563,
+      "eval_runtime": 31.688,
+      "eval_samples_per_second": 5.996,
+      "eval_steps_per_second": 1.515,
+      "step": 1200
+    },
+    {
+      "epoch": 6.84,
+      "eval_accuracy": 0.924766411843223,
+      "eval_f1": 0.9212297616881936,
+      "eval_loss": 0.327307790517807,
+      "eval_precision": 0.920392584514722,
+      "eval_recall": 0.922068463219228,
+      "eval_runtime": 31.8134,
+      "eval_samples_per_second": 5.972,
+      "eval_steps_per_second": 1.509,
+      "step": 1300
+    },
+    {
+      "epoch": 7.37,
+      "eval_accuracy": 0.9242810338551146,
+      "eval_f1": 0.9209428830462375,
+      "eval_loss": 0.32373183965682983,
+      "eval_precision": 0.9171180931744312,
+      "eval_recall": 0.9247997086671522,
+      "eval_runtime": 32.69,
+      "eval_samples_per_second": 5.812,
+      "eval_steps_per_second": 1.468,
+      "step": 1400
+    },
+    {
+      "epoch": 7.89,
+      "learning_rate": 9e-06,
+      "loss": 0.2736,
+      "step": 1500
+    },
+    {
+      "epoch": 7.89,
+      "eval_accuracy": 0.9282854022570076,
+      "eval_f1": 0.9256823709575714,
+      "eval_loss": 0.31131741404533386,
+      "eval_precision": 0.916027812444286,
+      "eval_recall": 0.9355426074289876,
+      "eval_runtime": 32.0518,
+      "eval_samples_per_second": 5.928,
+      "eval_steps_per_second": 1.498,
+      "step": 1500
+    },
+    {
+      "epoch": 8.42,
+      "eval_accuracy": 0.9254944788253853,
+      "eval_f1": 0.9237975139614484,
+      "eval_loss": 0.3146894872188568,
+      "eval_precision": 0.9140819964349376,
+      "eval_recall": 0.9337217771303714,
+      "eval_runtime": 32.0386,
+      "eval_samples_per_second": 5.93,
+      "eval_steps_per_second": 1.498,
+      "step": 1600
+    },
+    {
+      "epoch": 8.95,
+      "eval_accuracy": 0.9253731343283582,
+      "eval_f1": 0.9235605856462767,
+      "eval_loss": 0.3176872134208679,
+      "eval_precision": 0.9113632334692431,
+      "eval_recall": 0.9360888565185724,
+      "eval_runtime": 32.1667,
+      "eval_samples_per_second": 5.907,
+      "eval_steps_per_second": 1.492,
+      "step": 1700
+    },
+    {
+      "epoch": 9.47,
+      "eval_accuracy": 0.9262225458075476,
+      "eval_f1": 0.9248316120341267,
+      "eval_loss": 0.3194420337677002,
+      "eval_precision": 0.9124579124579124,
+      "eval_recall": 0.9375455207574654,
+      "eval_runtime": 32.186,
+      "eval_samples_per_second": 5.903,
+      "eval_steps_per_second": 1.491,
+      "step": 1800
+    },
+    {
+      "epoch": 10.0,
+      "eval_accuracy": 0.9293775027302512,
+      "eval_f1": 0.9273722955381991,
+      "eval_loss": 0.30748510360717773,
+      "eval_precision": 0.9146449442181689,
+      "eval_recall": 0.9404588492352512,
+      "eval_runtime": 31.8871,
+      "eval_samples_per_second": 5.959,
+      "eval_steps_per_second": 1.505,
+      "step": 1900
+    },
+    {
+      "epoch": 10.53,
+      "learning_rate": 8.666666666666668e-06,
+      "loss": 0.2045,
+      "step": 2000
+    },
+    {
+      "epoch": 10.53,
+      "eval_accuracy": 0.9305909477005218,
+      "eval_f1": 0.928266981046522,
+      "eval_loss": 0.3004026710987091,
+      "eval_precision": 0.9246612466124661,
+      "eval_recall": 0.9319009468317553,
+      "eval_runtime": 32.044,
+      "eval_samples_per_second": 5.929,
+      "eval_steps_per_second": 1.498,
+      "step": 2000
+    },
+    {
+      "epoch": 11.05,
+      "eval_accuracy": 0.9284067467540347,
+      "eval_f1": 0.9264534623309147,
+      "eval_loss": 0.3036515414714813,
+      "eval_precision": 0.9118321283724211,
+      "eval_recall": 0.9415513474144209,
+      "eval_runtime": 31.8687,
+      "eval_samples_per_second": 5.962,
+      "eval_steps_per_second": 1.506,
+      "step": 2100
+    },
+    {
+      "epoch": 11.58,
+      "eval_accuracy": 0.9307122921975488,
+      "eval_f1": 0.9279507603186097,
+      "eval_loss": 0.30012640357017517,
+      "eval_precision": 0.9226061915046796,
+      "eval_recall": 0.9333576110706482,
+      "eval_runtime": 31.9364,
+      "eval_samples_per_second": 5.949,
+      "eval_steps_per_second": 1.503,
+      "step": 2200
+    },
+    {
+      "epoch": 12.11,
+      "eval_accuracy": 0.9318043926707924,
+      "eval_f1": 0.9303956834532375,
+      "eval_loss": 0.3051491975784302,
+      "eval_precision": 0.9191542288557214,
+      "eval_recall": 0.9419155134741443,
+      "eval_runtime": 31.9279,
+      "eval_samples_per_second": 5.951,
+      "eval_steps_per_second": 1.503,
+      "step": 2300
+    },
+    {
+      "epoch": 12.63,
+      "eval_accuracy": 0.9299842252153865,
+      "eval_f1": 0.9288619723402333,
+      "eval_loss": 0.31714528799057007,
+      "eval_precision": 0.9222760725183988,
+      "eval_recall": 0.9355426074289876,
+      "eval_runtime": 32.3661,
+      "eval_samples_per_second": 5.87,
+      "eval_steps_per_second": 1.483,
+      "step": 2400
+    },
+    {
+      "epoch": 13.16,
+      "learning_rate": 8.333333333333334e-06,
+      "loss": 0.1678,
+      "step": 2500
+    },
+    {
+      "epoch": 13.16,
+      "eval_accuracy": 0.9279213687659265,
+      "eval_f1": 0.9271131805157592,
+      "eval_loss": 0.321869432926178,
+      "eval_precision": 0.9120859760394644,
+      "eval_recall": 0.9426438455935907,
+      "eval_runtime": 32.586,
+      "eval_samples_per_second": 5.831,
+      "eval_steps_per_second": 1.473,
+      "step": 2500
+    },
+    {
+      "epoch": 13.68,
+      "eval_accuracy": 0.9273146462807912,
+      "eval_f1": 0.9248711689720641,
+      "eval_loss": 0.3112223446369171,
+      "eval_precision": 0.9184772849703717,
+      "eval_recall": 0.9313546977421704,
+      "eval_runtime": 32.4269,
+      "eval_samples_per_second": 5.859,
+      "eval_steps_per_second": 1.48,
+      "step": 2600
+    },
+    {
+      "epoch": 14.21,
+      "eval_accuracy": 0.92901346923917,
+      "eval_f1": 0.9273105820584773,
+      "eval_loss": 0.3135831654071808,
+      "eval_precision": 0.922052205220522,
+      "eval_recall": 0.9326292789512017,
+      "eval_runtime": 31.2652,
+      "eval_samples_per_second": 6.077,
+      "eval_steps_per_second": 1.535,
+      "step": 2700
+    },
+    {
+      "epoch": 14.74,
+      "eval_accuracy": 0.9296201917243053,
+      "eval_f1": 0.9286809124515373,
+      "eval_loss": 0.3137897551059723,
+      "eval_precision": 0.9198071084122165,
+      "eval_recall": 0.937727603787327,
+      "eval_runtime": 31.2737,
+      "eval_samples_per_second": 6.075,
+      "eval_steps_per_second": 1.535,
+      "step": 2800
+    },
+    {
+      "epoch": 15.26,
+      "eval_accuracy": 0.93313918213809,
+      "eval_f1": 0.9307971014492753,
+      "eval_loss": 0.3145933747291565,
+      "eval_precision": 0.9260994953136266,
+      "eval_recall": 0.9355426074289876,
+      "eval_runtime": 32.3029,
+      "eval_samples_per_second": 5.882,
+      "eval_steps_per_second": 1.486,
+      "step": 2900
+    },
+    {
+      "epoch": 15.79,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 0.1404,
+      "step": 3000
+    },
+    {
+      "epoch": 15.79,
+      "eval_accuracy": 0.9296201917243053,
+      "eval_f1": 0.9290786516853932,
+      "eval_loss": 0.31863638758659363,
+      "eval_precision": 0.917450736729984,
+      "eval_recall": 0.9410050983248361,
+      "eval_runtime": 31.4449,
+      "eval_samples_per_second": 6.042,
+      "eval_steps_per_second": 1.526,
+      "step": 3000
+    },
+    {
+      "epoch": 16.32,
+      "eval_accuracy": 0.9314403591797112,
+      "eval_f1": 0.929384143037746,
+      "eval_loss": 0.3153584599494934,
+      "eval_precision": 0.9218917950555356,
+      "eval_recall": 0.9369992716678806,
+      "eval_runtime": 31.167,
+      "eval_samples_per_second": 6.096,
+      "eval_steps_per_second": 1.54,
+      "step": 3100
+    },
+    {
+      "epoch": 16.84,
+      "eval_accuracy": 0.9273146462807912,
+      "eval_f1": 0.926561797752809,
+      "eval_loss": 0.34182220697402954,
+      "eval_precision": 0.9149653825670158,
+      "eval_recall": 0.9384559359067735,
+      "eval_runtime": 31.6096,
+      "eval_samples_per_second": 6.011,
+      "eval_steps_per_second": 1.519,
+      "step": 3200
+    },
+    {
+      "epoch": 17.37,
+      "eval_accuracy": 0.925009100837277,
+      "eval_f1": 0.9233380986418871,
+      "eval_loss": 0.35532286763191223,
+      "eval_precision": 0.9064912280701755,
+      "eval_recall": 0.9408230152949745,
+      "eval_runtime": 32.7011,
+      "eval_samples_per_second": 5.81,
+      "eval_steps_per_second": 1.468,
+      "step": 3300
+    },
+    {
+      "epoch": 17.89,
+      "eval_accuracy": 0.9294988472272783,
+      "eval_f1": 0.9289882395188078,
+      "eval_loss": 0.339712530374527,
+      "eval_precision": 0.9162387108199044,
+      "eval_recall": 0.9420975965040058,
+      "eval_runtime": 31.1932,
+      "eval_samples_per_second": 6.091,
+      "eval_steps_per_second": 1.539,
+      "step": 3400
+    },
+    {
+      "epoch": 18.42,
+      "learning_rate": 7.666666666666667e-06,
+      "loss": 0.1174,
+      "step": 3500
+    },
+    {
+      "epoch": 18.42,
+      "eval_accuracy": 0.9322897706589006,
+      "eval_f1": 0.9322203066872333,
+      "eval_loss": 0.32590094208717346,
+      "eval_precision": 0.929101103273648,
+      "eval_recall": 0.935360524399126,
+      "eval_runtime": 31.3047,
+      "eval_samples_per_second": 6.069,
+      "eval_steps_per_second": 1.533,
+      "step": 3500
+    },
+    {
+      "epoch": 18.95,
+      "eval_accuracy": 0.9299842252153865,
+      "eval_f1": 0.9293038316243929,
+      "eval_loss": 0.330241322517395,
+      "eval_precision": 0.9182367579097049,
+      "eval_recall": 0.9406409322651129,
+      "eval_runtime": 31.3595,
+      "eval_samples_per_second": 6.059,
+      "eval_steps_per_second": 1.531,
+      "step": 3600
+    },
+    {
+      "epoch": 19.47,
+      "eval_accuracy": 0.9303482587064676,
+      "eval_f1": 0.9284752759580005,
+      "eval_loss": 0.33720284700393677,
+      "eval_precision": 0.9154132012033268,
+      "eval_recall": 0.9419155134741443,
+      "eval_runtime": 32.5529,
+      "eval_samples_per_second": 5.837,
+      "eval_steps_per_second": 1.475,
+      "step": 3700
+    },
+    {
+      "epoch": 20.0,
+      "eval_accuracy": 0.9305909477005218,
+      "eval_f1": 0.9294531038832327,
+      "eval_loss": 0.33688944578170776,
+      "eval_precision": 0.9199215266630997,
+      "eval_recall": 0.9391842680262199,
+      "eval_runtime": 31.1686,
+      "eval_samples_per_second": 6.096,
+      "eval_steps_per_second": 1.54,
+      "step": 3800
+    },
+    {
+      "epoch": 20.53,
+      "eval_accuracy": 0.9302269142094406,
+      "eval_f1": 0.9289775291038715,
+      "eval_loss": 0.33882710337638855,
+      "eval_precision": 0.9209160851672928,
+      "eval_recall": 0.9371813546977422,
+      "eval_runtime": 31.4744,
+      "eval_samples_per_second": 6.037,
+      "eval_steps_per_second": 1.525,
+      "step": 3900
+    },
+    {
+      "epoch": 21.05,
+      "learning_rate": 7.333333333333333e-06,
+      "loss": 0.0978,
+      "step": 4000
+    },
+    {
+      "epoch": 21.05,
+      "eval_accuracy": 0.9297415362213324,
+      "eval_f1": 0.9305718144979739,
+      "eval_loss": 0.34309884905815125,
+      "eval_precision": 0.9205415998574737,
+      "eval_recall": 0.9408230152949745,
+      "eval_runtime": 31.0896,
+      "eval_samples_per_second": 6.111,
+      "eval_steps_per_second": 1.544,
+      "step": 4000
+    },
+    {
+      "epoch": 21.58,
+      "eval_accuracy": 0.9313190146826842,
+      "eval_f1": 0.9312798342790237,
+      "eval_loss": 0.3530495762825012,
+      "eval_precision": 0.9214043842452326,
+      "eval_recall": 0.9413692643845594,
+      "eval_runtime": 32.1875,
+      "eval_samples_per_second": 5.903,
+      "eval_steps_per_second": 1.491,
+      "step": 4100
+    },
+    {
+      "epoch": 22.11,
+      "eval_accuracy": 0.9325324596529547,
+      "eval_f1": 0.932169845268082,
+      "eval_loss": 0.34749624133110046,
+      "eval_precision": 0.9212304409672831,
+      "eval_recall": 0.9433721777130372,
+      "eval_runtime": 31.1794,
+      "eval_samples_per_second": 6.094,
+      "eval_steps_per_second": 1.539,
+      "step": 4200
+    },
+    {
+      "epoch": 22.63,
+      "eval_accuracy": 0.9302269142094406,
+      "eval_f1": 0.9302577040908272,
+      "eval_loss": 0.35649344325065613,
+      "eval_precision": 0.9207991437745273,
+      "eval_recall": 0.9399126001456665,
+      "eval_runtime": 32.3866,
+      "eval_samples_per_second": 5.867,
+      "eval_steps_per_second": 1.482,
+      "step": 4300
+    },
+    {
+      "epoch": 23.16,
+      "eval_accuracy": 0.9326538041499818,
+      "eval_f1": 0.9312274368231047,
+      "eval_loss": 0.35333752632141113,
+      "eval_precision": 0.9232283464566929,
+      "eval_recall": 0.9393663510560816,
+      "eval_runtime": 31.2877,
+      "eval_samples_per_second": 6.073,
+      "eval_steps_per_second": 1.534,
+      "step": 4400
+    },
+    {
+      "epoch": 23.68,
+      "learning_rate": 7e-06,
+      "loss": 0.0822,
+      "step": 4500
+    },
+    {
+      "epoch": 23.68,
+      "eval_accuracy": 0.9333818711321441,
+      "eval_f1": 0.932110421724665,
+      "eval_loss": 0.3559868335723877,
+      "eval_precision": 0.9207674542547521,
+      "eval_recall": 0.9437363437727604,
+      "eval_runtime": 31.1716,
+      "eval_samples_per_second": 6.095,
+      "eval_steps_per_second": 1.54,
+      "step": 4500
+    },
+    {
+      "epoch": 24.21,
+      "eval_accuracy": 0.9337459046232254,
+      "eval_f1": 0.932995750067818,
+      "eval_loss": 0.3473477065563202,
+      "eval_precision": 0.9267109753906951,
+      "eval_recall": 0.9393663510560816,
+      "eval_runtime": 31.2203,
+      "eval_samples_per_second": 6.086,
+      "eval_steps_per_second": 1.537,
+      "step": 4600
+    },
+    {
+      "epoch": 24.74,
+      "eval_accuracy": 0.9287707802451158,
+      "eval_f1": 0.9275258194881005,
+      "eval_loss": 0.3723868131637573,
+      "eval_precision": 0.9151160730108099,
+      "eval_recall": 0.9402767662053897,
+      "eval_runtime": 32.4085,
+      "eval_samples_per_second": 5.863,
+      "eval_steps_per_second": 1.481,
+      "step": 4700
+    },
+    {
+      "epoch": 25.26,
+      "eval_accuracy": 0.9327751486470088,
+      "eval_f1": 0.9308232750855701,
+      "eval_loss": 0.3668186068534851,
+      "eval_precision": 0.9210338680926916,
+      "eval_recall": 0.9408230152949745,
+      "eval_runtime": 31.1822,
+      "eval_samples_per_second": 6.093,
+      "eval_steps_per_second": 1.539,
+      "step": 4800
+    },
+    {
+      "epoch": 25.79,
+      "eval_accuracy": 0.9342312826113336,
+      "eval_f1": 0.932900432900433,
+      "eval_loss": 0.35110706090927124,
+      "eval_precision": 0.9242315939957112,
+      "eval_recall": 0.9417334304442826,
+      "eval_runtime": 31.632,
+      "eval_samples_per_second": 6.007,
+      "eval_steps_per_second": 1.517,
+      "step": 4900
+    },
+    {
+      "epoch": 26.32,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 0.0685,
+      "step": 5000
+    },
+    {
+      "epoch": 26.32,
+      "eval_accuracy": 0.9322897706589006,
+      "eval_f1": 0.9301610276822869,
+      "eval_loss": 0.349013090133667,
+      "eval_precision": 0.9243078029485796,
+      "eval_recall": 0.9360888565185724,
+      "eval_runtime": 31.5377,
+      "eval_samples_per_second": 6.025,
+      "eval_steps_per_second": 1.522,
+      "step": 5000
+    }
+  ],
+  "max_steps": 15000,
+  "num_train_epochs": 79,
+  "total_flos": 5296004751249408.0,
+  "trial_name": null,
+  "trial_params": null
+}

models/checkpoint/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbf7fbdb5e11561aca8ea9af246554c6be453feb7f1de7fb0526ab1162afd7e0
+size 3643

models/vietocr/vgg_seq2seq.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0921503a41375a0584268e23ef3d414ea478a8fe8777865c7745d38f2d0bc5db
+size 89575371

models/vietocr/vgg_transformer.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:870475e8e72564e6732d896b79b3369faefa1440c8fa0ea68d11b2eefadd7721
+size 26877952

requirements.txt

@@ -0,0 +1,10 @@
+#paddleocr==2.9.1
+#paddlepaddle==2.6.2
+#torch==2.5.1
+#torchvision==0.20.1
+vietocr==0.3.13
+ultralytics==8.3.65
+huggingface-hub==0.27.1
+datasets==2.10.0
+seqeval==1.2.2
+transformers==4.48.1

utils.py

@@ -0,0 +1,140 @@
+import numpy as np
+from datasets import load_metric
+from PIL import ImageDraw, ImageFont
+import pandas as pd
+
+
+metric = load_metric("seqeval")
+
+
+def unnormalize_box(bbox, width, height):
+    return [
+        width * (bbox[0] / 1000),
+        height * (bbox[1] / 1000),
+        width * (bbox[2] / 1000),
+        height * (bbox[3] / 1000)
+    ]
+
+
+def normalize_box(bbox, width, height):
+    return [
+        int((bbox[0] / width) * 1000),
+        int((bbox[1] / height) * 1000),
+        int((bbox[2] / width) * 1000),
+        int((bbox[3] / height) * 1000)
+    ]
+
+
+def draw_output(image, true_predictions, true_boxes):
+    def iob_to_label(label):
+        label = label
+        if not label:
+            return 'other'
+        return label
+
+    # width, height = image.size
+
+    # predictions = logits.argmax(-1).squeeze().tolist()
+    # is_subword = np.array(offset_mapping)[:,0] != 0
+    # true_predictions = [id2label[pred] for idx, pred in enumerate(predictions) if not is_subword[idx]]
+    # true_boxes = [unnormalize_box(box, width, height) for idx, box in enumerate(token_boxes) if not is_subword[idx]]
+
+    # draw
+    draw = ImageDraw.Draw(image)
+    font = ImageFont.load_default()
+
+    for prediction, box in zip(true_predictions, true_boxes):
+        predicted_label = iob_to_label(prediction).lower()
+        draw.rectangle(box, outline='red')
+        draw.text((box[0] + 10, box[1] - 10),
+                  text=predicted_label, fill='red', font=font)
+
+    return image
+
+
+def create_df(true_texts,
+              true_predictions,
+              chosen_labels=['SHOP_NAME', 'ADDR', 'TITLE', 'PHONE',
+                             'PRODUCT_NAME', 'AMOUNT', 'UNIT', 'UPRICE', 'SUB_TPRICE', 'UDISCOUNT',
+                             'TAMOUNT', 'TPRICE', 'FPRICE', 'TDISCOUNT',
+                             'RECEMONEY', 'REMAMONEY',
+                             'BILLID', 'DATETIME', 'CASHIER']
+              ):
+
+    data = {'text': [], 'class_label': [], 'product_id': []}
+    product_id = -1
+    for text, prediction in zip(true_texts, true_predictions):
+        if prediction not in chosen_labels:
+            continue
+
+        if prediction == 'PRODUCT_NAME':
+            product_id += 1
+            
+
+        if prediction in ['AMOUNT', 'UNIT', 'UDISCOUNT', 'UPRICE', 'SUB_TPRICE',
+                          'UDISCOUNT', 'TAMOUNT', 'TPRICE', 'FPRICE', 'TDISCOUNT',
+                          'RECEMONEY', 'REMAMONEY']:
+            text = reformat(text)
+
+
+        if prediction in ['AMOUNT', 'SUB_TPRICE', 'UPRICE', 'PRODUCT_NAME']:
+            data['product_id'].append(product_id)
+        else:
+            data['product_id'].append('')
+
+
+        data['class_label'].append(prediction)
+        data['text'].append(text)
+
+
+    df = pd.DataFrame(data)
+
+    return df
+
+
+def reformat(text: str):
+    try:
+        text = text.replace('.', '').replace(',', '').replace(':', '').replace('/', '').replace('|', '').replace(
+            '\\', '').replace(')', '').replace('(', '').replace('-', '').replace(';', '').replace('_', '')
+        return int(text)
+    except:
+        return text
+
+def find_product(product_name, df):
+    product_name = product_name.lower()
+    product_df = df[df['class_label'] == 'PRODUCT_NAME']
+    mask = product_df['text'].str.lower().str.contains(product_name, case=False, na=False)
+    if mask.any():
+        product_id = product_df.loc[mask, 'product_id'].iloc[0]
+        product_info = df[df['product_id'] == product_id]
+        
+        prod_name = product_info.loc[product_info['class_label'] == 'PRODUCT_NAME', 'text'].iloc[0]
+            
+        try:
+            amount = product_info.loc[product_info['class_label'] == 'AMOUNT', 'text'].iloc[0]
+        except:
+            print("Error: cannot find amount")
+            amount = ''
+            
+        try:
+            uprice = product_info.loc[product_info['class_label'] == 'UPRICE', 'text'].iloc[0]
+        except:
+            print("Error: cannot find unit price")
+            uprice = ''
+            
+        try:
+            sub_tprice = product_info.loc[product_info['class_label'] == 'SUB_TPRICE', 'text'].iloc[0]
+        except:
+            print("Error: cannot find sub total price")
+            sub_tprice = ''
+            
+        #print("Sản phẩm: ", product_info.loc[product_info['class_label'] == 'PRODUCT_NAME', 'text'].iloc[0])
+        #print("Số lượng: ", product_info.loc[product_info['class_label'] == 'AMOUNT', 'text'].iloc[0])
+        #print("Đơn giá: ", product_info.loc[product_info['class_label'] == 'UPRICE', 'text'].iloc[0])
+        #print("Thành tiền: ", product_info.loc[product_info['class_label'] == 'SUB_TPRICE', 'text'].iloc[0])
+        return f"Sản phẩm: {prod_name}\n Số lượng: {amount}\n Đơn giá: {uprice}\n Thành tiền: {sub_tprice}"
+    else:
+        #print("Không tìm thấy item nào phù hợp.")
+        return "Không tìm thấy item nào phù hợp."
+    #return result = product_df['text'].str.contains(product_name, case=False, na=False).any()
+    #return product_df[product_df['text'].str.contains(product_name, case=False, na=False)]