SERVER_OCR.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "SERVER_OCR (2).ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "upb-kDwUkOVk",
        "colab_type": "text"
      },
      "source": [
        "Flask pour rest API et jsonpickle pour code la reponse avant de l'envoyer à l'application"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Gg3LBlPxqWA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install flask-ngrok\n",
        "!pip install jsonpickle"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JOtgchkAkWEi",
        "colab_type": "text"
      },
      "source": [
        "dans le cas des pancarte on utilise un detecteur de text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6K9PobplyO-t",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install psenet-text-detector"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "766KhG_QkmBf",
        "colab_type": "text"
      },
      "source": [
        "importer les packages nécessaire"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sMr8QIfIxrLj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import cv2\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import imutils\n",
        "import math\n",
        "from skimage.filters import threshold_sauvola, threshold_local\n",
        "from sklearn.externals import joblib \n",
        "from sklearn.preprocessing import StandardScaler\n",
        "import imutils\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from flask_ngrok import run_with_ngrok\n",
        "from flask import Flask, request, Response\n",
        "import jsonpickle"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kFihGb3YkqeZ",
        "colab_type": "text"
      },
      "source": [
        "connexion au Drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2hCr5t4HrNGQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive/')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7zRyfe9jkxwp",
        "colab_type": "text"
      },
      "source": [
        "pour les models on utilise CNN pour les doc imprimées et SVM pour les pancarte"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UEsGCnEkyTkA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "clf=tf.keras.models.load_model('/content/drive/My Drive/PFE licence_Groupe3/CNN3.h5') # doc imprimés\n",
        "clf2=joblib.load( '/content/drive/My Drive/Models/svm(norm).pkl')# pancartes"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MKreBfztlIwD",
        "colab_type": "text"
      },
      "source": [
        "importer les packages nécessaire"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RFtwQCCC3XIu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "outputId": "da506f8a-e78c-455a-b092-d8c8e40bd8f7"
      },
      "source": [
        "import psenet_text_detector as psenet\n",
        "from flask import Flask, request, Response\n",
        "from flask_ngrok import run_with_ngrok\n",
        "import io\n",
        "import base64 \n",
        "import numpy as np\n",
        "from PIL import Image\n",
        "import jsonpickle\n",
        "import numpy as np\n",
        "import cv2\n",
        "import matplotlib.pyplot as plt\n",
        "plt.rcParams['figure.figsize'] = [25,12]\n",
        "from skimage.filters import threshold_sauvola\n",
        "from sklearn.externals import joblib \n",
        "from sklearn.preprocessing import StandardScaler\n",
        "import imutils\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "import psenet_text_detector as psenet\n",
        "psenet_model = psenet.load_psenet_model()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "PSENet text detector weight will be downloaded to /root/psenet_text_detector/weights/psenet_best.pth\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1w2IgRkQXX49AbOARitO5xCr8-N93JHDd\n",
            "To: /root/psenet_text_detector/weights/psenet_best.pth\n",
            "229MB [00:02, 102MB/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aSvIVy6VyXrR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import psenet_text_detector as psenet"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IGfRjZhRyYG1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5ao2GkNDmCb-",
        "colab_type": "text"
      },
      "source": [
        "Prétraitement pour les pancarte"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k127FLBUldZf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def binarize(img):\n",
        "    tresh = threshold_sauvola(img.copy(), window_size=25)\n",
        "    \n",
        "    return ((img > tresh)*255).astype(np.uint8)\n",
        "def otsu(img):\n",
        "    return cv2.threshold(img, 0, 255,cv2.THRESH_BINARY | cv2.THRESH_OTSU)[1]\n",
        "def denoise2(img):\n",
        "    return cv2.medianBlur(img.copy(),5)\n",
        "def correcto(txt):\n",
        "  txt=txt.lower()\n",
        "  newtxt=list(txt)\n",
        "  for i in range(len(newtxt)):\n",
        "\n",
        "    if (newtxt[i]=='o'):\n",
        "      if(newtxt[i-1].isnumeric()):\n",
        "        newtxt[i]='0'\n",
        "\n",
        "    if (newtxt[i]=='0'):\n",
        "      if(not(newtxt[i-1].isnumeric())):\n",
        "        newtxt[i]='o'\n",
        "\n",
        "  newtxt=\"\".join(newtxt)\n",
        "  return newtxt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GyBA_XedmOnE",
        "colab_type": "text"
      },
      "source": [
        "**segmentation**, **extraction de caracteristiques** et **normalisation** de l'image pour **les pancartes**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uk90fYw5lwRY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def same_ligne(a,b):\n",
        "  if(abs(a[1]-b[1])<65):\n",
        "    return True\n",
        "  return False\n",
        "def list_c(boxes):\n",
        "  #fing image center\n",
        "  nb_boxes=boxes.shape[0]\n",
        "  list_C=np.zeros((nb_boxes,2))\n",
        "  for i in range(nb_boxes):\n",
        "    pts=boxes[i,:,:]\n",
        "    x1,y1 = pts.min(axis=0)\n",
        "    x2,y2 = pts.max(axis=0)\n",
        "    x_center=int((x1+x2)/2)\n",
        "    y_center=int((y1+y2)/2)\n",
        "    center=(x_center,y_center)\n",
        "    list_C[i,:]=center\n",
        "\n",
        "  return list_C\n",
        "def count_lig(vect):\n",
        "  #count words on the same line\n",
        "  k=1\n",
        "  ct=[]\n",
        "  for i in range(vect.shape[0]-1):\n",
        "    a=vect[i]\n",
        "    b=vect[i+1]\n",
        "    if(same_ligne(a,b)):\n",
        "      k+=1\n",
        "      continue\n",
        "    ct.append(k)\n",
        "    k=1 \n",
        "  ct.append(k)\n",
        "  return ct\n",
        "def sort_all(list_C,boxes):\n",
        "  #sort words \n",
        "  k=0\n",
        "  new_boxes=np.zeros((boxes.shape[0],4,2)).astype(int)\n",
        "  ct_r=count_lig(list_C)\n",
        "  for i in range(len(ct_r)):\n",
        "    tmp=ct_r[i]\n",
        "    if(tmp==1):\n",
        "      new_boxes[k,:,:]=boxes[k,:,:]\n",
        "      k+=1\n",
        "    else:\n",
        "      tmp_boxes=boxes[k:k+tmp,:,:]\n",
        "      sort_ind=np.argsort(list_C[k:k+tmp,0])\n",
        "      # for j in range(len(sort_ind)):\n",
        "      new_boxes[k:k+tmp,:,:]=tmp_boxes[sort_ind,:,:]\n",
        "      k=k+tmp\n",
        "\n",
        "  return new_boxes\n",
        "\n",
        "def correct(cont,img):\n",
        "    #perspective correction\n",
        "    lig=cont.shape[0]\n",
        "    cont=cont.reshape(lig,2)\n",
        "#     print(lig)\n",
        "    \n",
        "    rect = np.zeros((4,2), dtype=\"float32\")\n",
        "    \n",
        "    \n",
        "    #estimation du quadrilatère\n",
        "    s = np.sum(cont, axis=1)\n",
        "    rect[0] = cont[np.argmin(s)]\n",
        "    rect[2] = cont[np.argmax(s)]\n",
        "    \n",
        "    diff = np.diff(cont, axis=1)\n",
        "    rect[1] = cont[np.argmin(diff)]\n",
        "    rect[3] = cont[np.argmax(diff)]\n",
        "    \n",
        "\n",
        "    (A, B, C, D) = rect\n",
        "    \n",
        "    \n",
        "    #max(hauteur,largeur) du quadrilatère \n",
        "    widthA = np.sqrt((A[0] - B[0])**2 + (A[1] - B[1])**2 )\n",
        "    widthB = np.sqrt((D[0] - C[0])**2 + (D[1] - C[1])**2 )\n",
        "    maxWidth = max(int(widthA), int(widthB))\n",
        "\n",
        "    heightA = np.sqrt((A[0] - D[0])**2 + (A[1] - D[1])**2 )\n",
        "    heightB = np.sqrt((B[0] - C[0])**2 + (B[1] - C[1])**2 )\n",
        "    maxHeight = max(int(heightA), int(heightB))\n",
        "    \n",
        "    #quadrilatère de reference\n",
        "    dst = np.array([\n",
        "    [0,0],\n",
        "    [maxWidth-1, 0],\n",
        "    [maxWidth-1, maxHeight-1],\n",
        "    [0, maxHeight-1]], dtype=\"float32\")\n",
        "    \n",
        "    #matrice de transformation(quad d'origine --> quad de reference)\n",
        "    BansformMaBix = cv2.getPerspectiveTransform(rect, dst)\n",
        "    #transformation affine\n",
        "    # scan = cv2.warpPerspective(img.copy(), BansformMaBix, (maxWidth, maxHeight),borderMode=cv2.BORDER_REPLICATE)\n",
        "    \n",
        "    return cv2.warpPerspective(img.copy(), BansformMaBix, (maxWidth, maxHeight),borderMode=cv2.BORDER_REPLICATE)\n",
        "def pretseg(c):\n",
        "  #find the main line of the box \n",
        "  kernel = np.ones((1,180), np.uint8)\n",
        "  p = cv2.dilate(255-c, kernel, iterations=1)\n",
        "  _,contours, hierarchy = cv2.findContours(p, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)\n",
        "  area=[]\n",
        "  for cnt in contours: \n",
        "      rect = cv2.minAreaRect(cnt)\n",
        "      area.append(rect[1][0]*rect[1][1])\n",
        "      box = cv2.boxPoints(rect)\n",
        "      box = np.int0(box)\n",
        "  m=max(enumerate(area),key=lambda x: x[1])[0]\n",
        "  # cnt = max(cnts , key = cv2.contourArea)\n",
        "  rect = cv2.minAreaRect(contours[m])\n",
        "  box = cv2.boxPoints(rect)\n",
        "  box = np.int0(box)\n",
        "  if rect[2]==0:\n",
        "      return c\n",
        "  else:\n",
        "      return correct(box,c)   \n",
        "def normalisa(img):\n",
        "\n",
        "  pts = np.argwhere(img==0)\n",
        "  y1,x1 = pts.min(axis=0)\n",
        "  y2,x2 = pts.max(axis=0)\n",
        "  img = img.copy()[y1:y2+1, x1:x2+1]\n",
        "  img=cv2.resize(img,(40,80),interpolation=cv2.INTER_NEAREST)\n",
        "  [l,c]=img.shape\n",
        "  A=np.ones([128,128],dtype='uint8')*255\n",
        "  espH=128-c\n",
        "  espV=128-l\n",
        "  espH=espH//2\n",
        "  espV=espV//2\n",
        "  A[espV:espV+l,espH:espH+c]=img\n",
        "\n",
        "        \n",
        "  return A\n",
        "\n",
        "\n",
        "def caractere(n):\n",
        "    c='0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'\n",
        "    return c[n]\n",
        "def findlimit(bin1):\n",
        "  #find the start and the stop pixels of a caractere\n",
        "  bin1=(bin1/255).T\n",
        "  l,c=bin1.shape\n",
        "  b=np.ones([l,c])\n",
        "  h=c-np.sum(bin1,axis=1)\n",
        "  start=[]\n",
        "  stop=[]\n",
        "  for i in range(l-1):\n",
        "      if h[i]==0 and h[i+1]!=0 :\n",
        "          start.append(i+1)\n",
        "      else:\n",
        "          if h[i]!=0 and h[i+1]==0 :\n",
        "              stop.append(i+2)\n",
        "  return start,stop\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LYOhjnFmotcd",
        "colab_type": "text"
      },
      "source": [
        "**Extraction de caracteristiques**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gZp4mjQjlwgX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def extractHOG(size, frame):\n",
        "    winSize = size\n",
        "    blockSize = (16,16)\n",
        "    blockStride = (8,8)\n",
        "    cellSize = (16,16)\n",
        "    nbins = 9\n",
        "    derivAperture = 1\n",
        "    winSigma = -1.\n",
        "    histogramNormType = 0\n",
        "    L2HysThreshold = 0.2\n",
        "    gammaCorrection = 1\n",
        "    nlevels = 64\n",
        "    hog = cv2.HOGDescriptor(winSize,blockSize,blockStride,cellSize,nbins,derivAperture,winSigma,histogramNormType,L2HysThreshold,gammaCorrection,nlevels)\n",
        "    if frame is None:\n",
        "        return None\n",
        "    return hog.compute(frame)\n",
        "def caract(img):\n",
        "    hog=extractHOG((128,128),img)\n",
        "    hog=hog.T\n",
        "    hog=hog.ravel()\n",
        "    \n",
        "    return hog"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hifpfJC4oxTf",
        "colab_type": "text"
      },
      "source": [
        "**Classification**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NMPRCZfllwrL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def fcatR(txt,b,s,st):\n",
        "  #cut caracters of a word\n",
        "  for i in range(len(s)):\n",
        "    if ((b[:,s[i]:st[i]].size-float(cv2.countNonZero(b[:,s[i]:st[i]])))/(b[:,s[i]:st[i]].shape[0]*b[:,s[i]:st[i]].shape[1]))>0.1:\n",
        "      cropped = b[:,s[i]:st[i]]\n",
        "      vect=caract(normalisa(cropped))\n",
        "      vect= np.array(vect).reshape(-1,1)\n",
        "      car=clf2.predict(vect.T)\n",
        "      txt=txt+caractere(car[0])   \n",
        "  return txt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nhVkbuGfmXOg",
        "colab_type": "text"
      },
      "source": [
        "decodage de l'image \n",
        "image arrivée est codée en **base64** donc il faut le decoder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YqGXzP9KrV2V",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def stringToImage(base64_string):\n",
        "    imgdata = base64.b64decode(base64_string)\n",
        "    return Image.open(io.BytesIO(imgdata))\n",
        "var=np.zeros([20,20])\n",
        "# convert PIL Image to an RGB image( technically a numpy array ) that's compatible with opencv\n",
        "def toRGB(image):\n",
        "    return cv2.cvtColor(np.array(image), cv2.COLOR_BGR2RGB)\n",
        "    \n",
        "def correctRot(ind,img):\n",
        "  if(ind==6):\n",
        "    img=cv2.rotate(img, cv2.ROTATE_90_CLOCKWISE)\n",
        "  elif(ind==8):\n",
        "    img=cv2.rotate(img, cv2.ROTATE_90_COUNTERCLOCKWISE)\n",
        "  elif(ind==3):\n",
        "    img=cv2.rotate(img, cv2.ROTATE_180)\n",
        "  return img\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5voDmFXpmoh6",
        "colab_type": "text"
      },
      "source": [
        "**pretraitement** doc imprimée"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L4_5ZDwjxvrt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def readimg(path):\n",
        "    #converts rgb ro grayscale\n",
        "    img = cv2.imread(path,0)\n",
        "    return img\n",
        "def denoise(img):\n",
        "    blur = cv2.GaussianBlur(img.copy(),(5,5),0)\n",
        "    return blur\n",
        "def denoise2(img):\n",
        "    median = cv2.medianBlur(img.copy(),5)\n",
        "    return median\n",
        "def binarize(img):\n",
        "    tresh = threshold_sauvola(img.copy(), window_size=25)\n",
        "    bin1 = (img > tresh)*255\n",
        "    bin1=bin1.astype(np.uint8)\n",
        "    return bin1\n",
        "def otsu(img):\n",
        "    bin1=cv2.threshold(img, 0, 255,cv2.THRESH_BINARY | cv2.THRESH_OTSU)[1]\n",
        "    return bin1\n",
        "def canny(img):\n",
        "    edged = cv2.Canny(img.copy(), 75, 200)\n",
        "    return edged\n",
        "\n",
        "def detect_ctrs(img):\n",
        "    \n",
        "    blur=denoise(img.copy())\n",
        "    blur=denoise(blur)\n",
        "    ed1=canny(blur)\n",
        "    ed1=canny(ed1)\n",
        "    \n",
        "    return ed1\n",
        "def largest_cont(edg):\n",
        "    \n",
        "    _,cnts, hierarchy  = cv2.findContours(edg.copy(), cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)\n",
        "    \n",
        "    #selectionner le contour ayant la plus grande surface\n",
        "    cnt = max(cnts , key = cv2.contourArea)\n",
        "    \n",
        "    \n",
        "    #approximation du perimetre(True --> contour fermé)\n",
        "    peri = cv2.arcLength(cnt, True)\n",
        "    \n",
        "    #approcimation du polygone\n",
        "    approx = cv2.approxPolyDP(cnt, 0.02 * peri, True)\n",
        "    return approx\n",
        "\n",
        "def cont_area(edg):\n",
        "    _,cnts, hierarchy  = cv2.findContours(edg.copy(), cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)\n",
        "    \n",
        "    cnt = max(cnts , key = cv2.contourArea)\n",
        "    \n",
        "    return cv2.contourArea(cnt)\n",
        "def correct(cont,img):\n",
        "    \n",
        "    lig=cont.shape[0]\n",
        "    cont=cont.reshape(lig,2)\n",
        "#     print(lig)\n",
        "    \n",
        "    rect = np.zeros((4,2), dtype=\"float32\")\n",
        "    \n",
        "    \n",
        "    #estimation du quadrilatère\n",
        "    s = np.sum(cont, axis=1)\n",
        "    rect[0] = cont[np.argmin(s)]\n",
        "    rect[2] = cont[np.argmax(s)]\n",
        "    \n",
        "    diff = np.diff(cont, axis=1)\n",
        "    rect[1] = cont[np.argmin(diff)]\n",
        "    rect[3] = cont[np.argmax(diff)]\n",
        "    \n",
        "\n",
        "    (A, B, C, D) = rect\n",
        "    \n",
        "    \n",
        "    #max(hauteur,largeur) du quadrilatère \n",
        "    widthA = np.sqrt((A[0] - B[0])**2 + (A[1] - B[1])**2 )\n",
        "    widthB = np.sqrt((D[0] - C[0])**2 + (D[1] - C[1])**2 )\n",
        "    maxWidth = max(int(widthA), int(widthB))\n",
        "\n",
        "    heightA = np.sqrt((A[0] - D[0])**2 + (A[1] - D[1])**2 )\n",
        "    heightB = np.sqrt((B[0] - C[0])**2 + (B[1] - C[1])**2 )\n",
        "    maxHeight = max(int(heightA), int(heightB))\n",
        "    \n",
        "    #quadrilatère de reference\n",
        "    dst = np.array([\n",
        "    [0,0],\n",
        "    [maxWidth-1, 0],\n",
        "    [maxWidth-1, maxHeight-1],\n",
        "    [0, maxHeight-1]], dtype=\"float32\")\n",
        "    \n",
        "    #matrice de transformation(quad d'origine --> quad de reference)\n",
        "    BansformMaBix = cv2.getPerspectiveTransform(rect, dst)\n",
        "    #transformation affine\n",
        "    scan = cv2.warpPerspective(img.copy(), BansformMaBix, (maxWidth, maxHeight),borderMode=cv2.BORDER_REPLICATE)\n",
        "    \n",
        "    return scan\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bsbvsjYkm3zF",
        "colab_type": "text"
      },
      "source": [
        "**Segmentation** et normalisation pour les doc imprimées"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yq7TrnInm1VX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def text_only(img):\n",
        "    ed=detect_ctrs(img)\n",
        "    pts = np.argwhere(ed>0)\n",
        "    y1,x1 = pts.min(axis=0)\n",
        "    y2,x2 = pts.max(axis=0)\n",
        "    cropped = img.copy()[y1:y2, x1:x2]    \n",
        "    return cropped\n",
        "\n",
        "def pretseg(c):\n",
        "    kernel = np.ones((5,100), np.uint8)\n",
        "    p = cv2.dilate(255-c, kernel, iterations=1)\n",
        "    _,contours, hierarchy = cv2.findContours(p, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)\n",
        "    area=[]\n",
        "    im=c.copy()\n",
        "    for cnt in contours: \n",
        "        rect = cv2.minAreaRect(cnt)\n",
        "        area.append(rect[1][0]*rect[1][1])\n",
        "        box = cv2.boxPoints(rect)\n",
        "        box = np.int0(box)\n",
        "        im = cv2.drawContours(im,[box],0,(0,0,255),2)\n",
        "    m=max(enumerate(area),key=lambda x: x[1])[0]\n",
        "    rect = cv2.minAreaRect(contours[m])\n",
        "    box = cv2.boxPoints(rect)\n",
        "    box = np.int0(box)\n",
        "    if rect[2]==0:\n",
        "        return c\n",
        "    else:\n",
        "        return correct(box,c)      \n",
        "\n",
        "def findlimit(bin1):\n",
        "    bin1=bin1/255\n",
        "    bin1=bin1.T\n",
        "    l,c=bin1.shape\n",
        "    b=np.ones([l,c])\n",
        "    h=np.sum(bin1,axis=1)\n",
        "    h=c-h\n",
        "    start=[]\n",
        "    stop=[]\n",
        "    for i in range(l-1):\n",
        "        if h[i]==0 and h[i+1]!=0 :\n",
        "            start.append(i+1)\n",
        "        else:\n",
        "            if h[i]!=0 and h[i+1]==0 :\n",
        "                stop.append(i+2)\n",
        "    return start,stop \n",
        "\n",
        "def normalisa(img):\n",
        "    if (0 in img):\n",
        "        pts = np.argwhere(img==0)\n",
        "        y1,x1 = pts.min(axis=0)\n",
        "        y2,x2 = pts.max(axis=0)\n",
        "        img = img.copy()[y1:y2+1, x1:x2+1]\n",
        "\n",
        "        img=cv2.resize(img,(40,80),interpolation=cv2.INTER_NEAREST)\n",
        "\n",
        "        [l,c]=img.shape\n",
        "        A=np.ones([128,128],dtype='uint8')*255\n",
        "\n",
        "\n",
        "        espH=128-c\n",
        "        espV=128-l\n",
        "        espH=espH//2\n",
        "        espV=espV//2\n",
        "        A[espV:espV+l,espH:espH+c]=img\n",
        "\n",
        "        img=A\n",
        "\n",
        "        \n",
        "    else:\n",
        "        A=np.ones([128,128],dtype='uint8')*255\n",
        "        \n",
        "    return A\n",
        "\n",
        "\n",
        "\n",
        "def text_split(thresh):\n",
        "    im=thresh.copy()\n",
        "    #dilation\n",
        "    kernel = np.ones((5,100), np.uint8)\n",
        "    img_dilation = cv2.dilate(255-thresh, kernel, iterations=1)\n",
        "    l,c=im.shape\n",
        "    #find contours\n",
        "    _,ctrs, hier = cv2.findContours(img_dilation.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n",
        "    #sort contours\n",
        "    #contours = sorted(ctrs, key=lambda ctr: cv2.boundingRect(ctr)[0])\n",
        "    contours=np.flip(ctrs,axis=0)\n",
        "    mask = np.zeros(im.shape, dtype=np.uint8)\n",
        "    txt=''\n",
        "    for idx in range(len(contours)):\n",
        "        x, y, w, h = cv2.boundingRect(contours[idx])\n",
        "        # Getting ROI\n",
        "        roi = im[y:y+h, x:x+w]\n",
        "        cv2.drawContours(mask, contours, idx,255, -1)\n",
        "        # show ROI\n",
        "        r = float(cv2.countNonZero(mask[y:y+h, x:x+w])) / (w * h)\n",
        "        if r > 0.3 and w > 200 and h > 20 and h<0.4*l :\n",
        "            p=pretseg(roi)\n",
        "            a,b=findlimit(p)\n",
        "            ########################################## fcatR ou fcat\n",
        "            txt=fcat(txt,p,a,b) \n",
        "    return txt\n",
        "\n",
        "\n",
        "\n",
        "def draw_cont(img,edg):\n",
        "    _,cnts, hierarchy  = cv2.findContours(edg.copy(), cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)\n",
        "    \n",
        "    cnt = max(cnts , key = cv2.contourArea)\n",
        "    \n",
        "    return cv2.drawContours(img.copy(), [cnt], -1, (0,255,0), 3)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tcEvqGBjoRvx",
        "colab_type": "text"
      },
      "source": [
        "**extraction de caractéristique**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l98eo2wzoKwo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def extractHOG(size, frame):\n",
        "    winSize = size\n",
        "    blockSize = (16,16)\n",
        "    blockStride = (8,8)\n",
        "    cellSize = (16,16)\n",
        "    nbins = 9\n",
        "    derivAperture = 1\n",
        "    winSigma = -1.\n",
        "    histogramNormType = 0\n",
        "    L2HysThreshold = 0.2\n",
        "    gammaCorrection = 1\n",
        "    nlevels = 64\n",
        "    hog = cv2.HOGDescriptor(winSize,blockSize,blockStride,cellSize,nbins,derivAperture,winSigma,histogramNormType,L2HysThreshold,gammaCorrection,nlevels)\n",
        "    if frame is None:\n",
        "        return None\n",
        "    return hog.compute(frame)\n",
        "def caract(img):\n",
        "    vect=[]\n",
        "    hog=extractHOG((128,128),img)\n",
        "    hog=hog.T\n",
        "    hog=hog.ravel()\n",
        "    vect=np.hstack((vect,hog.T))\n",
        "    \n",
        "    return vect"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vG76LKpzoazS",
        "colab_type": "text"
      },
      "source": [
        "**classification**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ppEG7MxqoMXo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def fcat(txt,b,s,st):\n",
        "    k=1\n",
        "    space=[]\n",
        "    for i in range(len(s)-1):\n",
        "        space.append(s[i+1]-st[i])\n",
        "    if len(space)==0:\n",
        "      return txt\n",
        "    else:\n",
        "      m=sum(space)/len(space)\n",
        "    #print(m)\n",
        "    for i in range(len(s)-1):\n",
        "        cropped = b[:,s[i]:st[i]]\n",
        "        if (np.where(cropped==0)[0].size/cropped.size)>0.08 and 4<cropped.shape[1]<150 :\n",
        "            vect=caract(normalisa(cropped))\n",
        "            vect=vect.reshape(1,45,45,1)\n",
        "            car=clf.predict(vect)\n",
        "            txt=txt+caractere(np.argmax(car))\n",
        "            #print(txt)\n",
        "            k=k+1 \n",
        "        if s[i+1]-st[i]>m+8 :\n",
        "        #if s[i+1]-st[i]>20 :\n",
        "          #print(s[i+1]-st[i])\n",
        "          txt=txt+' '\n",
        "          k=k+1\n",
        "    cropped = b[:,s[-1]:st[-1]]\n",
        "    vect=caract(normalisa(cropped))\n",
        "    vect=vect.reshape(1,45,45,1)\n",
        "    car=clf.predict(vect)\n",
        "    txt=txt+caractere(np.argmax(car))\n",
        "    k=k+1\n",
        "    txt=txt+' '\n",
        "    return txt\n",
        "def caractere(n):\n",
        "    c=['0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z','a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z']\n",
        "    #c='0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'\n",
        "    return c[n]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L-GJhokppBj1",
        "colab_type": "text"
      },
      "source": [
        "REST API "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-nyDPwJTpIPQ",
        "colab_type": "text"
      },
      "source": [
        "apres execution de cette cellule ci-dessus, colab va generer un lien qui va lier l'application mobile et notre API"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qhC64Dctx5mg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "app = Flask(__name__)\n",
        "run_with_ngrok(app)\n",
        "@app.route('/api/doc', methods=['POST'])\n",
        "def ocr():\n",
        "  #r = request\n",
        "  # convert string of image data to uint8\n",
        "  #nparr = np.fromstring(r.data, np.uint8)\n",
        "  # decode image\n",
        "  #img = cv2.imdecode(nparr, cv2.IMREAD_COLOR)\n",
        "  r = request.get_json()\n",
        "  r=r['dt']\n",
        "  r=r['_parts']\n",
        "  r=r[0]\n",
        "  r=r[1]\n",
        "  r=str(r['uri'])\n",
        "  ind=int(r[0])\n",
        "  b64=r[1:]\n",
        "  img=stringToImage(b64)\n",
        "  # print(r[0])\n",
        "  # img=cv2.rotate(img, cv2.ROTATE_90_CLOCKWISE)\n",
        "  img=toRGB(img)\n",
        "  img=correctRot(ind,img)\n",
        "  # img=cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
        "  rgb=img\n",
        "  img=cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
        "  length=img.shape[0]\n",
        "\n",
        "  if(length>4000):\n",
        "      val=7.5\n",
        "  elif(length>1500):\n",
        "      val=5.5\n",
        "  elif(length>870):\n",
        "      val=4\n",
        "  else :\n",
        "      val=1\n",
        "\n",
        "  div=int(length/val)\n",
        "\n",
        "  im2 = imutils.resize(img, height = div)\n",
        "  rgb=imutils.resize(rgb, height = div)\n",
        "  ratio = img.shape[0] / div\n",
        "  vs=im2.copy()\n",
        "  ed=detect_ctrs(vs)\n",
        "  paper=largest_cont(ed)\n",
        "  # paper=largest_cont(paper,ed)\n",
        "  if(len(paper)==0):\n",
        "      res=img\n",
        "  else :\n",
        "      area=cont_area(ed)\n",
        "      draw=draw_cont(rgb,ed)\n",
        "\n",
        "      #print('area :',area)\n",
        "      if(area>400):\n",
        "          tmp=correct(paper*ratio,img)\n",
        "          co=1\n",
        "          res=tmp\n",
        "  #         res=denoise2(res)\n",
        "          bin1=binarize(res)\n",
        "          black=np.argwhere(bin1==0).size/2\n",
        "          white=np.argwhere(bin1==255).size/2\n",
        "          #print(\"back :\",black/bin1.size)\n",
        "          #print('white :',white/bin1.size)\n",
        "          #print(\"size :\",tmp.size/img.size)\n",
        "          if((black!=0 and black/bin1.size>0.86) or (white!=0 and white/bin1.size>0.995) or (tmp.size/img.size)<0.02):\n",
        "              tmp=img\n",
        "      else:\n",
        "          tmp=img\n",
        "\n",
        "      res=tmp\n",
        "  #     res=denoise2(res)\n",
        "\n",
        "  bin1=binarize(res)\n",
        "\n",
        "  #plt.subplot(1,3,1),plt.imshow(cv2.cvtColor(rgb, cv2.COLOR_BGR2RGB))\n",
        "  #if(len(paper)!=0):\n",
        "  #    plt.subplot(1,3,2),plt.imshow(cv2.cvtColor(draw, cv2.COLOR_BGR2RGB))\n",
        "  #plt.subplot(1,3,1),plt.imshow(bin1,'gray',interpolation='bicubic')\n",
        "  #plt.show()\n",
        "\n",
        "  #bin1=otsu(tmp)\n",
        "  l,c=bin1.shape\n",
        "  nbin=np.ones([l+2,c+2])*255\n",
        "  nbin[1:-1,1:-1]=bin1\n",
        "  nbin = nbin.astype(np.uint8)\n",
        "  txt = text_split(nbin)\n",
        "  response = { 'text':txt}\n",
        "                \n",
        "    # encode response using jsonpickle\n",
        "  response_pickled = jsonpickle.encode(response)\n",
        "\n",
        "  return Response(response=response_pickled, status=200, mimetype=\"application/json\")\n",
        " \n",
        "\n",
        "\n",
        "\n",
        "  ##################################### test appMobile\n",
        "\n",
        "@app.route('/api/pancarte', methods=['POST'])\n",
        "def ocr2():\n",
        "  ##################################### test appMobile\n",
        "  r = request.get_json()\n",
        "  r=r['dt']\n",
        "  r=r['_parts']\n",
        "  r=r[0]\n",
        "  r=r[1]\n",
        "  r=str(r['uri'])\n",
        "  ind=int(r[0])\n",
        "  b64=r[1:]\n",
        "  img=stringToImage(b64)\n",
        "  img=toRGB(img)\n",
        "  img=correctRot(ind,img)\n",
        "  ######################################## test clien\n",
        "  #r = request\n",
        "  #nparr = np.fromstring(r.data, np.uint8)\n",
        "  #img = cv2.imdecode(nparr, cv2.IMREAD_COLOR)\n",
        "  ######################################################\n",
        "  img=cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
        "# prediction_result = psenet.detect_text(image_path, output_dir, cuda=False)\n",
        "\n",
        "  prediction_result = psenet.get_prediction(image=img,\n",
        "                               \t\t  model=psenet_model,\n",
        "                                       \t  binary_th=1.0,\n",
        "                                       \t  kernel_num=3,\n",
        "                                       \t  upsample_scale=1,\n",
        "                                       \t  long_size=1280,\n",
        "                                       \t  min_kernel_area=10.0,\n",
        "                                       \t  min_area=100.0,\n",
        "                                       \t  min_score=0.93,\n",
        "                                       \t  )\n",
        "  boxes=prediction_result[\"boxes\"]\n",
        "  list_C=list_c(boxes)\n",
        "  new_boxes=sort_all(list_C,boxes)\n",
        "  image=img\n",
        "  nb_boxes=boxes.shape[0]\n",
        "  txt=''\n",
        "  for i in range(nb_boxes):\n",
        "    pts=new_boxes[i,:,:]\n",
        "    x1,y1 = pts.min(axis=0)\n",
        "    x2,y2 = pts.max(axis=0)\n",
        "    if(x1==0):\n",
        "      x1=2\n",
        "    if(y1==0):\n",
        "      y1=2\n",
        "    gray = cv2.cvtColor(image[y1-2:y2+2,x1-2:x2+2], cv2.COLOR_BGR2GRAY)\n",
        "\n",
        "    # sharpen image\n",
        "    sharpen = cv2.GaussianBlur(gray, (0,0), 3)\n",
        "    sharpen = cv2.addWeighted(gray, 1.5, sharpen, -0.5, 0)\n",
        "\n",
        "    # apply adaptive threshold to get black and white effect\n",
        "    thresh = otsu(sharpen)\n",
        "    #crop=binarize(gray)\n",
        "\n",
        "    #tranform all sort of image to an image with white backg\n",
        "    _,ctr,_=cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)\n",
        "    cntsSorted = sorted(ctr, key=lambda x: cv2.contourArea(x))\n",
        "    x, y,_,_ = cv2.boundingRect(cntsSorted[-1])\n",
        "    if x!=0 and y!=0:\n",
        "      thresh=255-thresh\n",
        "    #crop=thresh\n",
        "    \n",
        "    crop=pretseg(denoise2(denoise2(thresh)))\n",
        "    nbin=np.ones([crop.shape[0]+2,crop.shape[1]+2])*255\n",
        "    nbin[1:-1,1:-1]=crop\n",
        "    nbin = nbin.astype(np.uint8)\n",
        "\n",
        "    #find caracters's rectangle with height>image's height\n",
        "    a=np.zeros([1,4])\n",
        "    _,ctr,_=cv2.findContours(255-nbin, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)\n",
        "    contours = sorted(ctr, key=lambda ctrs: cv2.boundingRect(ctrs)[0])\n",
        "    for idx in range(len(contours)):\n",
        "      if cv2.boundingRect(contours[idx])[3]>nbin.shape[0]*0.3 :\n",
        "        a=np.vstack((a,np.array([cv2.boundingRect(contours[idx])[0],cv2.boundingRect(contours[idx])[1],cv2.boundingRect(contours[idx])[2],cv2.boundingRect(contours[idx])[3]])))\n",
        "    a=a[1:,:].astype(np.uint32)\n",
        "\n",
        "    #erase rectangles intersection \n",
        "    for r in range (a.shape[0]-1):\n",
        "      if a[r,0]+a[r,2]>= a[r+1,0]:\n",
        "        nbin[:,a[r+1,0]:a[r,0]+a[r,2]+1]=255\n",
        "    \n",
        "    s,st=findlimit(nbin)\n",
        "    txt=fcatR(txt,nbin,s,st)\n",
        "    txt=txt+' '\n",
        "    if i==0:\n",
        "      ddd=nbin\n",
        "\n",
        "  txt=txt.lower()\n",
        "  txt=correcto(txt)\n",
        "  response = {'text':txt}\n",
        "  response_pickled = jsonpickle.encode(response)\n",
        "\n",
        "  return Response(response=response_pickled, status=200, mimetype=\"application/json\")\n",
        "\n",
        "app.run()\n",
        "##https://reactnativecode.com/upload-image-to-server-using-php-mysql/\n",
        "app.run() "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qCKSuYgoL0Mm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}