A.I
OCR을 활용하여 글자 검출 본문
OCR¶
In [1]:
import os
path = os.path.join(os.getenv('HOME'),'aiffel/ocr')
os.chdir(path)
print(path)
/home/ssac24/aiffel/ocr
In [2]:
import tensorflow as tf
gpus = tf.config.experimental.list_physical_devices(device_type="GPU")
tf.config.experimental.set_visible_devices(devices=gpus[0], device_type="GPU")
tf.config.experimental.set_memory_growth(device=gpus[0], enable=True)
Recognition model (1)¶
- An End-to-End Trainable Neural Network for Image-based SequenceRecognition and Its Application to Scene Text Recognition
- 입력이미지를 Convolution Layer를 통해 Feature를 추출하고 추출된 Feature를 얻습니다.
- Recurrent Layer는 추출된 Feature의 전체적인 Context를 파악하고 다양한 output의 크기에 대응가능합니다.
- Transcription layer(Fully connected layer)는 step마다 어떤 character의 확률이 높은지 예측합니다.
In [3]:
# 영문 대문자와 숫자를 인식하기 위한 Class 개수로 36개가 필요하고, 문자가 없을 시 공백추가를 위하여 1개를 더 추가할 수 있다.
NUMBERS = "0123456789"
ENG_CHAR_UPPER = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
TARGET_CHARACTERS = ENG_CHAR_UPPER + NUMBERS
print(f"The total number of characters is {len(TARGET_CHARACTERS)}")
The total number of characters is 36
In [4]:
import re
import six
import math
import lmdb
import os
import numpy as np
import tensorflow as tf
from PIL import Image
from tensorflow.keras import layers
from tensorflow.keras.models import Model
from tensorflow.keras.utils import Sequence
from tensorflow.keras import backend as K
from tensorflow.keras.models import load_model
BATCH_SIZE = 128
HOME_DIR = os.getenv('HOME')+'/aiffel/ocr'
# 로컬 사용자
TRAIN_DATA_PATH = HOME_DIR+'/MJ/MJ_train'
VALID_DATA_PATH = HOME_DIR+'/MJ/MJ_valid'
TEST_DATA_PATH = HOME_DIR+'/MJ/MJ_test'
# 클라우드 사용자는 아래 주석을 사용해 주세요.
# TRAIN_DATA_PATH = HOME_DIR+'/data/MJ/MJ_train'
# VALID_DATA_PATH = HOME_DIR+'/data/MJ/MJ_valid'
# TEST_DATA_PATH = HOME_DIR+'/data/MJ/MJ_test'
print(TRAIN_DATA_PATH)
/home/ssac24/aiffel/ocr/MJ/MJ_train
Recognition model (2) Input Image¶
In [5]:
env = lmdb.open(TRAIN_DATA_PATH, max_readers=32, readonly=True, lock=False, readahead=False, meminit=False)
In [6]:
from IPython.display import display
with env.begin(write=False) as txn:
print(1)
for index in range(1, 5):
print(2)
label_key = 'label-%09d'.encode() % index
label = txn.get(label_key).decode('utf-8')
print(3)
img_key = 'image-%09d'.encode() % index
imgbuf = txn.get(img_key)
buf = six.BytesIO()
buf.write(imgbuf)
buf.seek(0)
print(4)
try:
img = Image.open(buf).convert('RGB')
except IOError:
img = Image.new('RGB', (100, 32))
label = '-'
width, height = img.size
print('original image width:{}, height:{}'.format(width, height))
target_width = min(int(width*32/height), 100)
target_img_size = (target_width,32 )
print('target_img_size:{}'.format(target_img_size))
img = np.array(img.resize(target_img_size)).transpose(1,0,2)
print('display img shape:{}'.format(img.shape))
print('label:{}'.format(label))
display(Image.fromarray(img.transpose(1,0,2).astype(np.uint8)))
1 2 3 4 original image width:72, height:31 target_img_size:(74, 32) display img shape:(74, 32, 3) label:Lube
2 3 4 original image width:82, height:31 target_img_size:(84, 32) display img shape:(84, 32, 3) label:Spencerian
2 3 4 original image width:115, height:31 target_img_size:(100, 32) display img shape:(100, 32, 3) label:accommodatingly
2 3 4 original image width:140, height:31 target_img_size:(100, 32) display img shape:(100, 32, 3) label:CARPENTER
In [7]:
class MJDatasetSequence(Sequence):
def __init__(self,
dataset_path,
label_converter,
batch_size=1,
img_size=(100,32),
max_text_len=22,
is_train=False,
character=''
):
self.label_converter = label_converter
self.batch_size = batch_size
self.img_size = img_size
self.max_text_len = max_text_len
self.character = character
self.is_train = is_train
self.divide_length = 100
self.env = lmdb.open(dataset_path, max_readers=32, readonly=True, lock=False, readahead=False, meminit=False)
with self.env.begin(write=False) as txn:
num_samples = int(txn.get('num-samples'.encode()))
self.num_samples = int(num_samples)
self.index_list = [index + 1 for index in range(self.num_samples)]
def __len__(self):
if self.is_train:
return math.ceil(self.num_samples/self.batch_size/self.divide_length)
return math.ceil(self.num_samples/self.batch_size/self.divide_length)
# index에 해당하는 image와 label을 가져오는 메소드
def _get_img_label(self, index):
with self.env.begin(write=False) as txn:
label_key = 'label-%09d'.encode() % index
label = txn.get(label_key).decode('utf-8')
img_key = 'image-%09d'.encode() % index
imgbuf = txn.get(img_key)
buf = six.BytesIO()
buf.write(imgbuf)
buf.seek(0)
try:
img = Image.open(buf).convert('RGB')
except IOError:
img = Image.new('RGB', self.img_size)
label = '-'
width, height = img.size
target_width = min(int(width*self.img_size[1]/height), self.img_size[0])
target_img_size = (target_width,self.img_size[1] )
img = np.array(img.resize(target_img_size)).transpose(1,0,2)
label = label.upper()[:self.max_text_len]
out_of_char = f'[^{self.character}]'
label = re.sub(out_of_char, '', label)
return (img, label)
# idx번째 배치를 가져오는 메소드
def __getitem__(self, idx):
batch_indicies = self.index_list[
idx*self.batch_size:
(idx+1)*self.batch_size
]
input_images = np.zeros([self.batch_size, *self.img_size, 3])
labels = np.zeros([self.batch_size, self.max_text_len], dtype='int64')
input_length = np.ones([self.batch_size], dtype='int64')*self.max_text_len
label_length = np.ones([self.batch_size], dtype='int64')
for i, index in enumerate(batch_indicies):
img, label = self._get_img_label(index)
encoded_label = self.label_converter.encode(label)
width = img.shape[0]
input_images[i,:width,:,:] = img
if len(encoded_label) > self.max_text_len:
continue
labels[i,0:len(encoded_label)] = encoded_label
label_length[i] = len(encoded_label)
inputs = {
'input_image': input_images,
'label': labels,
'input_length': input_length,
'label_length': label_length,
}
outputs = {'ctc': np.zeros([self.batch_size, 1])}
return inputs, outputs
def on_epoch_end(self):
self.index_list = [index + 1 for index in range(self.num_samples)]
if self.is_train :
np.random.shuffle(self.index_list)
return self.index_list
Recognition model (3) Encode¶
- init() 에서는 입력으로 받은 text를 self.dict에 각 character들이 어떤 index에 매핑되는지 저장합니다. 이 character와 index 정보를 통해 모델이 학습할 수 있는 output이 만들어집니다. 만약 character='ABCD'라면 'A'의 label은 1, 'B'의 label은 2가 됩니다.
- 공백(blank) 문자를 지정합니다. 여기서는 공백 문자를 뜻하기 위해 '-'를 활용하며, label은 0으로 지정합니다.
- decode()는 각 index를 다시 character로 변환한 후 이어주어 우리가 읽을 수 있는 text로 바꾸어줍니다.
In [8]:
class LabelConverter(object):
""" Convert between text-label and text-index """
def __init__(self, character):
self.character = "-" + character
self.label_map = dict()
for i, char in enumerate(self.character):
self.label_map[char] = i
def encode(self, text):
encoded_label = []
for i, char in enumerate(text):
if i > 0 and char == text[i - 1]:
encoded_label.append(0) # 같은 문자 사이에 공백 문자 label을 삽입
encoded_label.append(self.label_map[char])
return np.array(encoded_label)
def decode(self, encoded_label):
target_characters = list(self.character)
decoded_label = ""
for encode in encoded_label:
decoded_label += self.character[encode]
return decoded_label
In [9]:
label_converter = LabelConverter(TARGET_CHARACTERS)
encdoded_text = label_converter.encode('HELLO')
print("Encdoded_text: ", encdoded_text)
decoded_text = label_converter.decode(encdoded_text)
print("Decoded_text: ", decoded_text)
Encdoded_text: [ 8 5 12 0 12 15] Decoded_text: HEL-LO
동일한 영문 'L'이 연속되면 사이에 공백 문자가 포함된 것으로 확인됩니다.
Recognition model (4) Build CRNN model¶
ctc_batch_cost¶
- Tensorflow Tutorial - ctc_batch_cost
- CTC-Loss를 다루는 이유는 입력의 길이 T와 라벨의 길이 U의 단위가 일치하지 않을 때, 그래서 라벨은 APPLE이지만 모델이 출력한 결과는 AAAPPPPLLLLEE 처럼 나올 수 있기때문에 보정하는 로직을 추가해주었습니다.
- y_true: 실제 라벨 LUBE. 텍스트 라벨 그대로가 아니라, 각 글자를 One-hot 인코딩한 형태로서, max_string_length 값은 모델에서 22로 지정할 예정
- y_pred: 우리가 만들 RCNN 모델의 출력 결과. 그러나 길이는 4가 아니라 우리가 만들 RNN의 최종 출력 길이로서 24가 될 예정
- input_length tensor: 모델 입력 길이 T로서, 이 경우에는 텍스트의 width인 74
- label_length tensor: 라벨의 실제 정답 길이 U로서, 이 경우에는 4
In [10]:
def ctc_lambda_func(args): # CTC loss를 계산하기 위한 Lambda 함수
labels, y_pred, label_length, input_length = args
y_pred = y_pred[:, 2:, :]
return K.ctc_batch_cost(labels, y_pred, input_length, label_length)
In [11]:
def build_crnn_model(input_shape=(100,32,3), characters=TARGET_CHARACTERS):
num_chars = len(characters)+2
image_input = layers.Input(shape=input_shape, dtype='float32', name='input_image')
# Build CRNN model
conv = layers.Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(image_input)
conv = layers.MaxPooling2D(pool_size=(2, 2))(conv)
conv = layers.Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(conv)
conv = layers.MaxPooling2D(pool_size=(2, 2))(conv)
conv = layers.Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(conv)
conv = layers.Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(conv)
conv = layers.MaxPooling2D(pool_size=(1, 2))(conv)
conv = layers.Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(conv)
conv = layers.BatchNormalization()(conv)
conv = layers.Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(conv)
conv = layers.BatchNormalization()(conv)
conv = layers.MaxPooling2D(pool_size=(1, 2))(conv)
feature = layers.Conv2D(512, (2, 2), activation='relu', kernel_initializer='he_normal')(conv)
sequnce = layers.Reshape(target_shape=(24, 512))(feature)
sequnce = layers.Dense(64, activation='relu')(sequnce)
sequnce = layers.Bidirectional(layers.LSTM(256, return_sequences=True))(sequnce)
sequnce = layers.Bidirectional(layers.LSTM(256, return_sequences=True))(sequnce)
y_pred = layers.Dense(num_chars, activation='softmax', name='output')(sequnce)
labels = layers.Input(shape=[22], dtype='int64', name='label')
input_length = layers.Input(shape=[1], dtype='int64', name='input_length')
label_length = layers.Input(shape=[1], dtype='int64', name='label_length')
loss_out = layers.Lambda(ctc_lambda_func, output_shape=(1,), name="ctc")(
[labels, y_pred, label_length, input_length]
)
model_input = [image_input, labels, input_length, label_length]
model = Model(
inputs=model_input,
outputs=loss_out
)
y_func = tf.keras.backend.function(image_input, [y_pred])
return model, y_func
Recognition model (5) Train & Inference¶
In [12]:
train_set = MJDatasetSequence(TRAIN_DATA_PATH, label_converter, batch_size=BATCH_SIZE, character=TARGET_CHARACTERS, is_train=True)
val_set = MJDatasetSequence(VALID_DATA_PATH, label_converter, batch_size=BATCH_SIZE, character=TARGET_CHARACTERS)
checkpoint_path = HOME_DIR + '/model_checkpoint.hdf5'
model, y_func = build_crnn_model()
sgd = tf.keras.optimizers.Adadelta(lr=0.1, clipnorm=5)
model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd)
ckp = tf.keras.callbacks.ModelCheckpoint(
checkpoint_path, monitor='val_loss',
verbose=1, save_best_only=True, save_weights_only=True
)
earlystop = tf.keras.callbacks.EarlyStopping(
monitor='val_loss', min_delta=0, patience=10, verbose=0, mode='min'
)
model.fit(train_set,
steps_per_epoch=len(val_set),
epochs=100,
validation_data=val_set,
validation_steps=len(val_set),
callbacks=[ckp])
Epoch 1/100 63/63 [==============================] - ETA: 0s - loss: 29.4691 Epoch 00001: val_loss improved from inf to 27.13623, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 11s 168ms/step - loss: 29.4691 - val_loss: 27.1362 Epoch 2/100 63/63 [==============================] - ETA: 0s - loss: 26.5785 Epoch 00002: val_loss improved from 27.13623 to 26.23292, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 149ms/step - loss: 26.5785 - val_loss: 26.2329 Epoch 3/100 63/63 [==============================] - ETA: 0s - loss: 26.2466 Epoch 00003: val_loss did not improve from 26.23292 63/63 [==============================] - 9s 147ms/step - loss: 26.2466 - val_loss: 28.8256 Epoch 4/100 63/63 [==============================] - ETA: 0s - loss: 25.7218 Epoch 00004: val_loss improved from 26.23292 to 26.07353, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 152ms/step - loss: 25.7218 - val_loss: 26.0735 Epoch 5/100 63/63 [==============================] - ETA: 0s - loss: 25.4324 Epoch 00005: val_loss did not improve from 26.07353 63/63 [==============================] - 9s 143ms/step - loss: 25.4324 - val_loss: 27.7283 Epoch 6/100 63/63 [==============================] - ETA: 0s - loss: 25.1732 Epoch 00006: val_loss improved from 26.07353 to 25.98838, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 151ms/step - loss: 25.1732 - val_loss: 25.9884 Epoch 7/100 63/63 [==============================] - ETA: 0s - loss: 24.6509 Epoch 00007: val_loss improved from 25.98838 to 24.83633, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 150ms/step - loss: 24.6509 - val_loss: 24.8363 Epoch 8/100 63/63 [==============================] - ETA: 0s - loss: 24.2168 Epoch 00008: val_loss improved from 24.83633 to 24.58940, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 152ms/step - loss: 24.2168 - val_loss: 24.5894 Epoch 9/100 63/63 [==============================] - ETA: 0s - loss: 23.5130 Epoch 00009: val_loss improved from 24.58940 to 23.71060, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 149ms/step - loss: 23.5130 - val_loss: 23.7106 Epoch 10/100 63/63 [==============================] - ETA: 0s - loss: 22.8227 Epoch 00010: val_loss improved from 23.71060 to 22.67570, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 148ms/step - loss: 22.8227 - val_loss: 22.6757 Epoch 11/100 63/63 [==============================] - ETA: 0s - loss: 21.7462 Epoch 00011: val_loss improved from 22.67570 to 22.60368, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 148ms/step - loss: 21.7462 - val_loss: 22.6037 Epoch 12/100 63/63 [==============================] - ETA: 0s - loss: 20.2519 Epoch 00012: val_loss did not improve from 22.60368 63/63 [==============================] - 10s 155ms/step - loss: 20.2519 - val_loss: 23.5187 Epoch 13/100 63/63 [==============================] - ETA: 0s - loss: 18.2984 Epoch 00013: val_loss improved from 22.60368 to 17.62420, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 145ms/step - loss: 18.2984 - val_loss: 17.6242 Epoch 14/100 63/63 [==============================] - ETA: 0s - loss: 15.8871 Epoch 00014: val_loss improved from 17.62420 to 16.86154, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 149ms/step - loss: 15.8871 - val_loss: 16.8615 Epoch 15/100 63/63 [==============================] - ETA: 0s - loss: 13.6515 Epoch 00015: val_loss improved from 16.86154 to 13.66581, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 144ms/step - loss: 13.6515 - val_loss: 13.6658 Epoch 16/100 63/63 [==============================] - ETA: 0s - loss: 12.0305 Epoch 00016: val_loss improved from 13.66581 to 12.19383, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 142ms/step - loss: 12.0305 - val_loss: 12.1938 Epoch 17/100 63/63 [==============================] - ETA: 0s - loss: 10.9604 Epoch 00017: val_loss improved from 12.19383 to 11.29811, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 147ms/step - loss: 10.9604 - val_loss: 11.2981 Epoch 18/100 63/63 [==============================] - ETA: 0s - loss: 10.0328 Epoch 00018: val_loss did not improve from 11.29811 63/63 [==============================] - 9s 147ms/step - loss: 10.0328 - val_loss: 11.3832 Epoch 19/100 63/63 [==============================] - ETA: 0s - loss: 9.2965 Epoch 00019: val_loss improved from 11.29811 to 9.87942, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 149ms/step - loss: 9.2965 - val_loss: 9.8794 Epoch 20/100 63/63 [==============================] - ETA: 0s - loss: 8.7394 Epoch 00020: val_loss improved from 9.87942 to 8.87705, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 147ms/step - loss: 8.7394 - val_loss: 8.8770 Epoch 21/100 63/63 [==============================] - ETA: 0s - loss: 8.2013 Epoch 00021: val_loss improved from 8.87705 to 8.37091, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 147ms/step - loss: 8.2013 - val_loss: 8.3709 Epoch 22/100 63/63 [==============================] - ETA: 0s - loss: 7.9085 Epoch 00022: val_loss improved from 8.37091 to 8.09254, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 147ms/step - loss: 7.9085 - val_loss: 8.0925 Epoch 23/100 63/63 [==============================] - ETA: 0s - loss: 7.5398 Epoch 00023: val_loss did not improve from 8.09254 63/63 [==============================] - 9s 147ms/step - loss: 7.5398 - val_loss: 8.1931 Epoch 24/100 63/63 [==============================] - ETA: 0s - loss: 7.2093 Epoch 00024: val_loss improved from 8.09254 to 7.41894, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 145ms/step - loss: 7.2093 - val_loss: 7.4189 Epoch 25/100 63/63 [==============================] - ETA: 0s - loss: 6.8308 Epoch 00025: val_loss did not improve from 7.41894 63/63 [==============================] - 9s 148ms/step - loss: 6.8308 - val_loss: 7.4405 Epoch 26/100 63/63 [==============================] - ETA: 0s - loss: 6.6793 Epoch 00026: val_loss improved from 7.41894 to 6.74084, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 147ms/step - loss: 6.6793 - val_loss: 6.7408 Epoch 27/100 63/63 [==============================] - ETA: 0s - loss: 6.4860 Epoch 00027: val_loss improved from 6.74084 to 6.67241, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 146ms/step - loss: 6.4860 - val_loss: 6.6724 Epoch 28/100 63/63 [==============================] - ETA: 0s - loss: 6.3072 Epoch 00028: val_loss improved from 6.67241 to 6.45543, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 146ms/step - loss: 6.3072 - val_loss: 6.4554 Epoch 29/100 63/63 [==============================] - ETA: 0s - loss: 6.0907 Epoch 00029: val_loss did not improve from 6.45543 63/63 [==============================] - 9s 145ms/step - loss: 6.0907 - val_loss: 6.6606 Epoch 30/100 63/63 [==============================] - ETA: 0s - loss: 5.9973 Epoch 00030: val_loss improved from 6.45543 to 6.34712, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 143ms/step - loss: 5.9973 - val_loss: 6.3471 Epoch 31/100 63/63 [==============================] - ETA: 0s - loss: 5.9087 Epoch 00031: val_loss improved from 6.34712 to 5.97803, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 144ms/step - loss: 5.9087 - val_loss: 5.9780 Epoch 32/100 63/63 [==============================] - ETA: 0s - loss: 5.4937 Epoch 00032: val_loss improved from 5.97803 to 5.79475, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 140ms/step - loss: 5.4937 - val_loss: 5.7948 Epoch 33/100 63/63 [==============================] - ETA: 0s - loss: 5.5420 Epoch 00033: val_loss did not improve from 5.79475 63/63 [==============================] - 9s 143ms/step - loss: 5.5420 - val_loss: 5.9436 Epoch 34/100 63/63 [==============================] - ETA: 0s - loss: 5.3271 Epoch 00034: val_loss did not improve from 5.79475 63/63 [==============================] - 9s 144ms/step - loss: 5.3271 - val_loss: 5.8352 Epoch 35/100 63/63 [==============================] - ETA: 0s - loss: 5.3234 Epoch 00035: val_loss improved from 5.79475 to 5.53217, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 153ms/step - loss: 5.3234 - val_loss: 5.5322 Epoch 36/100 63/63 [==============================] - ETA: 0s - loss: 5.2053 Epoch 00036: val_loss improved from 5.53217 to 5.12386, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 151ms/step - loss: 5.2053 - val_loss: 5.1239 Epoch 37/100 63/63 [==============================] - ETA: 0s - loss: 5.1313 Epoch 00037: val_loss did not improve from 5.12386 63/63 [==============================] - 10s 151ms/step - loss: 5.1313 - val_loss: 5.4147 Epoch 38/100 63/63 [==============================] - ETA: 0s - loss: 4.9444 Epoch 00038: val_loss did not improve from 5.12386 63/63 [==============================] - 10s 153ms/step - loss: 4.9444 - val_loss: 5.3203 Epoch 39/100 63/63 [==============================] - ETA: 0s - loss: 4.7528 Epoch 00039: val_loss improved from 5.12386 to 5.07198, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 153ms/step - loss: 4.7528 - val_loss: 5.0720 Epoch 40/100 63/63 [==============================] - ETA: 0s - loss: 4.9064 Epoch 00040: val_loss improved from 5.07198 to 4.95759, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 150ms/step - loss: 4.9064 - val_loss: 4.9576 Epoch 41/100 63/63 [==============================] - ETA: 0s - loss: 4.7586 Epoch 00041: val_loss improved from 4.95759 to 4.76374, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 157ms/step - loss: 4.7586 - val_loss: 4.7637 Epoch 42/100 63/63 [==============================] - ETA: 0s - loss: 4.6515 Epoch 00042: val_loss did not improve from 4.76374 63/63 [==============================] - 10s 153ms/step - loss: 4.6515 - val_loss: 4.8551 Epoch 43/100 63/63 [==============================] - ETA: 0s - loss: 4.4771 Epoch 00043: val_loss improved from 4.76374 to 4.75639, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 154ms/step - loss: 4.4771 - val_loss: 4.7564 Epoch 44/100 63/63 [==============================] - ETA: 0s - loss: 4.4824 Epoch 00044: val_loss improved from 4.75639 to 4.56187, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 153ms/step - loss: 4.4824 - val_loss: 4.5619 Epoch 45/100 63/63 [==============================] - ETA: 0s - loss: 4.2294 Epoch 00045: val_loss did not improve from 4.56187 63/63 [==============================] - 10s 153ms/step - loss: 4.2294 - val_loss: 4.6618 Epoch 46/100 63/63 [==============================] - ETA: 0s - loss: 4.4671 Epoch 00046: val_loss did not improve from 4.56187 63/63 [==============================] - 10s 156ms/step - loss: 4.4671 - val_loss: 4.6246 Epoch 47/100 63/63 [==============================] - ETA: 0s - loss: 4.3131 Epoch 00047: val_loss improved from 4.56187 to 4.46177, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 157ms/step - loss: 4.3131 - val_loss: 4.4618 Epoch 48/100 63/63 [==============================] - ETA: 0s - loss: 4.3539 Epoch 00048: val_loss improved from 4.46177 to 4.43250, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 155ms/step - loss: 4.3539 - val_loss: 4.4325 Epoch 49/100 63/63 [==============================] - ETA: 0s - loss: 4.2040 Epoch 00049: val_loss improved from 4.43250 to 4.40153, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 154ms/step - loss: 4.2040 - val_loss: 4.4015 Epoch 50/100 63/63 [==============================] - ETA: 0s - loss: 4.1489 Epoch 00050: val_loss improved from 4.40153 to 4.25284, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 157ms/step - loss: 4.1489 - val_loss: 4.2528 Epoch 51/100 63/63 [==============================] - ETA: 0s - loss: 4.2695 Epoch 00051: val_loss did not improve from 4.25284 63/63 [==============================] - 9s 149ms/step - loss: 4.2695 - val_loss: 4.3561 Epoch 52/100 63/63 [==============================] - ETA: 0s - loss: 4.1450 Epoch 00052: val_loss improved from 4.25284 to 4.24516, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 153ms/step - loss: 4.1450 - val_loss: 4.2452 Epoch 53/100 63/63 [==============================] - ETA: 0s - loss: 3.9198 Epoch 00053: val_loss improved from 4.24516 to 4.17600, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 156ms/step - loss: 3.9198 - val_loss: 4.1760 Epoch 54/100 63/63 [==============================] - ETA: 0s - loss: 3.9327 Epoch 00054: val_loss improved from 4.17600 to 4.00178, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 155ms/step - loss: 3.9327 - val_loss: 4.0018 Epoch 55/100 63/63 [==============================] - ETA: 0s - loss: 3.9229 Epoch 00055: val_loss improved from 4.00178 to 3.97109, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 159ms/step - loss: 3.9229 - val_loss: 3.9711 Epoch 56/100 63/63 [==============================] - ETA: 0s - loss: 3.9287 Epoch 00056: val_loss did not improve from 3.97109 63/63 [==============================] - 10s 155ms/step - loss: 3.9287 - val_loss: 4.1734 Epoch 57/100 63/63 [==============================] - ETA: 0s - loss: 3.8066 Epoch 00057: val_loss did not improve from 3.97109 63/63 [==============================] - 10s 154ms/step - loss: 3.8066 - val_loss: 3.9714 Epoch 58/100 63/63 [==============================] - ETA: 0s - loss: 3.6586 Epoch 00058: val_loss did not improve from 3.97109 63/63 [==============================] - 10s 156ms/step - loss: 3.6586 - val_loss: 4.0335 Epoch 59/100 63/63 [==============================] - ETA: 0s - loss: 3.8364 Epoch 00059: val_loss did not improve from 3.97109 63/63 [==============================] - 10s 154ms/step - loss: 3.8364 - val_loss: 4.3106 Epoch 60/100 63/63 [==============================] - ETA: 0s - loss: 3.7454 Epoch 00060: val_loss improved from 3.97109 to 3.92563, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 157ms/step - loss: 3.7454 - val_loss: 3.9256 Epoch 61/100 63/63 [==============================] - ETA: 0s - loss: 3.7205 Epoch 00061: val_loss improved from 3.92563 to 3.83525, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 155ms/step - loss: 3.7205 - val_loss: 3.8352 Epoch 62/100 63/63 [==============================] - ETA: 0s - loss: 3.7276 Epoch 00062: val_loss improved from 3.83525 to 3.72713, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 146ms/step - loss: 3.7276 - val_loss: 3.7271 Epoch 63/100 63/63 [==============================] - ETA: 0s - loss: 3.5270 Epoch 00063: val_loss did not improve from 3.72713 63/63 [==============================] - 9s 149ms/step - loss: 3.5270 - val_loss: 3.8569 Epoch 64/100 63/63 [==============================] - ETA: 0s - loss: 3.5081 Epoch 00064: val_loss improved from 3.72713 to 3.65038, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 157ms/step - loss: 3.5081 - val_loss: 3.6504 Epoch 65/100 63/63 [==============================] - ETA: 0s - loss: 3.5176 Epoch 00065: val_loss did not improve from 3.65038 63/63 [==============================] - 9s 143ms/step - loss: 3.5176 - val_loss: 3.7427 Epoch 66/100 63/63 [==============================] - ETA: 0s - loss: 3.5056 Epoch 00066: val_loss did not improve from 3.65038 63/63 [==============================] - 10s 155ms/step - loss: 3.5056 - val_loss: 3.7221 Epoch 67/100 63/63 [==============================] - ETA: 0s - loss: 3.4142 Epoch 00067: val_loss improved from 3.65038 to 3.63731, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 151ms/step - loss: 3.4142 - val_loss: 3.6373 Epoch 68/100 63/63 [==============================] - ETA: 0s - loss: 3.4723 Epoch 00068: val_loss did not improve from 3.63731 63/63 [==============================] - 10s 153ms/step - loss: 3.4723 - val_loss: 3.7336 Epoch 69/100 63/63 [==============================] - ETA: 0s - loss: 3.5713 Epoch 00069: val_loss did not improve from 3.63731 63/63 [==============================] - 9s 150ms/step - loss: 3.5713 - val_loss: 3.7312 Epoch 70/100 63/63 [==============================] - ETA: 0s - loss: 3.4589 Epoch 00070: val_loss did not improve from 3.63731 63/63 [==============================] - 9s 150ms/step - loss: 3.4589 - val_loss: 3.7251 Epoch 71/100 63/63 [==============================] - ETA: 0s - loss: 3.3956 Epoch 00071: val_loss improved from 3.63731 to 3.57424, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 156ms/step - loss: 3.3956 - val_loss: 3.5742 Epoch 72/100 63/63 [==============================] - ETA: 0s - loss: 3.2909 Epoch 00072: val_loss improved from 3.57424 to 3.45337, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 155ms/step - loss: 3.2909 - val_loss: 3.4534 Epoch 73/100 63/63 [==============================] - ETA: 0s - loss: 3.2720 Epoch 00073: val_loss did not improve from 3.45337 63/63 [==============================] - 10s 153ms/step - loss: 3.2720 - val_loss: 3.5952 Epoch 74/100 63/63 [==============================] - ETA: 0s - loss: 3.2963 Epoch 00074: val_loss did not improve from 3.45337 63/63 [==============================] - 9s 148ms/step - loss: 3.2963 - val_loss: 3.5012 Epoch 75/100 63/63 [==============================] - ETA: 0s - loss: 3.2761 Epoch 00075: val_loss did not improve from 3.45337 63/63 [==============================] - 9s 144ms/step - loss: 3.2761 - val_loss: 3.5266 Epoch 76/100 63/63 [==============================] - ETA: 0s - loss: 3.4667 Epoch 00076: val_loss improved from 3.45337 to 3.33281, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 151ms/step - loss: 3.4667 - val_loss: 3.3328 Epoch 77/100 63/63 [==============================] - ETA: 0s - loss: 3.3475 Epoch 00077: val_loss did not improve from 3.33281 63/63 [==============================] - 10s 155ms/step - loss: 3.3475 - val_loss: 3.3598 Epoch 78/100 63/63 [==============================] - ETA: 0s - loss: 3.1759 Epoch 00078: val_loss did not improve from 3.33281 63/63 [==============================] - 10s 152ms/step - loss: 3.1759 - val_loss: 3.3337 Epoch 79/100 63/63 [==============================] - ETA: 0s - loss: 3.0986 Epoch 00079: val_loss improved from 3.33281 to 3.29266, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 9s 147ms/step - loss: 3.0986 - val_loss: 3.2927 Epoch 80/100 63/63 [==============================] - ETA: 0s - loss: 3.0322 Epoch 00080: val_loss did not improve from 3.29266 63/63 [==============================] - 9s 147ms/step - loss: 3.0322 - val_loss: 3.3252 Epoch 81/100 63/63 [==============================] - ETA: 0s - loss: 3.2318 Epoch 00081: val_loss did not improve from 3.29266 63/63 [==============================] - 10s 151ms/step - loss: 3.2318 - val_loss: 3.3663 Epoch 82/100 63/63 [==============================] - ETA: 0s - loss: 3.1674 Epoch 00082: val_loss improved from 3.29266 to 3.22915, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 154ms/step - loss: 3.1674 - val_loss: 3.2292 Epoch 83/100 63/63 [==============================] - ETA: 0s - loss: 3.0537 Epoch 00083: val_loss did not improve from 3.22915 63/63 [==============================] - 10s 155ms/step - loss: 3.0537 - val_loss: 3.2927 Epoch 84/100 63/63 [==============================] - ETA: 0s - loss: 3.2331 Epoch 00084: val_loss did not improve from 3.22915 63/63 [==============================] - 9s 148ms/step - loss: 3.2331 - val_loss: 3.2382 Epoch 85/100 63/63 [==============================] - ETA: 0s - loss: 3.1364 Epoch 00085: val_loss improved from 3.22915 to 3.14885, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 158ms/step - loss: 3.1364 - val_loss: 3.1488 Epoch 86/100 63/63 [==============================] - ETA: 0s - loss: 3.0854 Epoch 00086: val_loss did not improve from 3.14885 63/63 [==============================] - 10s 154ms/step - loss: 3.0854 - val_loss: 3.1535 Epoch 87/100 63/63 [==============================] - ETA: 0s - loss: 3.1131 Epoch 00087: val_loss improved from 3.14885 to 3.13276, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 157ms/step - loss: 3.1131 - val_loss: 3.1328 Epoch 88/100 63/63 [==============================] - ETA: 0s - loss: 3.0066 Epoch 00088: val_loss did not improve from 3.13276 63/63 [==============================] - 9s 151ms/step - loss: 3.0066 - val_loss: 3.3098 Epoch 89/100 63/63 [==============================] - ETA: 0s - loss: 3.1110 Epoch 00089: val_loss did not improve from 3.13276 63/63 [==============================] - 10s 152ms/step - loss: 3.1110 - val_loss: 3.2055 Epoch 90/100 63/63 [==============================] - ETA: 0s - loss: 2.9959 Epoch 00090: val_loss did not improve from 3.13276 63/63 [==============================] - 10s 155ms/step - loss: 2.9959 - val_loss: 3.1451 Epoch 91/100 63/63 [==============================] - ETA: 0s - loss: 2.9217 Epoch 00091: val_loss improved from 3.13276 to 3.06143, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 156ms/step - loss: 2.9217 - val_loss: 3.0614 Epoch 92/100 63/63 [==============================] - ETA: 0s - loss: 3.0291 Epoch 00092: val_loss improved from 3.06143 to 3.03809, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 153ms/step - loss: 3.0291 - val_loss: 3.0381 Epoch 93/100 63/63 [==============================] - ETA: 0s - loss: 3.0642 Epoch 00093: val_loss did not improve from 3.03809 63/63 [==============================] - 10s 157ms/step - loss: 3.0642 - val_loss: 3.0797 Epoch 94/100 63/63 [==============================] - ETA: 0s - loss: 2.9805 Epoch 00094: val_loss did not improve from 3.03809 63/63 [==============================] - 10s 153ms/step - loss: 2.9805 - val_loss: 3.1852 Epoch 95/100 63/63 [==============================] - ETA: 0s - loss: 2.7443 Epoch 00095: val_loss did not improve from 3.03809 63/63 [==============================] - 10s 153ms/step - loss: 2.7443 - val_loss: 3.0724 Epoch 96/100 63/63 [==============================] - ETA: 0s - loss: 2.9764 Epoch 00096: val_loss did not improve from 3.03809 63/63 [==============================] - 9s 150ms/step - loss: 2.9764 - val_loss: 3.0789 Epoch 97/100 63/63 [==============================] - ETA: 0s - loss: 2.8859 Epoch 00097: val_loss improved from 3.03809 to 2.96529, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 156ms/step - loss: 2.8859 - val_loss: 2.9653 Epoch 98/100 63/63 [==============================] - ETA: 0s - loss: 2.9004 Epoch 00098: val_loss did not improve from 2.96529 63/63 [==============================] - 9s 149ms/step - loss: 2.9004 - val_loss: 2.9847 Epoch 99/100 63/63 [==============================] - ETA: 0s - loss: 2.7298 Epoch 00099: val_loss improved from 2.96529 to 2.91299, saving model to /home/ssac24/aiffel/ocr/model_checkpoint.hdf5 63/63 [==============================] - 10s 157ms/step - loss: 2.7298 - val_loss: 2.9130 Epoch 100/100 63/63 [==============================] - ETA: 0s - loss: 2.7667 Epoch 00100: val_loss did not improve from 2.91299 63/63 [==============================] - 10s 154ms/step - loss: 2.7667 - val_loss: 2.9559
Out[12]:
<tensorflow.python.keras.callbacks.History at 0x7efde4541f50>
In [13]:
from IPython.display import display
test_set = MJDatasetSequence(TEST_DATA_PATH, label_converter, batch_size=BATCH_SIZE, character=TARGET_CHARACTERS)
model, y_func = build_crnn_model()
model.load_weights(checkpoint_path)
input_data = model.get_layer('input_image').output
y_pred = model.get_layer('output').output
model_pred = Model(inputs=input_data, outputs=y_pred)
def decode_predict_ctc(out, chars = TARGET_CHARACTERS, top_paths = 1):
results = []
beam_width = 5
if beam_width < top_paths:
beam_width = top_paths
for i in range(top_paths):
indexes = K.get_value(
K.ctc_decode(
out, input_length = np.ones(out.shape[0]) * out.shape[1],
greedy =False , beam_width = beam_width, top_paths = top_paths
)[0][i]
)[0]
text = ""
for index in indexes:
text += chars[index]
results.append(text)
return results
def check_inference(model, dataset, index = 5):
for i in range(index):
inputs, outputs = dataset[i]
img = dataset[i][0]['input_image'][0:1,:,:,:]
output = model_pred.predict(img)
result = decode_predict_ctc(output, chars="-"+TARGET_CHARACTERS)[0].replace('-','')
print("Result: \t", result)
display(Image.fromarray(img[0].transpose(1,0,2).astype(np.uint8)))
check_inference(model, test_set, index=10)
WARNING:tensorflow:From /home/ssac24/anaconda3/envs/aiffel/lib/python3.7/site-packages/tensorflow/python/util/dispatch.py:201: sparse_to_dense (from tensorflow.python.ops.sparse_ops) is deprecated and will be removed in a future version. Instructions for updating: Create a `tf.sparse.SparseTensor` and use `tf.sparse.to_dense` instead. Result: SLINKING9999999999999999
Result: ALEPPO99999999999999999
Result: FWANDA999999999999999999
Result: NATHANS99999999999999999
Result: HALLEO99999999999999999
Result: HURLS9999999999999999999
Result: DOWNSIZE9999999999999999
Result: ROBOTIC99999999999999999
Result: SLOPPY99999999999999999
Result: HERMITE99999999999999999
keras-ocr의 Detector¶
In [14]:
from keras_ocr.detection import Detector
SAMPLE_IMG_PATH = 'sample.jpg'
detector = Detector()
Looking for /home/ssac24/.keras-ocr/craft_mlt_25k.h5
In [23]:
def detect_text(img_path):
# TODO
# 이미지 불러오기
img_pil = Image.open(img_path)
img_pil = img_pil.resize((640, 640))
img_draw=ImageDraw.Draw(img_pil)
result_img = img_pil
img = tf.io.read_file(img_path)
img = tf.image.decode_jpeg(img)
img = tf.image.resize(img, (640, 640))
img = img[tf.newaxis, :, :, :]
# 배치 크기를 위해서 dimension을 확장해주고 kera-ocr의 입력 차원에 맞게 H,W,C로 변경합니다.
det_result = detector.detect(img.numpy())
# 배치의 첫 번째 결과만 가져옵니다.
ocr_result = det_result[0]
# 시각화를 위해서 x와 y좌표를 변경해줍니다. (앞선 h dimension으로 인해 y,x로 표기됨)
cropped_imgs = []
for text_result in ocr_result:
img_draw.polygon(text_result, outline='red')
x_min = text_result[:,0].min() - 5
x_max = text_result[:,0].max() + 5
y_min = text_result[:,1].min() - 5
y_max = text_result[:,1].max() + 5
word_box = [x_min, y_min, x_max, y_max]
cropped_imgs.append(img_pil.crop(word_box))
return result_img, cropped_imgs
In [24]:
from PIL import Image, ImageDraw
from IPython.display import display
img_pil, cropped_img = detect_text(SAMPLE_IMG_PATH)
display(img_pil)
In [25]:
def recognize_img(pil_img, input_img_size=(100,32)):
# TODO: 잘려진 단어 이미지를 인식하는 코드를 작성하세요!
pil_img = pil_img.resize(input_img_size)
np_img = np.array(pil_img)
np_img = np.transpose(np_img, (1, 0, 2))
np_img = np_img[np.newaxis, :, :, :]
output = model_pred.predict(np_img)
result = decode_predict_ctc(output, chars="-"+TARGET_CHARACTERS)[0].replace('-','')
print("Result: \t", result)
display(Image.fromarray(np.array(pil_img).astype(np.uint8)))
In [26]:
for _img in cropped_img:
recognize_img(_img)
Result: URUIGE999999999999999999
Result: SLEEPER9999999999999999
'Going Deeper' 카테고리의 다른 글
Human-in-the-loop & Active Learning (0) | 2021.05.05 |
---|---|
moviepy와 opencv로 동영상 처리(칼만필터 적용) (0) | 2021.05.05 |
동영상에 스티커 붙이기 (0) | 2021.05.05 |
Camera Sticker 붙여보기 (0) | 2021.04.26 |
OCR의 개요 (0) | 2021.04.24 |