Practical Machine Learning

by Sven Mayer

Lecture 11: Over and Underfitting

The material is licensed under the Creative Commons Attribution-Share Alike 4.0 (CC BY-SA) license: https://creativecommons.org/licenses/by-sa/4.0

import os
os.environ["CUDA_VISIBLE_DEVICES"]="1"

import sys
print("Python version: ", sys.version)
import numpy as np
print("numpy version", np.__version__)
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
print("matplotlib version", matplotlib.__version__)
import tensorflow as tf
print("TF:", tf.__version__)

physical_devices = tf.config.experimental.list_physical_devices('GPU')
if (len(physical_devices) > 0):
    tf.config.set_visible_devices(physical_devices[0], 'GPU')
    tf.config.experimental.set_memory_growth(physical_devices[0], True)
else:
    print("TF: No GPU found")

import pandas as pd

from tqdm import tqdm

Python version:  3.9.1 (default, Jan  8 2021, 17:17:17) 
[Clang 12.0.0 (clang-1200.0.32.28)]
numpy version 1.19.5
matplotlib version 3.3.3
INFO:tensorflow:Enabling eager execution
INFO:tensorflow:Enabling v2 tensorshape
INFO:tensorflow:Enabling resource variables
INFO:tensorflow:Enabling tensor equality
INFO:tensorflow:Enabling control flow v2
TF: 2.5.0-rc0
TF: No GPU found

Traditional Example

from sklearn.pipeline import make_pipeline
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import Ridge

x = np.linspace(-4,4,9*10)
r = np.random.RandomState(42)
y = np.sin(x) + r.normal(size=len(x)) * 0.5

reg = LinearRegression().fit(x.reshape(-1, 1), y)
plt.scatter (x, y)
plt.plot (x, reg.predict(x.reshape(-1, 1)), c="r")
plt.xlim(-4, 4)
plt.title("Underfitting")
plt.savefig("./figures/11_fit_u.png", dpi=500, bbox_inches = 'tight', pad_inches = 0)
plt.show()

model = make_pipeline(PolynomialFeatures(degree=3), Ridge())
model.fit(x.reshape(-1, 1), y.reshape(-1, 1))
plt.scatter (x, y)
plt.plot (x,  model.predict(x.reshape(-1, 1)), c="r")
plt.xlim(-4, 4)
plt.title("Good fit")
plt.savefig("./figures/11_fit_n.png", dpi=500, bbox_inches = 'tight', pad_inches = 0)
plt.show()

model = make_pipeline(PolynomialFeatures(degree=30), Ridge())
model.fit(x.reshape(-1, 1), y.reshape(-1, 1))
plt.scatter (x, y)
plt.plot (x,  model.predict(x.reshape(-1, 1)), c="r")
plt.xlim(-4, 4)
plt.title("Overfitting")
plt.savefig("./figures/11_fit_o.png", dpi=500, bbox_inches = 'tight', pad_inches = 0)
plt.show()

/usr/local/lib/python3.9/site-packages/sklearn/linear_model/_ridge.py:147: LinAlgWarning: Ill-conditioned matrix (rcond=2.81718e-37): result may not be accurate.
  return linalg.solve(A, Xy, sym_pos=True,

ML Model

df = pd.read_pickle("./dataKnckleTouch/knuckletouch_only3ps.pkl")

dfTrain = df[df.userID.isin([1,2])]
dfValidation = df[df.userID.isin([3])]

x_train = np.stack(dfTrain.Blob1D.to_list())
y_train = np.stack(dfTrain.Label.to_list())
print(x_train.shape, y_train.shape)

x_val = np.stack(dfValidation.Blob1D.to_list())
y_val = np.stack(dfValidation.Label.to_list())
print(x_val.shape, y_val.shape)

input_size = x_val.shape[1:]
print(input_size)

(14662, 400) (14662, 2)
(6342, 400) (6342, 2)
(400,)

Underfitting

def getModel():
    model = tf.keras.Sequential()

    model.add(tf.keras.layers.InputLayer(input_size, name = "InputLayer"))

    model.add(tf.keras.layers.Flatten())
    #model.add(tf.keras.layers.Dense(128, name = "HiddenLayer1", activation='relu'))
    model.add(tf.keras.layers.Dense(8, name = "HiddenLayer1", activation='relu'))
    model.add(tf.keras.layers.Dense(2, name = "OutputLayer", activation='softmax'))
    return model



model = getModel()

model.summary()

learning_rate = 0.0001
optimizer = tf.keras.optimizers.Adamax(learning_rate=learning_rate)
lossFunction = tf.keras.losses.CategoricalCrossentropy()
model.compile(optimizer=optimizer, loss=lossFunction, metrics=['accuracy'])
history = model.fit(x_train, y_train,
                validation_data = (x_val, y_val),
                epochs=25,
                verbose=1)

plt.plot(history.history['accuracy'], label="Train")
plt.plot(history.history['val_accuracy'], label="Validation")

plt.ylim(.5, 1)
#plt.text (0.1,.9, "$lr=0.001$", size=14)
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend(loc=4)
plt.savefig("./figures/11_fitting_under.png", dpi=500, bbox_inches = 'tight', pad_inches = 0)
plt.show()

Model: "sequential_5"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
flatten_5 (Flatten)          (None, 400)               0         
_________________________________________________________________
HiddenLayer1 (Dense)         (None, 8)                 3208      
_________________________________________________________________
OutputLayer (Dense)          (None, 2)                 18        
=================================================================
Total params: 3,226
Trainable params: 3,226
Non-trainable params: 0
_________________________________________________________________
Epoch 1/25
459/459 [==============================] - 1s 1ms/step - loss: 42.2298 - accuracy: 0.3397 - val_loss: 1.9345 - val_accuracy: 0.7512
Epoch 2/25
459/459 [==============================] - 0s 860us/step - loss: 3.0677 - accuracy: 0.5859 - val_loss: 1.0440 - val_accuracy: 0.8042
Epoch 3/25
459/459 [==============================] - 0s 887us/step - loss: 1.7666 - accuracy: 0.6200 - val_loss: 0.7640 - val_accuracy: 0.8206
Epoch 4/25
459/459 [==============================] - 0s 838us/step - loss: 1.0043 - accuracy: 0.6621 - val_loss: 0.9057 - val_accuracy: 0.7723
Epoch 5/25
459/459 [==============================] - 0s 830us/step - loss: 0.6701 - accuracy: 0.7608 - val_loss: 1.1810 - val_accuracy: 0.6991
Epoch 6/25
459/459 [==============================] - 0s 821us/step - loss: 0.5863 - accuracy: 0.8058 - val_loss: 1.3520 - val_accuracy: 0.6712
Epoch 7/25
459/459 [==============================] - 0s 827us/step - loss: 0.5254 - accuracy: 0.8302 - val_loss: 1.3638 - val_accuracy: 0.6709
Epoch 8/25
459/459 [==============================] - 0s 823us/step - loss: 0.4875 - accuracy: 0.8440 - val_loss: 1.3181 - val_accuracy: 0.6779
Epoch 9/25
459/459 [==============================] - 0s 834us/step - loss: 0.4541 - accuracy: 0.8489 - val_loss: 1.3490 - val_accuracy: 0.6791
Epoch 10/25
459/459 [==============================] - 0s 830us/step - loss: 0.4528 - accuracy: 0.8556 - val_loss: 1.2664 - val_accuracy: 0.6906
Epoch 11/25
459/459 [==============================] - 0s 833us/step - loss: 0.4090 - accuracy: 0.8658 - val_loss: 1.3198 - val_accuracy: 0.6835
Epoch 12/25
459/459 [==============================] - 0s 825us/step - loss: 0.4037 - accuracy: 0.8640 - val_loss: 1.1955 - val_accuracy: 0.7099
Epoch 13/25
459/459 [==============================] - 0s 862us/step - loss: 0.4263 - accuracy: 0.8705 - val_loss: 1.2192 - val_accuracy: 0.7047
Epoch 14/25
459/459 [==============================] - 0s 828us/step - loss: 0.3715 - accuracy: 0.8768 - val_loss: 1.2216 - val_accuracy: 0.7062
Epoch 15/25
459/459 [==============================] - 0s 837us/step - loss: 0.3764 - accuracy: 0.8742 - val_loss: 1.2153 - val_accuracy: 0.7066
Epoch 16/25
459/459 [==============================] - 0s 842us/step - loss: 0.3741 - accuracy: 0.8768 - val_loss: 1.1773 - val_accuracy: 0.7107
Epoch 17/25
459/459 [==============================] - 0s 829us/step - loss: 0.3552 - accuracy: 0.8821 - val_loss: 1.2782 - val_accuracy: 0.6985
Epoch 18/25
459/459 [==============================] - 0s 833us/step - loss: 0.3698 - accuracy: 0.8810 - val_loss: 1.1322 - val_accuracy: 0.7148
Epoch 19/25
459/459 [==============================] - 0s 835us/step - loss: 0.3474 - accuracy: 0.8870 - val_loss: 1.1909 - val_accuracy: 0.7088
Epoch 20/25
459/459 [==============================] - 0s 836us/step - loss: 0.3333 - accuracy: 0.8870 - val_loss: 1.1522 - val_accuracy: 0.7163
Epoch 21/25
459/459 [==============================] - 0s 829us/step - loss: 0.3377 - accuracy: 0.8866 - val_loss: 1.0892 - val_accuracy: 0.7214
Epoch 22/25
459/459 [==============================] - 0s 823us/step - loss: 0.3355 - accuracy: 0.8855 - val_loss: 1.1264 - val_accuracy: 0.7196
Epoch 23/25
459/459 [==============================] - 0s 829us/step - loss: 0.3589 - accuracy: 0.8831 - val_loss: 1.0957 - val_accuracy: 0.7226
Epoch 24/25
459/459 [==============================] - 0s 859us/step - loss: 0.3318 - accuracy: 0.8871 - val_loss: 1.0896 - val_accuracy: 0.7249
Epoch 25/25
459/459 [==============================] - 0s 852us/step - loss: 0.3427 - accuracy: 0.8883 - val_loss: 1.0232 - val_accuracy: 0.7356

Overfitting

def getModel():
    model = tf.keras.Sequential()

    model.add(tf.keras.layers.InputLayer(input_size, name = "InputLayer"))

    model.add(tf.keras.layers.Flatten())
    model.add(tf.keras.layers.Dense(512, name = "HiddenLayer1", activation='relu'))
    model.add(tf.keras.layers.Dense(256, name = "HiddenLayer2", activation='relu'))
    model.add(tf.keras.layers.Dense(2, name = "OutputLayer", activation='softmax'))
    return model



model = getModel()

model.summary()

learning_rate = 0.0001
optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
lossFunction = tf.keras.losses.CategoricalCrossentropy()
model.compile(optimizer=optimizer, loss=lossFunction, metrics=['accuracy'])
history = model.fit(x_train, y_train,
                validation_data = (x_val, y_val),
                epochs=50,
                verbose=1)

plt.plot(history.history['accuracy'], label="Train")
plt.plot(history.history['val_accuracy'], label="Validation")

plt.ylim(0.5, 1)
#plt.text (0.1,.9, f"$lr={learning_rate}$", size=12)
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend(loc=4)
plt.savefig("./figures/11_fitting_over.png", dpi=500, bbox_inches = 'tight', pad_inches = 0)
plt.show()

Model: "sequential_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
flatten_1 (Flatten)          (None, 400)               0         
_________________________________________________________________
HiddenLayer1 (Dense)         (None, 512)               205312    
_________________________________________________________________
HiddenLayer2 (Dense)         (None, 256)               131328    
_________________________________________________________________
OutputLayer (Dense)          (None, 2)                 514       
=================================================================
Total params: 337,154
Trainable params: 337,154
Non-trainable params: 0
_________________________________________________________________
Epoch 1/50
459/459 [==============================] - 2s 3ms/step - loss: 0.5944 - accuracy: 0.8424 - val_loss: 0.7794 - val_accuracy: 0.8091
Epoch 2/50
459/459 [==============================] - 1s 2ms/step - loss: 0.1903 - accuracy: 0.9298 - val_loss: 0.7946 - val_accuracy: 0.8023
Epoch 3/50
459/459 [==============================] - 1s 2ms/step - loss: 0.1690 - accuracy: 0.9420 - val_loss: 0.7389 - val_accuracy: 0.8089
Epoch 4/50
459/459 [==============================] - 1s 2ms/step - loss: 0.1447 - accuracy: 0.9468 - val_loss: 1.0870 - val_accuracy: 0.7674
Epoch 5/50
459/459 [==============================] - 1s 2ms/step - loss: 0.1334 - accuracy: 0.9526 - val_loss: 1.1763 - val_accuracy: 0.7327
Epoch 6/50
459/459 [==============================] - 1s 2ms/step - loss: 0.1295 - accuracy: 0.9511 - val_loss: 0.5640 - val_accuracy: 0.8373
Epoch 7/50
459/459 [==============================] - 1s 2ms/step - loss: 0.1376 - accuracy: 0.9503 - val_loss: 0.6187 - val_accuracy: 0.8461
Epoch 8/50
459/459 [==============================] - 1s 2ms/step - loss: 0.1108 - accuracy: 0.9586 - val_loss: 0.7486 - val_accuracy: 0.8284
Epoch 9/50
459/459 [==============================] - 1s 2ms/step - loss: 0.1103 - accuracy: 0.9592 - val_loss: 0.6249 - val_accuracy: 0.8499
Epoch 10/50
459/459 [==============================] - 1s 2ms/step - loss: 0.1024 - accuracy: 0.9620 - val_loss: 0.5297 - val_accuracy: 0.8761
Epoch 11/50
459/459 [==============================] - 1s 2ms/step - loss: 0.1073 - accuracy: 0.9601 - val_loss: 0.5906 - val_accuracy: 0.8729
Epoch 12/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0990 - accuracy: 0.9631 - val_loss: 0.4605 - val_accuracy: 0.8928
Epoch 13/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0934 - accuracy: 0.9661 - val_loss: 1.2620 - val_accuracy: 0.7638
Epoch 14/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0969 - accuracy: 0.9662 - val_loss: 0.5214 - val_accuracy: 0.8838
Epoch 15/50
459/459 [==============================] - 1s 2ms/step - loss: 0.1052 - accuracy: 0.9649 - val_loss: 0.6502 - val_accuracy: 0.8778
Epoch 16/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0804 - accuracy: 0.9691 - val_loss: 0.4316 - val_accuracy: 0.9060
Epoch 17/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0842 - accuracy: 0.9691 - val_loss: 0.7744 - val_accuracy: 0.8658
Epoch 18/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0826 - accuracy: 0.9712 - val_loss: 0.6128 - val_accuracy: 0.8895
Epoch 19/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0857 - accuracy: 0.9694 - val_loss: 0.4961 - val_accuracy: 0.9057
Epoch 20/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0729 - accuracy: 0.9732 - val_loss: 0.6437 - val_accuracy: 0.8896
Epoch 21/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0881 - accuracy: 0.9693 - val_loss: 0.8167 - val_accuracy: 0.8732
Epoch 22/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0661 - accuracy: 0.9769 - val_loss: 0.4701 - val_accuracy: 0.9197
Epoch 23/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0694 - accuracy: 0.9746 - val_loss: 0.4787 - val_accuracy: 0.9117
Epoch 24/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0742 - accuracy: 0.9745 - val_loss: 0.7375 - val_accuracy: 0.8937
Epoch 25/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0592 - accuracy: 0.9777 - val_loss: 0.9485 - val_accuracy: 0.8631
Epoch 26/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0608 - accuracy: 0.9771 - val_loss: 0.6857 - val_accuracy: 0.8981
Epoch 27/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0706 - accuracy: 0.9726 - val_loss: 0.7512 - val_accuracy: 0.8855
Epoch 28/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0580 - accuracy: 0.9796 - val_loss: 1.1550 - val_accuracy: 0.8374
Epoch 29/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0793 - accuracy: 0.9702 - val_loss: 0.7597 - val_accuracy: 0.8841
Epoch 30/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0567 - accuracy: 0.9784 - val_loss: 0.7789 - val_accuracy: 0.8833
Epoch 31/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0555 - accuracy: 0.9797 - val_loss: 0.9134 - val_accuracy: 0.8775
Epoch 32/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0501 - accuracy: 0.9815 - val_loss: 0.6042 - val_accuracy: 0.9084
Epoch 33/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0572 - accuracy: 0.9779 - val_loss: 0.6442 - val_accuracy: 0.9008
Epoch 34/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0570 - accuracy: 0.9774 - val_loss: 0.7024 - val_accuracy: 0.8918
Epoch 35/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0493 - accuracy: 0.9811 - val_loss: 0.8964 - val_accuracy: 0.8800
Epoch 36/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0400 - accuracy: 0.9858 - val_loss: 0.7889 - val_accuracy: 0.8830
Epoch 37/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0690 - accuracy: 0.9748 - val_loss: 0.9813 - val_accuracy: 0.8798
Epoch 38/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0417 - accuracy: 0.9850 - val_loss: 0.8416 - val_accuracy: 0.8933
Epoch 39/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0473 - accuracy: 0.9833 - val_loss: 0.8778 - val_accuracy: 0.8928
Epoch 40/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0421 - accuracy: 0.9842 - val_loss: 0.8554 - val_accuracy: 0.9041
Epoch 41/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0419 - accuracy: 0.9851 - val_loss: 0.7397 - val_accuracy: 0.9120
Epoch 42/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0446 - accuracy: 0.9830 - val_loss: 0.7996 - val_accuracy: 0.8970
Epoch 43/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0480 - accuracy: 0.9814 - val_loss: 0.8021 - val_accuracy: 0.9000
Epoch 44/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0469 - accuracy: 0.9836 - val_loss: 0.8742 - val_accuracy: 0.8903
Epoch 45/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0318 - accuracy: 0.9891 - val_loss: 0.8717 - val_accuracy: 0.8953
Epoch 46/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0344 - accuracy: 0.9870 - val_loss: 0.8852 - val_accuracy: 0.8955
Epoch 47/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0360 - accuracy: 0.9860 - val_loss: 0.8906 - val_accuracy: 0.8964
Epoch 48/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0366 - accuracy: 0.9860 - val_loss: 0.7045 - val_accuracy: 0.9095
Epoch 49/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0357 - accuracy: 0.9863 - val_loss: 0.9419 - val_accuracy: 0.8975
Epoch 50/50
459/459 [==============================] - 1s 2ms/step - loss: 0.0334 - accuracy: 0.9856 - val_loss: 0.8135 - val_accuracy: 0.9087

"Good" Fit

x_train_2 = np.stack(dfTrain.Blob1D.to_list()).reshape((-1, 20,20,1))
y_train_2 = np.stack(dfTrain.Label.to_list())
print(x_train.shape, y_train.shape)

x_val_2 = np.stack(dfValidation.Blob1D.to_list()).reshape((-1, 20,20,1))
y_val_2 = np.stack(dfValidation.Label.to_list())
print(x_val.shape, y_val.shape)

input_size = x_val_2.shape[1:]
print(input_size)

(14662, 400) (14662, 2)
(6342, 400) (6342, 2)
(20, 20, 1)

def getModel():
    model = tf.keras.Sequential()

    model.add(tf.keras.layers.InputLayer(input_size, name = "InputLayer"))
    
    model.add(tf.keras.layers.Conv2D(32, kernel_size=(3, 3), name = "Conv1", activation='relu'))
    
    model.add(tf.keras.layers.MaxPool2D())
    
    model.add(tf.keras.layers.Dropout(.5))

    model.add(tf.keras.layers.Flatten())
    model.add(tf.keras.layers.Dense(64, name = "HiddenLayer1", activation='relu', activity_regularizer=tf.keras.regularizers.L2(0.001)))
    model.add(tf.keras.layers.Dropout(.2))
    model.add(tf.keras.layers.Dense(2, name = "OutputLayer", activation='softmax'))
    return model



model = getModel()

model.summary()

learning_rate = 0.001
optimizer = tf.keras.optimizers.Adamax(learning_rate=learning_rate)
lossFunction = tf.keras.losses.CategoricalCrossentropy()
model.compile(optimizer=optimizer, loss=lossFunction, metrics=['accuracy'])
history = model.fit(x_train_2, y_train_2,
                validation_data = (x_val_2, y_val_2),
                epochs=50,
                verbose=1)

plt.plot(history.history['accuracy'], label="Train")
plt.plot(history.history['val_accuracy'], label="Validation")

plt.ylim(0.5, 1)
#plt.text (0.1,.9, f"$lr={learning_rate}$", size=12)
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend(loc=4)
plt.savefig("./figures/11_fitting.png", dpi=500, bbox_inches = 'tight', pad_inches = 0)
plt.show()

Model: "sequential_2"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
Conv1 (Conv2D)               (None, 18, 18, 32)        320       
_________________________________________________________________
max_pooling2d (MaxPooling2D) (None, 9, 9, 32)          0         
_________________________________________________________________
dropout (Dropout)            (None, 9, 9, 32)          0         
_________________________________________________________________
flatten_2 (Flatten)          (None, 2592)              0         
_________________________________________________________________
HiddenLayer1 (Dense)         (None, 64)                165952    
_________________________________________________________________
dropout_1 (Dropout)          (None, 64)                0         
_________________________________________________________________
OutputLayer (Dense)          (None, 2)                 130       
=================================================================
Total params: 166,402
Trainable params: 166,402
Non-trainable params: 0
_________________________________________________________________
Epoch 1/50
459/459 [==============================] - 3s 6ms/step - loss: 1.6792 - accuracy: 0.6303 - val_loss: 0.6392 - val_accuracy: 0.7029
Epoch 2/50
459/459 [==============================] - 2s 5ms/step - loss: 0.6118 - accuracy: 0.6781 - val_loss: 0.5991 - val_accuracy: 0.7029
Epoch 3/50
459/459 [==============================] - 2s 5ms/step - loss: 0.5344 - accuracy: 0.6889 - val_loss: 0.5315 - val_accuracy: 0.6805
Epoch 4/50
459/459 [==============================] - 2s 5ms/step - loss: 0.4738 - accuracy: 0.8112 - val_loss: 0.4411 - val_accuracy: 0.7939
Epoch 5/50
459/459 [==============================] - 2s 5ms/step - loss: 0.4034 - accuracy: 0.8459 - val_loss: 0.4073 - val_accuracy: 0.8409
Epoch 6/50
459/459 [==============================] - 3s 6ms/step - loss: 0.3492 - accuracy: 0.8735 - val_loss: 0.4161 - val_accuracy: 0.8396
Epoch 7/50
459/459 [==============================] - 3s 6ms/step - loss: 0.3101 - accuracy: 0.8911 - val_loss: 0.4146 - val_accuracy: 0.8370
Epoch 8/50
459/459 [==============================] - 3s 6ms/step - loss: 0.2993 - accuracy: 0.8992 - val_loss: 0.3605 - val_accuracy: 0.8619
Epoch 9/50
459/459 [==============================] - 3s 6ms/step - loss: 0.2593 - accuracy: 0.9104 - val_loss: 0.4159 - val_accuracy: 0.8259
Epoch 10/50
459/459 [==============================] - 3s 6ms/step - loss: 0.2558 - accuracy: 0.9141 - val_loss: 0.4069 - val_accuracy: 0.8411
Epoch 11/50
459/459 [==============================] - 3s 6ms/step - loss: 0.2313 - accuracy: 0.9265 - val_loss: 0.4030 - val_accuracy: 0.8466
Epoch 12/50
459/459 [==============================] - 3s 6ms/step - loss: 0.2328 - accuracy: 0.9192 - val_loss: 0.3977 - val_accuracy: 0.8433
Epoch 13/50
459/459 [==============================] - 3s 6ms/step - loss: 0.2196 - accuracy: 0.9254 - val_loss: 0.3795 - val_accuracy: 0.8546
Epoch 14/50
459/459 [==============================] - 3s 6ms/step - loss: 0.2141 - accuracy: 0.9259 - val_loss: 0.3912 - val_accuracy: 0.8541
Epoch 15/50
459/459 [==============================] - 3s 6ms/step - loss: 0.1993 - accuracy: 0.9346 - val_loss: 0.3762 - val_accuracy: 0.8619
Epoch 16/50
459/459 [==============================] - 3s 6ms/step - loss: 0.1980 - accuracy: 0.9328 - val_loss: 0.4238 - val_accuracy: 0.8398
Epoch 17/50
459/459 [==============================] - 3s 6ms/step - loss: 0.1902 - accuracy: 0.9367 - val_loss: 0.3767 - val_accuracy: 0.8584
Epoch 18/50
459/459 [==============================] - 3s 6ms/step - loss: 0.1900 - accuracy: 0.9367 - val_loss: 0.3179 - val_accuracy: 0.8776
Epoch 19/50
459/459 [==============================] - 3s 6ms/step - loss: 0.1827 - accuracy: 0.9396 - val_loss: 0.3329 - val_accuracy: 0.8707
Epoch 20/50
459/459 [==============================] - 3s 6ms/step - loss: 0.1845 - accuracy: 0.9394 - val_loss: 0.3568 - val_accuracy: 0.8619
Epoch 21/50
459/459 [==============================] - 3s 6ms/step - loss: 0.1912 - accuracy: 0.9353 - val_loss: 0.3477 - val_accuracy: 0.8679
Epoch 22/50
459/459 [==============================] - 3s 6ms/step - loss: 0.1752 - accuracy: 0.9415 - val_loss: 0.4046 - val_accuracy: 0.8529
Epoch 23/50
459/459 [==============================] - 3s 6ms/step - loss: 0.1720 - accuracy: 0.9442 - val_loss: 0.3343 - val_accuracy: 0.8726
Epoch 24/50
459/459 [==============================] - 3s 6ms/step - loss: 0.1628 - accuracy: 0.9470 - val_loss: 0.3318 - val_accuracy: 0.8757
Epoch 25/50
459/459 [==============================] - 3s 6ms/step - loss: 0.1695 - accuracy: 0.9442 - val_loss: 0.3494 - val_accuracy: 0.8688
Epoch 26/50
459/459 [==============================] - 3s 6ms/step - loss: 0.1671 - accuracy: 0.9454 - val_loss: 0.3204 - val_accuracy: 0.8797
Epoch 27/50
459/459 [==============================] - 3s 6ms/step - loss: 0.1552 - accuracy: 0.9468 - val_loss: 0.2935 - val_accuracy: 0.8862
Epoch 28/50
459/459 [==============================] - 3s 6ms/step - loss: 0.1681 - accuracy: 0.9421 - val_loss: 0.3235 - val_accuracy: 0.8791
Epoch 29/50
459/459 [==============================] - 3s 7ms/step - loss: 0.1585 - accuracy: 0.9460 - val_loss: 0.2822 - val_accuracy: 0.8934
Epoch 30/50
459/459 [==============================] - 3s 6ms/step - loss: 0.1595 - accuracy: 0.9442 - val_loss: 0.3230 - val_accuracy: 0.8819
Epoch 31/50
459/459 [==============================] - 3s 6ms/step - loss: 0.1622 - accuracy: 0.9443 - val_loss: 0.3175 - val_accuracy: 0.8836
Epoch 32/50
459/459 [==============================] - 3s 7ms/step - loss: 0.1486 - accuracy: 0.9504 - val_loss: 0.3376 - val_accuracy: 0.8761
Epoch 33/50
459/459 [==============================] - 3s 7ms/step - loss: 0.1552 - accuracy: 0.9465 - val_loss: 0.3919 - val_accuracy: 0.8627
Epoch 34/50
459/459 [==============================] - 3s 7ms/step - loss: 0.1511 - accuracy: 0.9479 - val_loss: 0.3240 - val_accuracy: 0.8825
Epoch 35/50
459/459 [==============================] - 3s 7ms/step - loss: 0.1608 - accuracy: 0.9459 - val_loss: 0.3595 - val_accuracy: 0.8728
Epoch 36/50
459/459 [==============================] - 3s 7ms/step - loss: 0.1381 - accuracy: 0.9545 - val_loss: 0.3561 - val_accuracy: 0.8720
Epoch 37/50
459/459 [==============================] - 3s 7ms/step - loss: 0.1488 - accuracy: 0.9506 - val_loss: 0.3532 - val_accuracy: 0.8698
Epoch 38/50
459/459 [==============================] - 3s 7ms/step - loss: 0.1468 - accuracy: 0.9518 - val_loss: 0.4087 - val_accuracy: 0.8611
Epoch 39/50
459/459 [==============================] - 3s 7ms/step - loss: 0.1356 - accuracy: 0.9558 - val_loss: 0.3703 - val_accuracy: 0.8746
Epoch 40/50
459/459 [==============================] - 3s 7ms/step - loss: 0.1540 - accuracy: 0.9470 - val_loss: 0.3393 - val_accuracy: 0.8780
Epoch 41/50
459/459 [==============================] - 3s 7ms/step - loss: 0.1501 - accuracy: 0.9491 - val_loss: 0.3827 - val_accuracy: 0.8631
Epoch 42/50
459/459 [==============================] - 3s 7ms/step - loss: 0.1410 - accuracy: 0.9526 - val_loss: 0.3587 - val_accuracy: 0.8687
Epoch 43/50
459/459 [==============================] - 3s 7ms/step - loss: 0.1470 - accuracy: 0.9500 - val_loss: 0.3404 - val_accuracy: 0.8757
Epoch 44/50
459/459 [==============================] - 3s 7ms/step - loss: 0.1422 - accuracy: 0.9504 - val_loss: 0.3568 - val_accuracy: 0.8710
Epoch 45/50
459/459 [==============================] - 4s 8ms/step - loss: 0.1389 - accuracy: 0.9529 - val_loss: 0.3049 - val_accuracy: 0.8858
Epoch 46/50
459/459 [==============================] - 3s 8ms/step - loss: 0.1474 - accuracy: 0.9512 - val_loss: 0.3118 - val_accuracy: 0.8808
Epoch 47/50
459/459 [==============================] - 4s 8ms/step - loss: 0.1427 - accuracy: 0.9516 - val_loss: 0.3703 - val_accuracy: 0.8655
Epoch 48/50
459/459 [==============================] - 3s 8ms/step - loss: 0.1418 - accuracy: 0.9514 - val_loss: 0.3847 - val_accuracy: 0.8666
Epoch 49/50
459/459 [==============================] - 4s 8ms/step - loss: 0.1444 - accuracy: 0.9513 - val_loss: 0.3433 - val_accuracy: 0.8751
Epoch 50/50
459/459 [==============================] - 3s 8ms/step - loss: 0.1338 - accuracy: 0.9559 - val_loss: 0.3474 - val_accuracy: 0.8746