Practical Machine Learning

by Sven Mayer

Lecture 10: Optimizer

The material is licensed under the Creative Commons Attribution-Share Alike 4.0 (CC BY-SA) license: https://creativecommons.org/licenses/by-sa/4.0

import os
os.environ["CUDA_VISIBLE_DEVICES"]="1"

import sys
print("Python version: ", sys.version)
import numpy as np
print("numpy version", np.__version__)
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
print("matplotlib version", matplotlib.__version__)
import tensorflow as tf
print("TF:", tf.__version__)

physical_devices = tf.config.experimental.list_physical_devices('GPU')
if (len(physical_devices) > 0):
    tf.config.set_visible_devices(physical_devices[0], 'GPU')
    tf.config.experimental.set_memory_growth(physical_devices[0], True)
else:
    print("TF: No GPU found")

import pandas as pd

from tqdm import tqdm

Python version:  3.9.1 (default, Jan  8 2021, 17:17:17) 
[Clang 12.0.0 (clang-1200.0.32.28)]
numpy version 1.19.5
matplotlib version 3.3.3
TF: 2.5.0-rc0
TF: No GPU found

(x_train, y_train), (x_val, y_val) = tf.keras.datasets.mnist.load_data()
x_train = x_train.reshape(-1, 28*28)[:30000,:]
x_val = x_val.reshape(-1, 28*28)[:5000,:]
y_train = tf.keras.utils.to_categorical(y_train)[:30000,:]
y_val = tf.keras.utils.to_categorical(y_val)[:5000,:]
input_size = x_val.shape[-1]
output_size = y_val.shape[-1]
print(input_size, output_size)

784 10

Optimizer Improvemetns

def getModel():
    model = tf.keras.Sequential()

    model.add(tf.keras.layers.InputLayer((input_size,), name = "InputLayer"))

    model.add(tf.keras.layers.Dense(256, name = "HiddenLayer1", activation = 'relu')) 
    model.add(tf.keras.layers.Dropout(.5)) 

    model.add(tf.keras.layers.Dense(128, name = "HiddenLayer2", activation = 'relu')) 
    model.add(tf.keras.layers.Dropout(.5)) 

    model.add(tf.keras.layers.Dense(output_size, name = "OutputLayer", activation = 'softmax'))
    return model

model = getModel()
model.summary()

Model: "sequential_4"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
HiddenLayer1 (Dense)         (None, 256)               200960    
_________________________________________________________________
dropout_8 (Dropout)          (None, 256)               0         
_________________________________________________________________
HiddenLayer2 (Dense)         (None, 128)               32896     
_________________________________________________________________
dropout_9 (Dropout)          (None, 128)               0         
_________________________________________________________________
OutputLayer (Dense)          (None, 10)                1290      
=================================================================
Total params: 235,146
Trainable params: 235,146
Non-trainable params: 0
_________________________________________________________________

lst = []
lossFunction = tf.keras.losses.CategoricalCrossentropy()
for optimizer in tqdm(["adadelta", "adagrad", "adam", "adamax", "ftrl", "nadam", "rmsprop", "sgd"]):
    model = getModel()
    model.compile(optimizer=optimizer, loss=lossFunction, metrics=['accuracy'])
    history = model.fit(x_train, y_train,
                    validation_data = (x_val, y_val),
                    epochs=25,
                       verbose=1)
    lst.append([optimizer, history])

  0%|          | 0/8 [00:00<?, ?it/s]

Epoch 1/25
938/938 [==============================] - 4s 4ms/step - loss: 228.2867 - accuracy: 0.1024 - val_loss: 84.1243 - val_accuracy: 0.1046
Epoch 2/25
938/938 [==============================] - 3s 4ms/step - loss: 194.1198 - accuracy: 0.1075 - val_loss: 63.6907 - val_accuracy: 0.1238
Epoch 3/25
938/938 [==============================] - 3s 4ms/step - loss: 169.3447 - accuracy: 0.1156 - val_loss: 50.5345 - val_accuracy: 0.1536
Epoch 4/25
938/938 [==============================] - 3s 4ms/step - loss: 150.3411 - accuracy: 0.1258 - val_loss: 41.3026 - val_accuracy: 0.1828
Epoch 5/25
938/938 [==============================] - 3s 4ms/step - loss: 135.1094 - accuracy: 0.1320 - val_loss: 34.7207 - val_accuracy: 0.2128
Epoch 6/25
938/938 [==============================] - 3s 4ms/step - loss: 122.5375 - accuracy: 0.1466 - val_loss: 29.9858 - val_accuracy: 0.2544
Epoch 7/25
938/938 [==============================] - 3s 4ms/step - loss: 112.8037 - accuracy: 0.1550 - val_loss: 26.5780 - val_accuracy: 0.2846
Epoch 8/25
938/938 [==============================] - 3s 4ms/step - loss: 104.6383 - accuracy: 0.1566 - val_loss: 23.9274 - val_accuracy: 0.3118
Epoch 9/25
938/938 [==============================] - 3s 4ms/step - loss: 96.5486 - accuracy: 0.1683 - val_loss: 21.8541 - val_accuracy: 0.3402
Epoch 10/25
938/938 [==============================] - 3s 4ms/step - loss: 90.0931 - accuracy: 0.1799 - val_loss: 20.1584 - val_accuracy: 0.3672
Epoch 11/25
938/938 [==============================] - 4s 4ms/step - loss: 85.5391 - accuracy: 0.1860 - val_loss: 18.7542 - val_accuracy: 0.3872
Epoch 12/25
938/938 [==============================] - 3s 4ms/step - loss: 80.1376 - accuracy: 0.1933 - val_loss: 17.5309 - val_accuracy: 0.4082
Epoch 13/25
938/938 [==============================] - 3s 4ms/step - loss: 75.9098 - accuracy: 0.2086 - val_loss: 16.4797 - val_accuracy: 0.4228
Epoch 14/25
938/938 [==============================] - 3s 4ms/step - loss: 69.9111 - accuracy: 0.2161 - val_loss: 15.5590 - val_accuracy: 0.4358
Epoch 15/25
938/938 [==============================] - 3s 3ms/step - loss: 67.8877 - accuracy: 0.2198 - val_loss: 14.7130 - val_accuracy: 0.4474
Epoch 16/25
938/938 [==============================] - 3s 4ms/step - loss: 64.4438 - accuracy: 0.2224 - val_loss: 13.9807 - val_accuracy: 0.4602
Epoch 17/25
938/938 [==============================] - 3s 3ms/step - loss: 61.7929 - accuracy: 0.2374 - val_loss: 13.3079 - val_accuracy: 0.4688
Epoch 18/25
938/938 [==============================] - 3s 3ms/step - loss: 58.5918 - accuracy: 0.2381 - val_loss: 12.7186 - val_accuracy: 0.4770
Epoch 19/25
938/938 [==============================] - 3s 4ms/step - loss: 56.2731 - accuracy: 0.2483 - val_loss: 12.1611 - val_accuracy: 0.4866
Epoch 20/25
938/938 [==============================] - 3s 3ms/step - loss: 54.6327 - accuracy: 0.2490 - val_loss: 11.6601 - val_accuracy: 0.4936
Epoch 21/25
938/938 [==============================] - 3s 4ms/step - loss: 51.9684 - accuracy: 0.2514 - val_loss: 11.2051 - val_accuracy: 0.5028
Epoch 22/25
938/938 [==============================] - 3s 4ms/step - loss: 50.0469 - accuracy: 0.2602 - val_loss: 10.8048 - val_accuracy: 0.5088
Epoch 23/25
938/938 [==============================] - 3s 4ms/step - loss: 48.3904 - accuracy: 0.2644 - val_loss: 10.4198 - val_accuracy: 0.5170
Epoch 24/25
938/938 [==============================] - 4s 4ms/step - loss: 46.4033 - accuracy: 0.2788 - val_loss: 10.0533 - val_accuracy: 0.5244
Epoch 25/25
938/938 [==============================] - 3s 4ms/step - loss: 44.4686 - accuracy: 0.2822 - val_loss: 9.7251 - val_accuracy: 0.5314

 12%|█▎        | 1/8 [01:25<09:59, 85.63s/it]

Epoch 1/25
938/938 [==============================] - 4s 4ms/step - loss: 61.8768 - accuracy: 0.3239 - val_loss: 4.2225 - val_accuracy: 0.7290
Epoch 2/25
938/938 [==============================] - 3s 4ms/step - loss: 16.7997 - accuracy: 0.4737 - val_loss: 2.9923 - val_accuracy: 0.7468
Epoch 3/25
938/938 [==============================] - 3s 3ms/step - loss: 11.3737 - accuracy: 0.4957 - val_loss: 2.3487 - val_accuracy: 0.7562
Epoch 4/25
938/938 [==============================] - 3s 4ms/step - loss: 8.2507 - accuracy: 0.5102 - val_loss: 2.0111 - val_accuracy: 0.7550
Epoch 5/25
938/938 [==============================] - 3s 4ms/step - loss: 6.9523 - accuracy: 0.5063 - val_loss: 1.7837 - val_accuracy: 0.7474
Epoch 6/25
938/938 [==============================] - 3s 4ms/step - loss: 5.6798 - accuracy: 0.4991 - val_loss: 1.6251 - val_accuracy: 0.7484
Epoch 7/25
938/938 [==============================] - 3s 4ms/step - loss: 4.9643 - accuracy: 0.4990 - val_loss: 1.5214 - val_accuracy: 0.7404
Epoch 8/25
938/938 [==============================] - 3s 4ms/step - loss: 4.3812 - accuracy: 0.4973 - val_loss: 1.4459 - val_accuracy: 0.7380
Epoch 9/25
938/938 [==============================] - 3s 4ms/step - loss: 3.8746 - accuracy: 0.4928 - val_loss: 1.3995 - val_accuracy: 0.7318
Epoch 10/25
938/938 [==============================] - 3s 4ms/step - loss: 3.5659 - accuracy: 0.4976 - val_loss: 1.3710 - val_accuracy: 0.7200
Epoch 11/25
938/938 [==============================] - 4s 4ms/step - loss: 3.3049 - accuracy: 0.4821 - val_loss: 1.3540 - val_accuracy: 0.7128
Epoch 12/25
938/938 [==============================] - 4s 4ms/step - loss: 3.1417 - accuracy: 0.4739 - val_loss: 1.3378 - val_accuracy: 0.7074
Epoch 13/25
938/938 [==============================] - 3s 4ms/step - loss: 2.9008 - accuracy: 0.4808 - val_loss: 1.3296 - val_accuracy: 0.7036
Epoch 14/25
938/938 [==============================] - 4s 4ms/step - loss: 2.8351 - accuracy: 0.4791 - val_loss: 1.3117 - val_accuracy: 0.6986
Epoch 15/25
938/938 [==============================] - 3s 4ms/step - loss: 2.6896 - accuracy: 0.4757 - val_loss: 1.3014 - val_accuracy: 0.7006
Epoch 16/25
938/938 [==============================] - 3s 3ms/step - loss: 2.5522 - accuracy: 0.4820 - val_loss: 1.2948 - val_accuracy: 0.6968
Epoch 17/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4695 - accuracy: 0.4799 - val_loss: 1.2925 - val_accuracy: 0.6914
Epoch 18/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4539 - accuracy: 0.4810 - val_loss: 1.2884 - val_accuracy: 0.6894
Epoch 19/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3694 - accuracy: 0.4846 - val_loss: 1.2815 - val_accuracy: 0.6898
Epoch 20/25
938/938 [==============================] - 3s 4ms/step - loss: 2.2842 - accuracy: 0.4821 - val_loss: 1.2722 - val_accuracy: 0.6882
Epoch 21/25
938/938 [==============================] - 3s 4ms/step - loss: 2.2401 - accuracy: 0.4896 - val_loss: 1.2666 - val_accuracy: 0.6902
Epoch 22/25
938/938 [==============================] - 3s 4ms/step - loss: 2.1478 - accuracy: 0.4832 - val_loss: 1.2665 - val_accuracy: 0.6880
Epoch 23/25
938/938 [==============================] - 3s 4ms/step - loss: 2.1229 - accuracy: 0.4924 - val_loss: 1.2602 - val_accuracy: 0.6898
Epoch 24/25
938/938 [==============================] - 3s 4ms/step - loss: 2.0884 - accuracy: 0.4856 - val_loss: 1.2575 - val_accuracy: 0.6892
Epoch 25/25
938/938 [==============================] - 4s 4ms/step - loss: 2.0618 - accuracy: 0.4926 - val_loss: 1.2537 - val_accuracy: 0.6872

 25%|██▌       | 2/8 [02:50<08:32, 85.47s/it]

Epoch 1/25
938/938 [==============================] - 4s 4ms/step - loss: 23.0249 - accuracy: 0.3933 - val_loss: 1.2977 - val_accuracy: 0.6142
Epoch 2/25
938/938 [==============================] - 3s 4ms/step - loss: 1.6226 - accuracy: 0.4890 - val_loss: 1.1082 - val_accuracy: 0.6544
Epoch 3/25
938/938 [==============================] - 4s 4ms/step - loss: 1.4652 - accuracy: 0.5291 - val_loss: 1.0229 - val_accuracy: 0.6978
Epoch 4/25
938/938 [==============================] - 3s 4ms/step - loss: 1.2967 - accuracy: 0.5817 - val_loss: 0.8542 - val_accuracy: 0.7634
Epoch 5/25
938/938 [==============================] - 3s 4ms/step - loss: 1.1609 - accuracy: 0.6280 - val_loss: 0.7236 - val_accuracy: 0.7848
Epoch 6/25
938/938 [==============================] - 3s 4ms/step - loss: 1.0263 - accuracy: 0.6633 - val_loss: 0.6754 - val_accuracy: 0.8510
Epoch 7/25
938/938 [==============================] - 3s 4ms/step - loss: 0.9843 - accuracy: 0.7000 - val_loss: 0.6490 - val_accuracy: 0.8564
Epoch 8/25
938/938 [==============================] - 3s 4ms/step - loss: 0.9049 - accuracy: 0.7209 - val_loss: 0.5512 - val_accuracy: 0.8634
Epoch 9/25
938/938 [==============================] - 3s 3ms/step - loss: 0.8416 - accuracy: 0.7398 - val_loss: 0.6269 - val_accuracy: 0.8744
Epoch 10/25
938/938 [==============================] - 3s 4ms/step - loss: 0.8239 - accuracy: 0.7510 - val_loss: 0.5348 - val_accuracy: 0.8734
Epoch 11/25
938/938 [==============================] - 3s 4ms/step - loss: 0.7494 - accuracy: 0.7773 - val_loss: 0.5090 - val_accuracy: 0.8842
Epoch 12/25
938/938 [==============================] - 3s 4ms/step - loss: 0.7211 - accuracy: 0.7885 - val_loss: 0.5211 - val_accuracy: 0.8710
Epoch 13/25
938/938 [==============================] - 3s 4ms/step - loss: 0.7212 - accuracy: 0.7896 - val_loss: 0.4888 - val_accuracy: 0.8756
Epoch 14/25
938/938 [==============================] - 3s 4ms/step - loss: 0.7074 - accuracy: 0.7982 - val_loss: 0.4655 - val_accuracy: 0.8882
Epoch 15/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6850 - accuracy: 0.7991 - val_loss: 0.4362 - val_accuracy: 0.8986
Epoch 16/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6829 - accuracy: 0.8075 - val_loss: 0.4827 - val_accuracy: 0.8896
Epoch 17/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6563 - accuracy: 0.8070 - val_loss: 0.4465 - val_accuracy: 0.8934
Epoch 18/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6751 - accuracy: 0.8037 - val_loss: 0.4550 - val_accuracy: 0.8938
Epoch 19/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6644 - accuracy: 0.8093 - val_loss: 0.4635 - val_accuracy: 0.9010
Epoch 20/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6430 - accuracy: 0.8158 - val_loss: 0.4949 - val_accuracy: 0.8866
Epoch 21/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6141 - accuracy: 0.8237 - val_loss: 0.4385 - val_accuracy: 0.8996
Epoch 22/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6391 - accuracy: 0.8194 - val_loss: 0.4578 - val_accuracy: 0.9002
Epoch 23/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6339 - accuracy: 0.8183 - val_loss: 0.4422 - val_accuracy: 0.9056
Epoch 24/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6135 - accuracy: 0.8228 - val_loss: 0.4332 - val_accuracy: 0.8958
Epoch 25/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6201 - accuracy: 0.8246 - val_loss: 0.4335 - val_accuracy: 0.9056

 38%|███▊      | 3/8 [04:17<07:08, 85.73s/it]

Epoch 1/25
938/938 [==============================] - 4s 4ms/step - loss: 37.0946 - accuracy: 0.3188 - val_loss: 1.7230 - val_accuracy: 0.4492
Epoch 2/25
938/938 [==============================] - 4s 4ms/step - loss: 2.5116 - accuracy: 0.2891 - val_loss: 1.5263 - val_accuracy: 0.5176
Epoch 3/25
938/938 [==============================] - 4s 4ms/step - loss: 1.9192 - accuracy: 0.3842 - val_loss: 1.4158 - val_accuracy: 0.5554
Epoch 4/25
938/938 [==============================] - 4s 4ms/step - loss: 1.7142 - accuracy: 0.4413 - val_loss: 1.2575 - val_accuracy: 0.6292
Epoch 5/25
938/938 [==============================] - 4s 4ms/step - loss: 1.5700 - accuracy: 0.4859 - val_loss: 1.1423 - val_accuracy: 0.6608
Epoch 6/25
938/938 [==============================] - 3s 4ms/step - loss: 1.4646 - accuracy: 0.5211 - val_loss: 1.0575 - val_accuracy: 0.6960
Epoch 7/25
938/938 [==============================] - 4s 4ms/step - loss: 1.3430 - accuracy: 0.5621 - val_loss: 1.0192 - val_accuracy: 0.7178
Epoch 8/25
938/938 [==============================] - 4s 4ms/step - loss: 1.2825 - accuracy: 0.5838 - val_loss: 0.9690 - val_accuracy: 0.7366
Epoch 9/25
938/938 [==============================] - 4s 4ms/step - loss: 1.2137 - accuracy: 0.6118 - val_loss: 0.9116 - val_accuracy: 0.7518
Epoch 10/25
938/938 [==============================] - 4s 4ms/step - loss: 1.1465 - accuracy: 0.6353 - val_loss: 0.8754 - val_accuracy: 0.7554
Epoch 11/25
938/938 [==============================] - 4s 4ms/step - loss: 1.0839 - accuracy: 0.6580 - val_loss: 0.8588 - val_accuracy: 0.7642
Epoch 12/25
938/938 [==============================] - 3s 4ms/step - loss: 1.0519 - accuracy: 0.6745 - val_loss: 0.8158 - val_accuracy: 0.7726
Epoch 13/25
938/938 [==============================] - 3s 4ms/step - loss: 1.0030 - accuracy: 0.6871 - val_loss: 0.8283 - val_accuracy: 0.7850
Epoch 14/25
938/938 [==============================] - 3s 4ms/step - loss: 0.9466 - accuracy: 0.6989 - val_loss: 0.7471 - val_accuracy: 0.7936
Epoch 15/25
938/938 [==============================] - 3s 3ms/step - loss: 0.8991 - accuracy: 0.7210 - val_loss: 0.7173 - val_accuracy: 0.7996
Epoch 16/25
938/938 [==============================] - 3s 4ms/step - loss: 0.8841 - accuracy: 0.7215 - val_loss: 0.7246 - val_accuracy: 0.8012
Epoch 17/25
938/938 [==============================] - 3s 4ms/step - loss: 0.8324 - accuracy: 0.7391 - val_loss: 0.6578 - val_accuracy: 0.8064
Epoch 18/25
938/938 [==============================] - 3s 4ms/step - loss: 0.8140 - accuracy: 0.7397 - val_loss: 0.5983 - val_accuracy: 0.8614
Epoch 19/25
938/938 [==============================] - 3s 4ms/step - loss: 0.7425 - accuracy: 0.7757 - val_loss: 0.5803 - val_accuracy: 0.8666
Epoch 20/25
938/938 [==============================] - 3s 4ms/step - loss: 0.7479 - accuracy: 0.7820 - val_loss: 0.5469 - val_accuracy: 0.8766
Epoch 21/25
938/938 [==============================] - 4s 4ms/step - loss: 0.7057 - accuracy: 0.7953 - val_loss: 0.5303 - val_accuracy: 0.8820
Epoch 22/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6905 - accuracy: 0.7996 - val_loss: 0.5148 - val_accuracy: 0.8828
Epoch 23/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6592 - accuracy: 0.8121 - val_loss: 0.4858 - val_accuracy: 0.8844
Epoch 24/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6543 - accuracy: 0.8111 - val_loss: 0.4660 - val_accuracy: 0.8948
Epoch 25/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6059 - accuracy: 0.8229 - val_loss: 0.4558 - val_accuracy: 0.8974

 50%|█████     | 4/8 [05:44<05:45, 86.48s/it]

Epoch 1/25
938/938 [==============================] - 4s 4ms/step - loss: 25.7898 - accuracy: 0.3212 - val_loss: 1.3089 - val_accuracy: 0.5770
Epoch 2/25
938/938 [==============================] - 3s 4ms/step - loss: 2.8397 - accuracy: 0.3841 - val_loss: 1.4453 - val_accuracy: 0.5892
Epoch 3/25
938/938 [==============================] - 4s 4ms/step - loss: 2.0668 - accuracy: 0.4088 - val_loss: 1.3539 - val_accuracy: 0.6456
Epoch 4/25
938/938 [==============================] - 3s 4ms/step - loss: 1.8442 - accuracy: 0.4385 - val_loss: 1.2275 - val_accuracy: 0.6728
Epoch 5/25
938/938 [==============================] - 3s 4ms/step - loss: 1.6870 - accuracy: 0.4674 - val_loss: 1.1356 - val_accuracy: 0.7090
Epoch 6/25
938/938 [==============================] - 3s 4ms/step - loss: 1.6167 - accuracy: 0.4849 - val_loss: 1.0585 - val_accuracy: 0.7384
Epoch 7/25
938/938 [==============================] - 4s 4ms/step - loss: 1.5243 - accuracy: 0.5096 - val_loss: 0.9989 - val_accuracy: 0.7596
Epoch 8/25
938/938 [==============================] - 4s 4ms/step - loss: 1.4556 - accuracy: 0.5326 - val_loss: 0.9612 - val_accuracy: 0.7690
Epoch 9/25
938/938 [==============================] - 4s 4ms/step - loss: 1.3804 - accuracy: 0.5492 - val_loss: 0.9006 - val_accuracy: 0.7892
Epoch 10/25
938/938 [==============================] - 4s 4ms/step - loss: 1.3364 - accuracy: 0.5624 - val_loss: 0.8630 - val_accuracy: 0.7918
Epoch 11/25
938/938 [==============================] - 4s 4ms/step - loss: 1.3164 - accuracy: 0.5713 - val_loss: 0.8380 - val_accuracy: 0.8010
Epoch 12/25
938/938 [==============================] - 4s 4ms/step - loss: 1.2850 - accuracy: 0.5805 - val_loss: 0.8191 - val_accuracy: 0.8066
Epoch 13/25
938/938 [==============================] - 4s 4ms/step - loss: 1.2382 - accuracy: 0.5940 - val_loss: 0.7951 - val_accuracy: 0.8122
Epoch 14/25
938/938 [==============================] - 4s 4ms/step - loss: 1.2140 - accuracy: 0.6069 - val_loss: 0.7710 - val_accuracy: 0.8202
Epoch 15/25
938/938 [==============================] - 4s 4ms/step - loss: 1.1926 - accuracy: 0.6115 - val_loss: 0.7577 - val_accuracy: 0.8232
Epoch 16/25
938/938 [==============================] - 4s 4ms/step - loss: 1.1659 - accuracy: 0.6193 - val_loss: 0.7367 - val_accuracy: 0.8276
Epoch 17/25
938/938 [==============================] - 4s 4ms/step - loss: 1.1395 - accuracy: 0.6223 - val_loss: 0.7130 - val_accuracy: 0.8362
Epoch 18/25
938/938 [==============================] - 4s 4ms/step - loss: 1.1098 - accuracy: 0.6395 - val_loss: 0.7025 - val_accuracy: 0.8358
Epoch 19/25
938/938 [==============================] - 4s 4ms/step - loss: 1.0937 - accuracy: 0.6417 - val_loss: 0.6857 - val_accuracy: 0.8390
Epoch 20/25
938/938 [==============================] - 4s 4ms/step - loss: 1.0891 - accuracy: 0.6457 - val_loss: 0.6741 - val_accuracy: 0.8418
Epoch 21/25
938/938 [==============================] - 4s 4ms/step - loss: 1.0785 - accuracy: 0.6480 - val_loss: 0.6620 - val_accuracy: 0.8466
Epoch 22/25
938/938 [==============================] - 4s 4ms/step - loss: 1.0405 - accuracy: 0.6649 - val_loss: 0.6519 - val_accuracy: 0.8486
Epoch 23/25
938/938 [==============================] - 4s 4ms/step - loss: 1.0244 - accuracy: 0.6684 - val_loss: 0.6391 - val_accuracy: 0.8484
Epoch 24/25
938/938 [==============================] - 4s 4ms/step - loss: 1.0191 - accuracy: 0.6708 - val_loss: 0.6302 - val_accuracy: 0.8502
Epoch 25/25
938/938 [==============================] - 4s 4ms/step - loss: 1.0015 - accuracy: 0.6780 - val_loss: 0.6236 - val_accuracy: 0.8538

 62%|██████▎   | 5/8 [07:14<04:23, 87.71s/it]

Epoch 1/25
938/938 [==============================] - 5s 5ms/step - loss: 23.5468 - accuracy: 0.4076 - val_loss: 1.2913 - val_accuracy: 0.6186
Epoch 2/25
938/938 [==============================] - 5s 5ms/step - loss: 1.6460 - accuracy: 0.4888 - val_loss: 1.0276 - val_accuracy: 0.7124
Epoch 3/25
938/938 [==============================] - 5s 5ms/step - loss: 1.3798 - accuracy: 0.5846 - val_loss: 0.9543 - val_accuracy: 0.7982
Epoch 4/25
938/938 [==============================] - 4s 5ms/step - loss: 1.2470 - accuracy: 0.6450 - val_loss: 0.7653 - val_accuracy: 0.8320
Epoch 5/25
938/938 [==============================] - 4s 5ms/step - loss: 1.0078 - accuracy: 0.7043 - val_loss: 0.6234 - val_accuracy: 0.8710
Epoch 6/25
938/938 [==============================] - 4s 5ms/step - loss: 0.9099 - accuracy: 0.7419 - val_loss: 0.5781 - val_accuracy: 0.8668
Epoch 7/25
938/938 [==============================] - 4s 5ms/step - loss: 0.8280 - accuracy: 0.7588 - val_loss: 0.5524 - val_accuracy: 0.8638
Epoch 8/25
938/938 [==============================] - 4s 5ms/step - loss: 0.7696 - accuracy: 0.7820 - val_loss: 0.4821 - val_accuracy: 0.8874
Epoch 9/25
938/938 [==============================] - 4s 5ms/step - loss: 0.7240 - accuracy: 0.7924 - val_loss: 0.4969 - val_accuracy: 0.8846
Epoch 10/25
938/938 [==============================] - 4s 5ms/step - loss: 0.6831 - accuracy: 0.8067 - val_loss: 0.4603 - val_accuracy: 0.8902
Epoch 11/25
938/938 [==============================] - 4s 5ms/step - loss: 0.6597 - accuracy: 0.8152 - val_loss: 0.4527 - val_accuracy: 0.8992
Epoch 12/25
938/938 [==============================] - 4s 5ms/step - loss: 0.6552 - accuracy: 0.8168 - val_loss: 0.4654 - val_accuracy: 0.8926
Epoch 13/25
938/938 [==============================] - 5s 5ms/step - loss: 0.6069 - accuracy: 0.8308 - val_loss: 0.4433 - val_accuracy: 0.8912
Epoch 14/25
938/938 [==============================] - 5s 5ms/step - loss: 0.6211 - accuracy: 0.8250 - val_loss: 0.4325 - val_accuracy: 0.9012
Epoch 15/25
938/938 [==============================] - 4s 5ms/step - loss: 0.5964 - accuracy: 0.8344 - val_loss: 0.4179 - val_accuracy: 0.9066
Epoch 16/25
938/938 [==============================] - 4s 5ms/step - loss: 0.6042 - accuracy: 0.8341 - val_loss: 0.4188 - val_accuracy: 0.9038
Epoch 17/25
938/938 [==============================] - 5s 5ms/step - loss: 0.5898 - accuracy: 0.8374 - val_loss: 0.4632 - val_accuracy: 0.9012
Epoch 18/25
938/938 [==============================] - 5s 5ms/step - loss: 0.5686 - accuracy: 0.8477 - val_loss: 0.4112 - val_accuracy: 0.9050
Epoch 19/25
938/938 [==============================] - 5s 5ms/step - loss: 0.5714 - accuracy: 0.8450 - val_loss: 0.4480 - val_accuracy: 0.9040
Epoch 20/25
938/938 [==============================] - 4s 5ms/step - loss: 0.5486 - accuracy: 0.8513 - val_loss: 0.4357 - val_accuracy: 0.9056
Epoch 21/25
938/938 [==============================] - 5s 5ms/step - loss: 0.5383 - accuracy: 0.8521 - val_loss: 0.4301 - val_accuracy: 0.9098
Epoch 22/25
938/938 [==============================] - 5s 5ms/step - loss: 0.5500 - accuracy: 0.8548 - val_loss: 0.4081 - val_accuracy: 0.9052
Epoch 23/25
938/938 [==============================] - 5s 5ms/step - loss: 0.5307 - accuracy: 0.8565 - val_loss: 0.4021 - val_accuracy: 0.9176
Epoch 24/25
938/938 [==============================] - 5s 5ms/step - loss: 0.5266 - accuracy: 0.8602 - val_loss: 0.3915 - val_accuracy: 0.9196
Epoch 25/25
938/938 [==============================] - 5s 5ms/step - loss: 0.5160 - accuracy: 0.8618 - val_loss: 0.4092 - val_accuracy: 0.9154

 75%|███████▌  | 6/8 [09:07<03:12, 96.46s/it]

Epoch 1/25
938/938 [==============================] - 5s 4ms/step - loss: 21.8982 - accuracy: 0.4233 - val_loss: 1.1722 - val_accuracy: 0.7324
Epoch 2/25
938/938 [==============================] - 4s 4ms/step - loss: 1.5385 - accuracy: 0.6577 - val_loss: 0.9902 - val_accuracy: 0.8450
Epoch 3/25
938/938 [==============================] - 4s 4ms/step - loss: 1.2129 - accuracy: 0.7531 - val_loss: 0.8162 - val_accuracy: 0.8680
Epoch 4/25
938/938 [==============================] - 4s 4ms/step - loss: 1.0152 - accuracy: 0.8022 - val_loss: 0.6704 - val_accuracy: 0.8784
Epoch 5/25
938/938 [==============================] - 4s 4ms/step - loss: 0.9106 - accuracy: 0.8254 - val_loss: 0.5631 - val_accuracy: 0.8974
Epoch 6/25
938/938 [==============================] - 4s 4ms/step - loss: 0.8769 - accuracy: 0.8446 - val_loss: 0.5368 - val_accuracy: 0.9026
Epoch 7/25
938/938 [==============================] - 4s 4ms/step - loss: 0.7959 - accuracy: 0.8527 - val_loss: 0.5552 - val_accuracy: 0.9062
Epoch 8/25
938/938 [==============================] - 4s 4ms/step - loss: 0.7854 - accuracy: 0.8622 - val_loss: 0.5520 - val_accuracy: 0.9088
Epoch 9/25
938/938 [==============================] - 4s 4ms/step - loss: 0.7897 - accuracy: 0.8616 - val_loss: 0.5578 - val_accuracy: 0.9100
Epoch 10/25
938/938 [==============================] - 4s 4ms/step - loss: 0.7788 - accuracy: 0.8618 - val_loss: 0.7030 - val_accuracy: 0.9066
Epoch 11/25
938/938 [==============================] - 4s 4ms/step - loss: 0.7964 - accuracy: 0.8706 - val_loss: 0.5945 - val_accuracy: 0.9250
Epoch 12/25
938/938 [==============================] - 4s 4ms/step - loss: 0.8072 - accuracy: 0.8693 - val_loss: 0.6467 - val_accuracy: 0.9148
Epoch 13/25
938/938 [==============================] - 4s 4ms/step - loss: 0.7747 - accuracy: 0.8746 - val_loss: 0.6521 - val_accuracy: 0.9104
Epoch 14/25
938/938 [==============================] - 4s 4ms/step - loss: 0.7647 - accuracy: 0.8807 - val_loss: 0.6803 - val_accuracy: 0.9182
Epoch 15/25
938/938 [==============================] - 4s 4ms/step - loss: 0.8611 - accuracy: 0.8793 - val_loss: 0.6059 - val_accuracy: 0.9218
Epoch 16/25
938/938 [==============================] - 4s 4ms/step - loss: 0.7894 - accuracy: 0.8774 - val_loss: 0.6659 - val_accuracy: 0.9216
Epoch 17/25
938/938 [==============================] - 4s 4ms/step - loss: 0.7804 - accuracy: 0.8754 - val_loss: 0.7020 - val_accuracy: 0.9164
Epoch 18/25
938/938 [==============================] - 4s 4ms/step - loss: 0.8081 - accuracy: 0.8840 - val_loss: 0.7614 - val_accuracy: 0.9144
Epoch 19/25
938/938 [==============================] - 4s 4ms/step - loss: 0.7600 - accuracy: 0.8799 - val_loss: 0.8110 - val_accuracy: 0.9202
Epoch 20/25
938/938 [==============================] - 4s 4ms/step - loss: 0.8383 - accuracy: 0.8825 - val_loss: 0.7603 - val_accuracy: 0.9176
Epoch 21/25
938/938 [==============================] - 4s 4ms/step - loss: 0.7708 - accuracy: 0.8820 - val_loss: 0.8897 - val_accuracy: 0.9246
Epoch 22/25
938/938 [==============================] - 4s 4ms/step - loss: 0.7944 - accuracy: 0.8835 - val_loss: 0.7569 - val_accuracy: 0.9264
Epoch 23/25
938/938 [==============================] - 4s 4ms/step - loss: 0.7874 - accuracy: 0.8887 - val_loss: 0.7993 - val_accuracy: 0.9208
Epoch 24/25
938/938 [==============================] - 4s 4ms/step - loss: 0.7910 - accuracy: 0.8908 - val_loss: 0.8054 - val_accuracy: 0.9250
Epoch 25/25
938/938 [==============================] - 4s 4ms/step - loss: 0.7529 - accuracy: 0.8904 - val_loss: 0.7839 - val_accuracy: 0.9238

 88%|████████▊ | 7/8 [10:47<01:37, 97.48s/it]

Epoch 1/25
938/938 [==============================] - 4s 4ms/step - loss: nan - accuracy: 0.0976 - val_loss: nan - val_accuracy: 0.0920
Epoch 2/25
938/938 [==============================] - 3s 3ms/step - loss: nan - accuracy: 0.0984 - val_loss: nan - val_accuracy: 0.0920
Epoch 3/25
938/938 [==============================] - 3s 3ms/step - loss: nan - accuracy: 0.0968 - val_loss: nan - val_accuracy: 0.0920
Epoch 4/25
938/938 [==============================] - 3s 3ms/step - loss: nan - accuracy: 0.0999 - val_loss: nan - val_accuracy: 0.0920
Epoch 5/25
938/938 [==============================] - 3s 3ms/step - loss: nan - accuracy: 0.0978 - val_loss: nan - val_accuracy: 0.0920
Epoch 6/25
938/938 [==============================] - 3s 3ms/step - loss: nan - accuracy: 0.0994 - val_loss: nan - val_accuracy: 0.0920
Epoch 7/25
938/938 [==============================] - 3s 3ms/step - loss: nan - accuracy: 0.0979 - val_loss: nan - val_accuracy: 0.0920
Epoch 8/25
938/938 [==============================] - 3s 3ms/step - loss: nan - accuracy: 0.0977 - val_loss: nan - val_accuracy: 0.0920
Epoch 9/25
938/938 [==============================] - 3s 3ms/step - loss: nan - accuracy: 0.0943 - val_loss: nan - val_accuracy: 0.0920
Epoch 10/25
938/938 [==============================] - 3s 4ms/step - loss: nan - accuracy: 0.1005 - val_loss: nan - val_accuracy: 0.0920
Epoch 11/25
938/938 [==============================] - 3s 3ms/step - loss: nan - accuracy: 0.0981 - val_loss: nan - val_accuracy: 0.0920
Epoch 12/25
938/938 [==============================] - 3s 3ms/step - loss: nan - accuracy: 0.0977 - val_loss: nan - val_accuracy: 0.0920
Epoch 13/25
938/938 [==============================] - 3s 3ms/step - loss: nan - accuracy: 0.0985 - val_loss: nan - val_accuracy: 0.0920
Epoch 14/25
938/938 [==============================] - 3s 3ms/step - loss: nan - accuracy: 0.1006 - val_loss: nan - val_accuracy: 0.0920
Epoch 15/25
938/938 [==============================] - 3s 3ms/step - loss: nan - accuracy: 0.0973 - val_loss: nan - val_accuracy: 0.0920
Epoch 16/25
938/938 [==============================] - 3s 4ms/step - loss: nan - accuracy: 0.0988 - val_loss: nan - val_accuracy: 0.0920
Epoch 17/25
938/938 [==============================] - 3s 4ms/step - loss: nan - accuracy: 0.0998 - val_loss: nan - val_accuracy: 0.0920
Epoch 18/25
938/938 [==============================] - 3s 4ms/step - loss: nan - accuracy: 0.0996 - val_loss: nan - val_accuracy: 0.0920
Epoch 19/25
938/938 [==============================] - 3s 4ms/step - loss: nan - accuracy: 0.0971 - val_loss: nan - val_accuracy: 0.0920
Epoch 20/25
938/938 [==============================] - 3s 4ms/step - loss: nan - accuracy: 0.0952 - val_loss: nan - val_accuracy: 0.0920
Epoch 21/25
938/938 [==============================] - 3s 4ms/step - loss: nan - accuracy: 0.0971 - val_loss: nan - val_accuracy: 0.0920
Epoch 22/25
938/938 [==============================] - 3s 4ms/step - loss: nan - accuracy: 0.0978 - val_loss: nan - val_accuracy: 0.0920
Epoch 23/25
938/938 [==============================] - 3s 4ms/step - loss: nan - accuracy: 0.0972 - val_loss: nan - val_accuracy: 0.0920
Epoch 24/25
938/938 [==============================] - 3s 4ms/step - loss: nan - accuracy: 0.0985 - val_loss: nan - val_accuracy: 0.0920
Epoch 25/25
938/938 [==============================] - 3s 4ms/step - loss: nan - accuracy: 0.0981 - val_loss: nan - val_accuracy: 0.0920

100%|██████████| 8/8 [12:10<00:00, 91.27s/it]

fig, ax = plt.subplots()
epochs = len(history.history['accuracy'])
c = ["g", "b", "r", "y", "orange", "purple", "pink", "#5050cc", "#1ddeb1", "#5ce30e"]
for i, (name, history) in enumerate(lst):
    plt.plot(list(range(1,epochs+1,1)),history.history['accuracy'], label=f"{name}", c=c[i])
#for i, (name, history) in enumerate(lst):
#    plt.plot(history.history['val_accuracy'], label=f"{name} - V", ls="--", c=c[i])
plt.ylim(0,1)
plt.xlim(1, len(history.history['accuracy']))
plt.xlabel("Epochs")
plt.ylabel("Accuracy")
plt.legend(ncol=3, loc=4)
plt.savefig("./figures/10_Optimizer.png", dpi=500, bbox_inches = 'tight', pad_inches = 0)
plt.show()

Learning Rate

lr_range = [0.000001, 0.00001, 0.0001, 0.001, 0.01, 0.1, 1, 10, 100]
lst_lr = []
for learning_rate in tqdm(lr_range):
    optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
    model = getModel()
    model.compile(optimizer=optimizer, loss=lossFunction, metrics=['accuracy'])
    history = model.fit(x_train, y_train,
                    validation_data = (x_val, y_val),
                    epochs=25,
                       verbose=1)
    lst_lr.append([learning_rate, history])

  0%|          | 0/9 [00:00<?, ?it/s]

Epoch 1/25
938/938 [==============================] - 4s 4ms/step - loss: 243.9518 - accuracy: 0.0945 - val_loss: 91.1107 - val_accuracy: 0.0820
Epoch 2/25
938/938 [==============================] - 3s 3ms/step - loss: 208.4988 - accuracy: 0.1017 - val_loss: 67.6521 - val_accuracy: 0.0980
Epoch 3/25
938/938 [==============================] - 3s 3ms/step - loss: 186.1036 - accuracy: 0.1062 - val_loss: 52.7131 - val_accuracy: 0.1220
Epoch 4/25
938/938 [==============================] - 3s 3ms/step - loss: 165.3556 - accuracy: 0.1208 - val_loss: 43.0138 - val_accuracy: 0.1652
Epoch 5/25
938/938 [==============================] - 3s 3ms/step - loss: 148.5613 - accuracy: 0.1309 - val_loss: 36.0763 - val_accuracy: 0.2130
Epoch 6/25
938/938 [==============================] - 3s 3ms/step - loss: 135.7601 - accuracy: 0.1403 - val_loss: 30.9300 - val_accuracy: 0.2602
Epoch 7/25
938/938 [==============================] - 3s 4ms/step - loss: 121.9539 - accuracy: 0.1513 - val_loss: 27.0454 - val_accuracy: 0.3098
Epoch 8/25
938/938 [==============================] - 3s 3ms/step - loss: 111.3787 - accuracy: 0.1627 - val_loss: 23.9479 - val_accuracy: 0.3510
Epoch 9/25
938/938 [==============================] - 3s 4ms/step - loss: 103.2496 - accuracy: 0.1749 - val_loss: 21.5144 - val_accuracy: 0.3910
Epoch 10/25
938/938 [==============================] - 3s 4ms/step - loss: 93.9318 - accuracy: 0.1893 - val_loss: 19.4618 - val_accuracy: 0.4248
Epoch 11/25
938/938 [==============================] - 3s 4ms/step - loss: 86.4679 - accuracy: 0.2039 - val_loss: 17.8174 - val_accuracy: 0.4460
Epoch 12/25
938/938 [==============================] - 3s 4ms/step - loss: 81.1615 - accuracy: 0.2186 - val_loss: 16.3563 - val_accuracy: 0.4666
Epoch 13/25
938/938 [==============================] - 3s 4ms/step - loss: 75.2178 - accuracy: 0.2215 - val_loss: 15.1100 - val_accuracy: 0.4870
Epoch 14/25
938/938 [==============================] - 3s 4ms/step - loss: 69.8658 - accuracy: 0.2420 - val_loss: 14.0155 - val_accuracy: 0.5048
Epoch 15/25
938/938 [==============================] - 3s 4ms/step - loss: 65.7864 - accuracy: 0.2482 - val_loss: 13.0565 - val_accuracy: 0.5188
Epoch 16/25
938/938 [==============================] - 3s 4ms/step - loss: 62.0768 - accuracy: 0.2595 - val_loss: 12.2378 - val_accuracy: 0.5340
Epoch 17/25
938/938 [==============================] - 3s 4ms/step - loss: 58.1441 - accuracy: 0.2719 - val_loss: 11.4903 - val_accuracy: 0.5470
Epoch 18/25
938/938 [==============================] - 3s 4ms/step - loss: 55.5214 - accuracy: 0.2805 - val_loss: 10.8255 - val_accuracy: 0.5598
Epoch 19/25
938/938 [==============================] - 3s 3ms/step - loss: 51.7639 - accuracy: 0.2938 - val_loss: 10.2490 - val_accuracy: 0.5740
Epoch 20/25
938/938 [==============================] - 3s 3ms/step - loss: 50.0873 - accuracy: 0.2962 - val_loss: 9.7401 - val_accuracy: 0.5864
Epoch 21/25
938/938 [==============================] - 3s 3ms/step - loss: 47.4058 - accuracy: 0.3010 - val_loss: 9.2684 - val_accuracy: 0.5972
Epoch 22/25
938/938 [==============================] - 3s 3ms/step - loss: 44.8103 - accuracy: 0.3113 - val_loss: 8.8538 - val_accuracy: 0.6032
Epoch 23/25
938/938 [==============================] - 3s 3ms/step - loss: 42.8749 - accuracy: 0.3230 - val_loss: 8.4545 - val_accuracy: 0.6128
Epoch 24/25
938/938 [==============================] - 3s 3ms/step - loss: 40.9563 - accuracy: 0.3254 - val_loss: 8.1143 - val_accuracy: 0.6208
Epoch 25/25
938/938 [==============================] - 3s 3ms/step - loss: 39.7053 - accuracy: 0.3295 - val_loss: 7.7847 - val_accuracy: 0.6280

 11%|█         | 1/9 [01:23<11:05, 83.15s/it]

Epoch 1/25
938/938 [==============================] - 4s 4ms/step - loss: 175.3493 - accuracy: 0.1376 - val_loss: 19.4197 - val_accuracy: 0.4708
Epoch 2/25
938/938 [==============================] - 3s 4ms/step - loss: 77.1935 - accuracy: 0.2342 - val_loss: 10.8960 - val_accuracy: 0.5850
Epoch 3/25
938/938 [==============================] - 3s 4ms/step - loss: 48.3444 - accuracy: 0.3086 - val_loss: 7.6217 - val_accuracy: 0.6508
Epoch 4/25
938/938 [==============================] - 3s 4ms/step - loss: 33.8272 - accuracy: 0.3590 - val_loss: 5.8310 - val_accuracy: 0.6828
Epoch 5/25
938/938 [==============================] - 4s 4ms/step - loss: 25.3931 - accuracy: 0.4005 - val_loss: 4.6031 - val_accuracy: 0.7112
Epoch 6/25
938/938 [==============================] - 4s 4ms/step - loss: 19.6583 - accuracy: 0.4275 - val_loss: 3.7793 - val_accuracy: 0.7260
Epoch 7/25
938/938 [==============================] - 3s 4ms/step - loss: 15.5532 - accuracy: 0.4505 - val_loss: 3.1282 - val_accuracy: 0.7376
Epoch 8/25
938/938 [==============================] - 3s 4ms/step - loss: 12.5905 - accuracy: 0.4630 - val_loss: 2.6417 - val_accuracy: 0.7368
Epoch 9/25
938/938 [==============================] - 3s 4ms/step - loss: 10.1353 - accuracy: 0.4605 - val_loss: 2.2400 - val_accuracy: 0.7344
Epoch 10/25
938/938 [==============================] - 3s 4ms/step - loss: 8.3346 - accuracy: 0.4582 - val_loss: 1.9354 - val_accuracy: 0.7180
Epoch 11/25
938/938 [==============================] - 3s 4ms/step - loss: 6.4930 - accuracy: 0.4561 - val_loss: 1.7242 - val_accuracy: 0.7094
Epoch 12/25
938/938 [==============================] - 4s 4ms/step - loss: 5.4076 - accuracy: 0.4654 - val_loss: 1.5754 - val_accuracy: 0.7170
Epoch 13/25
938/938 [==============================] - 4s 4ms/step - loss: 4.4290 - accuracy: 0.4678 - val_loss: 1.4891 - val_accuracy: 0.6986
Epoch 14/25
938/938 [==============================] - 3s 4ms/step - loss: 3.8750 - accuracy: 0.4549 - val_loss: 1.4355 - val_accuracy: 0.6830
Epoch 15/25
938/938 [==============================] - 3s 4ms/step - loss: 3.3087 - accuracy: 0.4481 - val_loss: 1.3973 - val_accuracy: 0.6688
Epoch 16/25
938/938 [==============================] - 3s 4ms/step - loss: 2.8922 - accuracy: 0.4434 - val_loss: 1.3781 - val_accuracy: 0.6612
Epoch 17/25
938/938 [==============================] - 3s 4ms/step - loss: 2.6125 - accuracy: 0.4346 - val_loss: 1.3672 - val_accuracy: 0.6562
Epoch 18/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4313 - accuracy: 0.4421 - val_loss: 1.3396 - val_accuracy: 0.6574
Epoch 19/25
938/938 [==============================] - 3s 3ms/step - loss: 2.2809 - accuracy: 0.4499 - val_loss: 1.3233 - val_accuracy: 0.6564
Epoch 20/25
938/938 [==============================] - 3s 4ms/step - loss: 2.1562 - accuracy: 0.4559 - val_loss: 1.3094 - val_accuracy: 0.6512
Epoch 21/25
938/938 [==============================] - 3s 4ms/step - loss: 2.0446 - accuracy: 0.4578 - val_loss: 1.2997 - val_accuracy: 0.6576
Epoch 22/25
938/938 [==============================] - 3s 4ms/step - loss: 1.9845 - accuracy: 0.4673 - val_loss: 1.2889 - val_accuracy: 0.6574
Epoch 23/25
938/938 [==============================] - 3s 4ms/step - loss: 1.9076 - accuracy: 0.4693 - val_loss: 1.2700 - val_accuracy: 0.6620
Epoch 24/25
938/938 [==============================] - 3s 4ms/step - loss: 1.8681 - accuracy: 0.4736 - val_loss: 1.2449 - val_accuracy: 0.6704
Epoch 25/25
938/938 [==============================] - 3s 4ms/step - loss: 1.8049 - accuracy: 0.4918 - val_loss: 1.2376 - val_accuracy: 0.6724

 22%|██▏       | 2/9 [02:49<09:53, 84.80s/it]

Epoch 1/25
938/938 [==============================] - 4s 4ms/step - loss: 87.2708 - accuracy: 0.2579 - val_loss: 3.0514 - val_accuracy: 0.7216
Epoch 2/25
938/938 [==============================] - 3s 4ms/step - loss: 9.4218 - accuracy: 0.4716 - val_loss: 1.6113 - val_accuracy: 0.6724
Epoch 3/25
938/938 [==============================] - 3s 4ms/step - loss: 3.4868 - accuracy: 0.4437 - val_loss: 1.4141 - val_accuracy: 0.6220
Epoch 4/25
938/938 [==============================] - 3s 4ms/step - loss: 2.2285 - accuracy: 0.4464 - val_loss: 1.3617 - val_accuracy: 0.6208
Epoch 5/25
938/938 [==============================] - 4s 4ms/step - loss: 1.8644 - accuracy: 0.4686 - val_loss: 1.2956 - val_accuracy: 0.6240
Epoch 6/25
938/938 [==============================] - 3s 4ms/step - loss: 1.6343 - accuracy: 0.5090 - val_loss: 1.2081 - val_accuracy: 0.6604
Epoch 7/25
938/938 [==============================] - 4s 4ms/step - loss: 1.5132 - accuracy: 0.5528 - val_loss: 1.0863 - val_accuracy: 0.7002
Epoch 8/25
938/938 [==============================] - 4s 4ms/step - loss: 1.3543 - accuracy: 0.6007 - val_loss: 0.9850 - val_accuracy: 0.7228
Epoch 9/25
938/938 [==============================] - 4s 4ms/step - loss: 1.2080 - accuracy: 0.6418 - val_loss: 0.9014 - val_accuracy: 0.7572
Epoch 10/25
938/938 [==============================] - 3s 4ms/step - loss: 1.1430 - accuracy: 0.6763 - val_loss: 0.8530 - val_accuracy: 0.8022
Epoch 11/25
938/938 [==============================] - 3s 3ms/step - loss: 1.0546 - accuracy: 0.6953 - val_loss: 0.8282 - val_accuracy: 0.8060
Epoch 12/25
938/938 [==============================] - 3s 4ms/step - loss: 0.9972 - accuracy: 0.7152 - val_loss: 0.7635 - val_accuracy: 0.8286
Epoch 13/25
938/938 [==============================] - 3s 4ms/step - loss: 0.9406 - accuracy: 0.7309 - val_loss: 0.6901 - val_accuracy: 0.8460
Epoch 14/25
938/938 [==============================] - 4s 4ms/step - loss: 0.8548 - accuracy: 0.7528 - val_loss: 0.6833 - val_accuracy: 0.8484
Epoch 15/25
938/938 [==============================] - 3s 4ms/step - loss: 0.8058 - accuracy: 0.7700 - val_loss: 0.6427 - val_accuracy: 0.8586
Epoch 16/25
938/938 [==============================] - 3s 4ms/step - loss: 0.8016 - accuracy: 0.7752 - val_loss: 0.5879 - val_accuracy: 0.8668
Epoch 17/25
938/938 [==============================] - 3s 4ms/step - loss: 0.7470 - accuracy: 0.7877 - val_loss: 0.5637 - val_accuracy: 0.8798
Epoch 18/25
938/938 [==============================] - 3s 4ms/step - loss: 0.7068 - accuracy: 0.8095 - val_loss: 0.5447 - val_accuracy: 0.8850
Epoch 19/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6620 - accuracy: 0.8141 - val_loss: 0.5199 - val_accuracy: 0.8904
Epoch 20/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6344 - accuracy: 0.8264 - val_loss: 0.4943 - val_accuracy: 0.8872
Epoch 21/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6161 - accuracy: 0.8303 - val_loss: 0.4881 - val_accuracy: 0.8922
Epoch 22/25
938/938 [==============================] - 3s 4ms/step - loss: 0.5681 - accuracy: 0.8438 - val_loss: 0.4473 - val_accuracy: 0.8954
Epoch 23/25
938/938 [==============================] - 3s 4ms/step - loss: 0.5433 - accuracy: 0.8512 - val_loss: 0.4338 - val_accuracy: 0.9006
Epoch 24/25
938/938 [==============================] - 3s 4ms/step - loss: 0.5260 - accuracy: 0.8573 - val_loss: 0.4346 - val_accuracy: 0.9042
Epoch 25/25
938/938 [==============================] - 3s 4ms/step - loss: 0.5019 - accuracy: 0.8654 - val_loss: 0.4120 - val_accuracy: 0.9120

 33%|███▎      | 3/9 [04:15<08:33, 85.54s/it]

Epoch 1/25
938/938 [==============================] - 4s 4ms/step - loss: 22.3174 - accuracy: 0.4027 - val_loss: 1.2778 - val_accuracy: 0.6102
Epoch 2/25
938/938 [==============================] - 3s 4ms/step - loss: 1.6408 - accuracy: 0.4873 - val_loss: 1.0991 - val_accuracy: 0.6738
Epoch 3/25
938/938 [==============================] - 3s 4ms/step - loss: 1.4861 - accuracy: 0.5354 - val_loss: 0.9836 - val_accuracy: 0.7650
Epoch 4/25
938/938 [==============================] - 3s 3ms/step - loss: 1.2966 - accuracy: 0.6060 - val_loss: 0.8332 - val_accuracy: 0.7990
Epoch 5/25
938/938 [==============================] - 3s 3ms/step - loss: 1.1221 - accuracy: 0.6708 - val_loss: 0.6324 - val_accuracy: 0.8444
Epoch 6/25
938/938 [==============================] - 3s 4ms/step - loss: 0.9424 - accuracy: 0.7195 - val_loss: 0.6106 - val_accuracy: 0.8652
Epoch 7/25
938/938 [==============================] - 3s 4ms/step - loss: 0.8767 - accuracy: 0.7410 - val_loss: 0.5581 - val_accuracy: 0.8724
Epoch 8/25
938/938 [==============================] - 3s 4ms/step - loss: 0.7957 - accuracy: 0.7690 - val_loss: 0.5304 - val_accuracy: 0.8824
Epoch 9/25
938/938 [==============================] - 3s 4ms/step - loss: 0.7214 - accuracy: 0.7928 - val_loss: 0.5235 - val_accuracy: 0.8828
Epoch 10/25
938/938 [==============================] - 3s 4ms/step - loss: 0.7272 - accuracy: 0.7892 - val_loss: 0.4685 - val_accuracy: 0.8880
Epoch 11/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6943 - accuracy: 0.8063 - val_loss: 0.4675 - val_accuracy: 0.8862
Epoch 12/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6592 - accuracy: 0.8078 - val_loss: 0.4745 - val_accuracy: 0.8868
Epoch 13/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6710 - accuracy: 0.8069 - val_loss: 0.4564 - val_accuracy: 0.8948
Epoch 14/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6568 - accuracy: 0.8147 - val_loss: 0.4621 - val_accuracy: 0.9028
Epoch 15/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6068 - accuracy: 0.8270 - val_loss: 0.4414 - val_accuracy: 0.8980
Epoch 16/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6208 - accuracy: 0.8246 - val_loss: 0.4143 - val_accuracy: 0.9012
Epoch 17/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6214 - accuracy: 0.8241 - val_loss: 0.4469 - val_accuracy: 0.8930
Epoch 18/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6146 - accuracy: 0.8247 - val_loss: 0.4473 - val_accuracy: 0.8986
Epoch 19/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6245 - accuracy: 0.8284 - val_loss: 0.4324 - val_accuracy: 0.8994
Epoch 20/25
938/938 [==============================] - 4s 4ms/step - loss: 0.5936 - accuracy: 0.8294 - val_loss: 0.4438 - val_accuracy: 0.9022
Epoch 21/25
938/938 [==============================] - 3s 4ms/step - loss: 0.6109 - accuracy: 0.8322 - val_loss: 0.4298 - val_accuracy: 0.9008
Epoch 22/25
938/938 [==============================] - 3s 4ms/step - loss: 0.5934 - accuracy: 0.8314 - val_loss: 0.4210 - val_accuracy: 0.9034
Epoch 23/25
938/938 [==============================] - 3s 4ms/step - loss: 0.5693 - accuracy: 0.8355 - val_loss: 0.4605 - val_accuracy: 0.8992
Epoch 24/25
938/938 [==============================] - 3s 4ms/step - loss: 0.5992 - accuracy: 0.8308 - val_loss: 0.4078 - val_accuracy: 0.9080
Epoch 25/25
938/938 [==============================] - 3s 4ms/step - loss: 0.5854 - accuracy: 0.8359 - val_loss: 0.4537 - val_accuracy: 0.9016

 44%|████▍     | 4/9 [05:41<07:07, 85.60s/it]

Epoch 1/25
938/938 [==============================] - 4s 4ms/step - loss: 20.4557 - accuracy: 0.1696 - val_loss: 2.3036 - val_accuracy: 0.1142
Epoch 2/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3253 - accuracy: 0.1159 - val_loss: 2.3023 - val_accuracy: 0.1142
Epoch 3/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3207 - accuracy: 0.1108 - val_loss: 2.3019 - val_accuracy: 0.1142
Epoch 4/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3061 - accuracy: 0.1136 - val_loss: 2.3024 - val_accuracy: 0.1142
Epoch 5/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3298 - accuracy: 0.1122 - val_loss: 2.3024 - val_accuracy: 0.1142
Epoch 6/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4232 - accuracy: 0.1146 - val_loss: 2.3011 - val_accuracy: 0.1142
Epoch 7/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3066 - accuracy: 0.1094 - val_loss: 2.3018 - val_accuracy: 0.1142
Epoch 8/25
938/938 [==============================] - 4s 4ms/step - loss: 2.3026 - accuracy: 0.1109 - val_loss: 2.3030 - val_accuracy: 0.1142
Epoch 9/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3027 - accuracy: 0.1132 - val_loss: 2.3021 - val_accuracy: 0.1142
Epoch 10/25
938/938 [==============================] - 3s 3ms/step - loss: 2.3192 - accuracy: 0.1124 - val_loss: 2.3030 - val_accuracy: 0.1142
Epoch 11/25
938/938 [==============================] - 3s 3ms/step - loss: 2.6163 - accuracy: 0.1088 - val_loss: 2.3016 - val_accuracy: 0.1142
Epoch 12/25
938/938 [==============================] - 3s 3ms/step - loss: 2.3100 - accuracy: 0.1131 - val_loss: 2.3029 - val_accuracy: 0.1142
Epoch 13/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3033 - accuracy: 0.1142 - val_loss: 2.3026 - val_accuracy: 0.1024
Epoch 14/25
938/938 [==============================] - 3s 3ms/step - loss: 2.3027 - accuracy: 0.1091 - val_loss: 2.3012 - val_accuracy: 0.1142
Epoch 15/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4116 - accuracy: 0.1133 - val_loss: 2.3033 - val_accuracy: 0.1142
Epoch 16/25
938/938 [==============================] - 3s 3ms/step - loss: 2.3173 - accuracy: 0.1107 - val_loss: 2.3042 - val_accuracy: 0.1142
Epoch 17/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3203 - accuracy: 0.1113 - val_loss: 2.3016 - val_accuracy: 0.1142
Epoch 18/25
938/938 [==============================] - 3s 3ms/step - loss: 2.3136 - accuracy: 0.1126 - val_loss: 2.3010 - val_accuracy: 0.1142
Epoch 19/25
938/938 [==============================] - 3s 3ms/step - loss: 2.3023 - accuracy: 0.1148 - val_loss: 2.3017 - val_accuracy: 0.1142
Epoch 20/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3025 - accuracy: 0.1118 - val_loss: 2.3016 - val_accuracy: 0.1142
Epoch 21/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3024 - accuracy: 0.1122 - val_loss: 2.3012 - val_accuracy: 0.1142
Epoch 22/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3055 - accuracy: 0.1089 - val_loss: 2.3023 - val_accuracy: 0.1142
Epoch 23/25
938/938 [==============================] - 3s 3ms/step - loss: 2.3058 - accuracy: 0.1084 - val_loss: 2.3035 - val_accuracy: 0.1142
Epoch 24/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3054 - accuracy: 0.1149 - val_loss: 2.3021 - val_accuracy: 0.1142
Epoch 25/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3021 - accuracy: 0.1147 - val_loss: 2.3019 - val_accuracy: 0.1142

 56%|█████▌    | 5/9 [07:05<05:39, 84.96s/it]

Epoch 1/25
938/938 [==============================] - 4s 4ms/step - loss: 318.4800 - accuracy: 0.1051 - val_loss: 2.3229 - val_accuracy: 0.0920
Epoch 2/25
938/938 [==============================] - 3s 3ms/step - loss: 3.8281 - accuracy: 0.1028 - val_loss: 2.3070 - val_accuracy: 0.1142
Epoch 3/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4062 - accuracy: 0.0993 - val_loss: 2.3154 - val_accuracy: 0.1142
Epoch 4/25
938/938 [==============================] - 3s 4ms/step - loss: 8.6324 - accuracy: 0.1045 - val_loss: 2.3085 - val_accuracy: 0.1142
Epoch 5/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3142 - accuracy: 0.1045 - val_loss: 2.3107 - val_accuracy: 0.0978
Epoch 6/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3143 - accuracy: 0.1037 - val_loss: 2.3144 - val_accuracy: 0.0924
Epoch 7/25
938/938 [==============================] - 3s 4ms/step - loss: 3.4071 - accuracy: 0.0988 - val_loss: 2.3131 - val_accuracy: 0.1060
Epoch 8/25
938/938 [==============================] - 3s 4ms/step - loss: 2.6983 - accuracy: 0.0976 - val_loss: 2.3099 - val_accuracy: 0.1040
Epoch 9/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3467 - accuracy: 0.1006 - val_loss: 2.3122 - val_accuracy: 0.0924
Epoch 10/25
938/938 [==============================] - 4s 4ms/step - loss: 2.3145 - accuracy: 0.1045 - val_loss: 2.3151 - val_accuracy: 0.1024
Epoch 11/25
938/938 [==============================] - 3s 4ms/step - loss: 3.2047 - accuracy: 0.1014 - val_loss: 2.3181 - val_accuracy: 0.1040
Epoch 12/25
938/938 [==============================] - 4s 4ms/step - loss: 2.3147 - accuracy: 0.1011 - val_loss: 2.3116 - val_accuracy: 0.1142
Epoch 13/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3130 - accuracy: 0.1033 - val_loss: 2.3190 - val_accuracy: 0.0924
Epoch 14/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3120 - accuracy: 0.1090 - val_loss: 2.3073 - val_accuracy: 0.1142
Epoch 15/25
938/938 [==============================] - 4s 4ms/step - loss: 2.4601 - accuracy: 0.1036 - val_loss: 2.3259 - val_accuracy: 0.1024
Epoch 16/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3179 - accuracy: 0.1047 - val_loss: 2.3145 - val_accuracy: 0.1000
Epoch 17/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4761 - accuracy: 0.0989 - val_loss: 2.3134 - val_accuracy: 0.0920
Epoch 18/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3148 - accuracy: 0.1005 - val_loss: 2.3081 - val_accuracy: 0.1142
Epoch 19/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3144 - accuracy: 0.0994 - val_loss: 2.3045 - val_accuracy: 0.1000
Epoch 20/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3136 - accuracy: 0.1021 - val_loss: 2.3078 - val_accuracy: 0.1142
Epoch 21/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3138 - accuracy: 0.1044 - val_loss: 2.3139 - val_accuracy: 0.1000
Epoch 22/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3134 - accuracy: 0.1010 - val_loss: 2.3108 - val_accuracy: 0.1000
Epoch 23/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3143 - accuracy: 0.1085 - val_loss: 2.3189 - val_accuracy: 0.0920
Epoch 24/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3165 - accuracy: 0.1045 - val_loss: 2.3092 - val_accuracy: 0.1040
Epoch 25/25
938/938 [==============================] - 3s 4ms/step - loss: 2.3127 - accuracy: 0.1004 - val_loss: 2.3071 - val_accuracy: 0.1142

 67%|██████▋   | 6/9 [08:30<04:15, 85.22s/it]

Epoch 1/25
938/938 [==============================] - 4s 4ms/step - loss: 220630.6226 - accuracy: 0.1054 - val_loss: 2.4432 - val_accuracy: 0.1000
Epoch 2/25
938/938 [==============================] - 3s 4ms/step - loss: 27.5078 - accuracy: 0.1054 - val_loss: 2.4129 - val_accuracy: 0.1000
Epoch 3/25
938/938 [==============================] - 3s 4ms/step - loss: 2.5075 - accuracy: 0.1026 - val_loss: 2.3585 - val_accuracy: 0.1142
Epoch 4/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4049 - accuracy: 0.0987 - val_loss: 2.4332 - val_accuracy: 0.0920
Epoch 5/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4272 - accuracy: 0.1016 - val_loss: 2.3557 - val_accuracy: 0.1000
Epoch 6/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4096 - accuracy: 0.0998 - val_loss: 2.4461 - val_accuracy: 0.0920
Epoch 7/25
938/938 [==============================] - 3s 4ms/step - loss: 133.4094 - accuracy: 0.1015 - val_loss: 2.4055 - val_accuracy: 0.1000
Epoch 8/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4220 - accuracy: 0.0998 - val_loss: 2.3537 - val_accuracy: 0.0978
Epoch 9/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4126 - accuracy: 0.0975 - val_loss: 2.3664 - val_accuracy: 0.1000
Epoch 10/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4060 - accuracy: 0.1062 - val_loss: 2.3741 - val_accuracy: 0.1040
Epoch 11/25
938/938 [==============================] - 3s 3ms/step - loss: 2.4082 - accuracy: 0.1000 - val_loss: 2.3629 - val_accuracy: 0.1142
Epoch 12/25
938/938 [==============================] - 3s 4ms/step - loss: 453.2622 - accuracy: 0.0998 - val_loss: 2.3698 - val_accuracy: 0.1142
Epoch 13/25
938/938 [==============================] - 3s 4ms/step - loss: 5.4387 - accuracy: 0.1014 - val_loss: 2.3726 - val_accuracy: 0.1142
Epoch 14/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4101 - accuracy: 0.0970 - val_loss: 2.3576 - val_accuracy: 0.1000
Epoch 15/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4086 - accuracy: 0.0988 - val_loss: 2.4158 - val_accuracy: 0.0920
Epoch 16/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4105 - accuracy: 0.1017 - val_loss: 2.3739 - val_accuracy: 0.0978
Epoch 17/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4005 - accuracy: 0.0989 - val_loss: 2.3933 - val_accuracy: 0.1024
Epoch 18/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4128 - accuracy: 0.1008 - val_loss: 2.3954 - val_accuracy: 0.1040
Epoch 19/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4199 - accuracy: 0.1026 - val_loss: 2.4326 - val_accuracy: 0.1000
Epoch 20/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4101 - accuracy: 0.0974 - val_loss: 2.4280 - val_accuracy: 0.1000
Epoch 21/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4055 - accuracy: 0.0988 - val_loss: 2.4141 - val_accuracy: 0.1142
Epoch 22/25
938/938 [==============================] - 3s 4ms/step - loss: 63.8223 - accuracy: 0.1034 - val_loss: 2.3647 - val_accuracy: 0.1060
Epoch 23/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4082 - accuracy: 0.1030 - val_loss: 2.4561 - val_accuracy: 0.1142
Epoch 24/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4043 - accuracy: 0.1002 - val_loss: 2.4729 - val_accuracy: 0.1024
Epoch 25/25
938/938 [==============================] - 3s 4ms/step - loss: 2.4149 - accuracy: 0.1027 - val_loss: 2.4434 - val_accuracy: 0.0920

 78%|███████▊  | 7/9 [09:56<02:50, 85.34s/it]

Epoch 1/25
938/938 [==============================] - 4s 4ms/step - loss: 169561410.3505 - accuracy: 0.1055 - val_loss: 2.6576 - val_accuracy: 0.1000
Epoch 2/25
938/938 [==============================] - 3s 4ms/step - loss: 307429.9418 - accuracy: 0.1017 - val_loss: 2.9726 - val_accuracy: 0.1024
Epoch 3/25
938/938 [==============================] - 3s 4ms/step - loss: 1245984.7608 - accuracy: 0.0990 - val_loss: 2.6934 - val_accuracy: 0.0920
Epoch 4/25
938/938 [==============================] - 3s 4ms/step - loss: 297662.7992 - accuracy: 0.1015 - val_loss: 3.4101 - val_accuracy: 0.0920
Epoch 5/25
938/938 [==============================] - 3s 4ms/step - loss: 52937.8988 - accuracy: 0.0964 - val_loss: 3.1202 - val_accuracy: 0.0924
Epoch 6/25
938/938 [==============================] - 3s 4ms/step - loss: 3.2518 - accuracy: 0.0996 - val_loss: 3.3321 - val_accuracy: 0.1000
Epoch 7/25
938/938 [==============================] - 3s 4ms/step - loss: 3.1897 - accuracy: 0.1002 - val_loss: 2.8414 - val_accuracy: 0.0924
Epoch 8/25
938/938 [==============================] - 4s 4ms/step - loss: 31498.3973 - accuracy: 0.1006 - val_loss: 3.2978 - val_accuracy: 0.1000
Epoch 9/25
938/938 [==============================] - 3s 4ms/step - loss: 3.1860 - accuracy: 0.0973 - val_loss: 3.8969 - val_accuracy: 0.1060
Epoch 10/25
938/938 [==============================] - 3s 4ms/step - loss: 306496.8085 - accuracy: 0.0984 - val_loss: 2.6580 - val_accuracy: 0.1040
Epoch 11/25
938/938 [==============================] - 4s 4ms/step - loss: 320392.0624 - accuracy: 0.1010 - val_loss: 3.4064 - val_accuracy: 0.1000
Epoch 12/25
938/938 [==============================] - 4s 4ms/step - loss: 3.2225 - accuracy: 0.1004 - val_loss: 2.9009 - val_accuracy: 0.1000
Epoch 13/25
938/938 [==============================] - 4s 4ms/step - loss: 273081.1942 - accuracy: 0.0973 - val_loss: 2.7147 - val_accuracy: 0.1142
Epoch 14/25
938/938 [==============================] - 3s 4ms/step - loss: 123.3137 - accuracy: 0.0995 - val_loss: 3.1362 - val_accuracy: 0.1142
Epoch 15/25
938/938 [==============================] - 4s 4ms/step - loss: 3.2720 - accuracy: 0.0959 - val_loss: 3.4643 - val_accuracy: 0.1040
Epoch 16/25
938/938 [==============================] - 3s 4ms/step - loss: 2274226.5199 - accuracy: 0.1053 - val_loss: 3.2348 - val_accuracy: 0.1024
Epoch 17/25
938/938 [==============================] - 3s 4ms/step - loss: 126646.9713 - accuracy: 0.1032 - val_loss: 3.1408 - val_accuracy: 0.1024
Epoch 18/25
938/938 [==============================] - 4s 4ms/step - loss: 3.2912 - accuracy: 0.0969 - val_loss: 3.0544 - val_accuracy: 0.1142
Epoch 19/25
938/938 [==============================] - 3s 4ms/step - loss: 3.3058 - accuracy: 0.0989 - val_loss: 3.4968 - val_accuracy: 0.0924
Epoch 20/25
938/938 [==============================] - 4s 4ms/step - loss: 3.3328 - accuracy: 0.0959 - val_loss: 3.0317 - val_accuracy: 0.1142
Epoch 21/25
938/938 [==============================] - 3s 4ms/step - loss: 3.3005 - accuracy: 0.1024 - val_loss: 3.5157 - val_accuracy: 0.0912
Epoch 22/25
938/938 [==============================] - 3s 4ms/step - loss: 3.2794 - accuracy: 0.0997 - val_loss: 2.6853 - val_accuracy: 0.1060
Epoch 23/25
938/938 [==============================] - 3s 4ms/step - loss: 3927.4691 - accuracy: 0.1026 - val_loss: 3.9686 - val_accuracy: 0.1024
Epoch 24/25
938/938 [==============================] - 3s 4ms/step - loss: 3.2818 - accuracy: 0.1001 - val_loss: 3.6366 - val_accuracy: 0.1040
Epoch 25/25
938/938 [==============================] - 3s 4ms/step - loss: 3.2329 - accuracy: 0.0942 - val_loss: 3.3682 - val_accuracy: 0.1024

 89%|████████▉ | 8/9 [11:23<01:25, 85.81s/it]

Epoch 1/25
938/938 [==============================] - 4s 4ms/step - loss: 109473295695.0886 - accuracy: 0.0990 - val_loss: 63.9248 - val_accuracy: 0.1000
Epoch 2/25
938/938 [==============================] - 3s 4ms/step - loss: 12104263962.0516 - accuracy: 0.1013 - val_loss: 55.7606 - val_accuracy: 0.0978
Epoch 3/25
938/938 [==============================] - 4s 4ms/step - loss: 762114952.5882 - accuracy: 0.0973 - val_loss: 54.4289 - val_accuracy: 0.1024
Epoch 4/25
938/938 [==============================] - 4s 4ms/step - loss: 603783672.0930 - accuracy: 0.0986 - val_loss: 30.8899 - val_accuracy: 0.1000
Epoch 5/25
938/938 [==============================] - 3s 4ms/step - loss: 161995417.6610 - accuracy: 0.0992 - val_loss: 58.8267 - val_accuracy: 0.1000
Epoch 6/25
938/938 [==============================] - 3s 4ms/step - loss: 142571550.5316 - accuracy: 0.1052 - val_loss: 44.6114 - val_accuracy: 0.1142
Epoch 7/25
938/938 [==============================] - 4s 4ms/step - loss: 56.2275 - accuracy: 0.1013 - val_loss: 54.0878 - val_accuracy: 0.0978
Epoch 8/25
938/938 [==============================] - 3s 4ms/step - loss: 50.7817 - accuracy: 0.1033 - val_loss: 48.3692 - val_accuracy: 0.1024
Epoch 9/25
938/938 [==============================] - 3s 4ms/step - loss: 133362130.6067 - accuracy: 0.0962 - val_loss: 65.1113 - val_accuracy: 0.1000
Epoch 10/25
938/938 [==============================] - 3s 4ms/step - loss: 10471252.1460 - accuracy: 0.0981 - val_loss: 31.1249 - val_accuracy: 0.0924
Epoch 11/25
938/938 [==============================] - 4s 4ms/step - loss: 55.1480 - accuracy: 0.0987 - val_loss: 25.8561 - val_accuracy: 0.1024
Epoch 12/25
938/938 [==============================] - 4s 4ms/step - loss: 466565821.0943 - accuracy: 0.1054 - val_loss: 54.7007 - val_accuracy: 0.0920
Epoch 13/25
938/938 [==============================] - 4s 4ms/step - loss: 55.8676 - accuracy: 0.0990 - val_loss: 43.2972 - val_accuracy: 0.0920
Epoch 14/25
938/938 [==============================] - 4s 4ms/step - loss: 55.1703 - accuracy: 0.0992 - val_loss: 51.8555 - val_accuracy: 0.1142
Epoch 15/25
938/938 [==============================] - 3s 4ms/step - loss: 52.6125 - accuracy: 0.0980 - val_loss: 58.2471 - val_accuracy: 0.1040
Epoch 16/25
938/938 [==============================] - 3s 4ms/step - loss: 276047414.7719 - accuracy: 0.1016 - val_loss: 46.0732 - val_accuracy: 0.0978
Epoch 17/25
938/938 [==============================] - 3s 4ms/step - loss: 48.7392 - accuracy: 0.0998 - val_loss: 68.3578 - val_accuracy: 0.1024
Epoch 18/25
938/938 [==============================] - 3s 4ms/step - loss: 56.8783 - accuracy: 0.0999 - val_loss: 30.9450 - val_accuracy: 0.1000
Epoch 19/25
938/938 [==============================] - 3s 4ms/step - loss: 959035222.6986 - accuracy: 0.1021 - val_loss: 37.2827 - val_accuracy: 0.0920
Epoch 20/25
938/938 [==============================] - 3s 4ms/step - loss: 253866554.1577 - accuracy: 0.1031 - val_loss: 30.9592 - val_accuracy: 0.0912
Epoch 21/25
938/938 [==============================] - 4s 4ms/step - loss: 49.5087 - accuracy: 0.1029 - val_loss: 58.7457 - val_accuracy: 0.1024
Epoch 22/25
938/938 [==============================] - 4s 4ms/step - loss: 54.9694 - accuracy: 0.1003 - val_loss: 61.8194 - val_accuracy: 0.0912
Epoch 23/25
938/938 [==============================] - 3s 4ms/step - loss: 1544796312.8225 - accuracy: 0.1014 - val_loss: 43.5872 - val_accuracy: 0.1142
Epoch 24/25
938/938 [==============================] - 4s 4ms/step - loss: 53.3446 - accuracy: 0.1033 - val_loss: 64.4411 - val_accuracy: 0.1000
Epoch 25/25
938/938 [==============================] - 3s 4ms/step - loss: 56.2068 - accuracy: 0.1000 - val_loss: 68.1033 - val_accuracy: 0.1024

100%|██████████| 9/9 [12:50<00:00, 85.64s/it]

fig, ax = plt.subplots()
epochs = len(history.history['accuracy'])
c = ["g", "b", "r", "y", "orange", "purple", "pink", "#5050cc", "#1ddeb1", "#5ce30e"]
for i, (name, history) in enumerate(lst_lr):
    plt.plot(list(range(1,epochs+1,1)),history.history['accuracy'], label=f"{name}", c=c[i])
#for i, (name, history) in enumerate(lst):
#    plt.plot(history.history['val_accuracy'], label=f"{name} - V", ls="--", c=c[i])
plt.ylim(0,1)
plt.xlim(1, len(history.history['accuracy']))
plt.xlabel("Epochs")
plt.ylabel("Accuracy")
plt.legend(ncol=3, loc=4)
plt.savefig("./figures/10_learning_rate.png", dpi=500, bbox_inches = 'tight', pad_inches = 0)
plt.show()

Loss

import sklearn.metrics
y_pred = np.linspace(-1.3,1.3,51)
y_true = np.array([0] * len(y_pred))

plt.plot(y_pred, np.abs(y_true- y_pred), label="L1")
plt.plot(y_pred, np.square(y_true-y_pred), label="L2")
plt.plot(y_pred, np.square(np.log(y_true+1)-np.log(y_pred+1)), label="Squared Log")
plt.ylim(0,2.5)
plt.xlim(np.min(y_pred),np.max(y_pred))
plt.xlabel('Predicted')
plt.ylabel('Loss')
plt.legend()
plt.savefig("./figures/10_loss_reg.png", dpi=500, bbox_inches = 'tight', pad_inches = 0)
plt.show()

<ipython-input-603-2c901e2c12aa>:6: RuntimeWarning: invalid value encountered in log
  plt.plot(y_pred, np.square(np.log(y_true+1)-np.log(y_pred+1)), label="Squared Log")

logits = np.linspace(0,1,100)
y_pred = np.zeros((len(logits), 2))
y_pred[:,0] = 1-logits
y_pred[:,1] = logits
#y_pred = scipy.special.softmax(y_pred)

labels = np.zeros((len(logits), 2))
labels[:,1] = 1
bce = tf.keras.losses.BinaryCrossentropy(from_logits=False)
y = []
for i in range(len(labels)):
    y.append(bce([labels[i]], [y_pred[i]]).numpy())
plt.plot(logits, y, label="Binary Crossentropy")

plt.xlabel('Predicted')
plt.ylabel('Loss')
plt.xlim(-.01, 1)

plt.ylim(0, 16)
plt.text (0.1,4, "$t=1$", size=14)
plt.legend()

plt.savefig("./figures/10_loss_class.png", dpi=500, bbox_inches = 'tight', pad_inches = 0)
plt.show()

def sigmoid(X):
    return 1/(1+np.exp(-X))

logits = np.linspace(-4,5,100)
y_pred = np.zeros((len(logits), 2))
y_pred[:,0] = 1-logits
y_pred[:,1] = logits

labels = np.zeros((len(logits), 2))
labels[:,1] = 1
bce = tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)
cce = tf.keras.losses.CategoricalCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)
plt.plot(logits, bce(labels, y_pred).numpy(), label="Binary Crossentropy")
plt.plot(logits, cce(labels, y_pred).numpy(), label="Categorical Crossentropy")

plt.xlabel('Predicted')
plt.ylabel('Loss')
plt.xlim(-3, 4)

plt.ylim(0, 3)
plt.legend()
plt.show()