Commit c95f3b2a authored by Oleh Astappiev's avatar Oleh Astappiev
Browse files

feat: move to embedding model to model classes

parent ffa4e7d0
......@@ -4,5 +4,5 @@ tensorboard~=2.9.0
matplotlib~=3.5.2
opencv-python~=4.5.5.64
tqdm~=4.64.0
tensorflow-addons~=0.16.1
tensorflow-addons~=0.17.0
scipy~=1.8.1
......@@ -5,7 +5,7 @@ import tensorflow as tf
from src.data.imagenette import load_dataset3, NUM_CLASSES
from src.utils.embeddings import save_embeddings, load_embeddings, project_embeddings, calc_vectors
from src.utils.common import get_modeldir
from src.model.alexnet import AlexNetModel, TARGET_SHAPE
from src.model.alexnet import AlexNetModel, TARGET_SHAPE, EMBEDDING_VECTOR_DIMENSION
from src.model.siamese import SiameseModel
model_name = 'imagenette_alexnet'
......@@ -34,7 +34,7 @@ for layer in model.layers:
layer.trainable = False
print('calculating embeddings...')
embedding_model = tf.keras.Model(inputs=model.input, outputs=model.layers[-2].output)
embedding_model = model.get_embedding_model()
embedding_model.summary()
emb_vectors, emb_labels = calc_vectors(comb_ds, embedding_model)
......@@ -43,7 +43,7 @@ save_embeddings(emb_vectors, emb_labels, embeddings_name)
# emb_vectors, emb_labels = load_embeddings(embeddings_name)
# siamese is the model we train
siamese = SiameseModel(embedding_vector_dimension=4096, image_vector_dimensions=3)
siamese = SiameseModel(embedding_vector_dimension=EMBEDDING_VECTOR_DIMENSION, image_vector_dimensions=3)
siamese.compile(loss_margin=0.05)
siamese.summary()
......
......@@ -5,7 +5,7 @@ import tensorflow as tf
from src.data.cifar10 import load_dataset3, NUM_CLASSES
from src.utils.embeddings import save_embeddings, load_embeddings, project_embeddings, calc_vectors
from src.utils.common import get_modeldir
from src.model.alexnet import AlexNetModel, TARGET_SHAPE
from src.model.alexnet import AlexNetModel, TARGET_SHAPE, EMBEDDING_VECTOR_DIMENSION
from src.model.siamese import SiameseModel
model_name = 'cifar10_alexnet'
......@@ -34,7 +34,7 @@ for layer in model.layers:
layer.trainable = False
print('calculating embeddings...')
embedding_model = tf.keras.Model(inputs=model.input, outputs=model.layers[-2].output)
embedding_model = model.get_embedding_model()
embedding_model.summary()
emb_vectors, emb_labels = calc_vectors(comb_ds, embedding_model)
......@@ -43,7 +43,7 @@ save_embeddings(emb_vectors, emb_labels, embeddings_name)
# emb_vectors, emb_labels = load_embeddings(embeddings_name)
# siamese is the model we train
siamese = SiameseModel(embedding_vector_dimension=4096, image_vector_dimensions=3)
siamese = SiameseModel(embedding_vector_dimension=EMBEDDING_VECTOR_DIMENSION, image_vector_dimensions=3)
siamese.compile(loss_margin=0.05)
siamese.summary()
......@@ -52,7 +52,7 @@ history = siamese.fit(ds, class_weight={0: 1 / NUM_CLASSES, 1: (NUM_CLASSES - 1)
# Build full inference model (from image to image vector):
inference_model = siamese.get_inference_model(embedding_model)
inference_model.save(get_modeldir(model_name + '_inference.tf'), save_format='tf', include_optimizer=False)
inference_model.save(get_modeldir(model_name + '_inference.tf'), save_format='tf', include_optimizer=False)\
# inference_model = tf.keras.models.load_model(get_modeldir(model_name + '_inference.tf'), compile=False)
......@@ -60,3 +60,6 @@ print('visualization')
# compute vectors of the images and their labels, store them in a tsv file for visualization
siamese_vectors, siamese_labels = calc_vectors(comb_ds, inference_model)
project_embeddings(siamese_vectors, siamese_labels, model_name + '_siamese')
projection_vectors = siamese.get_projection_model().predict(emb_vectors)
project_embeddings(projection_vectors, emb_labels, model_name + '_siamese2')
......@@ -24,7 +24,7 @@ save_embeddings(emb_vectors, emb_labels, embeddings_name)
# siamese is the model we train
siamese = SiameseModel(embedding_vector_dimension=1280, image_vector_dimensions=128)
siamese.compile(loss_margin=0.005)
siamese.compile(loss_margin=0.05)
siamese.summary()
ds = SiameseModel.prepare_dataset(emb_vectors, emb_labels)
......
......@@ -5,7 +5,7 @@ import tensorflow as tf
from src.data.imagenette import load_dataset3, NUM_CLASSES
from src.utils.embeddings import save_embeddings, project_embeddings, calc_vectors
from src.utils.common import get_modeldir
from src.model.mobilenet import MobileNetModel, PRETRAIN_EPOCHS, TARGET_SHAPE
from src.model.mobilenet import MobileNetModel, PRETRAIN_EPOCHS, TARGET_SHAPE, EMBEDDING_VECTOR_DIMENSION
from src.model.siamese import SiameseModel
model_name = 'imagenet_mobilenet'
......@@ -21,11 +21,11 @@ model.compile(optimizer=tf.keras.optimizers.RMSprop(tf.keras.optimizers.schedule
model.summary()
# load weights
# model.load_weights(get_modeldir(model_name + '.h5'))
model.load_weights(get_modeldir(model_name + '.h5'))
# train & save model
model.fit(train_ds, epochs=PRETRAIN_EPOCHS, validation_data=val_ds)
model.save_weights(get_modeldir(model_name + '.h5'))
# model.fit(train_ds, epochs=PRETRAIN_EPOCHS, validation_data=val_ds)
# model.save_weights(get_modeldir(model_name + '.h5'))
# evaluate
print('evaluating...')
......@@ -35,7 +35,7 @@ for layer in model.layers:
layer.trainable = False
print('calculating embeddings...')
embedding_model = tf.keras.Model(inputs=model.input, outputs=model.layers[-7].output)
embedding_model = model.get_embedding_model()
embedding_model.summary()
emb_vectors, emb_labels = calc_vectors(comb_ds, embedding_model)
......@@ -44,7 +44,7 @@ save_embeddings(emb_vectors, emb_labels, embeddings_name)
# emb_vectors, emb_labels = load_embeddings(embeddings_name)
# siamese is the model we train
siamese = SiameseModel(embedding_vector_dimension=1024, image_vector_dimensions=3)
siamese = SiameseModel(embedding_vector_dimension=EMBEDDING_VECTOR_DIMENSION, image_vector_dimensions=3)
siamese.compile(loss_margin=0.05)
siamese.summary()
......
......@@ -5,7 +5,7 @@ import tensorflow as tf
from src.data.cifar10 import load_dataset3, NUM_CLASSES
from src.utils.embeddings import save_embeddings, project_embeddings, calc_vectors
from src.utils.common import get_modeldir
from src.model.mobilenet import MobileNetModel, PRETRAIN_EPOCHS, TARGET_SHAPE
from src.model.mobilenet import MobileNetModel, PRETRAIN_EPOCHS, TARGET_SHAPE, EMBEDDING_VECTOR_DIMENSION
from src.model.siamese import SiameseModel
model_name = 'cifar10_mobilenet'
......@@ -34,7 +34,7 @@ for layer in model.layers:
layer.trainable = False
print('calculating embeddings...')
embedding_model = tf.keras.Model(inputs=model.input, outputs=model.layers[-7].output)
embedding_model = model.get_embedding_model()
embedding_model.summary()
emb_vectors, emb_labels = calc_vectors(comb_ds, embedding_model)
......@@ -43,7 +43,7 @@ save_embeddings(emb_vectors, emb_labels, embeddings_name)
# emb_vectors, emb_labels = load_embeddings(embeddings_name)
# siamese is the model we train
siamese = SiameseModel(embedding_vector_dimension=1024, image_vector_dimensions=3)
siamese = SiameseModel(embedding_vector_dimension=EMBEDDING_VECTOR_DIMENSION, image_vector_dimensions=3)
siamese.compile(loss_margin=0.05)
siamese.summary()
......
import tensorflow as tf
from tensorflow.keras import layers, callbacks, Sequential
from tensorflow.keras import layers, callbacks, Sequential, Model
from src.utils.common import get_logdir
tensorboard_cb = callbacks.TensorBoard(get_logdir('alexnet/fit'))
......@@ -8,6 +8,7 @@ BATCH_SIZE = 32
TARGET_SHAPE = (227, 227)
PRETRAIN_EPOCHS = 50
EMBEDDING_VECTOR_DIMENSION = 4096
class AlexNetModel(Sequential):
def __init__(self, classes=10):
......@@ -52,6 +53,9 @@ class AlexNetModel(Sequential):
def fit(self, x=None, y=None, batch_size=None, epochs=PRETRAIN_EPOCHS, callbacks=[tensorboard_cb], **kwargs):
return super().fit(x=x, y=y, batch_size=batch_size, epochs=epochs, callbacks=callbacks, **kwargs)
def get_embedding_model(self):
return Model(inputs=self.input, outputs=self.layers[-2].output)
@staticmethod
def preprocess_input(image, label):
# Normalize images to have a mean of 0 and standard deviation of 1
......
import tensorflow as tf
from tensorflow.keras import layers, callbacks, Model
from tensorflow.keras import layers, callbacks, Model, Sequential
from src.utils.common import get_logdir
tensorboard_cb = callbacks.TensorBoard(get_logdir('mobilenet/fit'))
......@@ -8,6 +8,7 @@ BATCH_SIZE = 32
TARGET_SHAPE = (224, 224)
PRETRAIN_EPOCHS = 20
EMBEDDING_VECTOR_DIMENSION = 1024
class MobileNetModel(Model):
def __init__(self):
......@@ -35,6 +36,13 @@ class MobileNetModel(Model):
def fit(self, x=None, y=None, batch_size=None, epochs=PRETRAIN_EPOCHS, callbacks=[tensorboard_cb], **kwargs):
return super().fit(x=x, y=y, batch_size=batch_size, epochs=epochs, callbacks=callbacks, **kwargs)
def get_embedding_model(self):
core = Model(inputs=self.input, outputs=self.layers[-7].output)
return Sequential([
core,
layers.Flatten(),
])
@staticmethod
def preprocess_input(image, label):
image = tf.cast(image, tf.float32)
......
......@@ -8,6 +8,7 @@ BATCH_SIZE = 32
TARGET_SHAPE = (224, 224)
PRETRAIN_EPOCHS = 20
EMBEDDING_VECTOR_DIMENSION = 4096
class VGG16Model(Model):
def __init__(self, input_shape=TARGET_SHAPE, weights=None, **kwargs):
......@@ -49,6 +50,9 @@ class VGG16Model(Model):
def fit(self, x=None, y=None, batch_size=None, epochs=PRETRAIN_EPOCHS, callbacks=[tensorboard_cb], **kwargs):
return super().fit(x=x, y=y, batch_size=batch_size, epochs=epochs, callbacks=callbacks, **kwargs)
def get_embedding_model(self):
return Model(inputs=self.input, outputs=self.layers[-2].output)
@staticmethod
def preprocess_input(image, label):
image = tf.keras.applications.vgg16.preprocess_input(image)
......
......@@ -8,7 +8,7 @@ from scipy.spatial import distance_matrix
from src.data.simple3 import load_dataset3, NUM_CLASSES
from src.utils.embeddings import project_embeddings, calc_vectors, save_embeddings
from src.utils.common import get_modeldir
from src.model.alexnet import AlexNetModel, TARGET_SHAPE
from src.model.alexnet import AlexNetModel, TARGET_SHAPE, EMBEDDING_VECTOR_DIMENSION
from src.model.siamese import SiameseModel
model_name = 'simple3_alexnet'
......@@ -37,7 +37,7 @@ for layer in model.layers:
layer.trainable = False
print('calculating embeddings...')
embedding_model = tf.keras.Model(inputs=model.input, outputs=model.layers[-2].output)
embedding_model = model.get_embedding_model()
embedding_model.summary()
emb_vectors, emb_labels = calc_vectors(comb_ds, embedding_model)
......@@ -47,7 +47,7 @@ save_embeddings(emb_vectors, emb_labels, embeddings_name)
# emb_vectors, emb_labels = load_embeddings(embeddings_name)
# siamese is the model we train
siamese = SiameseModel(embedding_vector_dimension=4096, image_vector_dimensions=3)
siamese = SiameseModel(embedding_vector_dimension=EMBEDDING_VECTOR_DIMENSION, image_vector_dimensions=3)
siamese.compile(loss_margin=0.1) # TODO: experiment with high value, e.g. 2
siamese.summary()
......
......@@ -5,7 +5,7 @@ import tensorflow as tf
from src.data.imagenette import load_dataset3, NUM_CLASSES
from src.utils.embeddings import save_embeddings, project_embeddings, calc_vectors
from src.utils.common import get_modeldir
from src.model.vgg16 import VGG16Model, PRETRAIN_EPOCHS, TARGET_SHAPE
from src.model.vgg16 import VGG16Model, PRETRAIN_EPOCHS, TARGET_SHAPE, EMBEDDING_VECTOR_DIMENSION
from src.model.siamese import SiameseModel
model_name = 'imagenet_vgg16'
......@@ -35,7 +35,7 @@ for layer in model.layers:
layer.trainable = False
print('calculating embeddings...')
embedding_model = tf.keras.Model(inputs=model.input, outputs=model.layers[-2].output)
embedding_model = model.get_embedding_model()
embedding_model.summary()
emb_vectors, emb_labels = calc_vectors(comb_ds, embedding_model)
......@@ -44,7 +44,7 @@ save_embeddings(emb_vectors, emb_labels, embeddings_name)
# emb_vectors, emb_labels = load_embeddings(embeddings_name)
# siamese is the model we train
siamese = SiameseModel(embedding_vector_dimension=4096, image_vector_dimensions=3)
siamese = SiameseModel(embedding_vector_dimension=EMBEDDING_VECTOR_DIMENSION, image_vector_dimensions=3)
siamese.compile(loss_margin=0.05)
siamese.summary()
......
......@@ -5,7 +5,7 @@ import tensorflow as tf
from src.data.cifar10 import load_dataset3, NUM_CLASSES
from src.utils.embeddings import save_embeddings, project_embeddings, calc_vectors
from src.utils.common import get_modeldir
from src.model.vgg16 import VGG16Model, PRETRAIN_EPOCHS
from src.model.vgg16 import VGG16Model, PRETRAIN_EPOCHS, EMBEDDING_VECTOR_DIMENSION
from src.model.siamese import SiameseModel
model_name = 'cifar10_vgg16'
......@@ -37,7 +37,7 @@ for layer in model.layers:
layer.trainable = False
print('calculating embeddings...')
embedding_model = tf.keras.Model(inputs=model.input, outputs=model.layers[-2].output)
embedding_model = model.get_embedding_model()
embedding_model.summary()
emb_vectors, emb_labels = calc_vectors(comb_ds, embedding_model)
......@@ -46,7 +46,7 @@ save_embeddings(emb_vectors, emb_labels, embeddings_name)
# emb_vectors, emb_labels = load_embeddings(embeddings_name)
# siamese is the model we train
siamese = SiameseModel(embedding_vector_dimension=4096, image_vector_dimensions=3)
siamese = SiameseModel(embedding_vector_dimension=EMBEDDING_VECTOR_DIMENSION, image_vector_dimensions=3)
siamese.compile(loss_margin=0.05)
siamese.summary()
......
......@@ -5,7 +5,7 @@ import tensorflow as tf
from src.data.imagenette import load_dataset3, NUM_CLASSES
from src.utils.embeddings import save_embeddings, project_embeddings, calc_vectors
from src.utils.common import get_modeldir
from src.model.vgg16 import VGG16Model, PRETRAIN_EPOCHS
from src.model.vgg16 import VGG16Model, PRETRAIN_EPOCHS, EMBEDDING_VECTOR_DIMENSION
from src.model.siamese import SiameseModel
model_name = 'imagenette_vgg16_small'
......@@ -18,16 +18,7 @@ comb_ds = train_ds.concatenate(val_ds).concatenate(test_ds)
PRETRAIN_TOTAL_STEPS = PRETRAIN_EPOCHS * len(train_ds)
# create model
model = tf.keras.applications.VGG16(
include_top=True,
input_shape=(32, 32, 3),
weights=None,
classes=10
)
PRETRAIN_EPOCHS = 20
PRETRAIN_TOTAL_STEPS = PRETRAIN_EPOCHS * len(train_ds)
model = VGG16Model(input_shape=TARGET_SHAPE)
model.compile(optimizer=tf.keras.optimizers.RMSprop(tf.keras.optimizers.schedules.CosineDecay(1e-3, PRETRAIN_TOTAL_STEPS)))
model.summary()
......@@ -46,7 +37,7 @@ for layer in model.layers:
layer.trainable = False
print('calculating embeddings...')
embedding_model = tf.keras.Model(inputs=model.input, outputs=model.layers[-2].output)
embedding_model = model.get_embedding_model()
embedding_model.summary()
emb_vectors, emb_labels = calc_vectors(comb_ds, embedding_model)
......@@ -55,7 +46,7 @@ save_embeddings(emb_vectors, emb_labels, embeddings_name)
# emb_vectors, emb_labels = load_embeddings(embeddings_name)
# siamese is the model we train
siamese = SiameseModel(embedding_vector_dimension=4096, image_vector_dimensions=3)
siamese = SiameseModel(embedding_vector_dimension=EMBEDDING_VECTOR_DIMENSION, image_vector_dimensions=3)
siamese.compile(loss_margin=0.05)
siamese.summary()
......
......@@ -25,7 +25,7 @@ save_embeddings(emb_vectors, emb_labels, embeddings_name)
# siamese is the model we train
siamese = SiameseModel(embedding_vector_dimension=384, image_vector_dimensions=512)
siamese.compile(loss_margin=0.005, optimizer=tf.keras.optimizers.Adam(learning_rate=0.001))
siamese.compile(loss_margin=0.05, optimizer=tf.keras.optimizers.Adam(learning_rate=0.001))
siamese.summary()
ds = SiameseModel.prepare_dataset(emb_vectors, emb_labels)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment