Commit 279c21eb authored by Oleh Astappiev's avatar Oleh Astappiev
Browse files

feat: update HSV and SIFT code

parent aa738388
......@@ -4,7 +4,7 @@ sys.path.append("..")
from src.model.alexnet import AlexNetModel, TARGET_SHAPE
from src.data.imagenette import Imagenette
from src.data.cifar10 import Cifar10
from src.utils.embeddings import project_embeddings, load_weights_of, get_embeddings_of, save_embeddings
from src.utils.embeddings import project_embeddings, load_weights_of, get_embeddings_of, save_vectors
from src.model.siamese import SiameseModel
dataset = Imagenette(image_size=TARGET_SHAPE, map_fn=AlexNetModel.preprocess_input)
......@@ -22,6 +22,6 @@ siamese.compile(loss_margin=0.05)
siamese.fit(emb_ds, num_classes=dataset.num_classes)
projection_vectors = siamese.projection_model.predict(emb_vectors)
save_embeddings(projection_vectors, emb_labels, dataset.name + '_' + siamese.name + '_vectors')
save_vectors(projection_vectors, emb_labels, dataset.name + '_' + siamese.name + '_vectors')
project_embeddings(projection_vectors, emb_labels, siamese.name + '_' + dataset.name)
print('Done!')
import sys
sys.path.append("..")
from src.utils.hsv import extract_hsv
from src.data.cifar10 import Cifar10
from src.data.imagenette import Imagenette
from src.utils.embeddings import calc_vectors_fn, save_vectors
TARGET_SHAPE = (227, 227)
FEATURES = [512, 1024, 2048, 4096]
dataset = Imagenette(batch_size=None, image_size=TARGET_SHAPE)
# dataset = Cifar10(batch_size=None, image_size=TARGET_SHAPE)
for feat in FEATURES:
print('Extracting ' + str(feat) + ' features of HSV')
emb_vectors, emb_labels = calc_vectors_fn(dataset.get_combined(), extract_hsv, feat)
save_vectors(emb_vectors, emb_labels, dataset.name + '_hsv_' + str(feat) + '_vectors')
print('Done')
import sys
sys.path.append("..")
from src.utils.sift import extract_sift
from src.data.cifar10 import Cifar10
from src.data.imagenette import Imagenette
from src.utils.embeddings import calc_vectors_fn, save_vectors
TARGET_SHAPE = (227, 227)
FEATURES = [512, 1024, 2048, 4096]
dataset = Imagenette(batch_size=None, image_size=TARGET_SHAPE)
# dataset = Cifar10(batch_size=None, image_size=TARGET_SHAPE)
for feat in FEATURES:
print('Extracting ' + str(feat) + ' features of SIFT')
emb_vectors, emb_labels = calc_vectors_fn(dataset.get_combined(), extract_sift, feat)
save_vectors(emb_vectors, emb_labels, dataset.name + '_sift_' + str(feat) + '_vectors')
print('Done')
......@@ -5,7 +5,7 @@ PRINT_SIZE = True
DEFAULT_BATCH_SIZE = 32
class BaseDataset(ABC):
def __init__(self, name: str, classes: List[str], image_size: Tuple[int, int], batch_size: int = DEFAULT_BATCH_SIZE, map_fn: Callable = None):
def __init__(self, name: str, classes: List[str], image_size: Tuple[int, int], batch_size: int = None, map_fn: Callable = None):
self.name = name
self.classes = classes
self.num_classes = len(classes)
......
import tensorflow as tf
from src.data.base import BaseDataset, DEFAULT_BATCH_SIZE
from src.data.base import BaseDataset
DEFAULT_BATCH_SIZE = 32
DEFAULT_IMAGE_SIZE = (32, 32)
CLASS_NAMES = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
......
import tensorflow as tf
from src.data.base import BaseDataset, DEFAULT_BATCH_SIZE
from src.data.base import BaseDataset
DEFAULT_BATCH_SIZE = 32
DEFAULT_IMAGE_SIZE = (400, 320)
CLASS_NAMES = ['fish', 'dog', 'player', 'saw', 'building', 'music', 'truck', 'gas', 'ball', 'parachute']
......
......@@ -4,7 +4,7 @@ sys.path.append("..")
from src.model.efficientnet import EfficientNetModel, TARGET_SHAPE, BATCH_SIZE
from src.data.imagenette import Imagenette
from src.data.cifar10 import Cifar10
from src.utils.embeddings import project_embeddings, load_weights_of, get_embeddings_of, save_embeddings
from src.utils.embeddings import project_embeddings, load_weights_of, get_embeddings_of, save_vectors
from src.model.siamese import SiameseModel
dataset = Imagenette(image_size=TARGET_SHAPE, batch_size=BATCH_SIZE, map_fn=EfficientNetModel.preprocess_input)
......@@ -22,6 +22,6 @@ siamese.compile(loss_margin=0.05)
siamese.fit(emb_ds, num_classes=dataset.num_classes)
projection_vectors = siamese.projection_model.predict(emb_vectors)
save_embeddings(projection_vectors, emb_labels, dataset.name + '_' + siamese.name + '_vectors')
save_vectors(projection_vectors, emb_labels, dataset.name + '_' + siamese.name + '_vectors')
project_embeddings(projection_vectors, emb_labels, siamese.name + '_' + dataset.name)
print('Done!')
import sys
sys.path.append("..")
import csv
from src.utils.hsv import *
from src.utils.sift import *
from src.data.cifar10 import Cifar10
train_ds, test_ds = Cifar10()
cifar10_vds = train_ds.concatenate(test_ds)
def export_hsv(bin0=256, bin1=256, bin2=256):
header = ['ID', 'Label', 'HSV vector']
with open('../data/hsv_' + str(bin0) + '.csv', 'w', encoding='UTF8', newline='') as f:
writer = csv.writer(f, delimiter=";")
writer.writerow(header)
for i, (image, label) in enumerate(cifar10_vds):
a, b, c, hist_array = extract_hsv(image.numpy(), bin0, bin1, bin2)
label_str = ','.join(map(str, label.numpy()))
value_str = ','.join(map(str, hist_array))
writer.writerow([i, label_str, value_str])
def export_sift(nfeatures=8):
header = ['ID', 'Label', 'SIFT descriptors']
with open('../data/sift_' + str(nfeatures) + '.csv', 'w', encoding='UTF8', newline='') as f:
writer = csv.writer(f, delimiter=";")
# write the header
writer.writerow(header)
for i, (image, label) in enumerate(cifar10_vds):
keypoints, features = extract_sift(image.numpy(), nfeatures)
label_str = ','.join(map(str, label.numpy()))
if features is not None:
value_str = ','.join(map(str, features.flatten()))
else:
value_str = 'None'
print('Unable to extract SIFT from image', i)
writer.writerow([i, label_str, value_str])
# HSV
# export_hsv(170, 171, 171) # 512
# export_hsv(340, 342, 342) # 1024
# export_hsv(682, 683, 683) # 2048
# export_hsv(1366, 1365, 1365) # 4096
# SIFT
# export_sift(4)
# export_sift(8)
# export_sift(16)
# export_sift(32)
# Siamese Embeddings
print('Done!')
......@@ -5,7 +5,7 @@ import tensorflow as tf
from src.model.mobilenet import MobileNetModel, PRETRAIN_EPOCHS, TARGET_SHAPE
from src.data.imagenette import Imagenette
from src.data.cifar10 import Cifar10
from src.utils.embeddings import project_embeddings, load_weights_of, get_embeddings_of, save_embeddings
from src.utils.embeddings import project_embeddings, load_weights_of, get_embeddings_of, save_vectors
from src.model.siamese import SiameseModel
dataset = Imagenette(image_size=TARGET_SHAPE, map_fn=MobileNetModel.preprocess_input)
......@@ -24,6 +24,6 @@ siamese.compile(loss_margin=0.05)
siamese.fit(emb_ds, num_classes=dataset.num_classes)
projection_vectors = siamese.projection_model.predict(emb_vectors)
save_embeddings(projection_vectors, emb_labels, dataset.name + '_' + siamese.name + '_vectors')
save_vectors(projection_vectors, emb_labels, dataset.name + '_' + siamese.name + '_vectors')
project_embeddings(projection_vectors, emb_labels, siamese.name + '_' + dataset.name)
print('Done!')
......@@ -12,21 +12,29 @@ from src.utils.common import get_datadir, get_modeldir, get_logdir_root
from src.data.base import BaseDataset
def save_embeddings(values, labels, name='embeddings'):
def _save_vectors_path(values, labels, path):
data = [values, labels]
with open(get_datadir(name + '.pkl'), 'wb') as outfile:
with open(path, 'wb') as outfile:
pickle.dump(data, outfile, -1)
def load_embeddings(name='embeddings'):
with open(get_datadir(name + '.pkl'), 'rb') as infile:
def _load_vectors_path(path):
with open(path, 'rb') as infile:
result = pickle.load(infile)
return result[0], result[1]
def export_embeddings(values, labels, name='embeddings'):
def load_vectors(name='embeddings'):
return _load_vectors_path(get_datadir(name + '.pkl'))
def save_vectors(values, labels, name='embeddings'):
return _save_vectors_path(values, labels, get_datadir(name + '.pkl'))
def export_vectors(values, labels, name='embeddings'):
header = ['Label', 'Embeddings']
with open(get_datadir(name + '.csv'), 'w', encoding='UTF8', newline='') as f:
writer = csv.writer(f, delimiter=";")
......@@ -41,8 +49,7 @@ def export_embeddings(values, labels, name='embeddings'):
def calc_vectors(ds, model):
ds_vectors = []
ds_labels = []
for ds_batch in tqdm(ds):
images, labels = ds_batch
for images, labels in tqdm(ds):
predictions = model(images)
ds_vectors.extend(predictions.numpy().tolist())
ds_labels.extend(labels.numpy().tolist())
......@@ -50,6 +57,18 @@ def calc_vectors(ds, model):
return np.array(ds_vectors), np.array(ds_labels)
def calc_vectors_fn(ds, fn, *args):
ds_vectors = []
ds_labels = []
for image, label in tqdm(ds.as_numpy_iterator(), total=ds.cardinality().numpy()):
vector = fn(image, *args)
if vector is not None:
ds_vectors.append(vector)
ds_labels.append(label)
return np.array(ds_vectors), np.array(ds_labels)
def evaluate_vectors(values, labels):
total = len(values)
match = 0
......@@ -116,24 +135,9 @@ def get_embeddings_of(model: tf.keras.Model, dataset: BaseDataset):
embedding_file = get_datadir(model.name + '_' + dataset.name + '.pkl')
if Path(embedding_file).exists():
with open(embedding_file, 'rb') as infile:
emb_vectors, emb_labels = pickle.load(infile)
return emb_vectors, emb_labels
return _load_vectors_path(embedding_file)
else:
print('calculating vectors...')
ds_vectors = []
ds_labels = []
for ds_batch in tqdm(dataset.get_combined()):
images, labels = ds_batch
predictions = model(images)
ds_vectors.extend(predictions.numpy().tolist())
ds_labels.extend(labels.numpy().tolist())
emb_vectors = np.array(ds_vectors)
emb_labels = np.array(ds_labels)
data = [emb_vectors, emb_labels]
with open(embedding_file, 'wb') as outfile:
pickle.dump(data, outfile, -1)
emb_vectors, emb_labels = calc_vectors(dataset.get_combined(), model)
_save_vectors_path(emb_vectors, emb_labels, embedding_file)
return emb_vectors, emb_labels
import numpy as np
import cv2
from src.utils.plot import *
def extract_hsv(image, bin0=256, bin1=256, bin2=256):
def hsv_histogram(image, bin_h=256, bin_s=256, bin_v=256):
"""Extract a 3 color channels histogram from the HSV"""
hsv = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)
# The ranges of the 3 HSV channels in opencv are 0-180, 0-256, 0-256 respectively
histh = cv2.calcHist([hsv], [0], None, [bin0], [0, 180])
hists = cv2.calcHist([hsv], [1], None, [bin1], [0, 256])
histv = cv2.calcHist([hsv], [2], None, [bin2], [0, 256])
hist_h = cv2.calcHist([hsv], [0], None, [bin_h], [0, 180])
hist_s = cv2.calcHist([hsv], [1], None, [bin_s], [0, 256])
hist_v = cv2.calcHist([hsv], [2], None, [bin_v], [0, 256])
# normalize the histogram
histh /= histh.sum()
hists /= hists.sum()
histv /= histv.sum()
hist_array = np.append(histh, hists)
hist_array = np.append(hist_array, histv)
# return the flattened histogram as the feature vector
return histh, hists, histv, hist_array.flatten()
hist_h /= hist_h.sum()
hist_s /= hist_s.sum()
hist_v /= hist_v.sum()
return hist_h, hist_s, hist_v
def plot_hsv(dataset, target_shape):
plt.figure(figsize=(20, 20))
for i, (image, label) in enumerate(dataset.take(3)):
# from smaller image only smaller number of key points can be extracted
img = cv2.resize(image.numpy(), target_shape)
def extract_hsv(image, features=512):
bin_size = int(features / 3)
bin_h = bin_size
bin_s = bin_size
bin_v = bin_size
subplot_image(3, 2, i * 2 + 1, img, "Original image")
remainder = features % 3
if remainder == 2:
bin_s += 1
bin_v += 1
elif remainder == 1:
bin_h += 1
hist0_s, hist1_s, hist2_s, hist_s = extract_hsv(img)
# print('the length of histogram of the sample', len(hist_s))
hist_h, hist_s, hist_v = hsv_histogram(image, bin_h, bin_s, bin_v)
# subplot_image(3, 2, i * 2 + 2, image, "HSV Histogram")
ax = plt.subplot(3, 2, i * 2 + 2)
# ax.imshow(image)
ax.set_title("HSV Histogram")
ax.plot(hist0_s, label='H')
ax.plot(hist1_s, label='S')
ax.plot(hist2_s, label='V')
plt.xlabel("Bins")
plt.ylabel("percentage of Pixels")
plt.legend()
# ax.axis('off')
plt.show()
# return the flattened histogram as the feature vector
hist_array = np.append(hist_h, hist_s)
hist_array = np.append(hist_array, hist_v)
return hist_array.flatten()
import cv2
import numpy as np
import matplotlib.pyplot as plt
from scipy.spatial import distance_matrix
......@@ -48,6 +49,32 @@ def plot_vectors(vectors):
plt.show()
def plot_hsv(hist_h, hist_s, hist_v):
plt.figure()
ax = plt.subplot()
ax.set_title("HSV Histogram")
ax.plot(hist_h, label='H')
ax.plot(hist_s, label='S')
ax.plot(hist_v, label='V')
plt.xlabel("Bins")
plt.ylabel("Percentage of Pixels")
plt.legend()
# ax.axis('off')
plt.show()
def plot_sift(image, keypoints):
plt.figure(figsize=(20, 20))
# from smaller image only smaller number of key points can be extracted
subplot_image(1, 2, 1, image, "Original image")
img_kp = image.copy()
cv2.drawKeypoints(img_kp, keypoints, img_kp, color=(255, 0, 0))
subplot_image(1, 2, 2, img_kp, "Keypoints")
plt.show()
def calc_under_margin(vectors, margin=0.1):
dm = distance_matrix(vectors, vectors)
print('Under the margin', (dm < margin).sum() / 2)
import matplotlib.pyplot as plt
import cv2
from src.utils.plot import *
def extract_sift(image, nfeatures=None):
# the result number of features is the number of keypoints * 128
def sift_features(image, nfeatures=None):
sift = cv2.SIFT_create(nfeatures)
# Calculate the keypoint and each point description of the image
keypoints, features = sift.detectAndCompute(image, None)
return keypoints, features
def plot_sift(dataset, target_shape):
plt.figure(figsize=(20, 20))
for i, (image, label) in enumerate(dataset.take(3)):
# from smaller image only smaller number of key points can be extracted
img = cv2.resize(image.numpy(), target_shape)
subplot_image(3, 2, i * 2 + 1, img, "Original image")
keypoints, features = extract_sift(img)
img_kp = img.copy()
cv2.drawKeypoints(img_kp, keypoints, img_kp, color=(255, 0, 0))
subplot_image(3, 2, i * 2 + 2, img_kp, "Keypoints")
plt.show()
def extract_sift(image, features=512):
# the result number of features is the number of keypoints * 128
nfeatures = int(features / 128)
keypoints, features = sift_features(image, nfeatures)
if features is None or len(features) < nfeatures:
return None
elif len(features) > nfeatures:
features = features[:nfeatures]
return features.flatten()
......@@ -5,7 +5,7 @@ import tensorflow as tf
from src.model.vgg16 import VGG16Model, PRETRAIN_EPOCHS, TARGET_SHAPE
from src.data.imagenette import Imagenette
from src.data.cifar10 import Cifar10
from src.utils.embeddings import project_embeddings, load_weights_of, get_embeddings_of, save_embeddings
from src.utils.embeddings import project_embeddings, load_weights_of, get_embeddings_of, save_vectors
from src.model.siamese import SiameseModel
dataset = Imagenette(image_size=TARGET_SHAPE, map_fn=VGG16Model.preprocess_input)
......@@ -24,6 +24,6 @@ siamese.compile(loss_margin=0.05)
siamese.fit(emb_ds, num_classes=dataset.num_classes)
projection_vectors = siamese.projection_model.predict(emb_vectors)
save_embeddings(projection_vectors, emb_labels, dataset.name + '_' + siamese.name + '_vectors')
save_vectors(projection_vectors, emb_labels, dataset.name + '_' + siamese.name + '_vectors')
project_embeddings(projection_vectors, emb_labels, siamese.name + '_' + dataset.name)
print('Done!')
......@@ -5,7 +5,7 @@ import tensorflow as tf
from src.model.vit import VitModel, TARGET_SHAPE, BATCH_SIZE
from src.data.imagenette import Imagenette
from src.data.cifar10 import Cifar10
from src.utils.embeddings import project_embeddings, load_weights_of, get_embeddings_of, save_embeddings
from src.utils.embeddings import project_embeddings, load_weights_of, get_embeddings_of, save_vectors
from src.model.siamese import SiameseModel
dataset = Imagenette(image_size=TARGET_SHAPE, batch_size=BATCH_SIZE, map_fn=VitModel.preprocess_input)
......@@ -23,6 +23,6 @@ siamese.compile(loss_margin=0.05, optimizer=tf.keras.optimizers.Adam(learning_ra
siamese.fit(emb_ds, num_classes=dataset.num_classes)
projection_vectors = siamese.projection_model.predict(emb_vectors)
save_embeddings(projection_vectors, emb_labels, dataset.name + '_' + siamese.name + '_vectors')
save_vectors(projection_vectors, emb_labels, dataset.name + '_' + siamese.name + '_vectors')
project_embeddings(projection_vectors, emb_labels, siamese.name + '_' + dataset.name)
print('Done!')
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment