first trained
Esse commit está contido em:
Charlie Hewitt
Arquivo binário não exibido.
+140
-97
@@ -1,51 +1,51 @@
|
||||
from keras.preprocessing import image
|
||||
from keras.models import Sequential
|
||||
from keras.layers import Conv2D, Conv3D, MaxPooling2D, MaxPooling3D, Activation, Dropout, Flatten, Dense, BatchNormalization, AveragePooling2D
|
||||
from keras.utils.np_utils import to_categorical
|
||||
from keras.optimizers import SGD, Adam
|
||||
from keras.callbacks import LearningRateScheduler, ModelCheckpoint
|
||||
from math import floor
|
||||
from sklearn.utils import class_weight
|
||||
import csv
|
||||
import numpy as np
|
||||
import os
|
||||
from math import floor
|
||||
|
||||
import numpy as np
|
||||
from keras.callbacks import LearningRateScheduler, ModelCheckpoint
|
||||
from keras.layers import (Activation, AveragePooling2D, BatchNormalization, Conv2D, Conv3D, Dense, Dropout, Flatten, MaxPooling2D, MaxPooling3D)
|
||||
from keras.models import Sequential, model_from_json
|
||||
from keras.optimizers import SGD, Adam
|
||||
from keras.preprocessing import image
|
||||
from keras.utils.np_utils import to_categorical
|
||||
from sklearn.utils import class_weight
|
||||
|
||||
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
|
||||
CLASSIFY = 0
|
||||
REGRESS = 1
|
||||
|
||||
|
||||
# OTPIONS #
|
||||
CLASSIFY_OR_REGRESS = CLASSIFY
|
||||
# OTPIONS #
|
||||
BATCH_SIZE = 256
|
||||
EPOCHS = 16
|
||||
|
||||
# LOADERS #
|
||||
def load_images(paths, labels, batch_size=32):
|
||||
def load_images(C_or_R, paths, labels, batch_size=32):
|
||||
batch_n = 0
|
||||
while True:
|
||||
batch_d = []
|
||||
batch_l = []
|
||||
for i in range(batch_size):
|
||||
if batch_n*batch_size + i > len(paths) - 1:
|
||||
if batch_n * batch_size + i > len(paths) - 1:
|
||||
batch_n = 0
|
||||
path = paths[batch_n*batch_size + i]
|
||||
path = paths[batch_n * batch_size + i]
|
||||
img = image.load_img(path, target_size=(96, 96))
|
||||
x = image.img_to_array(img)/255
|
||||
x = image.img_to_array(img) / 255
|
||||
x = image.random_rotation(x, 20)
|
||||
x = image.random_shift(x, 0.1, 0.1)
|
||||
if np.random.random() < 0.5:
|
||||
x = image.flip_axis(x, 1)
|
||||
y = labels[batch_n*batch_size + i]
|
||||
y = labels[batch_n * batch_size + i]
|
||||
batch_d.append(x)
|
||||
batch_l.append(y)
|
||||
batch_d = np.array(batch_d).reshape((batch_size, 96, 96, 3))
|
||||
if CLASSIFY_OR_REGRESS == CLASSIFY:
|
||||
if C_or_R == CLASSIFY:
|
||||
batch_l = np.array(batch_l).reshape((batch_size, 8))
|
||||
else:
|
||||
batch_l = np.array(batch_l).reshape((batch_size, 2))
|
||||
yield (batch_d, batch_l)
|
||||
batch_n += 1
|
||||
|
||||
|
||||
def load_paths(p):
|
||||
labels = []
|
||||
paths = []
|
||||
@@ -59,7 +59,7 @@ def load_paths(p):
|
||||
valence = float(row[-2])
|
||||
arousal = float(row[-1])
|
||||
emotion = int(row[-3])
|
||||
path = 'data_p/' + row[0]
|
||||
path = 'data_p/' + row[0]
|
||||
if emotion > 7 and (arousal == -2 or valence == -2):
|
||||
continue
|
||||
if not os.path.exists(path):
|
||||
@@ -72,12 +72,13 @@ def load_paths(p):
|
||||
print('Loaded:', count)
|
||||
return paths, labels
|
||||
|
||||
def process_data(paths, labels):
|
||||
|
||||
def process_data(C_or_R, paths, labels):
|
||||
labels_out = []
|
||||
paths_out = []
|
||||
count = 0
|
||||
for i, (emotion, valence, arousal) in enumerate(labels):
|
||||
if CLASSIFY_OR_REGRESS == CLASSIFY:
|
||||
if C_or_R == CLASSIFY:
|
||||
if emotion > 7:
|
||||
# ignore invalid emotions
|
||||
continue
|
||||
@@ -91,7 +92,7 @@ def process_data(paths, labels):
|
||||
paths_out.append(paths[i])
|
||||
count += 1
|
||||
print('Processed:', count, end='\r')
|
||||
if CLASSIFY_OR_REGRESS == CLASSIFY:
|
||||
if C_or_R == CLASSIFY:
|
||||
weights = class_weight.compute_class_weight('balanced', np.unique(labels_out), labels_out)
|
||||
weights = dict(enumerate(weights))
|
||||
labels_out = to_categorical(labels_out, num_classes=8)
|
||||
@@ -100,83 +101,125 @@ def process_data(paths, labels):
|
||||
print('Processed:', count)
|
||||
return paths_out, labels_out, weights
|
||||
|
||||
# MODEL #
|
||||
model = Sequential()
|
||||
# CONV BLOCK 1
|
||||
model.add(Conv2D(32, (3, 3), input_shape=(96, 96, 3)))
|
||||
model.add(BatchNormalization(axis=-1))
|
||||
model.add(Activation('relu'))
|
||||
model.add(Conv2D(32, (3, 3)))
|
||||
model.add(BatchNormalization(axis=-1))
|
||||
model.add(Activation('relu'))
|
||||
model.add(MaxPooling2D(pool_size=(2,2)))
|
||||
# CONV BLOCK 2
|
||||
model.add(Conv2D(64,(3, 3)))
|
||||
model.add(BatchNormalization(axis=-1))
|
||||
model.add(Activation('relu'))
|
||||
model.add(Conv2D(64, (3, 3)))
|
||||
model.add(BatchNormalization(axis=-1))
|
||||
model.add(Activation('relu'))
|
||||
model.add(MaxPooling2D(pool_size=(2,2)))
|
||||
# CONV BLOCK 3
|
||||
model.add(Conv2D(128,(3, 3)))
|
||||
model.add(BatchNormalization(axis=-1))
|
||||
model.add(Activation('relu'))
|
||||
model.add(Conv2D(128, (3, 3)))
|
||||
model.add(BatchNormalization(axis=-1))
|
||||
model.add(Activation('relu'))
|
||||
model.add(MaxPooling2D(pool_size=(2,2)))
|
||||
# FLATTEN
|
||||
model.add(Flatten())
|
||||
# DENSE 1
|
||||
model.add(Dense(128))
|
||||
model.add(BatchNormalization())
|
||||
model.add(Activation('relu'))
|
||||
model.add(Dropout(0.5))
|
||||
# DENSE 2
|
||||
model.add(Dense(128))
|
||||
model.add(BatchNormalization())
|
||||
model.add(Activation('relu'))
|
||||
model.add(Dropout(0.5))
|
||||
# OUTPUT
|
||||
if CLASSIFY_OR_REGRESS == CLASSIFY:
|
||||
model.add(Dense(8, activation='softmax'))
|
||||
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
|
||||
else:
|
||||
model.add(Dense(2, activation='linear'))
|
||||
model.compile(loss='mean_squared_error', optimizer='adam')
|
||||
|
||||
print('** LOADING DATA **')
|
||||
t_paths = np.load('training_paths.npy')
|
||||
t_labels = np.load('training_labels.npy')
|
||||
t_paths, t_labels, t_weights = process_data(t_paths, t_labels)
|
||||
v_paths = np.load('validation_paths.npy')
|
||||
v_labels = np.load('validation_labels.npy')
|
||||
v_paths, v_labels, v_weights = process_data(v_paths, v_labels)
|
||||
def base_model(C_or_R):
|
||||
model = Sequential()
|
||||
# CONV BLOCK 1
|
||||
model.add(Conv2D(32, (3, 3), input_shape=(96, 96, 3)))
|
||||
model.add(BatchNormalization(axis=-1))
|
||||
model.add(Activation('relu'))
|
||||
model.add(Conv2D(32, (3, 3)))
|
||||
model.add(BatchNormalization(axis=-1))
|
||||
model.add(Activation('relu'))
|
||||
model.add(MaxPooling2D(pool_size=(2, 2)))
|
||||
# CONV BLOCK 2
|
||||
model.add(Conv2D(64, (3, 3)))
|
||||
model.add(BatchNormalization(axis=-1))
|
||||
model.add(Activation('relu'))
|
||||
model.add(Conv2D(64, (3, 3)))
|
||||
model.add(BatchNormalization(axis=-1))
|
||||
model.add(Activation('relu'))
|
||||
model.add(MaxPooling2D(pool_size=(2, 2)))
|
||||
# CONV BLOCK 3
|
||||
model.add(Conv2D(128, (3, 3)))
|
||||
model.add(BatchNormalization(axis=-1))
|
||||
model.add(Activation('relu'))
|
||||
model.add(Conv2D(128, (3, 3)))
|
||||
model.add(BatchNormalization(axis=-1))
|
||||
model.add(Activation('relu'))
|
||||
model.add(MaxPooling2D(pool_size=(2, 2)))
|
||||
# FLATTEN
|
||||
model.add(Flatten())
|
||||
# DENSE 1
|
||||
model.add(Dense(128))
|
||||
model.add(BatchNormalization())
|
||||
model.add(Activation('relu'))
|
||||
model.add(Dropout(0.5))
|
||||
# DENSE 2
|
||||
model.add(Dense(128))
|
||||
model.add(BatchNormalization())
|
||||
model.add(Activation('relu'))
|
||||
model.add(Dropout(0.5))
|
||||
# OUTPUT
|
||||
if C_or_R == CLASSIFY:
|
||||
model.add(Dense(8, activation='softmax'))
|
||||
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
|
||||
else:
|
||||
model.add(Dense(2, activation='linear'))
|
||||
model.compile(loss='mean_squared_error', optimizer='adam')
|
||||
return model
|
||||
|
||||
print('** FITTING MODEL **')
|
||||
model.fit_generator(
|
||||
load_images(t_paths, t_labels, BATCH_SIZE),
|
||||
steps_per_epoch=len(t_labels)//BATCH_SIZE,
|
||||
|
||||
def fine_tune_model(C_or_R):
|
||||
# json_file = open('AFF_NET_'+str(CLASSIFY_OR_REGRESS)+'.json', 'r')
|
||||
# loaded_model_json = json_file.read()
|
||||
# json_file.close()
|
||||
# model = model_from_json(loaded_model_json)
|
||||
model = base_model(C_or_R)
|
||||
# for k in model.layers:
|
||||
# if type(k) is Dropout:
|
||||
# model.layers.remove(k)
|
||||
model.load_weights('AFF_NET.h5')
|
||||
# REMOVE DENSE LAYERS
|
||||
for i in range(9):
|
||||
model.layers.pop()
|
||||
for layer in model.layers:
|
||||
layer.trainable = False
|
||||
model.outputs = [model.layers[-1].output]
|
||||
model.layers[-1].outbound_nodes = []
|
||||
# DENSE 1
|
||||
model.add(Dense(512))
|
||||
model.add(BatchNormalization())
|
||||
model.add(Activation('relu'))
|
||||
model.add(Dropout(0.5))
|
||||
# DENSE 2
|
||||
model.add(Dense(512))
|
||||
model.add(BatchNormalization())
|
||||
model.add(Activation('relu'))
|
||||
model.add(Dropout(0.5))
|
||||
# OUTPUT
|
||||
if C_or_R == CLASSIFY:
|
||||
model.add(Dense(8, activation='softmax'))
|
||||
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
|
||||
else:
|
||||
model.add(Dense(2, activation='linear'))
|
||||
model.compile(loss='mean_squared_error', optimizer='adam')
|
||||
return model
|
||||
|
||||
|
||||
def train(C_or_R, mode=0):
|
||||
if mode == 0:
|
||||
model = base_model(C_or_R)
|
||||
else:
|
||||
model = fine_tune_model(C_or_R)
|
||||
|
||||
print('** LOADING DATA **')
|
||||
t_paths = np.load('training_paths.npy')
|
||||
t_labels = np.load('training_labels.npy')
|
||||
t_paths, t_labels, t_weights = process_data(C_or_R, t_paths, t_labels)
|
||||
v_paths = np.load('validation_paths.npy')
|
||||
v_labels = np.load('validation_labels.npy')
|
||||
v_paths, v_labels, v_weights = process_data(C_or_R, v_paths, v_labels)
|
||||
|
||||
print('** FITTING MODEL **')
|
||||
model.fit_generator(
|
||||
load_images(C_or_R, t_paths, t_labels, BATCH_SIZE),
|
||||
steps_per_epoch=len(t_labels) // BATCH_SIZE,
|
||||
class_weight=t_weights,
|
||||
epochs=EPOCHS,
|
||||
validation_data=load_images(v_paths, v_labels, BATCH_SIZE),
|
||||
validation_steps=len(v_labels)//BATCH_SIZE,
|
||||
callbacks=[ModelCheckpoint('AFF_NET_'+str(CLASSIFY_OR_REGRESS)+'_WIP.h5', save_best_only=True)])
|
||||
validation_data=load_images(C_or_R, v_paths, v_labels, BATCH_SIZE),
|
||||
validation_steps=len(v_labels) // BATCH_SIZE,
|
||||
callbacks=[ModelCheckpoint('AFF_NET_' + str(C_or_R) + '_WIP.h5', save_best_only=True)],
|
||||
use_multiprocessing=True,
|
||||
workers=4)
|
||||
|
||||
print('** EXPORTING MODEL **')
|
||||
for k in model.layers:
|
||||
if type(k) is keras.layers.Dropout:
|
||||
model.layers.remove(k)
|
||||
print('** EXPORTING MODEL **')
|
||||
# for k in model.layers:
|
||||
# if type(k) is Dropout:
|
||||
# model.layers.remove(k)
|
||||
model_json = model.to_json()
|
||||
with open('AFF_NET_' + str(C_or_R) + '.json', 'w') as json_file:
|
||||
json_file.write(model_json)
|
||||
model.save_weights('AFF_NET_' + str(C_or_R) + '.h5')
|
||||
|
||||
model.save_weights('AFF_NET_'+str(CLASSIFY_OR_REGRESS)+'.h5')
|
||||
|
||||
# idea is to train on emotion classification + fine tune for valence/arousal??
|
||||
# for finetuning
|
||||
# model.layers.pop()
|
||||
# model.outputs = [model.layers[-1].output]
|
||||
# model.layers[-1].outbound_nodes = []
|
||||
# model.add(Dense(2, activation='linear'))
|
||||
# for layer in model.layers[:TODO]:
|
||||
# layer.trainable = False
|
||||
# compile + fit
|
||||
train(CLASSIFY, mode=1)
|
||||
+26
-20
@@ -1,15 +1,15 @@
|
||||
import csv
|
||||
import os
|
||||
from math import sqrt
|
||||
|
||||
import os
|
||||
import cv2
|
||||
import numpy as np
|
||||
from scipy.stats import pearsonr
|
||||
from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor
|
||||
from sklearn.metrics import mean_squared_error, r2_score
|
||||
from sklearn.multioutput import MultiOutputRegressor
|
||||
from sklearn.svm import SVC, SVR
|
||||
from sklearn.tree import DecisionTreeRegressor
|
||||
from scipy.stats import pearsonr
|
||||
|
||||
|
||||
def get_subimage(image, centre, theta, w, h):
|
||||
@@ -23,34 +23,39 @@ def get_subimage(image, centre, theta, w, h):
|
||||
out = cv2.getRectSubPix(rotated, (w, h), centre)
|
||||
return out
|
||||
|
||||
|
||||
def calc_ccc(a, b):
|
||||
astd = np.std(a)
|
||||
bstd = np.std(b)
|
||||
am = np.mean(a)
|
||||
bm = np.mean(b)
|
||||
p = pearsonr(a,b)[0]
|
||||
o = (2*p*astd*bstd)/(pow(astd,2)+pow(bstd,2)+pow(am-bm,2))
|
||||
p = pearsonr(a, b)[0]
|
||||
o = (2 * p * astd * bstd) / (pow(astd, 2) + pow(bstd, 2) + pow(am - bm, 2))
|
||||
return round(o, 4)
|
||||
|
||||
def cacl_sagr(a,b):
|
||||
|
||||
def cacl_sagr(a, b):
|
||||
o = 0
|
||||
for i in range(len(a)):
|
||||
o += (a[i] > 0) == (b[i] > 0)
|
||||
o /= len(a)
|
||||
return round(o, 4)
|
||||
|
||||
def calc_rmse(a,b):
|
||||
return round(mean_squared_error(a,b), 4)
|
||||
|
||||
def calc_rmse(a, b):
|
||||
return round(mean_squared_error(a, b), 4)
|
||||
|
||||
|
||||
def print_res(p, c):
|
||||
p1 = p#[:, 0]
|
||||
c1 = c#[:, 0]
|
||||
p1 = p # [:, 0]
|
||||
c1 = c # [:, 0]
|
||||
# p2 = p#[:, 1]
|
||||
# c2 = c#[:, 1]
|
||||
print('Metric'.ljust(20), 'Valence'.ljust(20), 'Arousal')
|
||||
print('RMSE'.ljust(20), str(calc_rmse(p1,c1)).ljust(20))#, calc_rmse(p2,c2))
|
||||
print('CCC'.ljust(20), str(calc_ccc(p1,c1)).ljust(20))#, calc_ccc(p2,c2))
|
||||
print('SAGR'.ljust(20), str(cacl_sagr(p1,c1)).ljust(20))#, cacl_sagr(p2,c2))
|
||||
print('RMSE'.ljust(20), str(calc_rmse(p1, c1)).ljust(20)) # , calc_rmse(p2,c2))
|
||||
print('CCC'.ljust(20), str(calc_ccc(p1, c1)).ljust(20)) # , calc_ccc(p2,c2))
|
||||
print('SAGR'.ljust(20), str(cacl_sagr(p1, c1)).ljust(20)) # , cacl_sagr(p2,c2))
|
||||
|
||||
|
||||
def p_dist(a, b):
|
||||
return sqrt(pow(a[0] - b[0], 2) + pow(a[1] - b[1], 2))
|
||||
@@ -110,12 +115,12 @@ def get_hogs(image, nx, ny, bins=8):
|
||||
gy = cv2.Sobel(image, cv2.CV_32F, 0, 1, ksize=1)
|
||||
mag, angle = cv2.cartToPolar(gx, gy, angleInDegrees=True)
|
||||
h, w = image.shape[:2]
|
||||
sx, sy = int(w/nx), int(h/ny)
|
||||
sx, sy = int(w / nx), int(h / ny)
|
||||
fs = []
|
||||
for i in range(nx):
|
||||
for j in range(ny):
|
||||
m_hist = np.histogram(mag[j*sy:(j+1)*sy, i*sx:(i+1)*sx], bins=bins, density=True)[0]
|
||||
a_hist = np.histogram(angle[j*sy:(j+1)*sy, i*sx:(i+1)*sx], bins=bins, density=True)[0]
|
||||
m_hist = np.histogram(mag[j * sy:(j + 1) * sy, i * sx:(i + 1) * sx], bins=bins, density=True)[0]
|
||||
a_hist = np.histogram(angle[j * sy:(j + 1) * sy, i * sx:(i + 1) * sx], bins=bins, density=True)[0]
|
||||
fs.extend(m_hist)
|
||||
fs.extend(a_hist)
|
||||
return fs
|
||||
@@ -126,7 +131,7 @@ def extract_hog_features(path, landmarks, c, s):
|
||||
(x, y) = c
|
||||
(w, h) = s
|
||||
path = '/Volumes/Charlie Hewitt\'s HDD/Affective/Manually_Annotated_Images/' + path
|
||||
image = cv2.imread(path) # cut out face
|
||||
image = cv2.imread(path) # cut out face
|
||||
grey = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
|
||||
f = []
|
||||
# # EYES
|
||||
@@ -137,11 +142,12 @@ def extract_hog_features(path, landmarks, c, s):
|
||||
# f.extend(get_hogs(make_box(landmarks[22:27], grey), nx=2, ny=2))
|
||||
# # MOUTH
|
||||
# f.extend(get_hogs(make_box(landmarks[48:68], grey, b=2), nx=2, ny=2))
|
||||
face = grey[y:y+h, x:x+w]
|
||||
face = grey[y:y + h, x:x + w]
|
||||
cv2.resize(face, (256, 256))
|
||||
f.extend(get_hogs(face, nx=16, ny=16, bins=8))
|
||||
return f
|
||||
|
||||
|
||||
def mouth_int(path, landmarks):
|
||||
path = 'data/' + path
|
||||
image = cv2.imread(path)
|
||||
@@ -180,10 +186,10 @@ def load_data(p, limit=-1):
|
||||
# fs.extend(extract_hog_features(row[0], abs_landmarks, (int(x),int(y)), (int(w),int(h))))
|
||||
# featureset.append(fs)
|
||||
path = 'data/' + row[0]
|
||||
o_path = 'data_p/' + row[0]
|
||||
o_path = 'data_p/' + row[0]
|
||||
if not os.path.exists(o_path):
|
||||
image = cv2.imread(path)[int(y):int(y+h),int(x):int(x+w)]
|
||||
image = cv2.resize(image, (256,256))
|
||||
image = cv2.imread(path)[int(y):int(y + h), int(x):int(x + w)]
|
||||
image = cv2.resize(image, (256, 256))
|
||||
o_dir = '/'.join(o_path.split('/')[:-1])
|
||||
if not os.path.exists(o_dir):
|
||||
os.makedirs(o_dir)
|
||||
|
||||
Referência em uma Nova Issue
Bloquear um usuário