{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/ubuntu/.local/lib/python2.7/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n", " from ._conv import register_converters as _register_converters\n", "Using TensorFlow backend.\n" ] } ], "source": [ "import numpy as np\n", "import importlib \n", "import keras\n", "from keras.models import Sequential, model_from_json\n", "from keras.layers import Conv2DTranspose, Conv2D, MaxPooling2D, Flatten, Dense, Dropout, UpSampling2D, Reshape\n", "\n", "from keras.layers.normalization import BatchNormalization\n", "\n", "import pickle\n", "import os\n", "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n", "from sklearn.model_selection import train_test_split\n", "import h5py " ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['/job:localhost/replica:0/task:0/device:GPU:0']" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from keras import backend as K\n", "K.tensorflow_backend._get_available_gpus()" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting keras\n", " Using cached https://files.pythonhosted.org/packages/54/e8/eaff7a09349ae9bd40d3ebaf028b49f5e2392c771f294910f75bb608b241/Keras-2.1.6-py2.py3-none-any.whl\n", "Collecting pyyaml (from keras)\n", "Collecting six>=1.9.0 (from keras)\n", " Using cached https://files.pythonhosted.org/packages/67/4b/141a581104b1f6397bfa78ac9d43d8ad29a7ca43ea90a2d863fe3056e86a/six-1.11.0-py2.py3-none-any.whl\n", "Collecting scipy>=0.14 (from keras)\n", " Using cached https://files.pythonhosted.org/packages/2a/f3/de9c1bd16311982711209edaa8c6caa962db30ebb6a8cc6f1dcd2d3ef616/scipy-1.1.0-cp27-cp27mu-manylinux1_x86_64.whl\n", "Collecting h5py (from keras)\n", " Using cached https://files.pythonhosted.org/packages/24/9e/d68bd01058e748bd5e7c3c6368d1703b4cd882b669e5d993a0237c75af5a/h5py-2.7.1-cp27-cp27mu-manylinux1_x86_64.whl\n", "Collecting numpy>=1.9.1 (from keras)\n", " Using cached https://files.pythonhosted.org/packages/c0/e7/08f059a00367fd613e4f2875a16c70b6237268a1d6d166c6d36acada8301/numpy-1.14.3-cp27-cp27mu-manylinux1_x86_64.whl\n", "Installing collected packages: pyyaml, six, numpy, scipy, h5py, keras\n", "Successfully installed h5py-2.7.1 keras-2.1.6 numpy-1.14.3 pyyaml-3.12 scipy-1.1.0 six-1.11.0\n", "\u001b[33mYou are using pip version 8.1.1, however version 10.0.1 is available.\n", "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n" ] } ], "source": [ "!pip install keras" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "def read_h5_file(file_name, scaler = None, preprocess = False):\n", " h5_file = h5py.File(train_eeg_dir + file_name, 'r')\n", " a_group_key = list(h5_file.keys())[0]\n", " eeg_data = np.array(h5_file[a_group_key]).T\n", " if preprocess:\n", " eeg_data = scaler.transform(eeg_data)\n", " return eeg_data" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "def train_scaler(scaler, train_eeg_names, log = False):\n", " i = 0\n", " for eeg_name in train_eeg_names:\n", " if log:\n", " print(\"{} from {}\".format(i, len(train_eeg_names)))\n", " print(\"reading:{}\".format(eeg_name))\n", " data = read_h5_file(eeg_name)\n", " i = i+1\n", " scaler.fit(data)\n", " if log:\n", " print(\"trained on {}\".format(eeg_name))\n", " \n", "def save_scaler(path,scaler):\n", " pickle.dump(scaler, open(path, 'wb'))\n", "def load_scaler(path):\n", " scaler = pickle.load(open(path, 'rb'))\n", " return scaler" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "train_eeg_dir = \"./data/train/\"\n", "trained_scaler_path = None" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "('Number of EEG overall:', 32)\n" ] } ], "source": [ "train_eeg_dir = \"./data/train/\"\n", "all_train_eeg_names = [x for x in os.listdir(train_eeg_dir) \n", " if x[-3:] == \".h5\"]\n", "eeg_num = len(all_train_eeg_names)\n", "print(\"Number of EEG overall:\", eeg_num)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "('Params before training ', {'copy': True, 'with_mean': True, 'with_std': True})\n", "0 from 32\n", "reading:2003_ivanova_post_eeg_processed.h5\n", "trained on 2003_ivanova_post_eeg_processed.h5\n", "1 from 32\n", "reading:2403_kutuzova_posteeg_processed.h5\n", "trained on 2403_kutuzova_posteeg_processed.h5\n", "2 from 32\n", "reading:miloslavov_22_05_pre_eeg_processed.h5\n", "trained on miloslavov_22_05_pre_eeg_processed.h5\n", "3 from 32\n", "reading:2505_shirokova_post_eeg_processed.h5\n", "trained on 2505_shirokova_post_eeg_processed.h5\n", "4 from 32\n", "reading:2505_shirokova_processed.h5\n", "trained on 2505_shirokova_processed.h5\n", "5 from 32\n", "reading:zavrin_15021500_eyesclosed_post_eeg_processed.h5\n", "trained on zavrin_15021500_eyesclosed_post_eeg_processed.h5\n", "6 from 32\n", "reading:gorin_rest_eeg_post_31011200_processed.h5\n", "trained on gorin_rest_eeg_post_31011200_processed.h5\n", "7 from 32\n", "reading:zavrin_eyes_closed_eeg_15021500_processed.h5\n", "trained on zavrin_eyes_closed_eeg_15021500_processed.h5\n", "8 from 32\n", "reading:gorin_310117_rest_eeg_processed.h5\n", "trained on gorin_310117_rest_eeg_processed.h5\n", "9 from 32\n", "reading:gorbacheva_03021300_rest_eeg_processed.h5\n", "trained on gorbacheva_03021300_rest_eeg_processed.h5\n", "10 from 32\n", "reading:300120171600_dagaev_rest_eeg_processed.h5\n", "trained on 300120171600_dagaev_rest_eeg_processed.h5\n", "11 from 32\n", "reading:dagaev_post_rest_eeg_30011600_processed.h5\n", "trained on dagaev_post_rest_eeg_30011600_processed.h5\n", "12 from 32\n", "reading:2403_kutuzova_pre_eeg_processed.h5\n", "trained on 2403_kutuzova_pre_eeg_processed.h5\n", "13 from 32\n", "reading:zavrin_open_eyes_eeg_15021500_processed.h5\n", "trained on zavrin_open_eyes_eeg_15021500_processed.h5\n", "14 from 32\n", "reading:08021400_post_eeg_shuhova_processed.h5\n", "trained on 08021400_post_eeg_shuhova_processed.h5\n", "15 from 32\n", "reading:3103_petuhova_posteeg_processed.h5\n", "trained on 3103_petuhova_posteeg_processed.h5\n", "16 from 32\n", "reading:2103_kozunova_post_eeg_processed.h5\n", "trained on 2103_kozunova_post_eeg_processed.h5\n", "17 from 32\n", "reading:tsoy_pre_eeg_2504_processed.h5\n", "trained on tsoy_pre_eeg_2504_processed.h5\n", "18 from 32\n", "reading:pre_egg_petuhova3103_processed.h5\n", "trained on pre_egg_petuhova3103_processed.h5\n", "19 from 32\n", "reading:2704_zagirova_post_eeg_processed.h5\n", "trained on 2704_zagirova_post_eeg_processed.h5\n", "20 from 32\n", "reading:bagaeva_post_eeg_13031500_processed.h5\n", "trained on bagaeva_post_eeg_13031500_processed.h5\n", "21 from 32\n", "reading:2103_glebko_posteeg_processed.h5\n", "trained on 2103_glebko_posteeg_processed.h5\n", "22 from 32\n", "reading:glebko_2103_pre_eeg_processed.h5\n", "trained on glebko_2103_pre_eeg_processed.h5\n", "23 from 32\n", "reading:bagaeva_13021500_rest_eeg_processed.h5\n", "trained on bagaeva_13021500_rest_eeg_processed.h5\n", "24 from 32\n", "reading:2205_miloslavov_post_eeg_processed.h5\n", "trained on 2205_miloslavov_post_eeg_processed.h5\n", "25 from 32\n", "reading:post_eeg_tsoy_2504_processed.h5\n", "trained on post_eeg_tsoy_2504_processed.h5\n", "26 from 32\n", "reading:shuhova_08022017_rest_eeg_processed.h5\n", "trained on shuhova_08022017_rest_eeg_processed.h5\n", "27 from 32\n", "reading:200317_ivanova_pre_eeg_processed.h5\n", "trained on 200317_ivanova_pre_eeg_processed.h5\n", "28 from 32\n", "reading:kozunova_pre_eeg_2103_processed.h5\n", "trained on kozunova_pre_eeg_2103_processed.h5\n", "29 from 32\n", "reading:zavrib_post_eeg_eyesopen15021500_processed.h5\n", "trained on zavrib_post_eeg_eyesopen15021500_processed.h5\n", "30 from 32\n", "reading:zagirova_pre_eeg_2704_processed.h5\n", "trained on zagirova_pre_eeg_2704_processed.h5\n", "31 from 32\n", "reading:gorbacheva_03_02_2017_post_eeg_processed.h5\n", "trained on gorbacheva_03_02_2017_post_eeg_processed.h5\n", "('Params after training ', {'copy': True, 'with_mean': True, 'with_std': True})\n" ] } ], "source": [ "if trained_scaler_path:\n", " scaler = load_scaler(trained_scaler_path)\n", "else:\n", " scaler = StandardScaler()\n", " print(\"Params before training \", scaler.get_params())\n", " train_scaler(scaler, all_train_eeg_names, log = True)\n", " print(\"Params after training \", scaler.get_params())\n", " save_scaler(\"./StandardScaler.p\", scaler)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "_________________________________________________________________\n", "Layer (type) Output Shape Param # \n", "=================================================================\n", "conv2d_4 (Conv2D) (None, 6, 54, 5) 130 \n", "_________________________________________________________________\n", "dropout_4 (Dropout) (None, 6, 54, 5) 0 \n", "_________________________________________________________________\n", "max_pooling2d_4 (MaxPooling2 (None, 3, 27, 5) 0 \n", "_________________________________________________________________\n", "flatten_4 (Flatten) (None, 405) 0 \n", "_________________________________________________________________\n", "dense_8 (Dense) (None, 50) 20300 \n", "=================================================================\n", "Total params: 20,430\n", "Trainable params: 20,430\n", "Non-trainable params: 0\n", "_________________________________________________________________\n", "(None, 50)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:5: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(activation=\"relu\", input_shape=(10, 58, 1..., padding=\"valid\", filters=5, kernel_size=(5, 5))`\n", " \"\"\"\n" ] } ], "source": [ "window_size = 10\n", "encoding_dim = 50\n", "\n", "cnnencoder = Sequential((\n", " Conv2D(nb_filter=5, kernel_size=(5, 5), activation='relu', padding='valid', input_shape=(window_size, 58, 1)),\n", " Dropout(0.6),\n", " MaxPooling2D(),\n", " \n", " Flatten(),\n", " Dense(encoding_dim, activation='relu'),\n", "))\n", "cnnencoder.summary()\n", "print(cnnencoder.output_shape)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "_________________________________________________________________\n", "Layer (type) Output Shape Param # \n", "=================================================================\n", "dense_9 (Dense) (None, 405) 20655 \n", "_________________________________________________________________\n", "reshape_3 (Reshape) (None, 3, 27, 5) 0 \n", "_________________________________________________________________\n", "up_sampling2d_3 (UpSampling2 (None, 6, 54, 5) 0 \n", "_________________________________________________________________\n", "conv2d_transpose_3 (Conv2DTr (None, 10, 58, 5) 630 \n", "_________________________________________________________________\n", "dense_10 (Dense) (None, 10, 58, 1) 6 \n", "=================================================================\n", "Total params: 21,291\n", "Trainable params: 21,291\n", "Non-trainable params: 0\n", "_________________________________________________________________\n", "(None, 10, 58, 1)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:5: UserWarning: Update your `Conv2DTranspose` call to the Keras 2 API: `Conv2DTranspose(padding=\"valid\", activation=\"relu\", filters=5, kernel_size=(5, 5))`\n", " \"\"\"\n" ] } ], "source": [ "cnndecoder = Sequential((\n", " Dense(405, activation='relu', input_shape=(encoding_dim,)),\n", " Reshape((-1, 27, 5)),\n", " UpSampling2D(),\n", " Conv2DTranspose(nb_filter=5, kernel_size=(5, 5), activation='relu', padding='valid'),\n", " Dense(1, activation='relu')\n", "))\n", "cnndecoder.summary()\n", "print(cnndecoder.output_shape)" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "_________________________________________________________________\n", "Layer (type) Output Shape Param # \n", "=================================================================\n", "input_1 (InputLayer) (None, 10, 58, 1) 0 \n", "_________________________________________________________________\n", "sequential_6 (Sequential) (None, 50) 20430 \n", "_________________________________________________________________\n", "sequential_7 (Sequential) (None, 10, 58, 1) 21291 \n", "=================================================================\n", "Total params: 41,721\n", "Trainable params: 41,721\n", "Non-trainable params: 0\n", "_________________________________________________________________\n" ] } ], "source": [ "from keras.models import Model\n", "from keras.layers import Input\n", "\n", "input_ = Input(shape=(window_size, 58, 1))\n", "\n", "autoencoder = Model(input_, cnndecoder(cnnencoder(input_)), name=\"autoencoder\")\n", "autoencoder.compile(loss='mse', optimizer='adam') # .compile(optimizer='adadelta', loss='binary_crossentropy')\n", "\n", "autoencoder.summary()" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [], "source": [ "b1, b2, b3 = 'shuhova_08022017_rest_eeg_processed.h5', 'zavrib_post_eeg_eyesopen15021500_processed.h5', 'zavrin_15021500_eyesclosed_post_eeg_processed.h5'" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [], "source": [ "all_train_eeg_names = np.array(all_train_eeg_names)\n", "all_train_eeg_names = all_train_eeg_names[(all_train_eeg_names != b1) & (all_train_eeg_names != b2) & (all_train_eeg_names != b3)]" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "('test_eeg_name is ', 'tsoy_pre_eeg_2504_processed.h5')\n" ] } ], "source": [ "overall_epoch_num = 10\n", "file_epoch_num = 1\n", "history_path = \"train_hist\"\n", "\n", "test_eeg_name = np.random.choice(all_train_eeg_names)\n", "train_eeg_names = np.array(all_train_eeg_names)\n", "print(\"test_eeg_name is \", test_eeg_name)\n", "test_data = read_h5_file(test_eeg_name, scaler, True)" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "learn_file_length = 300000" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "import threading\n", "class threadsafe_iter:\n", " def __init__(self, it):\n", " self.it = it\n", " self.lock = threading.Lock()\n", " def __iter__(self):\n", " return self\n", " def __next__(self):\n", " with self.lock:\n", " return next(self.it)\n", "\n", "def threadsafe_generator(f):\n", " def g(*a, **kw):\n", " return threadsafe_iter(f(*a, **kw))\n", " return g" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [], "source": [ "batch_length = 10\n", "def generate_batch():\n", " while True:\n", " cur_files = [[] for _ in range(len(train_eeg_names))]\n", " batches = [[] for _ in range(len(train_eeg_names))]\n", " files_count = len(train_eeg_names)\n", " while files_count > 0:\n", " file_ind = np.random.choice(np.arange(len(train_eeg_names)))\n", " if len(cur_files[file_ind]) == 1 and cur_files[file_ind] == -1: continue\n", " elif len(cur_files[file_ind]) < 1:\n", " raw = read_h5_file(train_eeg_names[file_ind], scaler, True)\n", " cur_files[file_ind] = raw\n", " #print(raw.shape[0] // batch_length)\n", " batches[file_ind] = np.arange(raw.shape[0] // batch_length)\n", " \n", " begin = np.random.choice(np.arange(len(batches[file_ind])))\n", " \n", " data = cur_files[file_ind][begin:begin+batch_length, :]\n", " yield data.reshape(-1, window_size, 58, 1), data.reshape(-1, window_size, 58, 1) # add noise later\n", " \n", " batches[file_ind] = np.delete(batches[file_ind], begin)\n", " if len(batches[file_ind]) == 0:\n", " cur_files[file_ind] = -1\n", " files_count -= 1" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": false }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:5: UserWarning: The semantics of the Keras 2 argument `steps_per_epoch` is not the same as the Keras 1 argument `samples_per_epoch`. `steps_per_epoch` is the number of batches to draw from the generator at each epoch. Basically steps_per_epoch = samples_per_epoch/batch_size. Similarly `nb_val_samples`->`validation_steps` and `val_samples`->`steps` arguments have changed. Update your method calls accordingly.\n", " \"\"\"\n", "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:5: UserWarning: Update your `fit_generator` call to the Keras 2 API: `fit_generator(.] - ETA: 0s - loss: 0.8902" ] } ], "source": [ "history = autoencoder.fit_generator(generate_batch(), \n", " samples_per_epoch=30000, \n", " verbose=1,\n", " nb_epoch=10,\n", " validation_data=(test_data.reshape(-1, window_size, 58, 1), test_data.reshape(-1, window_size, 58, 1))\n", " )" ] }, { "cell_type": "code", "execution_count": 137, "metadata": {}, "outputs": [], "source": [ "history_path = \"train_hist_51.txt\"" ] }, { "cell_type": "code", "execution_count": 138, "metadata": {}, "outputs": [], "source": [ "cnnencoder.save('CNN_encoder50.p')\n", "autoencoder.save('CNN_autoencoder50.p')\n", "with open(history_path, 'wb') as file:\n", " pickle.dump(history.history, file)" ] }, { "cell_type": "code", "execution_count": 139, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "epoch: 0, file: 2003_ivanova_post_eeg_processed.h5\n", "Train on 30000 samples, validate on 60495 samples\n", "Epoch 1/1\n", "30000/30000 [==============================] - 35s 1ms/step - loss: 0.9979 - val_loss: 0.5644\n", "epoch: 0, file: 2403_kutuzova_posteeg_processed.h5\n", "Train on 30000 samples, validate on 60495 samples\n", "Epoch 1/1\n", "23240/30000 [======================>.......] - ETA: 4s - loss: 1.0314" ] }, { "ename": "KeyboardInterrupt", "evalue": "", "output_type": "error", "traceback": [ "\u001b[0;31m----------------------------------------------------------------------\u001b[0m", "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfile_epoch_num\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m validation_data=(test_data.reshape(-1, window_size, 58, 1), test_data.reshape(-1, window_size, 58, 1)))\n\u001b[0m\u001b[1;32m 12\u001b[0m \u001b[0mcnnencoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'CNN_encoder3.p'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0mautoencoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'CNN_autoencoder3.p'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/ubuntu/.local/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m 1703\u001b[0m \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1704\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1705\u001b[0;31m validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m 1706\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1707\u001b[0m def evaluate(self, x=None, y=None,\n", "\u001b[0;32m/home/ubuntu/.local/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m 1234\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1236\u001b[0;31m \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1237\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1238\u001b[0m \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/ubuntu/.local/lib/python2.7/site-packages/keras/backend/tensorflow_backend.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 2480\u001b[0m \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2481\u001b[0m updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[0;32m-> 2482\u001b[0;31m **self.session_kwargs)\n\u001b[0m\u001b[1;32m 2483\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2484\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 887\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 888\u001b[0m result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 889\u001b[0;31m run_metadata_ptr)\n\u001b[0m\u001b[1;32m 890\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 891\u001b[0m \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1118\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1119\u001b[0m results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1120\u001b[0;31m feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m 1121\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1122\u001b[0m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1315\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1316\u001b[0m return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1317\u001b[0;31m options, run_metadata)\n\u001b[0m\u001b[1;32m 1318\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1319\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m 1321\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1322\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1323\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1324\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1325\u001b[0m \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m 1300\u001b[0m return tf_session.TF_Run(session, options,\n\u001b[1;32m 1301\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1302\u001b[0;31m status, run_metadata)\n\u001b[0m\u001b[1;32m 1303\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1304\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], "source": [ "for epoch in range(overall_epoch_num//file_epoch_num):\n", " for name in train_eeg_names:\n", " train_data = read_h5_file(name, scaler, True)\n", " if len(train_data) > learn_file_length:\n", " train_data = train_data[:learn_file_length]\n", " print(\"epoch: {}, file: {}\".format(epoch, name))\n", " history = autoencoder.fit(train_data.reshape(-1, window_size, 58, 1), train_data.reshape(-1, window_size, 58, 1), \n", " verbose=1, \n", " epochs=file_epoch_num,\n", " batch_size = 10,\n", " validation_data=(test_data.reshape(-1, window_size, 58, 1), test_data.reshape(-1, window_size, 58, 1)))\n", "cnnencoder.save('CNN_encoder3.p')\n", "autoencoder.save('CNN_autoencoder3.p')\n", "with open(history_path, 'wb') as file:\n", " pickle.dump(history.history, file)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "test_data.reshape(-1, 60, 58, 1)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(604950, 58)" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test_data.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "'shuhova_08022017_rest_eeg_processed.h5', 'zavrib_post_eeg_eyesopen15021500_processed.h5', " ] } ], "metadata": { "kernelspec": { "display_name": "Python 2", "language": "python", "name": "python2" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", "version": "2.7.12" } }, "nbformat": 4, "nbformat_minor": 2 }