Prepare new PyPI release

set the default value of beta=1

.travis.yml

@@ -49,9 +49,9 @@ install:
 
   # install TensorFlow
   - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
-      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.9.0-cp27-none-linux_x86_64.whl;
+      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0-cp27-none-linux_x86_64.whl;
     elif [[ "$TRAVIS_PYTHON_VERSION" == "3.4" ]]; then
-      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.9.0-cp34-cp34m-linux_x86_64.whl;
+      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0-cp34-cp34m-linux_x86_64.whl;
     fi
 # command to run tests
 script:

README.md

@@ -1,19 +1,21 @@
-# Keras: Deep Learning library for Theano and TensorFlow
+# Keras: Deep Learning library for TensorFlow and Theano
 
 [![Build Status](https://travis-ci.org/fchollet/keras.svg?branch=master)](https://travis-ci.org/fchollet/keras)
 [![PyPI version](https://badge.fury.io/py/keras.svg)](https://badge.fury.io/py/keras)
 [![license](https://img.shields.io/github/license/mashape/apistatus.svg?maxAge=2592000)](https://github.com/fchollet/keras/blob/master/LICENSE)
+[![Join the chat at https://gitter.im/Keras-io/Lobby](https://badges.gitter.im/Keras-io/Lobby.svg)](https://gitter.im/Keras-io/Lobby)
+
 
 ## You have just found Keras.
 
-Keras is a minimalist, highly modular neural networks library, written in Python and capable of running on top of either [TensorFlow](https://github.com/tensorflow/tensorflow) or [Theano](https://github.com/Theano/Theano). It was developed with a focus on enabling fast experimentation. Being able to go from idea to result with the least possible delay is key to doing good research.
+Keras is a high-level neural networks library, written in Python and capable of running on top of either [TensorFlow](https://github.com/tensorflow/tensorflow) or [Theano](https://github.com/Theano/Theano). It was developed with a focus on enabling fast experimentation. *Being able to go from idea to result with the least possible delay is key to doing good research.*
 
 Use Keras if you need a deep learning library that:
 
-- allows for easy and fast prototyping (through total modularity, minimalism, and extensibility).
-- supports both convolutional networks and recurrent networks, as well as combinations of the two.
-- supports arbitrary connectivity schemes (including multi-input and multi-output training).
-- runs seamlessly on CPU and GPU.
+- Allows for easy and fast prototyping (through total modularity, minimalism, and extensibility).
+- Supports both convolutional networks and recurrent networks, as well as combinations of the two.
+- Supports arbitrary connectivity schemes (including multi-input and multi-output training).
+- Runs seamlessly on CPU and GPU.
 
 Read the documentation at [Keras.io](http://keras.io).
 
@@ -114,39 +116,43 @@ Keras uses the following dependencies:
 - HDF5 and h5py (optional, required if you use model saving/loading functions)
 - Optional but recommended if you use CNNs: cuDNN.
 
-*When using the Theano backend:*
-
-- Theano
-    - [See installation instructions](http://deeplearning.net/software/theano/install.html#install).
 
 *When using the TensorFlow backend:*
 
 - TensorFlow
     - [See installation instructions](https://github.com/tensorflow/tensorflow#download-and-setup).
 
+*When using the Theano backend:*
+
+- Theano
+    - [See installation instructions](http://deeplearning.net/software/theano/install.html#install).
+
 To install Keras, `cd` to the Keras folder and run the install command:
-```
+```sh
 sudo python setup.py install
 ```
 
 You can also install Keras from PyPI:
-```
+```sh
 sudo pip install keras
 ```
 
 ------------------
 
 
-## Switching from Theano to TensorFlow
+## Switching from TensorFlow to Theano
 
-By default, Keras will use Theano as its tensor manipulation library. [Follow these instructions](http://keras.io/backend/) to configure the Keras backend.
+By default, Keras will use TensorFlow as its tensor manipulation library. [Follow these instructions](http://keras.io/backend/) to configure the Keras backend.
 
 ------------------
 
 
 ## Support
 
-You can ask questions and join the development discussion on the [Keras Google group](https://groups.google.com/forum/#!forum/keras-users).
+You can ask questions and join the development discussion:
+
+- On the [Keras Google group](https://groups.google.com/forum/#!forum/keras-users).
+- On the [Keras Gitter channel](https://gitter.im/Keras-io/Lobby).
 
 You can also post bug reports and feature requests in [Github issues](https://github.com/fchollet/keras/issues). Make sure to read [our guidelines](https://github.com/fchollet/keras/blob/master/CONTRIBUTING.md) first.
 

docker/Dockerfile

@@ -0,0 +1,46 @@
+FROM nvidia/cuda:7.5-cudnn5-devel
+
+ENV CONDA_DIR /opt/conda
+ENV PATH $CONDA_DIR/bin:$PATH
+
+RUN mkdir -p $CONDA_DIR && \
+    echo export PATH=$CONDA_DIR/bin:'$PATH' > /etc/profile.d/conda.sh && \
+    apt-get update && \
+    apt-get install -y wget git libhdf5-dev g++ graphviz && \
+    wget --quiet https://repo.continuum.io/miniconda/Miniconda3-3.9.1-Linux-x86_64.sh && \
+    echo "6c6b44acdd0bc4229377ee10d52c8ac6160c336d9cdd669db7371aa9344e1ac3 *Miniconda3-3.9.1-Linux-x86_64.sh" | sha256sum -c - && \
+    /bin/bash /Miniconda3-3.9.1-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
+    rm Miniconda3-3.9.1-Linux-x86_64.sh
+
+ENV NB_USER keras
+ENV NB_UID 1000
+
+RUN useradd -m -s /bin/bash -N -u $NB_UID $NB_USER && \
+    mkdir -p $CONDA_DIR && \
+    chown keras $CONDA_DIR -R && \
+    mkdir -p /src && \
+    chown keras /src
+
+USER keras
+
+# Python
+ARG python_version=3.5.1
+ARG tensorflow_version=0.9.0rc0-cp35-cp35m
+RUN conda install -y python=${python_version} && \
+    pip install https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-${tensorflow_version}-linux_x86_64.whl && \
+    pip install git+git://github.com/Theano/Theano.git && \
+    pip install ipdb pytest pytest-cov python-coveralls coverage==3.7.1 pytest-xdist pep8 pytest-pep8 pydot_ng && \
+    conda install Pillow scikit-learn notebook pandas matplotlib nose pyyaml six h5py && \
+    pip install git+git://github.com/fchollet/keras.git && \
+    conda clean -yt
+
+ADD theanorc /home/keras/.theanorc
+
+ENV PYTHONPATH='/src/:$PYTHONPATH'
+
+WORKDIR /src
+
+EXPOSE 8888
+
+CMD jupyter notebook --port=8888 --ip=0.0.0.0
+

docker/Makefile

@@ -0,0 +1,26 @@
+help:
+	@cat Makefile
+
+DATA?="${HOME}/Data"
+GPU?=0
+DOCKER_FILE=Dockerfile
+DOCKER=GPU=$(GPU) nvidia-docker
+BACKEND=tensorflow
+TEST=tests/
+SRC=$(shell dirname `pwd`)
+
+build:
+	docker build -t keras --build-arg python_version=3.5 -f $(DOCKER_FILE) .
+
+bash: build
+	$(DOCKER) run -it -v $(SRC):/src -v $(DATA):/data --env KERAS_BACKEND=$(BACKEND) keras bash
+
+ipython: build
+	$(DOCKER) run -it -v $(SRC):/src -v $(DATA):/data --env KERAS_BACKEND=$(BACKEND) keras ipython
+
+notebook: build
+	$(DOCKER) run -it -v $(SRC):/src -v $(DATA):/data --net=host --env KERAS_BACKEND=$(BACKEND) keras
+
+test: build
+	$(DOCKER) run -it -v $(SRC):/src -v $(DATA):/data --env KERAS_BACKEND=$(BACKEND) keras py.test $(TEST)
+

docker/README.md

@@ -0,0 +1,58 @@
+# Using Keras via Docker
+
+This directory contains `Dockerfile` to make it easy to get up and running with
+Keras via [Docker](http://www.docker.com/).
+
+## Installing Docker
+
+General installation instructions are
+[on the Docker site](https://docs.docker.com/installation/), but we give some
+quick links here:
+
+* [OSX](https://docs.docker.com/installation/mac/): [docker toolbox](https://www.docker.com/toolbox)
+* [ubuntu](https://docs.docker.com/installation/ubuntulinux/)
+
+## Running the container
+
+We are using `Makefile` to simplify docker commands within make commands.
+
+Build the container and start a jupyter notebook
+
+    $ make notebook
+
+Build the container and start an iPython shell
+
+    $ make ipython
+
+Build the container and start a bash
+
+    $ make bash
+
+For GPU support install NVidia drivers (ideally latest) and
+[nvidia-docker](https://github.com/NVIDIA/nvidia-docker). Run using
+
+    $ make notebook GPU=0 # or [ipython, bash]
+
+Switch between Theano and TensorFlow
+
+    $ make notebook BACKEND=theano
+    $ make notebook BACKEND=tensorflow
+
+Mount a volume for external data sets
+
+    $ make DATA=~/mydata
+
+Prints all make tasks
+
+    $ make help
+
+You can change Theano parameters by editing `/docker/theanorc`.
+
+
+Note: If you would have a problem running nvidia-docker you may try the old way
+we have used. But it is not recommended. If you find a bug in the nvidia-docker report
+it there please and try using the nvidia-docker as described above.
+
+    $ export CUDA_SO=$(\ls /usr/lib/x86_64-linux-gnu/libcuda.* | xargs -I{} echo '-v {}:{}')
+    $ export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}')
+    $ docker run -it -p 8888:8888 $CUDA_SO $DEVICES gcr.io/tensorflow/tensorflow:latest-gpu

docker/theanorc

@@ -0,0 +1,5 @@
+[global]
+floatX = float32
+optimizer=None
+device = gpu
+

docs/autogen.py

@@ -40,6 +40,7 @@ Index
     Sequence preprocessing
 
 Objectives
+Metrics
 Optimizers
 Activations
 Callbacks
@@ -65,6 +66,8 @@ if sys.version[0] == '2':
     sys.setdefaultencoding('utf8')
 
 from keras.layers import convolutional
+from keras.layers import pooling
+from keras.layers import local
 from keras.layers import recurrent
 from keras.layers import core
 from keras.layers import noise
@@ -77,10 +80,15 @@ from keras import callbacks
 from keras import models
 from keras.engine import topology
 from keras import objectives
+from keras import metrics
 from keras import backend
 from keras import constraints
 from keras import activations
 from keras import regularizers
+from keras.utils import data_utils
+from keras.utils import io_utils
+from keras.utils import layer_utils
+from keras.utils import np_utils
 
 
 EXCLUDE = {
@@ -88,6 +96,7 @@ EXCLUDE = {
     'Wrapper',
     'get_session',
     'set_session',
+    'CallbackList',
 }
 
 PAGES = [
@@ -105,6 +114,7 @@ PAGES = [
             models.Sequential.predict_on_batch,
             models.Sequential.fit_generator,
             models.Sequential.evaluate_generator,
+            models.Sequential.predict_generator,
         ],
     },
     {
@@ -119,6 +129,7 @@ PAGES = [
             models.Model.predict_on_batch,
             models.Model.fit_generator,
             models.Model.evaluate_generator,
+            models.Model.predict_generator,
             models.Model.get_layer,
         ]
     },
@@ -128,6 +139,9 @@ PAGES = [
             core.Dense,
             core.Activation,
             core.Dropout,
+            core.SpatialDropout1D,
+            core.SpatialDropout2D,
+            core.SpatialDropout3D,
             core.Flatten,
             core.Reshape,
             core.Permute,
@@ -145,9 +159,15 @@ PAGES = [
         'page': 'layers/convolutional.md',
         'classes': [
             convolutional.Convolution1D,
+            convolutional.AtrousConvolution1D,
             convolutional.Convolution2D,
-            convolutional.AtrousConv2D,
+            convolutional.AtrousConvolution2D,
+            convolutional.SeparableConvolution2D,
+            convolutional.Deconvolution2D,
             convolutional.Convolution3D,
+            convolutional.Cropping1D,
+            convolutional.Cropping2D,
+            convolutional.Cropping3D,
             convolutional.UpSampling1D,
             convolutional.UpSampling2D,
             convolutional.UpSampling3D,
@@ -159,12 +179,23 @@ PAGES = [
     {
         'page': 'layers/pooling.md',
         'classes': [
-            convolutional.MaxPooling1D,
-            convolutional.MaxPooling2D,
-            convolutional.MaxPooling3D,
-            convolutional.AveragePooling1D,
-            convolutional.AveragePooling2D,
-            convolutional.AveragePooling3D,
+            pooling.MaxPooling1D,
+            pooling.MaxPooling2D,
+            pooling.MaxPooling3D,
+            pooling.AveragePooling1D,
+            pooling.AveragePooling2D,
+            pooling.AveragePooling3D,
+            pooling.GlobalMaxPooling1D,
+            pooling.GlobalAveragePooling1D,
+            pooling.GlobalMaxPooling2D,
+            pooling.GlobalAveragePooling2D,
+        ],
+    },
+    {
+        'page': 'layers/local.md',
+        'classes': [
+            local.LocallyConnected1D,
+            local.LocallyConnected2D,
         ],
     },
     {
@@ -200,8 +231,10 @@ PAGES = [
         'page': 'layers/wrappers.md',
         'all_module_classes': [wrappers],
     },
-
-
+    {
+        'page': 'metrics.md',
+        'all_module_functions': [metrics],
+    },
     {
         'page': 'optimizers.md',
         'all_module_classes': [optimizers],
@@ -214,6 +247,28 @@ PAGES = [
         'page': 'backend.md',
         'all_module_functions': [backend],
     },
+    {
+        'page': 'utils/data_utils.md',
+        'functions': [
+            data_utils.get_file,
+        ]
+    },
+    {
+        'page': 'utils/io_utils.md',
+        'classes': [
+            io_utils.HDF5Matrix
+        ],
+    },
+    {
+        'page': 'utils/layer_utils.md',
+        'functions': [
+            layer_utils.layer_from_config,
+        ]
+    },
+    {
+        'page': 'utils/np_utils.md',
+        'all_module_functions': [np_utils]
+    },
 ]
 
 ROOT = 'http://keras.io/'

docs/mkdocs.yml

@@ -25,6 +25,7 @@ pages:
   - Core Layers: layers/core.md
   - Convolutional Layers: layers/convolutional.md
   - Pooling Layers: layers/pooling.md
+  - Locally-connected Layers: layers/local.md
   - Recurrent Layers: layers/recurrent.md
   - Embedding Layers: layers/embeddings.md
   - Advanced Activations Layers: layers/advanced-activations.md
@@ -37,17 +38,23 @@ pages:
   - Text Preprocessing: preprocessing/text.md
   - Image Preprocessing: preprocessing/image.md
 - Objectives: objectives.md
+- Metrics: metrics.md
 - Optimizers: optimizers.md
 - Activations: activations.md
 - Callbacks: callbacks.md
 - Datasets: datasets.md
+- Applications: applications.md
 - Backend: backend.md
 - Initializations: initializations.md
 - Regularizers: regularizers.md
 - Constraints: constraints.md
 - Visualization: visualization.md
 - Scikit-learn API: scikit-learn-api.md
-
+- Utils:
+  - Data Utils: utils/data_utils.md
+  - I/O Utils: utils/io_utils.md
+  - Layer Utils: utils/layer_utils.md
+  - Numpy Utils: utils/np_utils.md
 
 
 

docs/templates/applications.md

@@ -0,0 +1,417 @@
+# Applications
+
+Keras Applications are deep learning models that are made available alongside pre-trained weights.
+These models can be used for prediction, feature extraction, and fine-tuning.
+
+Weights are downloaded automatically when instantiating a model. They are stored at `~/.keras/models/`.
+
+## Available models
+
+### Models for image classification with weights trained on ImageNet:
+
+- [Xception](#xception)
+- [VGG16](#vgg16)
+- [VGG19](#vgg19)
+- [ResNet50](#resnet50)
+- [InceptionV3](#inceptionv3)
+
+All of these architectures (except Xception) are compatible with both TensorFlow and Theano, and upon instantiation the models will be built according to the image dimension ordering set in your Keras configuration file at `~/.keras/keras.json`. For instance, if you have set `image_dim_ordering=tf`, then any model loaded from this repository will get built according to the TensorFlow dimension ordering convention, "Width-Height-Depth".
+
+The Xception model is only available for TensorFlow, due to its reliance on `SeparableConvolution` layers.
+
+### Model for music audio file auto-tagging (taking as input Mel-spectrograms):
+
+- [MusicTaggerCRNN](#musictaggercrnn)
+
+-----
+
+## Usage examples for image classification models
+
+### Classify ImageNet classes with ResNet50
+
+```python
+from keras.applications.resnet50 import ResNet50
+from keras.preprocessing import image
+from keras.applications.resnet50 import preprocess_input, decode_predictions
+import numpy as np
+
+model = ResNet50(weights='imagenet')
+
+img_path = 'elephant.jpg'
+img = image.load_img(img_path, target_size=(224, 224))
+x = image.img_to_array(img)
+x = np.expand_dims(x, axis=0)
+x = preprocess_input(x)
+
+preds = model.predict(x)
+# decode the results into a list of tuples (class, description, probability)
+# (one such list for each sample in the batch)
+print('Predicted:', decode_predictions(preds, top=3)[0])
+# Predicted: [(u'n02504013', u'Indian_elephant', 0.82658225), (u'n01871265', u'tusker', 0.1122357), (u'n02504458', u'African_elephant', 0.061040461)]
+```
+
+### Extract features with VGG16
+
+```python
+from keras.applications.vgg16 import VGG16
+from keras.preprocessing import image
+from keras.applications.vgg16 import preprocess_input
+import numpy as np
+
+model = VGG16(weights='imagenet', include_top=False)
+
+img_path = 'elephant.jpg'
+img = image.load_img(img_path, target_size=(224, 224))
+x = image.img_to_array(img)
+x = np.expand_dims(x, axis=0)
+x = preprocess_input(x)
+
+features = model.predict(x)
+```
+
+### Extract features from an arbitrary intermediate layer with VGG19
+
+```python
+from keras.applications.vgg19 import VGG19
+from keras.preprocessing import image
+from keras.applications.vgg19 import preprocess_input
+from keras.models import Model
+import numpy as np
+
+base_model = VGG19(weights='imagenet')
+model = Model(input=base_model.input, output=base_model.get_layer('block4_pool').output)
+
+img_path = 'elephant.jpg'
+img = image.load_img(img_path, target_size=(224, 224))
+x = image.img_to_array(img)
+x = np.expand_dims(x, axis=0)
+x = preprocess_input(x)
+
+block4_pool_features = model.predict(x)
+```
+
+### Fine-tune InceptionV3 on a new set of classes
+
+```python
+from keras.applications.inception_v3 import InceptionV3
+from keras.preprocessing import image
+from keras.models import Model
+from keras.layers import Dense, GlobalAveragePooling2D
+from keras import backend as K
+
+# create the base pre-trained model
+base_model = InceptionV3(weights='imagenet', include_top=False)
+
+# add a global spatial average pooling layer
+x = base_model.output
+x = GlobalAveragePooling2D()(x)
+# let's add a fully-connected layer
+x = Dense(1024, activation='relu')(x)
+# and a logistic layer -- let's say we have 200 classes
+predictions = Dense(200, activation='softmax')(x)
+
+# this is the model we will train
+model = Model(input=base_model.input, output=predictions)
+
+# first: train only the top layers (which were randomly initialized)
+# i.e. freeze all convolutional InceptionV3 layers
+for layer in base_model.layers:
+    layer.trainable = False
+
+# compile the model (should be done *after* setting layers to non-trainable)
+model.compile(optimizer='rmsprop', loss='categorical_crossentropy')
+
+# train the model on the new data for a few epochs
+model.fit_generator(...)
+
+# at this point, the top layers are well trained and we can start fine-tuning
+# convolutional layers from inception V3. We will freeze the bottom N layers
+# and train the remaining top layers.
+
+# let's visualize layer names and layer indices to see how many layers
+# we should freeze:
+for i, layer in enumerate(base_model.layers):
+   print(i, layer.name)
+
+# we chose to train the top 2 inception blocks, i.e. we will freeze
+# the first 172 layers and unfreeze the rest:
+for layer in model.layers[:172]:
+   layer.trainable = False
+for layer in model.layers[172:]:
+   layer.trainable = True
+
+# we need to recompile the model for these modifications to take effect
+# we use SGD with a low learning rate
+from keras.optimizers import SGD
+model.compile(optimizer=SGD(lr=0.0001, momentum=0.9), loss='categorical_crossentropy')
+
+# we train our model again (this time fine-tuning the top 2 inception blocks
+# alongside the top Dense layers
+model.fit_generator(...)
+```
+
+
+### Build InceptionV3 over a custom input tensor
+
+```python
+from keras.applications.inception_v3 import InceptionV3
+from keras.layers import Input
+
+# this could also be the output a different Keras model or layer
+input_tensor = Input(shape=(224, 224, 3))  # this assumes K.image_dim_ordering() == 'tf'
+
+model = InceptionV3(input_tensor=input_tensor, weights='imagenet', include_top=True)
+```
+
+-----
+
+# Documentation for individual models
+
+- [Xception](#xception)
+- [VGG16](#vgg16)
+- [VGG19](#vgg19)
+- [ResNet50](#resnet50)
+- [InceptionV3](#inceptionv3)
+- [MusicTaggerCRNN](#musictaggercrnn)
+
+-----
+
+
+## Xception
+
+
+```python
+keras.applications.xception.Xception(include_top=True, weights='imagenet', input_tensor=None)
+```
+
+Xception V1 model, with weights pre-trained on ImageNet.
+
+On ImageNet, this model gets to a top-1 validation accuracy of 0.790
+and a top-5 validation accuracy of 0.945.
+
+Note that this model is only available for the TensorFlow backend,
+due to its reliance on `SeparableConvolution` layers. Additionally it only supports
+the dimension ordering "tf" (width, height, channels).
+
+The default input size for this model is 299x299.
+
+### Arguments
+
+- include_top: whether to include the fully-connected layer at the top of the network.
+- weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+
+### Returns
+
+A Keras model instance.
+
+### References
+
+- [Xception: Deep Learning with Depthwise Separable Convolutions](https://arxiv.org/abs/1610.02357)
+
+### License
+
+These weights are trained by ourselves and are released under the MIT license.
+
+
+-----
+
+
+## VGG16
+
+```python
+keras.applications.vgg16.VGG16(include_top=True, weights='imagenet', input_tensor=None)
+```
+
+VGG16 model, with weights pre-trained on ImageNet.
+
+This model is available for both the Theano and TensorFlow backend, and can be built both
+with "th" dim ordering (channels, width, height) or "tf" dim ordering (width, height, channels).
+
+The default input size for this model is 224x224.
+
+### Arguments
+
+- include_top: whether to include the 3 fully-connected layers at the top of the network.
+- weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+
+### Returns
+
+A Keras model instance.
+
+### References
+
+- [Very Deep Convolutional Networks for Large-Scale Image Recognition](https://arxiv.org/abs/1409.1556): please cite this paper if you use the VGG models in your work.
+
+### License
+
+These weights are ported from the ones [released by VGG at Oxford](http://www.robots.ox.ac.uk/~vgg/research/very_deep/) under the [Creative Commons Attribution License](https://creativecommons.org/licenses/by/4.0/).
+
+-----
+
+## VGG19
+
+
+```python
+keras.applications.vgg19.VGG19(include_top=True, weights='imagenet', input_tensor=None)
+```
+
+
+VGG19 model, with weights pre-trained on ImageNet.
+
+This model is available for both the Theano and TensorFlow backend, and can be built both
+with "th" dim ordering (channels, width, height) or "tf" dim ordering (width, height, channels).
+
+The default input size for this model is 224x224.
+
+### Arguments
+
+- include_top: whether to include the 3 fully-connected layers at the top of the network.
+- weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+
+### Returns
+
+A Keras model instance.
+
+
+### References
+
+- [Very Deep Convolutional Networks for Large-Scale Image Recognition](https://arxiv.org/abs/1409.1556)
+
+### License
+
+These weights are ported from the ones [released by VGG at Oxford](http://www.robots.ox.ac.uk/~vgg/research/very_deep/) under the [Creative Commons Attribution License](https://creativecommons.org/licenses/by/4.0/).
+
+-----
+
+## ResNet50
+
+
+```python
+keras.applications.resnet50.ResNet50(include_top=True, weights='imagenet', input_tensor=None)
+```
+
+
+ResNet50 model, with weights pre-trained on ImageNet.
+
+This model is available for both the Theano and TensorFlow backend, and can be built both
+with "th" dim ordering (channels, width, height) or "tf" dim ordering (width, height, channels).
+
+The default input size for this model is 224x224.
+
+
+### Arguments
+
+- include_top: whether to include the fully-connected layer at the top of the network.
+- weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+
+### Returns
+
+A Keras model instance.
+
+### References
+
+- [Deep Residual Learning for Image Recognition](https://arxiv.org/abs/1512.03385)
+
+### License
+
+These weights are ported from the ones [released by Kaiming He](https://github.com/KaimingHe/deep-residual-networks) under the [MIT license](https://github.com/KaimingHe/deep-residual-networks/blob/master/LICENSE).
+
+-----
+
+## InceptionV3
+
+
+```python
+keras.applications.inception_v3.InceptionV3(include_top=True, weights='imagenet', input_tensor=None)
+```
+
+Inception V3 model, with weights pre-trained on ImageNet.
+
+This model is available for both the Theano and TensorFlow backend, and can be built both
+with "th" dim ordering (channels, width, height) or "tf" dim ordering (width, height, channels).
+
+The default input size for this model is 299x299.
+
+
+### Arguments
+
+- include_top: whether to include the fully-connected layer at the top of the network.
+- weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+
+### Returns
+
+A Keras model instance.
+
+### References
+
+- [Rethinking the Inception Architecture for Computer Vision](http://arxiv.org/abs/1512.00567)
+
+### License
+
+These weights are trained by ourselves and are released under the MIT license.
+
+-----
+
+## MusicTaggerCRNN
+
+
+```python
+keras.applications.music_tagger_crnn.MusicTaggerCRNN(weights='msd', input_tensor=None, include_top=True)
+```
+
+A convolutional-recurrent model taking as input a vectorized representation of the MelSpectrogram of a music track and capable of outputting the musical genre of the track. You can use `keras.applications.music_tagger_crnn.preprocess_input` to convert a sound file to a vectorized spectrogram. This requires to have installed the [Librosa](http://librosa.github.io/librosa/) library. See [the usage example](#music-tagging-and-feature-extraction-with-musictaggercrnn).
+
+### Arguments
+
+- weights: one of `None` (random initialization) or "msd" (pre-training on [Million Song Dataset](http://labrosa.ee.columbia.edu/millionsong/)).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+- include_top: whether to include the 1 fully-connected layer (output layer) at the top of the network. If False, the network outputs 32-dim features.
+
+### Returns
+
+A Keras model instance.
+
+### References
+
+- [Convolutional Recurrent Neural Networks for Music Classification](https://arxiv.org/abs/1609.04243)
+
+### License
+
+These weights are ported from the ones [released by Keunwoo Choi](https://github.com/keunwoochoi/music-auto_tagging-keras) under the [MIT license](https://github.com/keunwoochoi/music-auto_tagging-keras/blob/master/LICENSE.md).
+
+### Examples: music tagging and audio feature extraction
+
+```python
+from keras.applications.music_tagger_crnn import MusicTaggerCRNN
+from keras.applications.music_tagger_crnn import preprocess_input, decode_predictions
+import numpy as np
+
+# 1. Tagging
+model = MusicTaggerCRNN(weights='msd')
+
+audio_path = 'audio_file.mp3'
+melgram = preprocess_input(audio_path)
+melgrams = np.expand_dims(melgram, axis=0)
+
+preds = model.predict(melgrams)
+print('Predicted:')
+print(decode_predictions(preds))
+# print: ('Predicted:', [[('rock', 0.097071797), ('pop', 0.042456303), ('alternative', 0.032439161), ('indie', 0.024491295), ('female vocalists', 0.016455274)]])
+
+#. 2. Feature extraction
+model = MusicTaggerCRNN(weights='msd', include_top=False)
+
+audio_path = 'audio_file.mp3'
+melgram = preprocess_input(audio_path)
+melgrams = np.expand_dims(melgram, axis=0)
+
+feats = model.predict(melgrams)
+print('Features:')
+print(feats[0, :10])
+# print: ('Features:', [-0.19160545 0.94259131 -0.9991011 0.47644514 -0.19089699 0.99033844 0.1103896 -0.00340496 0.14823607 0.59856361])
+```

docs/templates/backend.md

@@ -4,10 +4,12 @@
 
 Keras is a model-level library, providing high-level building blocks for developing deep learning models. It does not handle itself low-level operations such as tensor products, convolutions and so on. Instead, it relies on a specialized, well-optimized tensor manipulation library to do so, serving as the "backend engine" of Keras. Rather than picking one single tensor library and making the implementation of Keras tied to that library, Keras handles the problem in a modular way, and several different backend engines can be plugged seamlessly into Keras.
 
-At this time, Keras has two backend implementations available: the **Theano** backend and the **TensorFlow** backend.
+At this time, Keras has two backend implementations available: the **TensorFlow** backend and the **Theano** backend.
 
-- [Theano](http://deeplearning.net/software/theano/) is an open-source symbolic tensor manipulation framework developed by LISA/MILA Lab at Université de Montréal.
 - [TensorFlow](http://www.tensorflow.org/) is an open-source symbolic tensor manipulation framework developed by Google, Inc.
+- [Theano](http://deeplearning.net/software/theano/) is an open-source symbolic tensor manipulation framework developed by LISA/MILA Lab at Université de Montréal.
+
+In the future, we are likely to add more backend options. If you are interested in developing a new backend, get in touch!
 
 ----
 
@@ -19,9 +21,16 @@ If you have run Keras at least once, you will find the Keras configuration file
 
 If it isn't there, you can create it.
 
-It probably looks like this:
+The default configuration file looks like this:
 
-`{"epsilon": 1e-07, "floatx": "float32", "backend": "theano"}`
+```
+{
+    "image_dim_ordering": "tf",
+    "epsilon": 1e-07,
+    "floatx": "float32",
+    "backend": "tensorflow"
+}
+```
 
 Simply change the field `backend` to either `"theano"` or `"tensorflow"`, and Keras will use the new configuration next time you run any Keras code.
 
@@ -29,9 +38,8 @@ You can also define the environment variable ``KERAS_BACKEND`` and this will
 override what is defined in your config file :
 
 ```bash
-KERAS_BACKEND=tensorflow python -c "from keras import backend; print backend._BACKEND"
+KERAS_BACKEND=tensorflow python -c "from keras import backend"
 Using TensorFlow backend.
-tensorflow
 ```
 
 ----

docs/templates/datasets.md

@@ -53,11 +53,14 @@ As a convention, "0" does not stand for a specific word, but instead is used to
 ```python
 from keras.datasets import imdb
 
-(X_train, y_train), (X_test, y_test) = imdb.load_data(path="imdb.pkl",
+(X_train, y_train), (X_test, y_test) = imdb.load_data(path="imdb_full.pkl",
                                                       nb_words=None,
                                                       skip_top=0,
                                                       maxlen=None,
-                                                      test_split=0.1)
+                                                      seed=113,
+                                                      start_char=1,
+                                                      oov_char=2,
+                                                      index_from=3)
 ```
 - __Return:__
     - 2 tuples:
@@ -70,8 +73,12 @@ from keras.datasets import imdb
     - __nb_words__: integer or None. Top most frequent words to consider. Any less frequent word will appear as 0 in the sequence data.
     - __skip_top__: integer. Top most frequent words to ignore (they will appear as 0s in the sequence data).
     - __maxlen__: int. Maximum sequence length. Any longer sequence will be truncated.
-    - __test_split__: float. Fraction of the dataset to be used as test data.
     - __seed__: int. Seed for reproducible data shuffling.
+    - __start_char__: char. The start of a sequence will be marked with this character.
+        Set to 1 because 0 is usually the padding character.
+    - __oov_char__: char. words that were cut out because of the `nb_words`
+        or `skip_top` limit will be replaced with this character.
+    - __index_from__: int. Index actual words with this index and higher.
 
 ---
 
@@ -88,10 +95,16 @@ from keras.datasets import reuters
                                                          nb_words=None,
                                                          skip_top=0,
                                                          maxlen=None,
-                                                         test_split=0.1)
+                                                         test_split=0.2,
+                                                         seed=113,
+                                                         start_char=1,
+                                                         oov_char=2,
+                                                         index_from=3)
 ```
 
-The specifications are the same as that of the IMDB dataset.
+The specifications are the same as that of the IMDB dataset, with the addition of:
+
+    - __test_split__: float. Fraction of the dataset to be used as test data.
 
 This dataset also makes available the word index used for encoding the sequences:
 

docs/templates/getting-started/faq.md

@@ -58,7 +58,31 @@ theano.config.floatX = 'float32'
 
 *It is not recommended to use pickle or cPickle to save a Keras model.*
 
-If you only need to save the architecture of a model, and not its weights, you can do:
+You can use `model.save(filepath)` to save a Keras model into a single HDF5 file which will contain:
+
+- the architecture of the model, allowing to re-create the model
+- the weights of the model
+- the training configuration (loss, optimizer)
+- the state of the optimizer, allowing to resume training exactly where you left off.
+
+You can then use `keras.models.load_model(filepath)` to reinstantiate your model.
+`load_model` will also take care of compiling the model using the saved training configuration
+(unless the model was never compiled in the first place).
+
+Example:
+
+```python
+from keras.models import load_model
+
+model.save('my_model.h5')  # creates a HDF5 file 'my_model.h5'
+del model  # deletes the existing model
+
+# returns a compiled model
+# identical to the previous one
+model = load_model('my_model.h5')
+```
+
+If you only need to save the **architecture of a model**, and not its weights or its training configuration, you can do:
 
 ```python
 # save as JSON
@@ -68,6 +92,8 @@ json_string = model.to_json()
 yaml_string = model.to_yaml()
 ```
 
+The generated JSON / YAML files are human-readable and can be manually edited if needed.
+
 You can then build a fresh model from this data:
 
 ```python
@@ -79,7 +105,7 @@ model = model_from_json(json_string)
 model = model_from_yaml(yaml_string)
 ```
 
-If you need to save the weights of a model, you can do so in HDF5 with the code below.
+If you need to save the **weights of a model**, you can do so in HDF5 with the code below.
 
 Note that you will first need to install HDF5 and the Python library h5py, which do not come bundled with Keras.
 
@@ -87,26 +113,37 @@ Note that you will first need to install HDF5 and the Python library h5py, which
 model.save_weights('my_model_weights.h5')
 ```
 
-Assuming you have code for instantiating your model, you can then load the weights you saved into a model with the same architecture:
+Assuming you have code for instantiating your model, you can then load the weights you saved into a model with the *same* architecture:
 
 ```python
 model.load_weights('my_model_weights.h5')
 ```
 
-This leads us to a way to save and reconstruct models from only serialized data:
-```python
-json_string = model.to_json()
-open('my_model_architecture.json', 'w').write(json_string)
-model.save_weights('my_model_weights.h5')
+If you need to load weights into a *different* architecture (with some layers in common), for instance for fine-tuning or transfer-learning, you can load weights by *layer name*:
 
-# elsewhere...
-model = model_from_json(open('my_model_architecture.json').read())
-model.load_weights('my_model_weights.h5')
+```python
+model.load_weights('my_model_weights.h5', by_name=True)
 ```
 
-Finally, before it can be used, the model shall be compiled.
+For example:
+
 ```python
-model.compile(optimizer='adagrad', loss='mse')
+"""
+Assume original model looks like this:
+    model = Sequential()
+    model.add(Dense(2, input_dim=3, name="dense_1"))
+    model.add(Dense(3, name="dense_2"))
+    ...
+    model.save_weights(fname)
+"""
+
+# new model
+model = Sequential()
+model.add(Dense(2, input_dim=3, name="dense_1"))  # will be loaded
+model.add(Dense(10, name="new_dense"))  # will not be loaded
+
+# load weights from first model; will only affect the first layer, dense_1.
+model.load_weights(fname, by_name=True)
 ```
 
 ---
@@ -321,13 +358,27 @@ print(len(model.layers))  # "1"
 
 Code and pre-trained weights are available for the following image classification models:
 
-- [VGG-16](https://gist.github.com/baraldilorenzo/07d7802847aaad0a35d3)
-- [VGG-19](https://gist.github.com/baraldilorenzo/8d096f48a1be4a2d660d)
-- [AlexNet](https://github.com/heuritech/convnets-keras)
+- VGG16
+- VGG19
+- ResNet50
+- Inception v3
 
-For an example of how to use such a pre-trained model for feature extraction or for fine-tuning, see [this blog post](http://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html).
+They can be imported from the module `keras.applications`:
 
-The VGG-16 model is also the basis for several Keras example scripts:
+```python
+from keras.applications.vgg16 import VGG16
+from keras.applications.vgg19 import VGG19
+from keras.applications.resnet50 import ResNet50
+from keras.applications.inception_v3 import InceptionV3
+
+model = VGG16(weights='imagenet', include_top=True)
+```
+
+For a few simple usage examples, see [the documentation for the Applications module](/applications).
+
+For a detailed example of how to use such a pre-trained model for feature extraction or for fine-tuning, see [this blog post](http://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html).
+
+The VGG16 model is also the basis for several Keras example scripts:
 
 - [Style transfer](https://github.com/fchollet/keras/blob/master/examples/neural_style_transfer.py)
 - [Feature visualization](https://github.com/fchollet/keras/blob/master/examples/conv_filter_visualization.py)

docs/templates/getting-started/functional-api-guide.md

@@ -75,7 +75,7 @@ The model will also be supervised via two loss functions. Using the main loss fu
 
 Here's what our model looks like:
 
-<img src="http://s3.amazonaws.com/keras.io/img/multi-input-multi-output-graph.png" alt="multi-input-multi-output-graph" style="width: 400px;"/>
+<img src="https://s3.amazonaws.com/keras.io/img/multi-input-multi-output-graph.png" alt="multi-input-multi-output-graph" style="width: 400px;"/>
 
 Let's implement it with the functional API.
 
@@ -102,7 +102,7 @@ lstm_out = LSTM(32)(x)
 Here we insert the auxiliary loss, allowing the LSTM and Embedding layer to be trained smoothly even though the main loss will be much higher in the model.
 
 ```python
-auxiliary_loss = Dense(1, activation='sigmoid', name='aux_output')(lstm_out)
+auxiliary_output = Dense(1, activation='sigmoid', name='aux_output')(lstm_out)
 ```
 
 At this point, we feed into the model our auxiliary input data by concatenating it with the LSTM output:
@@ -117,13 +117,13 @@ x = Dense(64, activation='relu')(x)
 x = Dense(64, activation='relu')(x)
 
 # and finally we add the main logistic regression layer
-main_loss = Dense(1, activation='sigmoid', name='main_output')(x)
+main_output = Dense(1, activation='sigmoid', name='main_output')(x)
 ```
 
 This defines a model with two inputs and two outputs:
 
 ```python
-model = Model(input=[main_input, auxiliary_input], output=[main_loss, auxiliary_loss])
+model = Model(input=[main_input, auxiliary_input], output=[main_output, auxiliary_output])
 ```
 
 We compile the model and assign a weight of 0.2 to the auxiliary loss.
@@ -310,7 +310,7 @@ from keras.layers import merge, Convolution2D, Input
 # input tensor for a 3-channel 256x256 image
 x = Input(shape=(3, 256, 256))
 # 3x3 conv with 3 output channels (same as input channels)
-y = Convolution2D(3, 3, 3, border_mode='same')
+y = Convolution2D(3, 3, 3, border_mode='same')(x)
 # this returns x + y.
 z = merge([x, y], mode='sum')
 ```

docs/templates/getting-started/sequential-model-guide.md

@@ -86,7 +86,7 @@ final_model.add(merged)
 final_model.add(Dense(10, activation='softmax'))
 ```
 
-<img src="http://s3.amazonaws.com/keras.io/img/two_branches_sequential_model.png" alt="two branch Sequential" style="width: 400px;"/>
+<img src="https://s3.amazonaws.com/keras.io/img/two_branches_sequential_model.png" alt="two branch Sequential" style="width: 400px;"/>
 
 Such a two-branch model can then be trained via e.g.:
 
@@ -107,7 +107,7 @@ The `Merge` layer supports a number of pre-defined modes:
 You can also pass a function as the `mode` argument, allowing for arbitrary transformations:
 
 ```python
-merged = Merge([left_branch, right_branch], mode=lambda x, y: x - y)
+merged = Merge([left_branch, right_branch], mode=lambda x: x[0] - x[1])
 ```
 
 Now you know enough to be able to define *almost* any model with Keras. For complex models that cannot be expressed via `Sequential` and `Merge`, you can use [the functional API](/getting-started/functional-api-guide).
@@ -121,7 +121,7 @@ Before training a model, you need to configure the learning process, which is do
 
 - an optimizer. This could be the string identifier of an existing optimizer (such as `rmsprop` or `adagrad`), or an instance of the `Optimizer` class. See: [optimizers](/optimizers).
 - a loss function. This is the objective that the model will try to minimize. It can be the string identifier of an existing loss function (such as `categorical_crossentropy` or `mse`), or it can be an objective function. See: [objectives](/objectives).
-- a list of metrics. For any classification problem you will want to set this to `metrics=['accuracy']`. A metric could be the string identifier of an existing metric (only `accuracy` is supported at this point), or a custom metric function.
+- a list of metrics. For any classification problem you will want to set this to `metrics=['accuracy']`. A metric could be the string identifier of an existing metric or a custom metric function.  Custom metric function should return either a single tensor value or a dict `metric_name -> metric_value`. See: [metrics](/metrics).
 
 ```python
 # for a multi-class classification problem
@@ -137,6 +137,24 @@ model.compile(optimizer='rmsprop',
 # for a mean squared error regression problem
 model.compile(optimizer='rmsprop',
               loss='mse')
+
+# for custom metrics
+import keras.backend as K
+
+def mean_pred(y_true, y_pred):
+    return K.mean(y_pred)
+
+def false_rates(y_true, y_pred):
+    false_neg = ...
+    false_pos = ...
+    return {
+        'false_neg': false_neg,
+        'false_pos': false_pos,
+    }
+
+model.compile(optimizer='rmsprop',
+              loss='binary_crossentropy',
+              metrics=['accuracy', mean_pred, false_rates])
 ```
 
 ----
@@ -149,7 +167,7 @@ Keras models are trained on Numpy arrays of input data and labels. For training
 # for a single-input model with 2 classes (binary):
 
 model = Sequential()
-model.add(Dense(1, input_dim=784, activation='softmax'))
+model.add(Dense(1, input_dim=784, activation='sigmoid'))
 model.compile(optimizer='rmsprop',
               loss='binary_crossentropy',
               metrics=['accuracy'])
@@ -418,7 +436,7 @@ The first two LSTMs return their full output sequences, but the last one only re
 the last step in its output sequence, thus dropping the temporal dimension
 (i.e. converting the input sequence into a single vector).
 
-<img src="http://keras.io/img/regular_stacked_lstm.png" alt="stacked LSTM" style="width: 300px;"/>
+<img src="https://keras.io/img/regular_stacked_lstm.png" alt="stacked LSTM" style="width: 300px;"/>
 
 ```python
 from keras.models import Sequential
@@ -507,7 +525,7 @@ In this model, two input sequences are encoded into vectors by two separate LSTM
 
 These two vectors are then concatenated, and a fully connected network is trained on top of the concatenated representations.
 
-<img src="http://keras.io/img/dual_lstm.png" alt="Dual LSTM" style="width: 600px;"/>
+<img src="https://keras.io/img/dual_lstm.png" alt="Dual LSTM" style="width: 600px;"/>
 
 ```python
 from keras.models import Sequential

docs/templates/index.md

@@ -2,14 +2,14 @@
 
 ## You have just found Keras.
 
-Keras is a minimalist, highly modular neural networks library, written in Python and capable of running on top of either [TensorFlow](https://github.com/tensorflow/tensorflow) or [Theano](https://github.com/Theano/Theano). It was developed with a focus on enabling fast experimentation. Being able to go from idea to result with the least possible delay is key to doing good research.
+Keras is a high-level neural networks library, written in Python and capable of running on top of either [TensorFlow](https://github.com/tensorflow/tensorflow) or [Theano](https://github.com/Theano/Theano). It was developed with a focus on enabling fast experimentation. *Being able to go from idea to result with the least possible delay is key to doing good research.*
 
 Use Keras if you need a deep learning library that:
 
-- allows for easy and fast prototyping (through total modularity, minimalism, and extensibility).
-- supports both convolutional networks and recurrent networks, as well as combinations of the two.
-- supports arbitrary connectivity schemes (including multi-input and multi-output training).
-- runs seamlessly on CPU and GPU.
+- Allows for easy and fast prototyping (through total modularity, minimalism, and extensibility).
+- Supports both convolutional networks and recurrent networks, as well as combinations of the two.
+- Supports arbitrary connectivity schemes (including multi-input and multi-output training).
+- Runs seamlessly on CPU and GPU.
 
 Read the documentation at [Keras.io](http://keras.io).
 
@@ -33,7 +33,6 @@ Keras is compatible with: __Python 2.7-3.5__.
 ------------------
 
 
-
 ## Getting started: 30 seconds to Keras
 
 The core data structure of Keras is a __model__, a way to organize layers. The main type of model is the [`Sequential`](http://keras.io/getting-started/sequential-model-guide) model, a linear stack of layers. For more complex architectures, you should use the [Keras functional API](http://keras.io/getting-started/functional-api-guide).
@@ -98,6 +97,7 @@ For a more in-depth tutorial about Keras, you can check out:
 
 In the [examples folder](https://github.com/fchollet/keras/tree/master/examples) of the repository, you will find more advanced models: question-answering with memory networks, text generation with stacked LSTMs, etc.
 
+
 ------------------
 
 
@@ -110,39 +110,43 @@ Keras uses the following dependencies:
 - HDF5 and h5py (optional, required if you use model saving/loading functions)
 - Optional but recommended if you use CNNs: cuDNN.
 
-*When using the Theano backend:*
-
-- Theano
-    - [See installation instructions](http://deeplearning.net/software/theano/install.html#install).
 
 *When using the TensorFlow backend:*
 
 - TensorFlow
     - [See installation instructions](https://github.com/tensorflow/tensorflow#download-and-setup).
 
+*When using the Theano backend:*
+
+- Theano
+    - [See installation instructions](http://deeplearning.net/software/theano/install.html#install).
+
 To install Keras, `cd` to the Keras folder and run the install command:
-```
+```sh
 sudo python setup.py install
 ```
 
 You can also install Keras from PyPI:
-```
+```sh
 sudo pip install keras
 ```
 
 ------------------
 
 
-## Switching from Theano to TensorFlow
+## Switching from TensorFlow to Theano
 
-By default, Keras will use Theano as its tensor manipulation library. [Follow these instructions](http://keras.io/backend/) to configure the Keras backend.
+By default, Keras will use TensorFlow as its tensor manipulation library. [Follow these instructions](http://keras.io/backend/) to configure the Keras backend.
 
 ------------------
 
 
 ## Support
 
-You can ask questions and join the development discussion on the [Keras Google group](https://groups.google.com/forum/#!forum/keras-users).
+You can ask questions and join the development discussion:
+
+- On the [Keras Google group](https://groups.google.com/forum/#!forum/keras-users).
+- On the [Keras Gitter channel](https://gitter.im/Keras-io/Lobby).
 
 You can also post bug reports and feature requests in [Github issues](https://github.com/fchollet/keras/issues). Make sure to read [our guidelines](https://github.com/fchollet/keras/blob/master/CONTRIBUTING.md) first.
 

docs/templates/layers/writing-your-own-keras-layers.md

@@ -4,7 +4,7 @@ For simple, stateless custom operations, you are probably better off using `laye
 
 Here is the skeleton of a Keras layer. There are only three methods you need to implement:
 
-- `build(input_shape)`: this is where you will define your weights. Trainable weights should be added to the list `self.trainable_weights`. Other attributes of note are: `self.non_trainable_weights` (list) and `self.updates` (list of update tuples (tensor, new_tensor)). For an example of how to use `non_trainable_weights` and `updates`, see the code for the `BatchNormalization` layer.
+- `build(input_shape)`: this is where you will define your weights. Trainable weights should be added to the list `self.trainable_weights`. Other attributes of note are: `self.non_trainable_weights` (list) and `self.updates` (list of update tuples (tensor, new_tensor)). For an example of how to use `non_trainable_weights` and `updates`, see the code for the `BatchNormalization` layer.  This method must set `self.built = True`, which can be done by calling `super([Layer], self).build()`.
 - `call(x)`: this is where the layer's logic lives. Unless you want your layer to support masking, you only have to care about the first argument passed to `call`: the input tensor.
 - `get_output_shape_for(input_shape)`: in case your layer modifies the shape of its input, you should specify here the shape transformation logic. This allows Keras to do automatic shape inference.
 
@@ -23,6 +23,7 @@ class MyLayer(Layer):
         initial_weight_value = np.random.random((input_dim, output_dim))
         self.W = K.variable(initial_weight_value)
         self.trainable_weights = [self.W]
+        super(MyLayer, self).build()  # be sure you call this somewhere!
 
     def call(self, x, mask=None):
         return K.dot(x, self.W)
@@ -31,4 +32,4 @@ class MyLayer(Layer):
         return (input_shape[0], self.output_dim)
 ```
 
-The existing Keras layers provide ample examples of how to implement almost anything. Never hesitate to read the source code!
+The existing Keras layers provide ample examples of how to implement almost anything. Never hesitate to read the source code!

docs/templates/metrics.md

@@ -0,0 +1,51 @@
+
+## Usage of metrics
+
+A metric is a function that is used to judge the performance of your model. Metric functions are to be supplied in the `metrics` parameter when a model is compiled.
+
+A metric function is similar to an [objective function](/objectives), except that the results from evaluating a metric are not used when training the model.
+
+You can either pass the name of an existing metric, or pass a Theano/TensorFlow symbolic function (see [Custom metrics](#custom-metrics)).
+
+#### Arguments
+  - __y_true__: True labels. Theano/TensorFlow tensor.
+  -  __y_pred__: Predictions. Theano/TensorFlow tensor of the same shape as y_true.
+
+#### Returns
+  Single tensor value representing the mean of the output array across all
+  datapoints.
+
+----
+
+## Available metrics
+
+
+{{autogenerated}}
+
+----
+
+## Custom metrics
+
+Custom metrics can be defined and passed via the compilation step. The
+function would need to take `(y_true, y_pred)` as arguments and return
+either a single tensor value or a dict `metric_name -> metric_value`.
+
+```python
+# for custom metrics
+import keras.backend as K
+
+def mean_pred(y_true, y_pred):
+    return K.mean(y_pred)
+
+def false_rates(y_true, y_pred):
+    false_neg = ...
+    false_pos = ...
+    return {
+        'false_neg': false_neg,
+        'false_pos': false_pos,
+    }
+
+model.compile(optimizer='rmsprop',
+              loss='binary_crossentropy',
+              metrics=['accuracy', mean_pred, false_rates])
+```

docs/templates/models/about-keras-models.md

@@ -30,4 +30,4 @@ yaml_string = model.to_yaml()
 model = model_from_yaml(yaml_string)
 ```
 - `model.save_weights(filepath)`: saves the weights of the model as a HDF5 file.
-- `model.load_weights(filepath)`: loads the weights of the model from a HDF5 file (created by `save_weights`).
+- `model.load_weights(filepath, by_name=False)`: loads the weights of the model from a HDF5 file (created by `save_weights`). By default, the architecture is expected to be unchanged. To load weights into a different architecture (with some layers in common), use `by_name=True` to load only those layers with the same name.

docs/templates/objectives.md

@@ -30,3 +30,11 @@ For a few examples of such functions, check out the [objectives source](https://
 - __kullback_leibler_divergence__ / __kld__: Information gain from a predicted probability distribution Q to a true probability distribution P. Gives a measure of difference between both distributions.
 - __poisson__: Mean of `(predictions - targets * log(predictions))`
 - __cosine_proximity__: The opposite (negative) of the mean cosine proximity between predictions and targets.
+
+**Note**: when using the `categorical_crossentropy` objective, your targets should be in categorical format (e.g. if you have 10 classes, the target for each sample should be a 10-dimensional vector that is all-zeros expect for a 1 at the index corresponding to the class of the sample). In order to convert *integer targets* into *categorical targets*, you can use the Keras utility `to_categorical`:
+
+```python
+from keras.utils.np_utils import to_categorical
+
+categorical_labels = to_categorical(int_labels, nb_classes=None)
+```

docs/templates/optimizers.md

@@ -9,7 +9,7 @@ model.add(Dense(64, init='uniform', input_dim=10))
 model.add(Activation('tanh'))
 model.add(Activation('softmax'))
 
-sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
+sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
 model.compile(loss='mean_squared_error', optimizer=sgd)
 ```
 
@@ -22,4 +22,23 @@ model.compile(loss='mean_squared_error', optimizer='sgd')
 
 ---
 
+## Parameters common to all Keras optimizers
+
+The parameters `clipnorm` and `clipvalue` can be used with all optimizers to control gradient clipping:
+
+```python
+# all parameter gradients will be clipped to
+# a maximum norm of 1.
+sgd = SGD(lr=0.01, clipnorm=1.)
+```
+
+```python
+# all parameter gradients will be clipped to
+# a maximum value of 0.5 and
+# a minimum value of -0.5.
+sgd = SGD(lr=0.01, clipvalue=0.5)
+```
+
+---
+
 {{autogenerated}}

docs/templates/preprocessing/image.md

@@ -47,7 +47,7 @@ Generate batches of tensor image data with real-time data augmentation. The data
         "th" mode means that the images should have shape `(samples, channels, width, height)`.
         It defaults to the `image_dim_ordering` value found in your
         Keras config file at `~/.keras/keras.json`.
-        If you never set it, then it will be "th".
+        If you never set it, then it will be "tf".
 
 - __Methods__:
     - __fit(X)__: Compute the internal data stats related to the data-dependent transformations, based on an array of sample data.
@@ -56,20 +56,22 @@ Generate batches of tensor image data with real-time data augmentation. The data
             - __X__: sample data.
             - __augment__: Boolean (default: False). Whether to fit on randomly augmented samples.
             - __rounds__: int (default: 1). If augment, how many augmentation passes over the data to use.
+            - __seed__: int (default: None). Random seed.
     - __flow(X, y)__: Takes numpy data & label arrays, and generates batches of augmented/normalized data. Yields batches indefinitely, in an infinite loop.
         - __Arguments__:
             - __X__: data.
             - __y__: labels.
             - __batch_size__: int (default: 32).
-            - __shuffle__: boolean (defaut: False).
+            - __shuffle__: boolean (defaut: True).
+            - __seed__: int (default: None).
             - __save_to_dir__: None or str (default: None). This allows you to optimally specify a directory to which to save the augmented pictures being generated (useful for visualizing what you are doing).
             - __save_prefix__: str (default: `''`). Prefix to use for filenames of saved pictures (only relevant if `save_to_dir` is set).
             - __save_format__: one of "png", "jpeg" (only relevant if `save_to_dir` is set). Default: "jpeg".
-        - ___yields__: Tuples of `(x, y)` where `x` is a numpy array of image data and `y` is a numpy array of corresponding labels.
+        - __yields__: Tuples of `(x, y)` where `x` is a numpy array of image data and `y` is a numpy array of corresponding labels.
             The generator loops indefinitely.
     - __flow_from_directory(directory)__: Takes the path to a directory, and generates batches of augmented/normalized data. Yields batches indefinitely, in an infinite loop.
         - __Arguments__:
-            - __directory: path to the target directory. It should contain one subdirectory per class,
+            - __directory__: path to the target directory. It should contain one subdirectory per class,
                 and the subdirectories should contain PNG or JPG images. See [this script](https://gist.github.com/fchollet/0830affa1f7f19fd47b06d4cf89ed44d) for more details.
             - __target_size__: tuple of integers, default: `(256, 256)`. The dimensions to which all images found will be resized.
             - __color_mode__: one of "grayscale", "rbg". Default: "rgb". Whether the images will be converted to have 1 or 3 color channels.
@@ -77,7 +79,7 @@ Generate batches of tensor image data with real-time data augmentation. The data
             - __class_mode__: one of "categorical", "binary", "sparse" or None. Default: "categorical". Determines the type of label arrays that are returned: "categorical" will be 2D one-hot encoded labels, "binary" will be 1D binary labels, "sparse" will be 1D integer labels. If None, no labels are returned (the generator will only yield batches of image data, which is useful to use `model.predict_generator()`, `model.evaluate_generator()`, etc.).
             - __batch_size__: size of the batches of data (default: 32).
             - __shuffle__: whether to shuffle the data (default: True)
-            - __seed__: optional random seed for shuffling.
+            - __seed__: optional random seed for shuffling and transformations.
             - __save_to_dir__: None or str (default: None). This allows you to optimally specify a directory to which to save the augmented pictures being generated (useful for visualizing what you are doing).
             - __save_prefix__: str. Prefix to use for filenames of saved pictures (only relevant if `save_to_dir` is set).
             - __save_format__: one of "png", "jpeg" (only relevant if `save_to_dir` is set). Default: "jpeg".
@@ -88,7 +90,7 @@ Generate batches of tensor image data with real-time data augmentation. The data
 Example of using `.flow(X, y)`:
 
 ```python
-(X_train, y_train), (X_test, y_test) = cifar10.load_data(test_split=0.1)
+(X_train, y_train), (X_test, y_test) = cifar10.load_data()
 Y_train = np_utils.to_categorical(y_train, nb_classes)
 Y_test = np_utils.to_categorical(y_test, nb_classes)
 
@@ -151,3 +153,40 @@ model.fit_generator(
         validation_data=validation_generator,
         nb_val_samples=800)
 ```
+
+Example of transforming images and masks together.
+
+```python
+# we create two instances with the same arguments
+data_gen_args = dict(featurewise_center=True,
+                     featurewise_std_normalization=True,
+                     rotation_range=90.,
+                     width_shift_range=0.1,
+                     height_shift_range=0.1,
+                     zoom_range=0.2)
+image_datagen = ImageDataGenerator(**data_gen_args)
+mask_datagen = ImageDataGenerator(**data_gen_args)
+
+# Provide the same seed and keyword arguments to the fit and flow methods
+seed = 1
+image_datagen.fit(images, augment=True, seed=seed)
+mask_datagen.fit(masks, augment=True, seed=seed)
+
+image_generator = image_datagen.flow_from_directory(
+    'data/images',
+    class_mode=None,
+    seed=seed)
+
+mask_generator = mask_datagen.flow_from_directory(
+    'data/masks',
+    class_mode=None,
+    seed=seed)
+
+# combine generators into one which yields image and masks
+train_generator = zip(image_generator, mask_generator)
+
+model.fit_generator(
+    train_generator,
+    samples_per_epoch=2000,
+    nb_epoch=50)
+```

docs/templates/scikit-learn-api.md

@@ -1,12 +1,12 @@
 # Wrappers for the Scikit-Learn API
 
-You can use `Sequential` Keras models (single-input only) as part of your Scikit-Learn workflow via the wrappers found at `keras.wrappers.sklearn.py`.
+You can use `Sequential` Keras models (single-input only) as part of your Scikit-Learn workflow via the wrappers found at `keras.wrappers.scikit_learn.py`.
 
 There are two wrappers available:
 
-`keras.wrappers.sklearn.KerasClassifier(build_fn=None, **sk_params)`, which implements the sklearn classifier interface,
+`keras.wrappers.scikit_learn.KerasClassifier(build_fn=None, **sk_params)`, which implements the Scikit-Learn classifier interface,
 
-`keras.wrappers.sklearn.KerasRegressor(build_fn=None, **sk_params)`, which implements the sklearn regressor interface.
+`keras.wrappers.scikit_learn.KerasRegressor(build_fn=None, **sk_params)`, which implements the Scikit-Learn regressor interface.
 
 ### Arguments
 

examples/README.md

@@ -0,0 +1,97 @@
+# Keras examples directory
+
+[addition_rnn.py](addition_rnn.py)
+Implementation of sequence to sequence learning for performing addition of two numbers (as strings).
+
+[antirectifier.py](antirectifier.py)
+Demonstrates how to write custom layers for Keras.
+
+[babi_memnn.py](babi_memnn.py)
+Trains a memory network on the bAbI dataset for reading comprehension.
+
+[babi_rnn.py](babi_rnn.py)
+Trains a two-branch recurrent network on the bAbI dataset for reading comprehension.
+
+[cifar10_cnn.py](cifar10_cnn.py)
+Trains a simple deep CNN on the CIFAR10 small images dataset.
+
+[conv_filter_visualization.py](conv_filter_visualization.py)
+Visualization of the filters of VGG16, via gradient ascent in input space.
+
+[conv_lstm.py](conv_lstm.py)
+Demonstrates the use of a convolutional LSTM network.
+
+[deep_dream.py](deep_dream.py)
+Deep Dreams in Keras.
+
+[image_ocr.py](image_ocr.py)
+Trains a convolutional stack followed by a recurrent stack and a CTC logloss function to perform optical character recognition (OCR).
+
+[imdb_bidirectional_lstm.py](imdb_bidirectional_lstm.py)
+Trains a Bidirectional LSTM on the IMDB sentiment classification task.
+
+[imdb_cnn.py](imdb_cnn.py)
+Demonstrates the use of Convolution1D for text classification.
+
+[imdb_cnn_lstm.py](imdb_cnn_lstm.py)
+Trains a convolutional stack followed by a recurrent stack network on the IMDB sentiment classification task.
+
+[imdb_fasttext.py](imdb_fasttext.py)
+Trains a FastText model on the IMDB sentiment classification task.
+
+[imdb_lstm.py](imdb_lstm.py)
+Trains a LSTM on the IMDB sentiment classification task.
+
+[lstm_benchmark.py](lstm_benchmark.py)
+Compares different LSTM implementations on the IMDB sentiment classification task.
+
+[lstm_text_generation.py](lstm_text_generation.py)
+Generates text from Nietzsche's writings.
+
+[mnist_cnn.py](mnist_cnn.py)
+Trains a simple convnet on the MNIST dataset.
+
+[mnist_hierarchical_rnn.py](mnist_hierarchical_rnn.py)
+Trains a Hierarchical RNN (HRNN) to classify MNIST digits.
+
+[mnist_irnn.py](mnist_irnn.py)
+Reproduction of the IRNN experiment with pixel-by-pixel sequential MNIST in "A Simple Way to Initialize Recurrent Networks of Rectified Linear Units" by Le et al.
+
+[mnist_mlp.py](mnist_mlp.py)
+Trains a simple deep multi-layer perceptron on the MNIST dataset.
+
+[mnist_net2net.py](mnist_net2net.py)
+Reproduction of the Net2Net experiment with MNIST in "Net2Net: Accelerating Learning via Knowledge Transfer".
+
+[mnist_siamese_graph.py](mnist_siamese_graph.py)
+Trains a Siamese multi-layer perceptron on pairs of digits from the MNIST dataset.
+
+[mnist_sklearn_wrapper.py](mnist_sklearn_wrapper.py)
+Demonstrates how to use the sklearn wrapper.
+
+[mnist_swwae.py](mnist_swwae.py)
+Trains a Stacked What-Where AutoEncoder built on residual blocks on the MNIST dataset.
+
+[mnist_transfer_cnn.py](mnist_transfer_cnn.py)
+Transfer learning toy example.
+
+[neural_doodle.py](neural_doodle.py)
+Neural doodle.
+
+[neural_style_transfer.py](neural_style_transfer.py)
+Neural style transfer.
+
+[pretrained_word_embeddings.py](pretrained_word_embeddings.py)
+Loads pre-trained word embeddings (GloVe embeddings) into a frozen Keras Embedding layer, and uses it to train a text classification model on the 20 Newsgroup dataset.
+
+[reuters_mlp.py](reuters_mlp.py)
+Trains and evaluate a simple MLP on the Reuters newswire topic classification task.
+
+[stateful_lstm.py](stateful_lstm.py)
+Demonstrates how to use stateful RNNs to model long sequences efficiently.
+
+[variational_autoencoder.py](variational_autoencoder.py)
+Demonstrates how to build a variational autoencoder.
+
+[variational_autoencoder_deconv.py](variational_autoencoder_deconv.py)
+Demonstrates how to build a variational autoencoder with Keras using deconvolution layers.

examples/babi_memnn.py

@@ -95,7 +95,7 @@ def vectorize_stories(data, word_idx, story_maxlen, query_maxlen):
 
 
 try:
-    path = get_file('babi-tasks-v1-2.tar.gz', origin='http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz')
+    path = get_file('babi-tasks-v1-2.tar.gz', origin='https://s3.amazonaws.com/text-datasets/babi_tasks_1-20_v1-2.tar.gz')
 except:
     print('Error downloading dataset, please download it manually:\n'
           '$ wget http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz\n'
@@ -173,6 +173,7 @@ match = Sequential()
 match.add(Merge([input_encoder_m, question_encoder],
                 mode='dot',
                 dot_axes=[2, 2]))
+match.add(Activation('softmax'))
 # output: (samples, story_maxlen, query_maxlen)
 # embed the input into a single vector with size = story_maxlen:
 input_encoder_c = Sequential()

examples/babi_rnn.py

@@ -147,7 +147,7 @@ EPOCHS = 40
 print('RNN / Embed / Sent / Query = {}, {}, {}, {}'.format(RNN, EMBED_HIDDEN_SIZE, SENT_HIDDEN_SIZE, QUERY_HIDDEN_SIZE))
 
 try:
-    path = get_file('babi-tasks-v1-2.tar.gz', origin='http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz')
+    path = get_file('babi-tasks-v1-2.tar.gz', origin='https://s3.amazonaws.com/text-datasets/babi_tasks_1-20_v1-2.tar.gz')
 except:
     print('Error downloading dataset, please download it manually:\n'
           '$ wget http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz\n'

examples/cifar10_cnn.py

@@ -43,7 +43,7 @@ Y_test = np_utils.to_categorical(y_test, nb_classes)
 model = Sequential()
 
 model.add(Convolution2D(32, 3, 3, border_mode='same',
-                        input_shape=(img_channels, img_rows, img_cols)))
+                        input_shape=X_train.shape[1:]))
 model.add(Activation('relu'))
 model.add(Convolution2D(32, 3, 3))
 model.add(Activation('relu'))

examples/conv_filter_visualization.py

@@ -3,32 +3,21 @@
 This script can run on CPU in a few minutes (with the TensorFlow backend).
 
 Results example: http://i.imgur.com/4nj4KjN.jpg
-
-Before running this script, download the weights for the VGG16 model at:
-https://drive.google.com/file/d/0Bz7KyqmuGsilT0J5dmRCM0ROVHc/view?usp=sharing
-(source: https://gist.github.com/baraldilorenzo/07d7802847aaad0a35d3)
-and make sure the variable `weights_path` in this script matches the location of the file.
 '''
 from __future__ import print_function
 from scipy.misc import imsave
 import numpy as np
 import time
-import os
-import h5py
-
-from keras.models import Sequential
-from keras.layers import Convolution2D, ZeroPadding2D, MaxPooling2D
+from keras.applications import vgg16
 from keras import backend as K
 
 # dimensions of the generated pictures for each filter.
 img_width = 128
 img_height = 128
 
-# path to the model weights file.
-weights_path = 'vgg16_weights.h5'
-
-# the name of the layer we want to visualize (see model definition below)
-layer_name = 'conv5_1'
+# the name of the layer we want to visualize
+# (see model definition at keras/applications/vgg16.py)
+layer_name = 'block5_conv1'
 
 # util function to convert a tensor into a valid image
 def deprocess_image(x):
@@ -43,70 +32,22 @@ def deprocess_image(x):
 
     # convert to RGB array
     x *= 255
-    x = x.transpose((1, 2, 0))
+    if K.image_dim_ordering() == 'th':
+        x = x.transpose((1, 2, 0))
     x = np.clip(x, 0, 255).astype('uint8')
     return x
 
-# build the VGG16 network
-model = Sequential()
-model.add(ZeroPadding2D((1, 1), batch_input_shape=(1, 3, img_width, img_height)))
-first_layer = model.layers[-1]
-# this is a placeholder tensor that will contain our generated images
-input_img = first_layer.input
-
-model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_2'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_2'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_2'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_3'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_3'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_2'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_3'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-# load the weights of the VGG16 networks
-# (trained on ImageNet, won the ILSVRC competition in 2014)
-# note: when there is a complete match between your model definition
-# and your weight savefile, you can simply call model.load_weights(filename)
-assert os.path.exists(weights_path), 'Model weights not found (see "weights_path" variable in script).'
-f = h5py.File(weights_path)
-for k in range(f.attrs['nb_layers']):
-    if k >= len(model.layers):
-        # we don't look at the last (fully-connected) layers in the savefile
-        break
-    g = f['layer_{}'.format(k)]
-    weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]
-    model.layers[k].set_weights(weights)
-f.close()
+# build the VGG16 network with ImageNet weights
+model = vgg16.VGG16(weights='imagenet', include_top=False)
 print('Model loaded.')
 
+model.summary()
+
+# this is the placeholder for the input images
+input_img = model.input
+
 # get the symbolic outputs of each "key" layer (we gave them unique names).
-layer_dict = dict([(layer.name, layer) for layer in model.layers])
+layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
 
 
 def normalize(x):
@@ -124,7 +65,10 @@ for filter_index in range(0, 200):
     # we build a loss function that maximizes the activation
     # of the nth filter of the layer considered
     layer_output = layer_dict[layer_name].output
-    loss = K.mean(layer_output[:, filter_index, :, :])
+    if K.image_dim_ordering() == 'th':
+        loss = K.mean(layer_output[:, filter_index, :, :])
+    else:
+        loss = K.mean(layer_output[:, :, :, filter_index])
 
     # we compute the gradient of the input picture wrt this loss
     grads = K.gradients(loss, input_img)[0]
@@ -139,7 +83,11 @@ for filter_index in range(0, 200):
     step = 1.
 
     # we start from a gray image with some random noise
-    input_img_data = np.random.random((1, 3, img_width, img_height)) * 20 + 128.
+    if K.image_dim_ordering() == 'th':
+        input_img_data = np.random.random((1, 3, img_width, img_height))
+    else:
+        input_img_data = np.random.random((1, img_width, img_height, 3))
+    input_img_data = (input_img_data - 0.5) * 20 + 128
 
     # we run gradient ascent for 20 steps
     for i in range(20):

examples/conv_lstm.py

@@ -0,0 +1,142 @@
+""" This script demonstrates the use of a convolutional LSTM network.
+This network is used to predict the next frame of an artificially
+generated movie which contains moving squares.
+"""
+from keras.models import Sequential
+from keras.layers.convolutional import Convolution3D
+from keras.layers.convolutional_recurrent import ConvLSTM2D
+from keras.layers.normalization import BatchNormalization
+import numpy as np
+import pylab as plt
+
+# We create a layer which take as input movies of shape
+# (n_frames, width, height, channels) and returns a movie
+# of identical shape.
+
+seq = Sequential()
+seq.add(ConvLSTM2D(nb_filter=40, nb_row=3, nb_col=3,
+                   input_shape=(None, 40, 40, 1),
+                   border_mode='same', return_sequences=True))
+seq.add(BatchNormalization())
+
+seq.add(ConvLSTM2D(nb_filter=40, nb_row=3, nb_col=3,
+                   border_mode='same', return_sequences=True))
+seq.add(BatchNormalization())
+
+seq.add(ConvLSTM2D(nb_filter=40, nb_row=3, nb_col=3,
+                   border_mode='same', return_sequences=True))
+seq.add(BatchNormalization())
+
+seq.add(ConvLSTM2D(nb_filter=40, nb_row=3, nb_col=3,
+                   border_mode='same', return_sequences=True))
+seq.add(BatchNormalization())
+
+seq.add(Convolution3D(nb_filter=1, kernel_dim1=1, kernel_dim2=3,
+                      kernel_dim3=3, activation='sigmoid',
+                      border_mode='same', dim_ordering='tf'))
+
+seq.compile(loss='binary_crossentropy', optimizer='adadelta')
+
+
+# Artificial data generation:
+# Generate movies with 3 to 7 moving squares inside.
+# The squares are of shape 1x1 or 2x2 pixels,
+# which move linearly over time.
+# For convenience we first create movies with bigger width and height (80x80)
+# and at the end we select a 40x40 window.
+
+def generate_movies(n_samples=1200, n_frames=15):
+    row = 80
+    col = 80
+    noisy_movies = np.zeros((n_samples, n_frames, row, col, 1), dtype=np.float)
+    shifted_movies = np.zeros((n_samples, n_frames, row, col, 1),
+                              dtype=np.float)
+
+    for i in range(n_samples):
+        # Add 3 to 7 moving squares
+        n = np.random.randint(3, 8)
+
+        for j in range(n):
+            # Initial position
+            xstart = np.random.randint(20, 60)
+            ystart = np.random.randint(20, 60)
+            # Direction of motion
+            directionx = np.random.randint(0, 3) - 1
+            directiony = np.random.randint(0, 3) - 1
+
+            # Size of the square
+            w = np.random.randint(2, 4)
+
+            for t in range(n_frames):
+                x_shift = xstart + directionx * t
+                y_shift = ystart + directiony * t
+                noisy_movies[i, t, x_shift - w: x_shift + w,
+                             y_shift - w: y_shift + w, 0] += 1
+
+                # Make it more robust by adding noise.
+                # The idea is that if during inference,
+                # the value of the pixel is not exactly one,
+                # we need to train the network to be robust and still
+                # consider it as a pixel belonging to a square.
+                if np.random.randint(0, 2):
+                    noise_f = (-1)**np.random.randint(0, 2)
+                    noisy_movies[i, t,
+                                 x_shift - w - 1: x_shift + w + 1,
+                                 y_shift - w - 1: y_shift + w + 1,
+                                 0] += noise_f * 0.1
+
+                # Shift the ground truth by 1
+                x_shift = xstart + directionx * (t + 1)
+                y_shift = ystart + directiony * (t + 1)
+                shifted_movies[i, t, x_shift - w: x_shift + w,
+                               y_shift - w: y_shift + w, 0] += 1
+
+    # Cut to a 40x40 window
+    noisy_movies = noisy_movies[::, ::, 20:60, 20:60, ::]
+    shifted_movies = shifted_movies[::, ::, 20:60, 20:60, ::]
+    noisy_movies[noisy_movies >= 1] = 1
+    shifted_movies[shifted_movies >= 1] = 1
+    return noisy_movies, shifted_movies
+
+# Train the network
+noisy_movies, shifted_movies = generate_movies(n_samples=1200)
+seq.fit(noisy_movies[:1000], shifted_movies[:1000], batch_size=10,
+        nb_epoch=300, validation_split=0.05)
+
+# Testing the network on one movie
+# feed it with the first 7 positions and then
+# predict the new positions
+which = 1004
+track = noisy_movies[which][:7, ::, ::, ::]
+
+for j in range(16):
+    new_pos = seq.predict(track[np.newaxis, ::, ::, ::, ::])
+    new = new_pos[::, -1, ::, ::, ::]
+    track = np.concatenate((track, new), axis=0)
+
+
+# And then compare the predictions
+# to the ground truth
+track2 = noisy_movies[which][::, ::, ::, ::]
+for i in range(15):
+    fig = plt.figure(figsize=(10, 5))
+
+    ax = fig.add_subplot(121)
+
+    if i >= 7:
+        ax.text(1, 3, 'Predictions !', fontsize=20, color='w')
+    else:
+        ax.text(1, 3, 'Inital trajectory', fontsize=20)
+
+    toplot = track[i, ::, ::, 0]
+
+    plt.imshow(toplot)
+    ax = fig.add_subplot(122)
+    plt.text(1, 3, 'Ground truth', fontsize=20)
+
+    toplot = track2[i, ::, ::, 0]
+    if i >= 2:
+        toplot = shifted_movies[which][i - 1, ::, ::, 0]
+
+    plt.imshow(toplot)
+    plt.savefig('%i_animate.png' % (i + 1))

examples/deep_dream.py

@@ -15,17 +15,16 @@ If running on CPU, prefer the TensorFlow backend (much faster).
 Example results: http://i.imgur.com/FX6ROg9.jpg
 '''
 from __future__ import print_function
-from scipy.misc import imread, imresize, imsave
+from keras.preprocessing.image import load_img, img_to_array
 import numpy as np
+from scipy.misc import imsave
 from scipy.optimize import fmin_l_bfgs_b
 import time
 import argparse
-import h5py
-import os
 
-from keras.models import Sequential
-from keras.layers import Convolution2D, ZeroPadding2D, MaxPooling2D
+from keras.applications import vgg16
 from keras import backend as K
+from keras.layers import Input
 
 parser = argparse.ArgumentParser(description='Deep Dreams with Keras.')
 parser.add_argument('base_image_path', metavar='base', type=str,
@@ -46,14 +45,14 @@ weights_path = 'vgg16_weights.h5'
 
 # some settings we found interesting
 saved_settings = {
-    'bad_trip': {'features': {'conv4_1': 0.05,
-                              'conv4_2': 0.01,
-                              'conv4_3': 0.01},
+    'bad_trip': {'features': {'block4_conv1': 0.05,
+                              'block4_conv2': 0.01,
+                              'block4_conv3': 0.01},
                  'continuity': 0.1,
                  'dream_l2': 0.8,
                  'jitter': 5},
-    'dreamy': {'features': {'conv5_1': 0.05,
-                            'conv5_2': 0.02},
+    'dreamy': {'features': {'block5_conv1': 0.05,
+                            'block5_conv2': 0.02},
                'continuity': 0.1,
                'dream_l2': 0.02,
                'jitter': 0},
@@ -63,73 +62,39 @@ settings = saved_settings['dreamy']
 
 # util function to open, resize and format pictures into appropriate tensors
 def preprocess_image(image_path):
-    img = imresize(imread(image_path), (img_width, img_height))
-    img = img.transpose((2, 0, 1)).astype('float64')
+    img = load_img(image_path, target_size=(img_width, img_height))
+    img = img_to_array(img)
     img = np.expand_dims(img, axis=0)
+    img = vgg16.preprocess_input(img)
     return img
 
 # util function to convert a tensor into a valid image
 def deprocess_image(x):
-    x = x.transpose((1, 2, 0))
+    if K.image_dim_ordering() == 'th':
+        x = x.reshape((3, img_width, img_height))
+        x = x.transpose((1, 2, 0))
+    else:
+        x = x.reshape((img_width, img_height, 3))
+    # Remove zero-center by mean pixel
+    x[:, :, 0] += 103.939
+    x[:, :, 1] += 116.779
+    x[:, :, 2] += 123.68
+    # 'BGR'->'RGB'
+    x = x[:, :, ::-1]
     x = np.clip(x, 0, 255).astype('uint8')
     return x
 
-# build the VGG16 network
-model = Sequential()
-model.add(ZeroPadding2D((1, 1), batch_input_shape=(1, 3, img_width, img_height)))
-first_layer = model.layers[-1]
-# this is a placeholder tensor that will contain our generated images
-dream = first_layer.input
+if K.image_dim_ordering() == 'th':
+    img_size = (3, img_width, img_height)
+else:
+    img_size = (img_width, img_height, 3)
+# this will contain our generated image
+dream = Input(batch_shape=(1,) + img_size)
 
-model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_2'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_2'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_2'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_3'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_3'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_2'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_3'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-# load the weights of the VGG16 networks
-# (trained on ImageNet, won the ILSVRC competition in 2014)
-# note: when there is a complete match between your model definition
-# and your weight savefile, you can simply call model.load_weights(filename)
-assert os.path.exists(weights_path), 'Model weights not found (see "weights_path" variable in script).'
-f = h5py.File(weights_path)
-for k in range(f.attrs['nb_layers']):
-    if k >= len(model.layers):
-        # we don't look at the last (fully-connected) layers in the savefile
-        break
-    g = f['layer_{}'.format(k)]
-    weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]
-    model.layers[k].set_weights(weights)
-f.close()
+# build the VGG16 network with our placeholder
+# the model will be loaded with pre-trained ImageNet weights
+model = vgg16.VGG16(input_tensor=dream,
+                    weights='imagenet', include_top=False)
 print('Model loaded.')
 
 # get the symbolic outputs of each "key" layer (we gave them unique names).
@@ -138,8 +103,16 @@ layer_dict = dict([(layer.name, layer) for layer in model.layers])
 # continuity loss util function
 def continuity_loss(x):
     assert K.ndim(x) == 4
-    a = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, 1:, :img_height-1])
-    b = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, :img_width-1, 1:])
+    if K.image_dim_ordering() == 'th':
+        a = K.square(x[:, :, :img_width - 1, :img_height - 1] -
+                     x[:, :, 1:, :img_height - 1])
+        b = K.square(x[:, :, :img_width - 1, :img_height - 1] -
+                     x[:, :, :img_width - 1, 1:])
+    else:
+        a = K.square(x[:, :img_width - 1, :img_height-1, :] -
+                     x[:, 1:, :img_height - 1, :])
+        b = K.square(x[:, :img_width - 1, :img_height-1, :] -
+                     x[:, :img_width - 1, 1:, :])
     return K.sum(K.pow(a + b, 1.25))
 
 # define the loss
@@ -151,12 +124,15 @@ for layer_name in settings['features']:
     x = layer_dict[layer_name].output
     shape = layer_dict[layer_name].output_shape
     # we avoid border artifacts by only involving non-border pixels in the loss
-    loss -= coeff * K.sum(K.square(x[:, :, 2: shape[2]-2, 2: shape[3]-2])) / np.prod(shape[1:])
+    if K.image_dim_ordering() == 'th':
+        loss -= coeff * K.sum(K.square(x[:, :, 2: shape[2] - 2, 2: shape[3] - 2])) / np.prod(shape[1:])
+    else:
+        loss -= coeff * K.sum(K.square(x[:, 2: shape[1] - 2, 2: shape[2] - 2, :])) / np.prod(shape[1:])
 
 # add continuity loss (gives image local coherence, can result in an artful blur)
-loss += settings['continuity'] * continuity_loss(dream) / (3 * img_width * img_height)
+loss += settings['continuity'] * continuity_loss(dream) / np.prod(img_size)
 # add image L2 norm to loss (prevents pixels from taking very high values, makes image darker)
-loss += settings['dream_l2'] * K.sum(K.square(dream)) / (3 * img_width * img_height)
+loss += settings['dream_l2'] * K.sum(K.square(dream)) / np.prod(img_size)
 
 # feel free to further modify the loss as you see fit, to achieve new effects...
 
@@ -171,7 +147,7 @@ else:
 
 f_outputs = K.function([dream], outputs)
 def eval_loss_and_grads(x):
-    x = x.reshape((1, 3, img_width, img_height))
+    x = x.reshape((1,) + img_size)
     outs = f_outputs([x])
     loss_value = outs[0]
     if len(outs[1:]) == 1:
@@ -215,7 +191,7 @@ for i in range(5):
     start_time = time.time()
 
     # add a random jitter to the initial image. This will be reverted at decoding time
-    random_jitter = (settings['jitter'] * 2) * (np.random.random((3, img_width, img_height)) - 0.5)
+    random_jitter = (settings['jitter'] * 2) * (np.random.random(img_size) - 0.5)
     x += random_jitter
 
     # run L-BFGS for 7 steps
@@ -223,9 +199,9 @@ for i in range(5):
                                      fprime=evaluator.grads, maxfun=7)
     print('Current loss value:', min_val)
     # decode the dream and save it
-    x = x.reshape((3, img_width, img_height))
+    x = x.reshape(img_size)
     x -= random_jitter
-    img = deprocess_image(x)
+    img = deprocess_image(np.copy(x))
     fname = result_prefix + '_at_iteration_%d.png' % i
     imsave(fname, img)
     end_time = time.time()

examples/image_ocr.py

@@ -0,0 +1,470 @@
+'''This example uses a convolutional stack followed by a recurrent stack
+and a CTC logloss function to perform optical character recognition
+of generated text images. I have no evidence of whether it actually
+learns general shapes of text, or just is able to recognize all
+the different fonts thrown at it...the purpose is more to demonstrate CTC
+inside of Keras.  Note that the font list may need to be updated
+for the particular OS in use.
+
+This starts off with 4 letter words. After 10 or so epochs, CTC
+learns translational invariance, so longer words and groups of words
+with spaces are gradually fed in.  This gradual increase in difficulty
+is handled using the TextImageGenerator class which is both a generator
+class for test/train data and a Keras callback class. Every 10 epochs
+the wordlist that the generator draws from increases in difficulty.
+
+The table below shows normalized edit distance values. Theano uses
+a slightly different CTC implementation, so some Theano-specific
+hyperparameter tuning would be needed to get it to match Tensorflow.
+
+            Norm. ED
+Epoch |   TF   |   TH
+------------------------
+    10   0.072    0.272
+    20   0.032    0.115
+    30   0.024    0.098
+    40   0.023    0.108
+
+This requires cairo and editdistance packages:
+pip install cairocffi
+pip install editdistance
+
+Due to the use of a dummy loss function, Theano requires the following flags:
+on_unused_input='ignore'
+
+Created by Mike Henry
+https://github.com/mbhenry/
+'''
+
+import os
+import itertools
+import re
+import datetime
+import cairocffi as cairo
+import editdistance
+import numpy as np
+from scipy import ndimage
+import pylab
+from keras import backend as K
+from keras.layers.convolutional import Convolution2D, MaxPooling2D
+from keras.layers import Input, Layer, Dense, Activation, Flatten
+from keras.layers import Reshape, Lambda, merge, Permute, TimeDistributed
+from keras.models import Model
+from keras.layers.recurrent import GRU
+from keras.optimizers import SGD
+from keras.utils import np_utils
+from keras.utils.data_utils import get_file
+from keras.preprocessing import image
+import keras.callbacks
+
+OUTPUT_DIR = "image_ocr"
+
+np.random.seed(55)
+
+
+# this creates larger "blotches" of noise which look
+# more realistic than just adding gaussian noise
+# assumes greyscale with pixels ranging from 0 to 1
+
+def speckle(img):
+    severity = np.random.uniform(0, 0.6)
+    blur = ndimage.gaussian_filter(np.random.randn(*img.shape) * severity, 1)
+    img_speck = (img + blur)
+    img_speck[img_speck > 1] = 1
+    img_speck[img_speck <= 0] = 0
+    return img_speck
+
+
+# paints the string in a random location the bounding box
+# also uses a random font, a slight random rotation,
+# and a random amount of speckle noise
+
+def paint_text(text, w, h):
+    surface = cairo.ImageSurface(cairo.FORMAT_RGB24, w, h)
+    with cairo.Context(surface) as context:
+        context.set_source_rgb(1, 1, 1)  # White
+        context.paint()
+        # this font list works in Centos 7
+        fonts = ['Century Schoolbook', 'Courier', 'STIX', 'URW Chancery L', 'FreeMono']
+        context.select_font_face(np.random.choice(fonts), cairo.FONT_SLANT_NORMAL,
+                                 np.random.choice([cairo.FONT_WEIGHT_BOLD, cairo.FONT_WEIGHT_NORMAL]))
+        context.set_font_size(40)
+        box = context.text_extents(text)
+        if box[2] > w or box[3] > h:
+            raise IOError('Could not fit string into image. Max char count is too large for given image width.')
+
+        # teach the RNN translational invariance by
+        # fitting text box randomly on canvas, with some room to rotate
+        border_w_h = (10, 16)
+        max_shift_x = w - box[2] - border_w_h[0]
+        max_shift_y = h - box[3] - border_w_h[1]
+        top_left_x = np.random.randint(0, int(max_shift_x))
+        top_left_y = np.random.randint(0, int(max_shift_y))
+
+        context.move_to(top_left_x - int(box[0]), top_left_y - int(box[1]))
+        context.set_source_rgb(0, 0, 0)
+        context.show_text(text)
+
+    buf = surface.get_data()
+    a = np.frombuffer(buf, np.uint8)
+    a.shape = (h, w, 4)
+    a = a[:, :, 0]  # grab single channel
+    a = a.astype(np.float32) / 255
+    a = np.expand_dims(a, 0)
+    a = speckle(a)
+    a = image.random_rotation(a, 3 * (w - top_left_x) / w + 1)
+
+    return a
+
+
+def shuffle_mats_or_lists(matrix_list, stop_ind=None):
+    ret = []
+    assert all([len(i) == len(matrix_list[0]) for i in matrix_list])
+    len_val = len(matrix_list[0])
+    if stop_ind is None:
+        stop_ind = len_val
+    assert stop_ind <= len_val
+
+    a = range(stop_ind)
+    np.random.shuffle(a)
+    a += range(stop_ind, len_val)
+    for mat in matrix_list:
+        if isinstance(mat, np.ndarray):
+            ret.append(mat[a])
+        elif isinstance(mat, list):
+            ret.append([mat[i] for i in a])
+        else:
+            raise TypeError('shuffle_mats_or_lists only supports '
+                            'numpy.array and list objects')
+    return ret
+
+
+def text_to_labels(text, num_classes):
+    ret = []
+    for char in text:
+        if char >= 'a' and char <= 'z':
+            ret.append(ord(char) - ord('a'))
+        elif char == ' ':
+            ret.append(26)
+    return ret
+
+
+# only a-z and space..probably not to difficult
+# to expand to uppercase and symbols
+
+def is_valid_str(in_str):
+    search = re.compile(r'[^a-z\ ]').search
+    return not bool(search(in_str))
+
+
+# Uses generator functions to supply train/test with
+# data. Image renderings are text are created on the fly
+# each time with random perturbations
+
+class TextImageGenerator(keras.callbacks.Callback):
+
+    def __init__(self, monogram_file, bigram_file, minibatch_size,
+                 img_w, img_h, downsample_width, val_split,
+                 absolute_max_string_len=16):
+
+        self.minibatch_size = minibatch_size
+        self.img_w = img_w
+        self.img_h = img_h
+        self.monogram_file = monogram_file
+        self.bigram_file = bigram_file
+        self.downsample_width = downsample_width
+        self.val_split = val_split
+        self.blank_label = self.get_output_size() - 1
+        self.absolute_max_string_len = absolute_max_string_len
+
+    def get_output_size(self):
+        return 28
+
+    # num_words can be independent of the epoch size due to the use of generators
+    # as max_string_len grows, num_words can grow
+    def build_word_list(self, num_words, max_string_len=None, mono_fraction=0.5):
+        assert max_string_len <= self.absolute_max_string_len
+        assert num_words % self.minibatch_size == 0
+        assert (self.val_split * num_words) % self.minibatch_size == 0
+        self.num_words = num_words
+        self.string_list = []
+        self.max_string_len = max_string_len
+        self.Y_data = np.ones([self.num_words, self.absolute_max_string_len]) * -1
+        self.X_text = []
+        self.Y_len = [0] * self.num_words
+
+        # monogram file is sorted by frequency in english speech
+        with open(self.monogram_file, 'rt') as f:
+            for line in f:
+                if len(self.string_list) == int(self.num_words * mono_fraction):
+                    break
+                word = line.rstrip()
+                if max_string_len == -1 or max_string_len is None or len(word) <= max_string_len:
+                    self.string_list.append(word)
+
+        # bigram file contains common word pairings in english speech
+        with open(self.bigram_file, 'rt') as f:
+            lines = f.readlines()
+            for line in lines:
+                if len(self.string_list) == self.num_words:
+                    break
+                columns = line.lower().split()
+                word = columns[0] + ' ' + columns[1]
+                if is_valid_str(word) and \
+                        (max_string_len == -1 or max_string_len is None or len(word) <= max_string_len):
+                    self.string_list.append(word)
+        if len(self.string_list) != self.num_words:
+            raise IOError('Could not pull enough words from supplied monogram and bigram files. ')
+
+        for i, word in enumerate(self.string_list):
+            self.Y_len[i] = len(word)
+            self.Y_data[i, 0:len(word)] = text_to_labels(word, self.get_output_size())
+            self.X_text.append(word)
+        self.Y_len = np.expand_dims(np.array(self.Y_len), 1)
+
+        self.cur_val_index = self.val_split
+        self.cur_train_index = 0
+
+    # each time an image is requested from train/val/test, a new random
+    # painting of the text is performed
+    def get_batch(self, index, size, train):
+        if K.image_dim_ordering() == 'th':
+            X_data = np.ones([size, 1, self.img_h, self.img_w])
+        else:
+            X_data = np.ones([size, self.img_h, self.img_w, 1])
+        labels = np.ones([size, self.absolute_max_string_len])
+        input_length = np.zeros([size, 1])
+        label_length = np.zeros([size, 1])
+        source_str = []
+
+        for i in range(0, size):
+            # Mix in some blank inputs.  This seems to be important for
+            # achieving translational invariance
+            if train and i > size - 4:
+                if K.image_dim_ordering() == 'th':
+                    X_data[i, 0, :, :] = paint_text('', self.img_w, self.img_h)
+                else:
+                    X_data[i, :, :, 0] = paint_text('', self.img_w, self.img_h)
+                labels[i, 0] = self.blank_label
+                input_length[i] = self.downsample_width
+                label_length[i] = 1
+                source_str.append('')
+            else:
+                if K.image_dim_ordering() == 'th':
+                    X_data[i, 0, :, :] = paint_text(self.X_text[index + i], self.img_w, self.img_h)
+                else:
+                    X_data[i, :, :, 0] = paint_text(self.X_text[index + i], self.img_w, self.img_h)
+                labels[i, :] = self.Y_data[index + i]
+                input_length[i] = self.downsample_width
+                label_length[i] = self.Y_len[index + i]
+                source_str.append(self.X_text[index + i])
+
+        inputs = {'the_input': X_data,
+                  'the_labels': labels,
+                  'input_length': input_length,
+                  'label_length': label_length,
+                  'source_str': source_str  # used for visualization only
+                  }
+        outputs = {'ctc': np.zeros([size])}  # dummy data for dummy loss function
+        return (inputs, outputs)
+
+    def next_train(self):
+        while 1:
+            ret = self.get_batch(self.cur_train_index, self.minibatch_size, train=True)
+            self.cur_train_index += self.minibatch_size
+            if self.cur_train_index >= self.val_split:
+                self.cur_train_index = self.cur_train_index % 32
+                (self.X_text, self.Y_data, self.Y_len) = shuffle_mats_or_lists(
+                    [self.X_text, self.Y_data, self.Y_len], self.val_split)
+            yield ret
+
+    def next_val(self):
+        while 1:
+            ret = self.get_batch(self.cur_val_index, self.minibatch_size, train=False)
+            self.cur_val_index += self.minibatch_size
+            if self.cur_val_index >= self.num_words:
+                self.cur_val_index = self.val_split + self.cur_val_index % 32
+            yield ret
+
+    def on_train_begin(self, logs={}):
+        # translational invariance seems to be the hardest thing
+        # for the RNN to learn, so start with <= 4 letter words.
+        self.build_word_list(16000, 4, 1)
+
+    def on_epoch_begin(self, epoch, logs={}):
+        # After 10 epochs, translational invariance should be learned
+        # so start feeding longer words and eventually multiple words with spaces
+        if epoch == 10:
+            self.build_word_list(32000, 8, 1)
+        if epoch == 20:
+            self.build_word_list(32000, 8, 0.6)
+        if epoch == 30:
+            self.build_word_list(64000, 12, 0.5)
+
+
+# the actual loss calc occurs here despite it not being
+# an internal Keras loss function
+
+def ctc_lambda_func(args):
+    y_pred, labels, input_length, label_length = args
+    # the 2 is critical here since the first couple outputs of the RNN
+    # tend to be garbage:
+    y_pred = y_pred[:, 2:, :]
+    return K.ctc_batch_cost(labels, y_pred, input_length, label_length)
+
+
+# For a real OCR application, this should be beam search with a dictionary
+# and language model.  For this example, best path is sufficient.
+
+def decode_batch(test_func, word_batch):
+    out = test_func([word_batch])[0]
+    ret = []
+    for j in range(out.shape[0]):
+        out_best = list(np.argmax(out[j, 2:], 1))
+        out_best = [k for k, g in itertools.groupby(out_best)]
+        # 26 is space, 27 is CTC blank char
+        outstr = ''
+        for c in out_best:
+            if c >= 0 and c < 26:
+                outstr += chr(c + ord('a'))
+            elif c == 26:
+                outstr += ' '
+        ret.append(outstr)
+    return ret
+
+
+class VizCallback(keras.callbacks.Callback):
+
+    def __init__(self, test_func, text_img_gen, num_display_words=6):
+        self.test_func = test_func
+        self.output_dir = os.path.join(
+            OUTPUT_DIR, datetime.datetime.now().strftime('%A, %d. %B %Y %I.%M%p'))
+        self.text_img_gen = text_img_gen
+        self.num_display_words = num_display_words
+        os.makedirs(self.output_dir)
+
+    def show_edit_distance(self, num):
+        num_left = num
+        mean_norm_ed = 0.0
+        mean_ed = 0.0
+        while num_left > 0:
+            word_batch = next(self.text_img_gen)[0]
+            num_proc = min(word_batch['the_input'].shape[0], num_left)
+            decoded_res = decode_batch(self.test_func, word_batch['the_input'][0:num_proc])
+            for j in range(0, num_proc):
+                edit_dist = editdistance.eval(decoded_res[j], word_batch['source_str'][j])
+                mean_ed += float(edit_dist)
+                mean_norm_ed += float(edit_dist) / len(word_batch['source_str'][j])
+            num_left -= num_proc
+        mean_norm_ed = mean_norm_ed / num
+        mean_ed = mean_ed / num
+        print('\nOut of %d samples:  Mean edit distance: %.3f Mean normalized edit distance: %0.3f'
+              % (num, mean_ed, mean_norm_ed))
+
+    def on_epoch_end(self, epoch, logs={}):
+        self.model.save_weights(os.path.join(self.output_dir, 'weights%02d.h5' % epoch))
+        self.show_edit_distance(256)
+        word_batch = next(self.text_img_gen)[0]
+        res = decode_batch(self.test_func, word_batch['the_input'][0:self.num_display_words])
+
+        for i in range(self.num_display_words):
+            pylab.subplot(self.num_display_words, 1, i + 1)
+            if K.image_dim_ordering() == 'th':
+                the_input = word_batch['the_input'][i, 0, :, :]
+            else:
+                the_input = word_batch['the_input'][i, :, :, 0]
+            pylab.imshow(the_input, cmap='Greys_r')
+            pylab.xlabel('Truth = \'%s\' Decoded = \'%s\'' % (word_batch['source_str'][i], res[i]))
+        fig = pylab.gcf()
+        fig.set_size_inches(10, 12)
+        pylab.savefig(os.path.join(self.output_dir, 'e%02d.png' % epoch))
+        pylab.close()
+
+# Input Parameters
+img_h = 64
+img_w = 512
+nb_epoch = 50
+minibatch_size = 32
+words_per_epoch = 16000
+val_split = 0.2
+val_words = int(words_per_epoch * (val_split))
+
+# Network parameters
+conv_num_filters = 16
+filter_size = 3
+pool_size_1 = 4
+pool_size_2 = 2
+time_dense_size = 32
+rnn_size = 512
+time_steps = img_w // (pool_size_1 * pool_size_2)
+
+if K.image_dim_ordering() == 'th':
+    input_shape = (1, img_h, img_w)
+else:
+    input_shape = (img_h, img_w, 1)
+
+fdir = os.path.dirname(get_file('wordlists.tgz',
+                                origin='http://www.isosemi.com/datasets/wordlists.tgz', untar=True))
+
+img_gen = TextImageGenerator(monogram_file=os.path.join(fdir, 'wordlist_mono_clean.txt'),
+                             bigram_file=os.path.join(fdir, 'wordlist_bi_clean.txt'),
+                             minibatch_size=32,
+                             img_w=img_w,
+                             img_h=img_h,
+                             downsample_width=img_w // (pool_size_1 * pool_size_2) - 2,
+                             val_split=words_per_epoch - val_words)
+
+act = 'relu'
+input_data = Input(name='the_input', shape=input_shape, dtype='float32')
+inner = Convolution2D(conv_num_filters, filter_size, filter_size, border_mode='same',
+                      activation=act, name='conv1')(input_data)
+inner = MaxPooling2D(pool_size=(pool_size_1, pool_size_1), name='max1')(inner)
+inner = Convolution2D(conv_num_filters, filter_size, filter_size, border_mode='same',
+                      activation=act, name='conv2')(inner)
+inner = MaxPooling2D(pool_size=(pool_size_2, pool_size_2), name='max2')(inner)
+
+conv_to_rnn_dims = ((img_h // (pool_size_1 * pool_size_2)) * conv_num_filters, img_w // (pool_size_1 * pool_size_2))
+inner = Reshape(target_shape=conv_to_rnn_dims, name='reshape')(inner)
+inner = Permute(dims=(2, 1), name='permute')(inner)
+
+# cuts down input size going into RNN:
+inner = TimeDistributed(Dense(time_dense_size, activation=act, name='dense1'))(inner)
+
+# Two layers of bidirecitonal GRUs
+# GRU seems to work as well, if not better than LSTM:
+gru_1 = GRU(rnn_size, return_sequences=True, name='gru1')(inner)
+gru_1b = GRU(rnn_size, return_sequences=True, go_backwards=True, name='gru1_b')(inner)
+gru1_merged = merge([gru_1, gru_1b], mode='sum')
+gru_2 = GRU(rnn_size, return_sequences=True, name='gru2')(gru1_merged)
+gru_2b = GRU(rnn_size, return_sequences=True, go_backwards=True)(gru1_merged)
+
+# transforms RNN output to character activations:
+inner = TimeDistributed(Dense(img_gen.get_output_size(), name='dense2'))(merge([gru_2, gru_2b], mode='concat'))
+y_pred = Activation('softmax', name='softmax')(inner)
+Model(input=[input_data], output=y_pred).summary()
+
+labels = Input(name='the_labels', shape=[img_gen.absolute_max_string_len], dtype='float32')
+input_length = Input(name='input_length', shape=[1], dtype='int64')
+label_length = Input(name='label_length', shape=[1], dtype='int64')
+# Keras doesn't currently support loss funcs with extra parameters
+# so CTC loss is implemented in a lambda layer
+loss_out = Lambda(ctc_lambda_func, output_shape=(1,), name="ctc")([y_pred, labels, input_length, label_length])
+
+lr = 0.03
+# clipnorm seems to speeds up convergence
+clipnorm = 5
+sgd = SGD(lr=lr, decay=3e-7, momentum=0.9, nesterov=True, clipnorm=clipnorm)
+
+model = Model(input=[input_data, labels, input_length, label_length], output=[loss_out])
+
+# the loss calc occurs elsewhere, so use a dummy lambda func for the loss
+model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd)
+
+# captures output of softmax so we can decode the output during visualization
+test_func = K.function([input_data], [y_pred])
+
+viz_cb = VizCallback(test_func, img_gen.next_val())
+
+model.fit_generator(generator=img_gen.next_train(), samples_per_epoch=(words_per_epoch - val_words),
+                    nb_epoch=nb_epoch, validation_data=img_gen.next_val(), nb_val_samples=val_words,
+                    callbacks=[viz_cb, img_gen])

examples/imdb_bidirectional_lstm.py

@@ -9,8 +9,8 @@ import numpy as np
 np.random.seed(1337)  # for reproducibility
 
 from keras.preprocessing import sequence
-from keras.models import Model
-from keras.layers import Dense, Dropout, Embedding, LSTM, Input, merge
+from keras.models import Sequential
+from keras.layers import Dense, Dropout, Embedding, LSTM, Input, Bidirectional
 from keras.datasets import imdb
 
 
@@ -19,8 +19,7 @@ maxlen = 100  # cut texts after this number of words (among top max_features mos
 batch_size = 32
 
 print('Loading data...')
-(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features,
-                                                      test_split=0.2)
+(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features)
 print(len(X_train), 'train sequences')
 print(len(X_test), 'test sequences')
 
@@ -32,24 +31,11 @@ print('X_test shape:', X_test.shape)
 y_train = np.array(y_train)
 y_test = np.array(y_test)
 
-
-# this is the placeholder tensor for the input sequences
-sequence = Input(shape=(maxlen,), dtype='int32')
-# this embedding layer will transform the sequences of integers
-# into vectors of size 128
-embedded = Embedding(max_features, 128, input_length=maxlen)(sequence)
-
-# apply forwards LSTM
-forwards = LSTM(64)(embedded)
-# apply backwards LSTM
-backwards = LSTM(64, go_backwards=True)(embedded)
-
-# concatenate the outputs of the 2 LSTMs
-merged = merge([forwards, backwards], mode='concat', concat_axis=-1)
-after_dp = Dropout(0.5)(merged)
-output = Dense(1, activation='sigmoid')(after_dp)
-
-model = Model(input=sequence, output=output)
+model = Sequential()
+model.add(Embedding(max_features, 128, input_length=maxlen))
+model.add(Bidirectional(LSTM(64)))
+model.add(Dropout(0.5))
+model.add(Dense(1, activation='sigmoid'))
 
 # try using different optimizers and different optimizer configs
 model.compile('adam', 'binary_crossentropy', metrics=['accuracy'])

examples/imdb_cnn.py

@@ -1,6 +1,6 @@
 '''This example demonstrates the use of Convolution1D for text classification.
 
-Gets to 0.88 test accuracy after 2 epochs. 
+Gets to 0.89 test accuracy after 2 epochs.
 90s/epoch on Intel i5 2.4Ghz CPU.
 10s/epoch on Tesla K40 GPU.
 
@@ -12,9 +12,9 @@ np.random.seed(1337)  # for reproducibility
 
 from keras.preprocessing import sequence
 from keras.models import Sequential
-from keras.layers import Dense, Dropout, Activation, Lambda
+from keras.layers import Dense, Dropout, Activation
 from keras.layers import Embedding
-from keras.layers import Convolution1D
+from keras.layers import Convolution1D, GlobalMaxPooling1D
 from keras.datasets import imdb
 from keras import backend as K
 
@@ -30,8 +30,7 @@ hidden_dims = 250
 nb_epoch = 2
 
 print('Loading data...')
-(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features,
-                                                      test_split=0.2)
+(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features)
 print(len(X_train), 'train sequences')
 print(len(X_test), 'test sequences')
 
@@ -58,13 +57,8 @@ model.add(Convolution1D(nb_filter=nb_filter,
                         border_mode='valid',
                         activation='relu',
                         subsample_length=1))
-
-# we use max over time pooling by defining a python function to use
-# in a Lambda layer
-def max_1d(X):
-    return K.max(X, axis=1)
-
-model.add(Lambda(max_1d, output_shape=(nb_filter,)))
+# we use max pooling:
+model.add(GlobalMaxPooling1D())
 
 # We add a vanilla hidden layer:
 model.add(Dense(hidden_dims))

examples/imdb_cnn_lstm.py

@@ -11,7 +11,7 @@ from keras.preprocessing import sequence
 from keras.models import Sequential
 from keras.layers import Dense, Dropout, Activation
 from keras.layers import Embedding
-from keras.layers import LSTM, GRU, SimpleRNN
+from keras.layers import LSTM
 from keras.layers import Convolution1D, MaxPooling1D
 from keras.datasets import imdb
 
@@ -22,9 +22,9 @@ maxlen = 100
 embedding_size = 128
 
 # Convolution
-filter_length = 3
+filter_length = 5
 nb_filter = 64
-pool_length = 2
+pool_length = 4
 
 # LSTM
 lstm_output_size = 70
@@ -40,7 +40,7 @@ Only 2 epochs are needed as the dataset is very small.
 '''
 
 print('Loading data...')
-(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features, test_split=0.2)
+(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features)
 print(len(X_train), 'train sequences')
 print(len(X_test), 'test sequences')
 

examples/imdb_fasttext.py

@@ -0,0 +1,136 @@
+'''This example demonstrates the use of fasttext for text classification
+
+Based on Joulin et al's paper:
+
+Bags of Tricks for Efficient Text Classification
+https://arxiv.org/abs/1607.01759
+
+Results on IMDB datasets with uni and bi-gram embeddings:
+    Uni-gram: 0.8813 test accuracy after 5 epochs. 8s/epoch on i7 cpu.
+    Bi-gram : 0.9056 test accuracy after 5 epochs. 2s/epoch on GTX 980M gpu.
+'''
+
+from __future__ import print_function
+import numpy as np
+np.random.seed(1337)  # for reproducibility
+
+from keras.preprocessing import sequence
+from keras.models import Sequential
+from keras.layers import Dense
+from keras.layers import Embedding
+from keras.layers import GlobalAveragePooling1D
+from keras.datasets import imdb
+
+
+def create_ngram_set(input_list, ngram_value=2):
+    """
+    Extract a set of n-grams from a list of integers.
+
+    >>> create_ngram_set([1, 4, 9, 4, 1, 4], ngram_value=2)
+    {(4, 9), (4, 1), (1, 4), (9, 4)}
+
+    >>> create_ngram_set([1, 4, 9, 4, 1, 4], ngram_value=3)
+    [(1, 4, 9), (4, 9, 4), (9, 4, 1), (4, 1, 4)]
+    """
+    return set(zip(*[input_list[i:] for i in range(ngram_value)]))
+
+
+def add_ngram(sequences, token_indice, ngram_range=2):
+    """
+    Augment the input list of list (sequences) by appending n-grams values.
+
+    Example: adding bi-gram
+    >>> sequences = [[1, 3, 4, 5], [1, 3, 7, 9, 2]]
+    >>> token_indice = {(1, 3): 1337, (9, 2): 42, (4, 5): 2017}
+    >>> add_ngram(sequences, token_indice, ngram_range=2)
+    [[1, 3, 4, 5, 1337, 2017], [1, 3, 7, 9, 2, 1337, 42]]
+
+    Example: adding tri-gram
+    >>> sequences = [[1, 3, 4, 5], [1, 3, 7, 9, 2]]
+    >>> token_indice = {(1, 3): 1337, (9, 2): 42, (4, 5): 2017, (7, 9, 2): 2018}
+    >>> add_ngram(sequences, token_indice, ngram_range=3)
+    [[1, 3, 4, 5, 1337], [1, 3, 7, 9, 2, 1337, 2018]]
+    """
+    new_sequences = []
+    for input_list in sequences:
+        new_list = input_list[:]
+        for i in range(len(new_list)-ngram_range+1):
+            for ngram_value in range(2, ngram_range+1):
+                ngram = tuple(new_list[i:i+ngram_value])
+                if ngram in token_indice:
+                    new_list.append(token_indice[ngram])
+        new_sequences.append(new_list)
+
+    return new_sequences
+
+# Set parameters:
+# ngram_range = 2 will add bi-grams features
+ngram_range = 1
+max_features = 20000
+maxlen = 400
+batch_size = 32
+embedding_dims = 50
+nb_epoch = 5
+
+print('Loading data...')
+(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features)
+print(len(X_train), 'train sequences')
+print(len(X_test), 'test sequences')
+print('Average train sequence length: {}'.format(np.mean(list(map(len, X_train)), dtype=int)))
+print('Average test sequence length: {}'.format(np.mean(list(map(len, X_test)), dtype=int)))
+
+if ngram_range > 1:
+    print('Adding {}-gram features'.format(ngram_range))
+    # Create set of unique n-gram from the training set.
+    ngram_set = set()
+    for input_list in X_train:
+        for i in range(2, ngram_range+1):
+            set_of_ngram = create_ngram_set(input_list, ngram_value=i)
+            ngram_set.update(set_of_ngram)
+
+    # Dictionary mapping n-gram token to a unique integer.
+    # Integer values are greater than max_features in order
+    # to avoid collision with existing features.
+    start_index = max_features + 1
+    token_indice = {v: k+start_index for k, v in enumerate(ngram_set)}
+    indice_token = {token_indice[k]: k for k in token_indice}
+
+    # max_features is the highest integer that could be found in the dataset.
+    max_features = np.max(list(indice_token.keys())) + 1
+
+    # Augmenting X_train and X_test with n-grams features
+    X_train = add_ngram(X_train, token_indice, ngram_range)
+    X_test = add_ngram(X_test, token_indice, ngram_range)
+    print('Average train sequence length: {}'.format(np.mean(list(map(len, X_train)), dtype=int)))
+    print('Average test sequence length: {}'.format(np.mean(list(map(len, X_test)), dtype=int)))
+
+print('Pad sequences (samples x time)')
+X_train = sequence.pad_sequences(X_train, maxlen=maxlen)
+X_test = sequence.pad_sequences(X_test, maxlen=maxlen)
+print('X_train shape:', X_train.shape)
+print('X_test shape:', X_test.shape)
+
+print('Build model...')
+model = Sequential()
+
+# we start off with an efficient embedding layer which maps
+# our vocab indices into embedding_dims dimensions
+model.add(Embedding(max_features,
+                    embedding_dims,
+                    input_length=maxlen))
+
+# we add a GlobalAveragePooling1D, which will average the embeddings
+# of all words in the document
+model.add(GlobalAveragePooling1D())
+
+# We project onto a single unit output layer, and squash it with a sigmoid:
+model.add(Dense(1, activation='sigmoid'))
+
+model.compile(loss='binary_crossentropy',
+              optimizer='adam',
+              metrics=['accuracy'])
+
+model.fit(X_train, y_train,
+          batch_size=batch_size,
+          nb_epoch=nb_epoch,
+          validation_data=(X_test, y_test))

examples/imdb_lstm.py

@@ -1,8 +1,6 @@
 '''Trains a LSTM on the IMDB sentiment classification task.
-
 The dataset is actually too small for LSTM to be of any advantage
-compared to simpler, much faster methods such as TF-IDF+LogReg.
-
+compared to simpler, much faster methods such as TF-IDF + LogReg.
 Notes:
 
 - RNNs are tricky. Choice of batch size is important,
@@ -28,8 +26,7 @@ maxlen = 80  # cut texts after this number of words (among top max_features most
 batch_size = 32
 
 print('Loading data...')
-(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features,
-                                                      test_split=0.2)
+(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features)
 print(len(X_train), 'train sequences')
 print(len(X_test), 'test sequences')
 
@@ -41,7 +38,7 @@ print('X_test shape:', X_test.shape)
 
 print('Build model...')
 model = Sequential()
-model.add(Embedding(max_features, 128, input_length=maxlen, dropout=0.2))
+model.add(Embedding(max_features, 128, dropout=0.2))
 model.add(LSTM(128, dropout_W=0.2, dropout_U=0.2))  # try using a GRU instead, for fun
 model.add(Dense(1))
 model.add(Activation('sigmoid'))
@@ -52,8 +49,6 @@ model.compile(loss='binary_crossentropy',
               metrics=['accuracy'])
 
 print('Train...')
-print(X_train.shape)
-print(y_train.shape)
 model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=15,
           validation_data=(X_test, y_test))
 score, acc = model.evaluate(X_test, y_test,

examples/inception_v3.py

@@ -1,290 +0,0 @@
-'''This script demonstrates how to build the Inception v3 architecture
-using the Keras functional API.
-We are not actually training it here, for lack of appropriate data.
-
-For more information about this architecture, see:
-
-"Rethinking the Inception Architecture for Computer Vision"
-Christian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens, Zbigniew Wojna
-http://arxiv.org/abs/1512.00567
-'''
-from keras.layers import Convolution2D, MaxPooling2D, AveragePooling2D
-from keras.layers import BatchNormalization, Flatten, Dense, Dropout
-from keras.layers import Input, merge
-from keras.models import Model
-from keras import regularizers
-
-
-# global constants
-NB_CLASS = 1000  # number of classes
-DIM_ORDERING = 'th'  # 'th' (channels, width, height) or 'tf' (width, height, channels)
-WEIGHT_DECAY = 0.  # L2 regularization factor
-USE_BN = False  # whether to use batch normalization
-
-
-def conv2D_bn(x, nb_filter, nb_row, nb_col,
-              border_mode='same', subsample=(1, 1),
-              activation='relu', batch_norm=USE_BN,
-              weight_decay=WEIGHT_DECAY, dim_ordering=DIM_ORDERING):
-    '''Utility function to apply to a tensor a module conv + BN
-    with optional weight decay (L2 weight regularization).
-    '''
-    if weight_decay:
-        W_regularizer = regularizers.l2(weight_decay)
-        b_regularizer = regularizers.l2(weight_decay)
-    else:
-        W_regularizer = None
-        b_regularizer = None
-    x = Convolution2D(nb_filter, nb_row, nb_col,
-                      subsample=subsample,
-                      activation=activation,
-                      border_mode=border_mode,
-                      W_regularizer=W_regularizer,
-                      b_regularizer=b_regularizer,
-                      dim_ordering=dim_ordering)(x)
-    if batch_norm:
-        x = BatchNormalization()(x)
-    return x
-
-# Define image input layer
-
-if DIM_ORDERING == 'th':
-    img_input = Input(shape=(3, 299, 299))
-    CONCAT_AXIS = 1
-elif DIM_ORDERING == 'tf':
-    img_input = Input(shape=(299, 299, 3))
-    CONCAT_AXIS = 3
-else:
-    raise Exception('Invalid dim ordering: ' + str(DIM_ORDERING))
-
-# Entry module
-
-x = conv2D_bn(img_input, 32, 3, 3, subsample=(2, 2), border_mode='valid')
-x = conv2D_bn(x, 32, 3, 3, border_mode='valid')
-x = conv2D_bn(x, 64, 3, 3)
-x = MaxPooling2D((3, 3), strides=(2, 2), dim_ordering=DIM_ORDERING)(x)
-
-x = conv2D_bn(x, 80, 1, 1, border_mode='valid')
-x = conv2D_bn(x, 192, 3, 3, border_mode='valid')
-x = MaxPooling2D((3, 3), strides=(2, 2), dim_ordering=DIM_ORDERING)(x)
-
-# mixed: 35 x 35 x 256
-
-branch1x1 = conv2D_bn(x, 64, 1, 1)
-
-branch5x5 = conv2D_bn(x, 48, 1, 1)
-branch5x5 = conv2D_bn(branch5x5, 64, 5, 5)
-
-branch3x3dbl = conv2D_bn(x, 64, 1, 1)
-branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)
-branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)
-
-branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
-branch_pool = conv2D_bn(branch_pool, 32, 1, 1)
-x = merge([branch1x1, branch5x5, branch3x3dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
-
-# mixed_1: 35 x 35 x 288
-
-branch1x1 = conv2D_bn(x, 64, 1, 1)
-
-branch5x5 = conv2D_bn(x, 48, 1, 1)
-branch5x5 = conv2D_bn(branch5x5, 64, 5, 5)
-
-branch3x3dbl = conv2D_bn(x, 64, 1, 1)
-branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)
-branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)
-
-branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
-branch_pool = conv2D_bn(branch_pool, 64, 1, 1)
-x = merge([branch1x1, branch5x5, branch3x3dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
-
-# mixed2: 35 x 35 x 288
-
-branch1x1 = conv2D_bn(x, 64, 1, 1)
-
-branch5x5 = conv2D_bn(x, 48, 1, 1)
-branch5x5 = conv2D_bn(branch5x5, 64, 5, 5)
-
-branch3x3dbl = conv2D_bn(x, 64, 1, 1)
-branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)
-branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)
-
-branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
-branch_pool = conv2D_bn(branch_pool, 64, 1, 1)
-x = merge([branch1x1, branch5x5, branch3x3dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
-
-# mixed3: 17 x 17 x 768
-
-branch3x3 = conv2D_bn(x, 384, 3, 3, subsample=(2, 2), border_mode='valid')
-
-branch3x3dbl = conv2D_bn(x, 64, 1, 1)
-branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3)
-branch3x3dbl = conv2D_bn(branch3x3dbl, 96, 3, 3, subsample=(2, 2), border_mode='valid')
-
-branch_pool = MaxPooling2D((3, 3), strides=(2, 2), dim_ordering=DIM_ORDERING)(x)
-x = merge([branch3x3, branch3x3dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
-
-# mixed4: 17 x 17 x 768
-
-branch1x1 = conv2D_bn(x, 192, 1, 1)
-
-branch7x7 = conv2D_bn(x, 128, 1, 1)
-branch7x7 = conv2D_bn(branch7x7, 128, 1, 7)
-branch7x7 = conv2D_bn(branch7x7, 192, 7, 1)
-
-branch7x7dbl = conv2D_bn(x, 128, 1, 1)
-branch7x7dbl = conv2D_bn(branch7x7dbl, 128, 7, 1)
-branch7x7dbl = conv2D_bn(branch7x7dbl, 128, 1, 7)
-branch7x7dbl = conv2D_bn(branch7x7dbl, 128, 7, 1)
-branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)
-
-branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
-branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
-x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
-
-# mixed5: 17 x 17 x 768
-
-branch1x1 = conv2D_bn(x, 192, 1, 1)
-
-branch7x7 = conv2D_bn(x, 160, 1, 1)
-branch7x7 = conv2D_bn(branch7x7, 160, 1, 7)
-branch7x7 = conv2D_bn(branch7x7, 192, 7, 1)
-
-branch7x7dbl = conv2D_bn(x, 160, 1, 1)
-branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 7, 1)
-branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 1, 7)
-branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 7, 1)
-branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)
-
-branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
-branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
-x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
-
-# mixed5: 17 x 17 x 768
-
-branch1x1 = conv2D_bn(x, 192, 1, 1)
-
-branch7x7 = conv2D_bn(x, 160, 1, 1)
-branch7x7 = conv2D_bn(branch7x7, 160, 1, 7)
-branch7x7 = conv2D_bn(branch7x7, 192, 7, 1)
-
-branch7x7dbl = conv2D_bn(x, 160, 1, 1)
-branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 7, 1)
-branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 1, 7)
-branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 7, 1)
-branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)
-
-branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
-branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
-x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
-
-# mixed6: 17 x 17 x 768
-
-branch1x1 = conv2D_bn(x, 192, 1, 1)
-
-branch7x7 = conv2D_bn(x, 160, 1, 1)
-branch7x7 = conv2D_bn(branch7x7, 160, 1, 7)
-branch7x7 = conv2D_bn(branch7x7, 192, 7, 1)
-
-branch7x7dbl = conv2D_bn(x, 160, 1, 1)
-branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 7, 1)
-branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)
-branch7x7dbl = conv2D_bn(branch7x7dbl, 160, 7, 1)
-branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)
-
-branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
-branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
-x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
-
-# mixed7: 17 x 17 x 768
-
-branch1x1 = conv2D_bn(x, 192, 1, 1)
-
-branch7x7 = conv2D_bn(x, 192, 1, 1)
-branch7x7 = conv2D_bn(branch7x7, 192, 1, 7)
-branch7x7 = conv2D_bn(branch7x7, 192, 7, 1)
-
-branch7x7dbl = conv2D_bn(x, 160, 1, 1)
-branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 7, 1)
-branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)
-branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 7, 1)
-branch7x7dbl = conv2D_bn(branch7x7dbl, 192, 1, 7)
-
-branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
-branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
-x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
-
-# Auxiliary head
-
-aux_logits = AveragePooling2D((5, 5), strides=(3, 3), dim_ordering=DIM_ORDERING)(x)
-aux_logits = conv2D_bn(aux_logits, 128, 1, 1)
-aux_logits = conv2D_bn(aux_logits, 728, 5, 5, border_mode='valid')
-aux_logits = Flatten()(aux_logits)
-aux_preds = Dense(NB_CLASS, activation='softmax')(aux_logits)
-
-# mixed8: 8 x 8 x 1280
-
-branch3x3 = conv2D_bn(x, 192, 1, 1)
-branch3x3 = conv2D_bn(branch3x3, 320, 3, 3, subsample=(2, 2), border_mode='valid')
-
-branch7x7x3 = conv2D_bn(x, 192, 1, 1)
-branch7x7x3 = conv2D_bn(branch7x7x3, 192, 1, 7)
-branch7x7x3 = conv2D_bn(branch7x7x3, 192, 7, 1)
-branch7x7x3 = conv2D_bn(branch7x7x3, 192, 3, 3, subsample=(2, 2), border_mode='valid')
-
-branch_pool = AveragePooling2D((3, 3), strides=(2, 2), dim_ordering=DIM_ORDERING)(x)
-x = merge([branch3x3, branch7x7x3, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
-
-# mixed9: 8 x 8 x 2048
-
-branch1x1 = conv2D_bn(x, 320, 1, 1)
-
-branch3x3 = conv2D_bn(x, 384, 1, 1)
-branch3x3_1 = conv2D_bn(branch3x3, 384, 1, 3)
-branch3x3_2 = conv2D_bn(branch3x3, 384, 3, 1)
-branch3x3 = merge([branch3x3_1, branch3x3_2], mode='concat', concat_axis=CONCAT_AXIS)
-
-branch3x3dbl = conv2D_bn(x, 448, 1, 1)
-branch3x3dbl = conv2D_bn(branch3x3dbl, 384, 3, 3)
-branch3x3dbl_1 = conv2D_bn(branch3x3dbl, 384, 1, 3)
-branch3x3dbl_2 = conv2D_bn(branch3x3dbl, 384, 3, 1)
-branch3x3dbl = merge([branch3x3dbl_1, branch3x3dbl_2], mode='concat', concat_axis=CONCAT_AXIS)
-
-branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
-branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
-x = merge([branch1x1, branch3x3, branch3x3dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
-
-# mixed10: 8 x 8 x 2048
-
-branch1x1 = conv2D_bn(x, 320, 1, 1)
-
-branch3x3 = conv2D_bn(x, 384, 1, 1)
-branch3x3_1 = conv2D_bn(branch3x3, 384, 1, 3)
-branch3x3_2 = conv2D_bn(branch3x3, 384, 3, 1)
-branch3x3 = merge([branch3x3_1, branch3x3_2], mode='concat', concat_axis=CONCAT_AXIS)
-
-branch3x3dbl = conv2D_bn(x, 448, 1, 1)
-branch3x3dbl = conv2D_bn(branch3x3dbl, 384, 3, 3)
-branch3x3dbl_1 = conv2D_bn(branch3x3dbl, 384, 1, 3)
-branch3x3dbl_2 = conv2D_bn(branch3x3dbl, 384, 3, 1)
-branch3x3dbl = merge([branch3x3dbl_1, branch3x3dbl_2], mode='concat', concat_axis=CONCAT_AXIS)
-
-branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same', dim_ordering=DIM_ORDERING)(x)
-branch_pool = conv2D_bn(branch_pool, 192, 1, 1)
-x = merge([branch1x1, branch3x3, branch3x3dbl, branch_pool], mode='concat', concat_axis=CONCAT_AXIS)
-
-# Final pooling and prediction
-
-x = AveragePooling2D((8, 8), strides=(1, 1), dim_ordering=DIM_ORDERING)(x)
-x = Dropout(0.5)(x)
-x = Flatten()(x)
-preds = Dense(NB_CLASS, activation='softmax')(x)
-
-# Define model
-
-model = Model(input=img_input, output=[preds, aux_preds])
-model.compile('rmsprop', 'categorical_crossentropy')
-
-# train via e.g. `model.fit(x_train, [y_train] * 2, batch_size=32, nb_epoch=100)`
-# Note that for a large dataset it would be preferable
-# to train using `fit_generator` (see Keras docs).

examples/lstm_text_generation.py

@@ -14,6 +14,7 @@ from __future__ import print_function
 from keras.models import Sequential
 from keras.layers import Dense, Activation, Dropout
 from keras.layers import LSTM
+from keras.optimizers import RMSprop
 from keras.utils.data_utils import get_file
 import numpy as np
 import random
@@ -47,23 +48,25 @@ for i, sentence in enumerate(sentences):
     y[i, char_indices[next_chars[i]]] = 1
 
 
-# build the model: 2 stacked LSTM
+# build the model: a single LSTM
 print('Build model...')
 model = Sequential()
-model.add(LSTM(512, return_sequences=True, input_shape=(maxlen, len(chars))))
-model.add(LSTM(512, return_sequences=False))
-model.add(Dropout(0.2))
+model.add(LSTM(128, input_shape=(maxlen, len(chars))))
 model.add(Dense(len(chars)))
 model.add(Activation('softmax'))
 
-model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
+optimizer = RMSprop(lr=0.01)
+model.compile(loss='categorical_crossentropy', optimizer=optimizer)
 
 
-def sample(a, temperature=1.0):
+def sample(preds, temperature=1.0):
     # helper function to sample an index from a probability array
-    a = np.log(a) / temperature
-    a = np.exp(a) / np.sum(np.exp(a))
-    return np.argmax(np.random.multinomial(1, a, 1))
+    preds = np.asarray(preds).astype('float64')
+    preds = np.log(preds) / temperature
+    exp_preds = np.exp(preds)
+    preds = exp_preds / np.sum(exp_preds)
+    probas = np.random.multinomial(1, preds, 1)
+    return np.argmax(probas)
 
 # train the model, output generated text after each iteration
 for iteration in range(1, 60):

examples/mnist_acgan.py

@@ -0,0 +1,314 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Train an Auxiliary Classifier Generative Adversarial Network (ACGAN) on the
+MNIST dataset. See https://arxiv.org/abs/1610.09585 for more details.
+
+You should start to see reasonable images after ~5 epochs, and good images
+by ~15 epochs. You should use a GPU, as the convolution-heavy operations are
+very slow on the CPU. Prefer the TensorFlow backend if you plan on iterating, as
+the compilation time can be a blocker using Theano.
+
+Timings:
+
+Hardware           | Backend | Time / Epoch
+-------------------------------------------
+ CPU               | TF      | 3 hrs
+ Titan X (maxwell) | TF      | 4 min
+ Titan X (maxwell) | TH      | 7 min
+
+Consult https://github.com/lukedeo/keras-acgan for more information and
+example output
+"""
+from __future__ import print_function
+
+from collections import defaultdict
+import cPickle as pickle
+from PIL import Image
+
+from six.moves import range
+
+import keras.backend as K
+from keras.datasets import mnist
+from keras.layers import Input, Dense, Reshape, Flatten, Embedding, merge, Dropout
+from keras.layers.advanced_activations import LeakyReLU
+from keras.layers.convolutional import UpSampling2D, Convolution2D
+from keras.models import Sequential, Model
+from keras.optimizers import Adam
+from keras.utils.generic_utils import Progbar
+import numpy as np
+
+np.random.seed(1337)
+
+K.set_image_dim_ordering('th')
+
+
+def build_generator(latent_size):
+    # we will map a pair of (z, L), where z is a latent vector and L is a
+    # label drawn from P_c, to image space (..., 1, 28, 28)
+    cnn = Sequential()
+
+    cnn.add(Dense(1024, input_dim=latent_size, activation='relu'))
+    cnn.add(Dense(128 * 7 * 7, activation='relu'))
+    cnn.add(Reshape((128, 7, 7)))
+
+    # upsample to (..., 14, 14)
+    cnn.add(UpSampling2D(size=(2, 2)))
+    cnn.add(Convolution2D(256, 5, 5, border_mode='same',
+                          activation='relu', init='glorot_normal'))
+
+    # upsample to (..., 28, 28)
+    cnn.add(UpSampling2D(size=(2, 2)))
+    cnn.add(Convolution2D(128, 5, 5, border_mode='same',
+                          activation='relu', init='glorot_normal'))
+
+    # take a channel axis reduction
+    cnn.add(Convolution2D(1, 2, 2, border_mode='same',
+                          activation='tanh', init='glorot_normal'))
+
+    # this is the z space commonly refered to in GAN papers
+    latent = Input(shape=(latent_size, ))
+
+    # this will be our label
+    image_class = Input(shape=(1,), dtype='int32')
+
+    # 10 classes in MNIST
+    cls = Flatten()(Embedding(10, latent_size,
+                              init='glorot_normal')(image_class))
+
+    # hadamard product between z-space and a class conditional embedding
+    h = merge([latent, cls], mode='mul')
+
+    fake_image = cnn(h)
+
+    return Model(input=[latent, image_class], output=fake_image)
+
+
+def build_discriminator():
+    # build a relatively standard conv net, with LeakyReLUs as suggested in
+    # the reference paper
+    cnn = Sequential()
+
+    cnn.add(Convolution2D(32, 3, 3, border_mode='same', subsample=(2, 2),
+                          input_shape=(1, 28, 28)))
+    cnn.add(LeakyReLU())
+    cnn.add(Dropout(0.3))
+
+    cnn.add(Convolution2D(64, 3, 3, border_mode='same', subsample=(1, 1)))
+    cnn.add(LeakyReLU())
+    cnn.add(Dropout(0.3))
+
+    cnn.add(Convolution2D(128, 3, 3, border_mode='same', subsample=(2, 2)))
+    cnn.add(LeakyReLU())
+    cnn.add(Dropout(0.3))
+
+    cnn.add(Convolution2D(256, 3, 3, border_mode='same', subsample=(1, 1)))
+    cnn.add(LeakyReLU())
+    cnn.add(Dropout(0.3))
+
+    cnn.add(Flatten())
+
+    image = Input(shape=(1, 28, 28))
+
+    features = cnn(image)
+
+    # first output (name=generation) is whether or not the discriminator
+    # thinks the image that is being shown is fake, and the second output
+    # (name=auxiliary) is the class that the discriminator thinks the image
+    # belongs to.
+    fake = Dense(1, activation='sigmoid', name='generation')(features)
+    aux = Dense(10, activation='softmax', name='auxiliary')(features)
+
+    return Model(input=image, output=[fake, aux])
+
+if __name__ == '__main__':
+
+    # batch and latent size taken from the paper
+    nb_epochs = 50
+    batch_size = 100
+    latent_size = 100
+
+    # Adam parameters suggested in https://arxiv.org/abs/1511.06434
+    adam_lr = 0.0002
+    adam_beta_1 = 0.5
+
+    # build the discriminator
+    discriminator = build_discriminator()
+    discriminator.compile(
+        optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1),
+        loss=['binary_crossentropy', 'sparse_categorical_crossentropy']
+    )
+
+    # build the generator
+    generator = build_generator(latent_size)
+    generator.compile(optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1),
+                      loss='binary_crossentropy')
+
+    latent = Input(shape=(latent_size, ))
+    image_class = Input(shape=(1,), dtype='int32')
+
+    # get a fake image
+    fake = generator([latent, image_class])
+
+    # we only want to be able to train generation for the combined model
+    discriminator.trainable = False
+    fake, aux = discriminator(fake)
+    combined = Model(input=[latent, image_class], output=[fake, aux])
+
+    combined.compile(
+        optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1),
+        loss=['binary_crossentropy', 'sparse_categorical_crossentropy']
+    )
+
+    discriminator.trainable = True
+
+    # get our mnist data, and force it to be of shape (..., 1, 28, 28) with
+    # range [-1, 1]
+    (X_train, y_train), (X_test, y_test) = mnist.load_data()
+    X_train = (X_train.astype(np.float32) - 127.5) / 127.5
+    X_train = np.expand_dims(X_train, axis=1)
+
+    X_test = (X_test.astype(np.float32) - 127.5) / 127.5
+    X_test = np.expand_dims(X_test, axis=1)
+
+    nb_train, nb_test = X_train.shape[0], X_test.shape[0]
+
+    train_history = defaultdict(list)
+    test_history = defaultdict(list)
+
+    for epoch in range(nb_epochs):
+        print('Epoch {} of {}'.format(epoch + 1, nb_epochs))
+
+        nb_batches = int(X_train.shape[0] / batch_size)
+        progress_bar = Progbar(target=nb_batches)
+
+        epoch_gen_loss = []
+        epoch_disc_loss = []
+
+        for index in range(nb_batches):
+            progress_bar.update(index)
+            # generate a new batch of noise
+            noise = np.random.uniform(-1, 1, (batch_size, latent_size))
+
+            # get a batch of real images
+            image_batch = X_train[index * batch_size:(index + 1) * batch_size]
+            label_batch = y_train[index * batch_size:(index + 1) * batch_size]
+
+            # sample some labels from p_c
+            sampled_labels = np.random.randint(0, 10, batch_size)
+
+            # generate a batch of fake images, using the generated labels as a
+            # conditioner. We reshape the sampled labels to be
+            # (batch_size, 1) so that we can feed them into the embedding
+            # layer as a length one sequence
+            generated_images = generator.predict(
+                [noise, sampled_labels.reshape((-1, 1))], verbose=0)
+
+            X = np.concatenate((image_batch, generated_images))
+            y = np.array([1] * batch_size + [0] * batch_size)
+            aux_y = np.concatenate((label_batch, sampled_labels), axis=0)
+
+            # see if the discriminator can figure itself out...
+            epoch_disc_loss.append(discriminator.train_on_batch(X, [y, aux_y]))
+
+            # make new noise. we generate 2 * batch size here such that we have
+            # the generator optimize over an identical number of images as the
+            # discriminator
+            noise = np.random.uniform(-1, 1, (2 * batch_size, latent_size))
+            sampled_labels = np.random.randint(0, 10, 2 * batch_size)
+
+            # we want to fix the discriminator and let the generator train to
+            # trick it
+            discriminator.trainable = False
+
+            # For the generator, we want all the {fake, not-fake} labels to say
+            # not-fake
+            trick = np.ones(2 * batch_size)
+
+            epoch_gen_loss.append(combined.train_on_batch(
+                [noise, sampled_labels.reshape((-1, 1))], [trick, sampled_labels]))
+
+            discriminator.trainable = True
+
+        print('\nTesting for epoch {}:'.format(epoch + 1))
+
+        # evaluate the testing loss here
+
+        # generate a new batch of noise
+        noise = np.random.uniform(-1, 1, (nb_test, latent_size))
+
+        # sample some labels from p_c and generate images from them
+        sampled_labels = np.random.randint(0, 10, nb_test)
+        generated_images = generator.predict(
+            [noise, sampled_labels.reshape((-1, 1))], verbose=False)
+
+        X = np.concatenate((X_test, generated_images))
+        y = np.array([1] * nb_test + [0] * nb_test)
+        aux_y = np.concatenate((y_test, sampled_labels), axis=0)
+
+        # see if the discriminator can figure itself out...
+        discriminator_test_loss = discriminator.evaluate(
+            X, [y, aux_y], verbose=False)
+
+        discriminator_train_loss = np.mean(np.array(epoch_disc_loss), axis=0)
+
+        # make new noise
+        noise = np.random.uniform(-1, 1, (2 * nb_test, latent_size))
+        sampled_labels = np.random.randint(0, 10, 2 * nb_test)
+
+        trick = np.ones(2 * nb_test)
+
+        generator_test_loss = combined.evaluate(
+            [noise, sampled_labels.reshape((-1, 1))],
+            [trick, sampled_labels], verbose=False)
+
+        generator_train_loss = np.mean(np.array(epoch_gen_loss), axis=0)
+
+        # generate an epoch report on performance
+        train_history['generator'].append(generator_train_loss)
+        train_history['discriminator'].append(discriminator_train_loss)
+
+        test_history['generator'].append(generator_test_loss)
+        test_history['discriminator'].append(discriminator_test_loss)
+
+        print('{0:<22s} | {1:4s} | {2:15s} | {3:5s}'.format(
+            'component', *discriminator.metrics_names))
+        print('-' * 65)
+
+        ROW_FMT = '{0:<22s} | {1:<4.2f} | {2:<15.2f} | {3:<5.2f}'
+        print(ROW_FMT.format('generator (train)',
+                             *train_history['generator'][-1]))
+        print(ROW_FMT.format('generator (test)',
+                             *test_history['generator'][-1]))
+        print(ROW_FMT.format('discriminator (train)',
+                             *train_history['discriminator'][-1]))
+        print(ROW_FMT.format('discriminator (test)',
+                             *test_history['discriminator'][-1]))
+
+        # save weights every epoch
+        generator.save_weights(
+            'params_generator_epoch_{0:03d}.hdf5'.format(epoch), True)
+        discriminator.save_weights(
+            'params_discriminator_epoch_{0:03d}.hdf5'.format(epoch), True)
+
+        # generate some digits to display
+        noise = np.random.uniform(-1, 1, (100, latent_size))
+
+        sampled_labels = np.array([
+            [i] * 10 for i in range(10)
+        ]).reshape(-1, 1)
+
+        # get a batch to display
+        generated_images = generator.predict(
+            [noise, sampled_labels], verbose=0)
+
+        # arrange them into a grid
+        img = (np.concatenate([r.reshape(-1, 28)
+                               for r in np.split(generated_images, 10)
+                               ], axis=-1) * 127.5 + 127.5).astype(np.uint8)
+
+        Image.fromarray(img).save(
+            'plot_epoch_{0:03d}_generated.png'.format(epoch))
+
+    pickle.dump({'train': train_history, 'test': test_history},
+                open('acgan-history.pkl', 'wb'))

examples/mnist_cnn.py

@@ -14,6 +14,7 @@ from keras.models import Sequential
 from keras.layers import Dense, Dropout, Activation, Flatten
 from keras.layers import Convolution2D, MaxPooling2D
 from keras.utils import np_utils
+from keras import backend as K
 
 batch_size = 128
 nb_classes = 10
@@ -24,15 +25,22 @@ img_rows, img_cols = 28, 28
 # number of convolutional filters to use
 nb_filters = 32
 # size of pooling area for max pooling
-nb_pool = 2
+pool_size = (2, 2)
 # convolution kernel size
-nb_conv = 3
+kernel_size = (3, 3)
 
 # the data, shuffled and split between train and test sets
 (X_train, y_train), (X_test, y_test) = mnist.load_data()
 
-X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)
-X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)
+if K.image_dim_ordering() == 'th':
+    X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)
+    X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)
+    input_shape = (1, img_rows, img_cols)
+else:
+    X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1)
+    X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)
+    input_shape = (img_rows, img_cols, 1)
+
 X_train = X_train.astype('float32')
 X_test = X_test.astype('float32')
 X_train /= 255
@@ -47,13 +55,13 @@ Y_test = np_utils.to_categorical(y_test, nb_classes)
 
 model = Sequential()
 
-model.add(Convolution2D(nb_filters, nb_conv, nb_conv,
+model.add(Convolution2D(nb_filters, kernel_size[0], kernel_size[1],
                         border_mode='valid',
-                        input_shape=(1, img_rows, img_cols)))
+                        input_shape=input_shape))
 model.add(Activation('relu'))
-model.add(Convolution2D(nb_filters, nb_conv, nb_conv))
+model.add(Convolution2D(nb_filters, kernel_size[0], kernel_size[1]))
 model.add(Activation('relu'))
-model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
+model.add(MaxPooling2D(pool_size=pool_size))
 model.add(Dropout(0.25))
 
 model.add(Flatten())

examples/mnist_hierarchical_rnn.py

@@ -0,0 +1,87 @@
+"""This is an example of using Hierarchical RNN (HRNN) to classify MNIST digits.
+
+HRNNs can learn across multiple levels of temporal hiearchy over a complex sequence.
+Usually, the first recurrent layer of an HRNN encodes a sentence (e.g. of word vectors)
+into a  sentence vector. The second recurrent layer then encodes a sequence of
+such vectors (encoded by the first layer) into a document vector. This
+document vector is considered to preserve both the word-level and
+sentence-level structure of the context.
+
+# References
+    - [A Hierarchical Neural Autoencoder for Paragraphs and Documents](https://web.stanford.edu/~jurafsky/pubs/P15-1107.pdf)
+        Encodes paragraphs and documents with HRNN.
+        Results have shown that HRNN outperforms standard
+        RNNs and may play some role in more sophisticated generation tasks like
+        summarization or question answering.
+    - [Hierarchical recurrent neural network for skeleton based action recognition](http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=7298714)
+        Achieved state-of-the-art results on skeleton based action recognition with 3 levels
+        of bidirectional HRNN combined with fully connected layers.
+
+In the below MNIST example the first LSTM layer first encodes every
+column of pixels of shape (28, 1) to a column vector of shape (128,). The second LSTM
+layer encodes then these 28 column vectors of shape (28, 128) to a image vector
+representing the whole image. A final Dense layer is added for prediction.
+
+After 5 epochs: train acc: 0.9858, val acc: 0.9864
+"""
+from __future__ import print_function
+
+from keras.datasets import mnist
+from keras.models import Sequential, Model
+from keras.layers import Input, Dense, TimeDistributed
+from keras.layers import LSTM
+from keras.utils import np_utils
+
+# Training parameters.
+batch_size = 32
+nb_classes = 10
+nb_epochs = 5
+
+# Embedding dimensions.
+row_hidden = 128
+col_hidden = 128
+
+# The data, shuffled and split between train and test sets.
+(X_train, y_train), (X_test, y_test) = mnist.load_data()
+
+# Reshapes data to 4D for Hierarchical RNN.
+X_train = X_train.reshape(X_train.shape[0], 28, 28, 1)
+X_test = X_test.reshape(X_test.shape[0], 28, 28, 1)
+X_train = X_train.astype('float32')
+X_test = X_test.astype('float32')
+X_train /= 255
+X_test /= 255
+print('X_train shape:', X_train.shape)
+print(X_train.shape[0], 'train samples')
+print(X_test.shape[0], 'test samples')
+
+# Converts class vectors to binary class matrices.
+Y_train = np_utils.to_categorical(y_train, nb_classes)
+Y_test = np_utils.to_categorical(y_test, nb_classes)
+
+row, col, pixel = X_train.shape[1:]
+
+# 4D input.
+x = Input(shape=(row, col, pixel))
+
+# Encodes a row of pixels using TimeDistributed Wrapper.
+encoded_rows = TimeDistributed(LSTM(output_dim=row_hidden))(x)
+
+# Encodes columns of encoded rows.
+encoded_columns = LSTM(col_hidden)(encoded_rows)
+
+# Final predictions and model.
+prediction = Dense(nb_classes, activation='softmax')(encoded_columns)
+model = Model(input=x, output=prediction)
+model.compile(loss='categorical_crossentropy',
+              optimizer='rmsprop',
+              metrics=['accuracy'])
+
+# Training.
+model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epochs,
+          verbose=1, validation_data=(X_test, Y_test))
+
+# Evaluation.
+scores = model.evaluate(X_test, Y_test, verbose=0)
+print('Test loss:', scores[0])
+print('Test accuracy:', scores[1])

examples/mnist_net2net.py

@@ -0,0 +1,384 @@
+'''This is an implementation of Net2Net experiment with MNIST in
+'Net2Net: Accelerating Learning via Knowledge Transfer'
+by Tianqi Chen, Ian Goodfellow, and Jonathon Shlens
+
+arXiv:1511.05641v4 [cs.LG] 23 Apr 2016
+http://arxiv.org/abs/1511.05641
+
+Notes
+- What:
+  + Net2Net is a group of methods to transfer knowledge from a teacher neural
+    net to a student net,so that the student net can be trained faster than
+    from scratch.
+  + The paper discussed two specific methods of Net2Net, i.e. Net2WiderNet
+    and Net2DeeperNet.
+  + Net2WiderNet replaces a model with an equivalent wider model that has
+    more units in each hidden layer.
+  + Net2DeeperNet replaces a model with an equivalent deeper model.
+  + Both are based on the idea of 'function-preserving transformations of
+    neural nets'.
+- Why:
+  + Enable fast exploration of multiple neural nets in experimentation and
+    design process,by creating a series of wider and deeper models with
+    transferable knowledge.
+  + Enable 'lifelong learning system' by gradually adjusting model complexity
+    to data availability,and reusing transferable knowledge.
+
+Experiments
+- Teacher model: a basic CNN model trained on MNIST for 3 epochs.
+- Net2WiderNet exepriment:
+  + Student model has a wider Conv2D layer and a wider FC layer.
+  + Comparison of 'random-padding' vs 'net2wider' weight initialization.
+  + With both methods, student model should immediately perform as well as
+    teacher model, but 'net2wider' is slightly better.
+- Net2DeeperNet experiment:
+  + Student model has an extra Conv2D layer and an extra FC layer.
+  + Comparison of 'random-init' vs 'net2deeper' weight initialization.
+  + Starting performance of 'net2deeper' is better than 'random-init'.
+- Hyper-parameters:
+  + SGD with momentum=0.9 is used for training teacher and student models.
+  + Learning rate adjustment: it's suggested to reduce learning rate
+    to 1/10 for student model.
+  + Addition of noise in 'net2wider' is used to break weight symmetry
+    and thus enable full capacity of student models. It is optional
+    when a Dropout layer is used.
+
+Results
+- Tested with 'Theano' backend and 'th' image_dim_ordering.
+- Running on GPU GeForce GTX 980M
+- Performance Comparisons - validation loss values during first 3 epochs:
+(1) teacher_model:             0.075    0.041    0.041
+(2) wider_random_pad:          0.036    0.034    0.032
+(3) wider_net2wider:           0.032    0.030    0.030
+(4) deeper_random_init:        0.061    0.043    0.041
+(5) deeper_net2deeper:         0.032    0.031    0.029
+'''
+
+from __future__ import print_function
+import numpy as np
+np.random.seed(1337)
+
+from keras.models import Sequential
+from keras.layers import Conv2D, MaxPooling2D, Dense, Flatten
+from keras.optimizers import SGD
+from keras.utils import np_utils
+from keras.datasets import mnist
+
+input_shape = (1, 28, 28)  # image shape
+nb_class = 10  # number of class
+
+
+# load and pre-process data
+def preprocess_input(x):
+    return x.reshape((-1, ) + input_shape) / 255.
+
+
+def preprocess_output(y):
+    return np_utils.to_categorical(y)
+
+(train_x, train_y), (validation_x, validation_y) = mnist.load_data()
+train_x, validation_x = map(preprocess_input, [train_x, validation_x])
+train_y, validation_y = map(preprocess_output, [train_y, validation_y])
+print('Loading MNIST data...')
+print('train_x shape:', train_x.shape, 'train_y shape:', train_y.shape)
+print('validation_x shape:', validation_x.shape,
+      'validation_y shape', validation_y.shape)
+
+
+# knowledge transfer algorithms
+def wider2net_conv2d(teacher_w1, teacher_b1, teacher_w2, new_width, init):
+    '''Get initial weights for a wider conv2d layer with a bigger nb_filter,
+    by 'random-padding' or 'net2wider'.
+
+    # Arguments
+        teacher_w1: `weight` of conv2d layer to become wider,
+          of shape (nb_filter1, nb_channel1, kh1, kw1)
+        teacher_b1: `bias` of conv2d layer to become wider,
+          of shape (nb_filter1, )
+        teacher_w2: `weight` of next connected conv2d layer,
+          of shape (nb_filter2, nb_channel2, kh2, kw2)
+        new_width: new `nb_filter` for the wider conv2d layer
+        init: initialization algorithm for new weights,
+          either 'random-pad' or 'net2wider'
+    '''
+    assert teacher_w1.shape[0] == teacher_w2.shape[1], (
+        'successive layers from teacher model should have compatible shapes')
+    assert teacher_w1.shape[0] == teacher_b1.shape[0], (
+        'weight and bias from same layer should have compatible shapes')
+    assert new_width > teacher_w1.shape[0], (
+        'new width (nb_filter) should be bigger than the existing one')
+
+    n = new_width - teacher_w1.shape[0]
+    if init == 'random-pad':
+        new_w1 = np.random.normal(0, 0.1, size=(n, ) + teacher_w1.shape[1:])
+        new_b1 = np.ones(n) * 0.1
+        new_w2 = np.random.normal(0, 0.1, size=(
+            teacher_w2.shape[0], n) + teacher_w2.shape[2:])
+    elif init == 'net2wider':
+        index = np.random.randint(teacher_w1.shape[0], size=n)
+        factors = np.bincount(index)[index] + 1.
+        new_w1 = teacher_w1[index, :, :, :]
+        new_b1 = teacher_b1[index]
+        new_w2 = teacher_w2[:, index, :, :] / factors.reshape((1, -1, 1, 1))
+    else:
+        raise ValueError('Unsupported weight initializer: %s' % init)
+
+    student_w1 = np.concatenate((teacher_w1, new_w1), axis=0)
+    if init == 'random-pad':
+        student_w2 = np.concatenate((teacher_w2, new_w2), axis=1)
+    elif init == 'net2wider':
+        # add small noise to break symmetry, so that student model will have
+        # full capacity later
+        noise = np.random.normal(0, 5e-2 * new_w2.std(), size=new_w2.shape)
+        student_w2 = np.concatenate((teacher_w2, new_w2 + noise), axis=1)
+        student_w2[:, index, :, :] = new_w2
+    student_b1 = np.concatenate((teacher_b1, new_b1), axis=0)
+
+    return student_w1, student_b1, student_w2
+
+
+def wider2net_fc(teacher_w1, teacher_b1, teacher_w2, new_width, init):
+    '''Get initial weights for a wider fully connected (dense) layer
+       with a bigger nout, by 'random-padding' or 'net2wider'.
+
+    # Arguments
+        teacher_w1: `weight` of fc layer to become wider,
+          of shape (nin1, nout1)
+        teacher_b1: `bias` of fc layer to become wider,
+          of shape (nout1, )
+        teacher_w2: `weight` of next connected fc layer,
+          of shape (nin2, nout2)
+        new_width: new `nout` for the wider fc layer
+        init: initialization algorithm for new weights,
+          either 'random-pad' or 'net2wider'
+    '''
+    assert teacher_w1.shape[1] == teacher_w2.shape[0], (
+        'successive layers from teacher model should have compatible shapes')
+    assert teacher_w1.shape[1] == teacher_b1.shape[0], (
+        'weight and bias from same layer should have compatible shapes')
+    assert new_width > teacher_w1.shape[1], (
+        'new width (nout) should be bigger than the existing one')
+
+    n = new_width - teacher_w1.shape[1]
+    if init == 'random-pad':
+        new_w1 = np.random.normal(0, 0.1, size=(teacher_w1.shape[0], n))
+        new_b1 = np.ones(n) * 0.1
+        new_w2 = np.random.normal(0, 0.1, size=(n, teacher_w2.shape[1]))
+    elif init == 'net2wider':
+        index = np.random.randint(teacher_w1.shape[1], size=n)
+        factors = np.bincount(index)[index] + 1.
+        new_w1 = teacher_w1[:, index]
+        new_b1 = teacher_b1[index]
+        new_w2 = teacher_w2[index, :] / factors[:, np.newaxis]
+    else:
+        raise ValueError('Unsupported weight initializer: %s' % init)
+
+    student_w1 = np.concatenate((teacher_w1, new_w1), axis=1)
+    if init == 'random-pad':
+        student_w2 = np.concatenate((teacher_w2, new_w2), axis=0)
+    elif init == 'net2wider':
+        # add small noise to break symmetry, so that student model will have
+        # full capacity later
+        noise = np.random.normal(0, 5e-2 * new_w2.std(), size=new_w2.shape)
+        student_w2 = np.concatenate((teacher_w2, new_w2 + noise), axis=0)
+        student_w2[index, :] = new_w2
+    student_b1 = np.concatenate((teacher_b1, new_b1), axis=0)
+
+    return student_w1, student_b1, student_w2
+
+
+def deeper2net_conv2d(teacher_w):
+    '''Get initial weights for a deeper conv2d layer by net2deeper'.
+
+    # Arguments
+        teacher_w: `weight` of previous conv2d layer,
+          of shape (nb_filter, nb_channel, kh, kw)
+    '''
+    nb_filter, nb_channel, kh, kw = teacher_w.shape
+    student_w = np.zeros((nb_filter, nb_filter, kh, kw))
+    for i in xrange(nb_filter):
+        student_w[i, i, (kh - 1) / 2, (kw - 1) / 2] = 1.
+    student_b = np.zeros(nb_filter)
+    return student_w, student_b
+
+
+def copy_weights(teacher_model, student_model, layer_names):
+    '''Copy weights from teacher_model to student_model,
+     for layers with names listed in layer_names
+    '''
+    for name in layer_names:
+        weights = teacher_model.get_layer(name=name).get_weights()
+        student_model.get_layer(name=name).set_weights(weights)
+
+
+# methods to construct teacher_model and student_models
+def make_teacher_model(train_data, validation_data, nb_epoch=3):
+    '''Train a simple CNN as teacher model.
+    '''
+    model = Sequential()
+    model.add(Conv2D(64, 3, 3, input_shape=input_shape,
+                     border_mode='same', name='conv1'))
+    model.add(MaxPooling2D(name='pool1'))
+    model.add(Conv2D(64, 3, 3, border_mode='same', name='conv2'))
+    model.add(MaxPooling2D(name='pool2'))
+    model.add(Flatten(name='flatten'))
+    model.add(Dense(64, activation='relu', name='fc1'))
+    model.add(Dense(nb_class, activation='softmax', name='fc2'))
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=SGD(lr=0.01, momentum=0.9),
+                  metrics=['accuracy'])
+
+    train_x, train_y = train_data
+    history = model.fit(train_x, train_y, nb_epoch=nb_epoch,
+                        validation_data=validation_data)
+    return model, history
+
+
+def make_wider_student_model(teacher_model, train_data,
+                             validation_data, init, nb_epoch=3):
+    '''Train a wider student model based on teacher_model,
+       with either 'random-pad' (baseline) or 'net2wider'
+    '''
+    new_conv1_width = 128
+    new_fc1_width = 128
+
+    model = Sequential()
+    # a wider conv1 compared to teacher_model
+    model.add(Conv2D(new_conv1_width, 3, 3, input_shape=input_shape,
+                     border_mode='same', name='conv1'))
+    model.add(MaxPooling2D(name='pool1'))
+    model.add(Conv2D(64, 3, 3, border_mode='same', name='conv2'))
+    model.add(MaxPooling2D(name='pool2'))
+    model.add(Flatten(name='flatten'))
+    # a wider fc1 compared to teacher model
+    model.add(Dense(new_fc1_width, activation='relu', name='fc1'))
+    model.add(Dense(nb_class, activation='softmax', name='fc2'))
+
+    # The weights for other layers need to be copied from teacher_model
+    # to student_model, except for widened layers
+    # and their immediate downstreams, which will be initialized separately.
+    # For this example there are no other layers that need to be copied.
+
+    w_conv1, b_conv1 = teacher_model.get_layer('conv1').get_weights()
+    w_conv2, b_conv2 = teacher_model.get_layer('conv2').get_weights()
+    new_w_conv1, new_b_conv1, new_w_conv2 = wider2net_conv2d(
+        w_conv1, b_conv1, w_conv2, new_conv1_width, init)
+    model.get_layer('conv1').set_weights([new_w_conv1, new_b_conv1])
+    model.get_layer('conv2').set_weights([new_w_conv2, b_conv2])
+
+    w_fc1, b_fc1 = teacher_model.get_layer('fc1').get_weights()
+    w_fc2, b_fc2 = teacher_model.get_layer('fc2').get_weights()
+    new_w_fc1, new_b_fc1, new_w_fc2 = wider2net_fc(
+        w_fc1, b_fc1, w_fc2, new_fc1_width, init)
+    model.get_layer('fc1').set_weights([new_w_fc1, new_b_fc1])
+    model.get_layer('fc2').set_weights([new_w_fc2, b_fc2])
+
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=SGD(lr=0.001, momentum=0.9),
+                  metrics=['accuracy'])
+
+    train_x, train_y = train_data
+    history = model.fit(train_x, train_y, nb_epoch=nb_epoch,
+                        validation_data=validation_data)
+    return model, history
+
+
+def make_deeper_student_model(teacher_model, train_data,
+                              validation_data, init, nb_epoch=3):
+    '''Train a deeper student model based on teacher_model,
+       with either 'random-init' (baseline) or 'net2deeper'
+    '''
+    model = Sequential()
+    model.add(Conv2D(64, 3, 3, input_shape=input_shape,
+                     border_mode='same', name='conv1'))
+    model.add(MaxPooling2D(name='pool1'))
+    model.add(Conv2D(64, 3, 3, border_mode='same', name='conv2'))
+    # add another conv2d layer to make original conv2 deeper
+    if init == 'net2deeper':
+        prev_w, _ = model.get_layer('conv2').get_weights()
+        new_weights = deeper2net_conv2d(prev_w)
+        model.add(Conv2D(64, 3, 3, border_mode='same',
+                         name='conv2-deeper', weights=new_weights))
+    elif init == 'random-init':
+        model.add(Conv2D(64, 3, 3, border_mode='same', name='conv2-deeper'))
+    else:
+        raise ValueError('Unsupported weight initializer: %s' % init)
+    model.add(MaxPooling2D(name='pool2'))
+    model.add(Flatten(name='flatten'))
+    model.add(Dense(64, activation='relu', name='fc1'))
+    # add another fc layer to make original fc1 deeper
+    if init == 'net2deeper':
+        # net2deeper for fc layer with relu, is just an identity initializer
+        model.add(Dense(64, init='identity',
+                        activation='relu', name='fc1-deeper'))
+    elif init == 'random-init':
+        model.add(Dense(64, activation='relu', name='fc1-deeper'))
+    else:
+        raise ValueError('Unsupported weight initializer: %s' % init)
+    model.add(Dense(nb_class, activation='softmax', name='fc2'))
+
+    # copy weights for other layers
+    copy_weights(teacher_model, model, layer_names=[
+                 'conv1', 'conv2', 'fc1', 'fc2'])
+
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=SGD(lr=0.001, momentum=0.9),
+                  metrics=['accuracy'])
+
+    train_x, train_y = train_data
+    history = model.fit(train_x, train_y, nb_epoch=nb_epoch,
+                        validation_data=validation_data)
+    return model, history
+
+
+# experiments setup
+def net2wider_experiment():
+    '''Benchmark performances of
+    (1) a teacher model,
+    (2) a wider student model with `random_pad` initializer
+    (3) a wider student model with `Net2WiderNet` initializer
+    '''
+    train_data = (train_x, train_y)
+    validation_data = (validation_x, validation_y)
+    print('\nExperiment of Net2WiderNet ...')
+    print('\nbuilding teacher model ...')
+    teacher_model, _ = make_teacher_model(train_data,
+                                          validation_data,
+                                          nb_epoch=3)
+
+    print('\nbuilding wider student model by random padding ...')
+    make_wider_student_model(teacher_model, train_data,
+                             validation_data, 'random-pad',
+                             nb_epoch=3)
+    print('\nbuilding wider student model by net2wider ...')
+    make_wider_student_model(teacher_model, train_data,
+                             validation_data, 'net2wider',
+                             nb_epoch=3)
+
+
+def net2deeper_experiment():
+    '''Benchmark performances of
+    (1) a teacher model,
+    (2) a deeper student model with `random_init` initializer
+    (3) a deeper student model with `Net2DeeperNet` initializer
+    '''
+    train_data = (train_x, train_y)
+    validation_data = (validation_x, validation_y)
+    print('\nExperiment of Net2DeeperNet ...')
+    print('\nbuilding teacher model ...')
+    teacher_model, _ = make_teacher_model(train_data,
+                                          validation_data,
+                                          nb_epoch=3)
+
+    print('\nbuilding deeper student model by random init ...')
+    make_deeper_student_model(teacher_model, train_data,
+                              validation_data, 'random-init',
+                              nb_epoch=3)
+    print('\nbuilding deeper student model by net2deeper ...')
+    make_deeper_student_model(teacher_model, train_data,
+                              validation_data, 'net2deeper',
+                              nb_epoch=3)
+
+# run the experiments
+net2wider_experiment()
+net2deeper_experiment()

examples/mnist_swwae.py

@@ -0,0 +1,167 @@
+'''Trains a stacked what-where autoencoder built on residual blocks on the
+MNIST dataset.  It exemplifies two influential methods that have been developed
+in the past few years.
+
+The first is the idea of properly "unpooling." During any max pool, the
+exact location (the "where") of the maximal value in a pooled receptive field
+is lost, however it can be very useful in the overall reconstruction of an
+input image.  Therefore, if the "where" is handed from the encoder
+to the corresponding decoder layer, features being decoded can be "placed" in
+the right location, allowing for reconstructions of much higher fidelity.
+
+References:
+[1]
+"Visualizing and Understanding Convolutional Networks"
+Matthew D Zeiler, Rob Fergus
+https://arxiv.org/abs/1311.2901v3
+
+[2]
+"Stacked What-Where Auto-encoders"
+Junbo Zhao, Michael Mathieu, Ross Goroshin, Yann LeCun
+https://arxiv.org/abs/1506.02351v8
+
+The second idea exploited here is that of residual learning.  Residual blocks
+ease the training process by allowing skip connections that give the network
+the ability to be as linear (or non-linear) as the data sees fit.  This allows
+for much deep networks to be easily trained.  The residual element seems to
+be advantageous in the context of this example as it allows a nice symmetry
+between the encoder and decoder.  Normally, in the decoder, the final
+projection to the space where the image is reconstructed is linear, however
+this does not have to be the case for a residual block as the degree to which
+its output is linear or non-linear is determined by the data it is fed.
+However, in order to cap the reconstruction in this example, a hard softmax is
+applied as a bias because we know the MNIST digits are mapped to [0,1].
+
+References:
+[3]
+"Deep Residual Learning for Image Recognition"
+Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun
+https://arxiv.org/abs/1512.03385v1
+
+[4]
+"Identity Mappings in Deep Residual Networks"
+Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun
+https://arxiv.org/abs/1603.05027v3
+
+'''
+
+from __future__ import print_function
+import numpy as np
+np.random.seed(1337)  # for reproducibility
+
+from keras.datasets import mnist
+from keras.models import Model
+from keras.layers import Activation, merge
+from keras.layers import UpSampling2D, Convolution2D, MaxPooling2D
+from keras.layers import Input, BatchNormalization
+import matplotlib.pyplot as plt
+import keras.backend as K
+
+
+def convresblock(x, nfeats=8, ksize=3, nskipped=2):
+    ''' The proposed residual block from [4]'''
+    y0 = Convolution2D(nfeats, ksize, ksize, border_mode='same')(x)
+    y = y0
+    for i in range(nskipped):
+        y = BatchNormalization(mode=0, axis=1)(y)
+        y = Activation('relu')(y)
+        y = Convolution2D(nfeats, ksize, ksize, border_mode='same')(y)
+    return merge([y0, y], mode='sum')
+
+
+def getwhere(x):
+    ''' Calculate the "where" mask that contains switches indicating which
+    index contained the max value when MaxPool2D was applied.  Using the
+    gradient of the sum is a nice trick to keep everything high level.'''
+    y_prepool, y_postpool = x
+    return K.gradients(K.sum(y_postpool), y_prepool)
+
+# input image dimensions
+img_rows, img_cols = 28, 28
+
+# the data, shuffled and split between train and test sets
+(X_train, _), (X_test, _) = mnist.load_data()
+
+X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)
+X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)
+X_train = X_train.astype('float32')
+X_test = X_test.astype('float32')
+X_train /= 255
+X_test /= 255
+print('X_train shape:', X_train.shape)
+print(X_train.shape[0], 'train samples')
+print(X_test.shape[0], 'test samples')
+
+# The size of the kernel used for the MaxPooling2D
+pool_size = 2
+# The total number of feature maps at each layer
+nfeats = [8, 16, 32, 64, 128]
+# The sizes of the pooling kernel at each layer
+pool_sizes = np.array([1, 1, 1, 1, 1]) * pool_size
+# The convolution kernel size
+ksize = 3
+# Number of epochs to train for
+nb_epoch = 5
+# Batch size during training
+batch_size = 128
+
+if pool_size == 2:
+    # if using a 5 layer net of pool_size = 2
+    X_train = np.pad(X_train, [[0, 0], [0, 0], [2, 2], [2, 2]],
+                     mode='constant')
+    X_test = np.pad(X_test, [[0, 0], [0, 0], [2, 2], [2, 2]], mode='constant')
+    nlayers = 5
+elif pool_size == 3:
+    # if using a 3 layer net of pool_size = 3
+    X_train = X_train[:, :, :-1, :-1]
+    X_test = X_test[:, :, :-1, :-1]
+    nlayers = 3
+else:
+    import sys
+    sys.exit("Script supports pool_size of 2 and 3.")
+
+# Shape of input to train on (note that model is fully convolutional however)
+input_shape = X_train.shape[1:]
+# The final list of the size of axis=1 for all layers, including input
+nfeats_all = [input_shape[0]] + nfeats
+
+# First build the encoder, all the while keeping track of the "where" masks
+img_input = Input(shape=input_shape)
+
+# We push the "where" masks to the following list
+wheres = [None] * nlayers
+y = img_input
+for i in range(nlayers):
+    y_prepool = convresblock(y, nfeats=nfeats_all[i + 1], ksize=ksize)
+    y = MaxPooling2D(pool_size=(pool_sizes[i], pool_sizes[i]))(y_prepool)
+    wheres[i] = merge([y_prepool, y], mode=getwhere,
+                      output_shape=lambda x: x[0])
+
+# Now build the decoder, and use the stored "where" masks to place the features
+for i in range(nlayers):
+    ind = nlayers - 1 - i
+    y = UpSampling2D(size=(pool_sizes[ind], pool_sizes[ind]))(y)
+    y = merge([y, wheres[ind]], mode='mul')
+    y = convresblock(y, nfeats=nfeats_all[ind], ksize=ksize)
+
+# Use hard_simgoid to clip range of reconstruction
+y = Activation('hard_sigmoid')(y)
+
+# Define the model and it's mean square error loss, and compile it with Adam
+model = Model(img_input, y)
+model.compile('adam', 'mse')
+
+# Fit the model
+model.fit(X_train, X_train, validation_data=(X_test, X_test),
+          batch_size=batch_size, nb_epoch=nb_epoch)
+
+# Plot
+X_recon = model.predict(X_test[:25])
+X_plot = np.concatenate((X_test[:25], X_recon), axis=1)
+X_plot = X_plot.reshape((5, 10, input_shape[-2], input_shape[-1]))
+X_plot = np.vstack([np.hstack(x) for x in X_plot])
+plt.figure()
+plt.axis('off')
+plt.title('Test Samples: Originals/Reconstructions')
+plt.imshow(X_plot, interpolation='none', cmap='gray')
+plt.savefig('reconstructions.png')

examples/mnist_transfer_cnn.py

@@ -22,7 +22,7 @@ from keras.models import Sequential
 from keras.layers import Dense, Dropout, Activation, Flatten
 from keras.layers import Convolution2D, MaxPooling2D
 from keras.utils import np_utils
-
+from keras import backend as K
 
 now = datetime.datetime.now
 
@@ -35,14 +35,19 @@ img_rows, img_cols = 28, 28
 # number of convolutional filters to use
 nb_filters = 32
 # size of pooling area for max pooling
-nb_pool = 2
+pool_size = 2
 # convolution kernel size
-nb_conv = 3
+kernel_size = 3
+
+if K.image_dim_ordering() == 'th':
+    input_shape = (1, img_rows, img_cols)
+else:
+    input_shape = (img_rows, img_cols, 1)
 
 
 def train_model(model, train, test, nb_classes):
-    X_train = train[0].reshape(train[0].shape[0], 1, img_rows, img_cols)
-    X_test = test[0].reshape(test[0].shape[0], 1, img_rows, img_cols)
+    X_train = train[0].reshape((train[0].shape[0],) + input_shape)
+    X_test = test[0].reshape((test[0].shape[0],) + input_shape)
     X_train = X_train.astype('float32')
     X_test = X_test.astype('float32')
     X_train /= 255
@@ -86,13 +91,13 @@ y_test_gte5 = y_test[y_test >= 5] - 5
 
 # define two groups of layers: feature (convolutions) and classification (dense)
 feature_layers = [
-    Convolution2D(nb_filters, nb_conv, nb_conv,
+    Convolution2D(nb_filters, kernel_size, kernel_size,
                   border_mode='valid',
-                  input_shape=(1, img_rows, img_cols)),
+                  input_shape=input_shape),
     Activation('relu'),
-    Convolution2D(nb_filters, nb_conv, nb_conv),
+    Convolution2D(nb_filters, kernel_size, kernel_size),
     Activation('relu'),
-    MaxPooling2D(pool_size=(nb_pool, nb_pool)),
+    MaxPooling2D(pool_size=(pool_size, pool_size)),
     Dropout(0.25),
     Flatten(),
 ]
@@ -105,9 +110,7 @@ classification_layers = [
 ]
 
 # create complete model
-model = Sequential()
-for l in feature_layers + classification_layers:
-    model.add(l)
+model = Sequential(feature_layers + classification_layers)
 
 # train model for 5-digit classification [0..4]
 train_model(model,

examples/neural_doodle.py

@@ -0,0 +1,366 @@
+'''Neural doodle with Keras
+
+Script Usage:
+    # Arguments:
+    ```
+    --nlabels:              # of regions (colors) in mask images
+    --style-image:          image to learn style from
+    --style-mask:           semantic labels for style image
+    --target-mask:          semantic labels for target image (your doodle)
+    --content-image:        optional image to learn content from
+    --target-image-prefix:  path prefix for generated target images
+    ```
+
+    # Example 1: doodle using a style image, style mask
+    and target mask.
+    ```
+    python neural_doodle.py --nlabels 4 --style-image Monet/style.png \
+    --style-mask Monet/style_mask.png --target-mask Monet/target_mask.png \
+    --target-image-prefix generated/monet
+    ```
+
+    # Example 2: doodle using a style image, style mask,
+    target mask and an optional content image.
+    ```
+    python neural_doodle.py --nlabels 4 --style-image Renoir/style.png \
+    --style-mask Renoir/style_mask.png --target-mask Renoir/target_mask.png \
+    --content-image Renoir/creek.jpg \
+    --target-image-prefix generated/renoir
+    ```
+
+References:
+[Dmitry Ulyanov's blog on fast-neural-doodle](http://dmitryulyanov.github.io/feed-forward-neural-doodle/)
+[Torch code for fast-neural-doodle](https://github.com/DmitryUlyanov/fast-neural-doodle)
+[Torch code for online-neural-doodle](https://github.com/DmitryUlyanov/online-neural-doodle)
+[Paper Texture Networks: Feed-forward Synthesis of Textures and Stylized Images](http://arxiv.org/abs/1603.03417)
+[Discussion on parameter tuning](https://github.com/fchollet/keras/issues/3705)
+
+Resources:
+Example images can be downloaded from
+https://github.com/DmitryUlyanov/fast-neural-doodle/tree/master/data
+'''
+from __future__ import print_function
+import time
+import argparse
+import numpy as np
+from scipy.optimize import fmin_l_bfgs_b
+from scipy.misc import imread, imsave
+
+from keras import backend as K
+from keras.layers import Input, Convolution2D, MaxPooling2D, AveragePooling2D
+from keras.models import Model
+from keras.preprocessing.image import load_img, img_to_array
+from keras.applications import vgg19
+
+# Command line arguments
+parser = argparse.ArgumentParser(description='Keras neural doodle example')
+parser.add_argument('--nlabels', type=int,
+                    help='number of semantic labels'
+                    ' (regions in differnet colors)'
+                    ' in style_mask/target_mask')
+parser.add_argument('--style-image', type=str,
+                    help='path to image to learn style from')
+parser.add_argument('--style-mask', type=str,
+                    help='path to semantic mask of style image')
+parser.add_argument('--target-mask', type=str,
+                    help='path to semantic mask of target image')
+parser.add_argument('--content-image', type=str, default=None,
+                    help='path to optional content image')
+parser.add_argument('--target-image-prefix', type=str,
+                    help='path prefix for generated results')
+args = parser.parse_args()
+
+style_img_path = args.style_image
+style_mask_path = args.style_mask
+target_mask_path = args.target_mask
+content_img_path = args.content_image
+target_img_prefix = args.target_image_prefix
+use_content_img = content_img_path is not None
+
+nb_labels = args.nlabels
+nb_colors = 3  # RGB
+# determine image sizes based on target_mask
+ref_img = imread(target_mask_path)
+img_nrows, img_ncols = ref_img.shape[:2]
+
+total_variation_weight = 50.
+style_weight = 1.
+content_weight = 0.1 if use_content_img else 0
+
+content_feature_layers = ['block5_conv2']
+# To get better generation qualities, use more conv layers for style features
+style_feature_layers = ['block1_conv1', 'block2_conv1', 'block3_conv1',
+                        'block4_conv1', 'block5_conv1']
+
+
+# helper functions for reading/processing images
+def preprocess_image(image_path):
+    img = load_img(image_path, target_size=(img_nrows, img_ncols))
+    img = img_to_array(img)
+    img = np.expand_dims(img, axis=0)
+    img = vgg19.preprocess_input(img)
+    return img
+
+
+def deprocess_image(x):
+    if K.image_dim_ordering() == 'th':
+        x = x.reshape((3, img_nrows, img_ncols))
+        x = x.transpose((1, 2, 0))
+    else:
+        x = x.reshape((img_nrows, img_ncols, 3))
+    # Remove zero-center by mean pixel
+    x[:, :, 0] += 103.939
+    x[:, :, 1] += 116.779
+    x[:, :, 2] += 123.68
+    # 'BGR'->'RGB'
+    x = x[:, :, ::-1]
+    x = np.clip(x, 0, 255).astype('uint8')
+    return x
+
+
+def kmeans(xs, k):
+    assert xs.ndim == 2
+    try:
+        from sklearn.cluster import k_means
+        _, labels, _ = k_means(xs.astype("float64"), k)
+    except ImportError:
+        from scipy.cluster.vq import kmeans2
+        _, labels = kmeans2(xs, k, missing='raise')
+    return labels
+
+
+def load_mask_labels():
+    '''Load both target and style masks.
+    A mask image (nr x nc) with m labels/colors will be loaded
+    as a 4D boolean tensor: (1, m, nr, nc) for 'th' or (1, nr, nc, m) for 'tf'
+    '''
+    target_mask_img = load_img(target_mask_path,
+                               target_size=(img_nrows, img_ncols))
+    target_mask_img = img_to_array(target_mask_img)
+    style_mask_img = load_img(style_mask_path,
+                              target_size=(img_nrows, img_ncols))
+    style_mask_img = img_to_array(style_mask_img)
+    if K.image_dim_ordering() == 'th':
+        mask_vecs = np.vstack([style_mask_img.reshape((3, -1)).T,
+                               target_mask_img.reshape((3, -1)).T])
+    else:
+        mask_vecs = np.vstack([style_mask_img.reshape((-1, 3)),
+                               target_mask_img.reshape((-1, 3))])
+
+    labels = kmeans(mask_vecs, nb_labels)
+    style_mask_label = labels[:img_nrows *
+                              img_ncols].reshape((img_nrows, img_ncols))
+    target_mask_label = labels[img_nrows *
+                               img_ncols:].reshape((img_nrows, img_ncols))
+
+    stack_axis = 0 if K.image_dim_ordering() == 'th' else -1
+    style_mask = np.stack([style_mask_label == r for r in xrange(nb_labels)],
+                          axis=stack_axis)
+    target_mask = np.stack([target_mask_label == r for r in xrange(nb_labels)],
+                           axis=stack_axis)
+
+    return (np.expand_dims(style_mask, axis=0),
+            np.expand_dims(target_mask, axis=0))
+
+# Create tensor variables for images
+if K.image_dim_ordering() == 'th':
+    shape = (1, nb_colors, img_nrows, img_ncols)
+else:
+    shape = (1, img_nrows, img_ncols, nb_colors)
+
+style_image = K.variable(preprocess_image(style_img_path))
+target_image = K.placeholder(shape=shape)
+if use_content_img:
+    content_image = K.variable(preprocess_image(content_img_path))
+else:
+    content_image = K.zeros(shape=shape)
+
+images = K.concatenate([style_image, target_image, content_image], axis=0)
+
+# Create tensor variables for masks
+raw_style_mask, raw_target_mask = load_mask_labels()
+style_mask = K.variable(raw_style_mask.astype("float32"))
+target_mask = K.variable(raw_target_mask.astype("float32"))
+masks = K.concatenate([style_mask, target_mask], axis=0)
+
+# index constants for images and tasks variables
+STYLE, TARGET, CONTENT = 0, 1, 2
+
+# Build image model, mask model and use layer outputs as features
+# image model as VGG19
+image_model = vgg19.VGG19(include_top=False, input_tensor=images)
+
+# mask model as a series of pooling
+mask_input = Input(tensor=masks, shape=(None, None, None), name="mask_input")
+x = mask_input
+for layer in image_model.layers[1:]:
+    name = 'mask_%s' % layer.name
+    if 'conv' in layer.name:
+        x = AveragePooling2D((3, 3), strides=(
+            1, 1), name=name, border_mode="same")(x)
+    elif 'pool' in layer.name:
+        x = AveragePooling2D((2, 2), name=name)(x)
+mask_model = Model(mask_input, x)
+
+# Collect features from image_model and task_model
+image_features = {}
+mask_features = {}
+for img_layer, mask_layer in zip(image_model.layers, mask_model.layers):
+    if 'conv' in img_layer.name:
+        assert 'mask_' + img_layer.name == mask_layer.name
+        layer_name = img_layer.name
+        img_feat, mask_feat = img_layer.output, mask_layer.output
+        image_features[layer_name] = img_feat
+        mask_features[layer_name] = mask_feat
+
+
+# Define loss functions
+def gram_matrix(x):
+    assert K.ndim(x) == 3
+    features = K.batch_flatten(x)
+    gram = K.dot(features, K.transpose(features))
+    return gram
+
+
+def region_style_loss(style_image, target_image, style_mask, target_mask):
+    '''Calculate style loss between style_image and target_image,
+    for one common region specified by their (boolean) masks
+    '''
+    assert 3 == K.ndim(style_image) == K.ndim(target_image)
+    assert 2 == K.ndim(style_mask) == K.ndim(target_mask)
+    if K.image_dim_ordering() == 'th':
+        masked_style = style_image * style_mask
+        masked_target = target_image * target_mask
+        nb_channels = K.shape(style_image)[0]
+    else:
+        masked_style = K.permute_dimensions(
+            style_image, (2, 0, 1)) * style_mask
+        masked_target = K.permute_dimensions(
+            target_image, (2, 0, 1)) * target_mask
+        nb_channels = K.shape(style_image)[-1]
+    s = gram_matrix(masked_style) / K.mean(style_mask) / nb_channels
+    c = gram_matrix(masked_target) / K.mean(target_mask) / nb_channels
+    return K.mean(K.square(s - c))
+
+
+def style_loss(style_image, target_image, style_masks, target_masks):
+    '''Calculate style loss between style_image and target_image,
+    in all regions.
+    '''
+    assert 3 == K.ndim(style_image) == K.ndim(target_image)
+    assert 3 == K.ndim(style_masks) == K.ndim(target_masks)
+    loss = K.variable(0)
+    for i in xrange(nb_labels):
+        if K.image_dim_ordering() == 'th':
+            style_mask = style_masks[i, :, :]
+            target_mask = target_masks[i, :, :]
+        else:
+            style_mask = style_masks[:, :, i]
+            target_mask = target_masks[:, :, i]
+        loss += region_style_loss(style_image,
+                                  target_image, style_mask, target_mask)
+    return loss
+
+
+def content_loss(content_image, target_image):
+    return K.sum(K.square(target_image - content_image))
+
+
+def total_variation_loss(x):
+    assert 4 == K.ndim(x)
+    if K.image_dim_ordering() == 'th':
+        a = K.square(x[:, :, :img_nrows - 1, :img_ncols - 1] -
+                     x[:, :, 1:, :img_ncols - 1])
+        b = K.square(x[:, :, :img_nrows - 1, :img_ncols - 1] -
+                     x[:, :, :img_nrows - 1, 1:])
+    else:
+        a = K.square(x[:, :img_nrows - 1, :img_ncols - 1, :] -
+                     x[:, 1:, :img_ncols - 1, :])
+        b = K.square(x[:, :img_nrows - 1, :img_ncols - 1, :] -
+                     x[:, :img_nrows - 1, 1:, :])
+    return K.sum(K.pow(a + b, 1.25))
+
+# Overall loss is the weighted sum of content_loss, style_loss and tv_loss
+# Each individual loss uses features from image/mask models.
+loss = K.variable(0)
+for layer in content_feature_layers:
+    content_feat = image_features[layer][CONTENT, :, :, :]
+    target_feat = image_features[layer][TARGET, :, :, :]
+    loss += content_weight * content_loss(content_feat, target_feat)
+
+for layer in style_feature_layers:
+    style_feat = image_features[layer][STYLE, :, :, :]
+    target_feat = image_features[layer][TARGET, :, :, :]
+    style_masks = mask_features[layer][STYLE, :, :, :]
+    target_masks = mask_features[layer][TARGET, :, :, :]
+    sl = style_loss(style_feat, target_feat, style_masks, target_masks)
+    loss += (style_weight / len(style_feature_layers)) * sl
+
+loss += total_variation_weight * total_variation_loss(target_image)
+loss_grads = K.gradients(loss, target_image)
+
+# Evaluator class for computing efficiency
+outputs = [loss]
+if type(loss_grads) in {list, tuple}:
+    outputs += loss_grads
+else:
+    outputs.append(loss_grads)
+
+f_outputs = K.function([target_image], outputs)
+
+
+def eval_loss_and_grads(x):
+    if K.image_dim_ordering() == 'th':
+        x = x.reshape((1, 3, img_nrows, img_ncols))
+    else:
+        x = x.reshape((1, img_nrows, img_ncols, 3))
+    outs = f_outputs([x])
+    loss_value = outs[0]
+    if len(outs[1:]) == 1:
+        grad_values = outs[1].flatten().astype('float64')
+    else:
+        grad_values = np.array(outs[1:]).flatten().astype('float64')
+    return loss_value, grad_values
+
+
+class Evaluator(object):
+
+    def __init__(self):
+        self.loss_value = None
+        self.grads_values = None
+
+    def loss(self, x):
+        assert self.loss_value is None
+        loss_value, grad_values = eval_loss_and_grads(x)
+        self.loss_value = loss_value
+        self.grad_values = grad_values
+        return self.loss_value
+
+    def grads(self, x):
+        assert self.loss_value is not None
+        grad_values = np.copy(self.grad_values)
+        self.loss_value = None
+        self.grad_values = None
+        return grad_values
+
+evaluator = Evaluator()
+
+# Generate images by iterative optimization
+if K.image_dim_ordering() == 'th':
+    x = np.random.uniform(0, 255, (1, 3, img_nrows, img_ncols)) - 128.
+else:
+    x = np.random.uniform(0, 255, (1, img_nrows, img_ncols, 3)) - 128.
+
+for i in range(50):
+    print('Start of iteration', i)
+    start_time = time.time()
+    x, min_val, info = fmin_l_bfgs_b(evaluator.loss, x.flatten(),
+                                     fprime=evaluator.grads, maxfun=20)
+    print('Current loss value:', min_val)
+    # save current generated image
+    img = deprocess_image(x.copy())
+    fname = target_img_prefix + '_at_iteration_%d.png' % i
+    imsave(fname, img)
+    end_time = time.time()
+    print('Image saved as', fname)
+    print('Iteration %d completed in %ds' % (i, end_time - start_time))

examples/neural_style_transfer.py

@@ -1,10 +1,5 @@
 '''Neural style transfer with Keras.
 
-Before running this script, download the weights for the VGG16 model at:
-https://drive.google.com/file/d/0Bz7KyqmuGsilT0J5dmRCM0ROVHc/view?usp=sharing
-(source: https://gist.github.com/baraldilorenzo/07d7802847aaad0a35d3)
-and make sure the variable `weights_path` in this script matches the location of the file.
-
 Run the script with:
 ```
 python neural_style_transfer.py path_to_your_base_image.jpg path_to_your_reference.jpg prefix_for_results
@@ -15,7 +10,6 @@ python neural_style_transfer.py img/tuebingen.jpg img/starry_night.jpg results/m
 ```
 
 It is preferable to run this script on GPU, for speed.
-If running on CPU, prefer the TensorFlow backend (much faster).
 
 Example result: https://twitter.com/fchollet/status/686631033085677568
 
@@ -49,16 +43,14 @@ keeping the generated image close enough to the original one.
 '''
 
 from __future__ import print_function
-from scipy.misc import imread, imresize, imsave
+from keras.preprocessing.image import load_img, img_to_array
+from scipy.misc import imsave
 import numpy as np
 from scipy.optimize import fmin_l_bfgs_b
 import time
-import os
 import argparse
-import h5py
 
-from keras.models import Sequential
-from keras.layers import Convolution2D, ZeroPadding2D, MaxPooling2D
+from keras.applications import vgg16
 from keras import backend as K
 
 parser = argparse.ArgumentParser(description='Neural style transfer with Keras.')
@@ -73,7 +65,6 @@ args = parser.parse_args()
 base_image_path = args.base_image_path
 style_reference_image_path = args.style_reference_image_path
 result_prefix = args.result_prefix
-weights_path = 'vgg16_weights.h5'
 
 # these are the weights of the different loss components
 total_variation_weight = 1.
@@ -81,20 +72,31 @@ style_weight = 1.
 content_weight = 0.025
 
 # dimensions of the generated picture.
-img_width = 400
-img_height = 400
-assert img_height == img_width, 'Due to the use of the Gram matrix, width and height must match.'
+img_nrows = 400
+img_ncols = 400
+assert img_ncols == img_nrows, 'Due to the use of the Gram matrix, width and height must match.'
 
 # util function to open, resize and format pictures into appropriate tensors
 def preprocess_image(image_path):
-    img = imresize(imread(image_path), (img_width, img_height))
-    img = img.transpose((2, 0, 1)).astype('float64')
+    img = load_img(image_path, target_size=(img_nrows, img_ncols))
+    img = img_to_array(img)
     img = np.expand_dims(img, axis=0)
+    img = vgg16.preprocess_input(img)
     return img
 
 # util function to convert a tensor into a valid image
 def deprocess_image(x):
-    x = x.transpose((1, 2, 0))
+    if K.image_dim_ordering() == 'th':
+        x = x.reshape((3, img_nrows, img_ncols))
+        x = x.transpose((1, 2, 0))
+    else:
+        x = x.reshape((img_nrows, img_ncols, 3))
+    # Remove zero-center by mean pixel
+    x[:, :, 0] += 103.939
+    x[:, :, 1] += 116.779
+    x[:, :, 2] += 123.68
+    # 'BGR'->'RGB'
+    x = x[:, :, ::-1]
     x = np.clip(x, 0, 255).astype('uint8')
     return x
 
@@ -103,7 +105,10 @@ base_image = K.variable(preprocess_image(base_image_path))
 style_reference_image = K.variable(preprocess_image(style_reference_image_path))
 
 # this will contain our generated image
-combination_image = K.placeholder((1, 3, img_width, img_height))
+if K.image_dim_ordering() == 'th':
+    combination_image = K.placeholder((1, 3, img_nrows, img_ncols))
+else:
+    combination_image = K.placeholder((1, img_nrows, img_ncols, 3))
 
 # combine the 3 images into a single Keras tensor
 input_tensor = K.concatenate([base_image,
@@ -111,60 +116,9 @@ input_tensor = K.concatenate([base_image,
                               combination_image], axis=0)
 
 # build the VGG16 network with our 3 images as input
-first_layer = ZeroPadding2D((1, 1))
-first_layer.set_input(input_tensor, shape=(3, 3, img_width, img_height))
-
-model = Sequential()
-model.add(first_layer)
-model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(64, 3, 3, activation='relu'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(128, 3, 3, activation='relu'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-# load the weights of the VGG16 networks
-# (trained on ImageNet, won the ILSVRC competition in 2014)
-# note: when there is a complete match between your model definition
-# and your weight savefile, you can simply call model.load_weights(filename)
-assert os.path.exists(weights_path), 'Model weights not found (see "weights_path" variable in script).'
-f = h5py.File(weights_path)
-for k in range(f.attrs['nb_layers']):
-    if k >= len(model.layers):
-        # we don't look at the last (fully-connected) layers in the savefile
-        break
-    g = f['layer_{}'.format(k)]
-    weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]
-    model.layers[k].set_weights(weights)
-f.close()
+# the model will be loaded with pre-trained ImageNet weights
+model = vgg16.VGG16(input_tensor=input_tensor,
+                    weights='imagenet', include_top=False)
 print('Model loaded.')
 
 # get the symbolic outputs of each "key" layer (we gave them unique names).
@@ -176,7 +130,10 @@ outputs_dict = dict([(layer.name, layer.output) for layer in model.layers])
 # the gram matrix of an image tensor (feature-wise outer product)
 def gram_matrix(x):
     assert K.ndim(x) == 3
-    features = K.batch_flatten(x)
+    if K.image_dim_ordering() == 'th':
+        features = K.batch_flatten(x)
+    else:
+        features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1)))
     gram = K.dot(features, K.transpose(features))
     return gram
 
@@ -191,7 +148,7 @@ def style_loss(style, combination):
     S = gram_matrix(style)
     C = gram_matrix(combination)
     channels = 3
-    size = img_width * img_height
+    size = img_nrows * img_ncols
     return K.sum(K.square(S - C)) / (4. * (channels ** 2) * (size ** 2))
 
 # an auxiliary loss function
@@ -204,19 +161,25 @@ def content_loss(base, combination):
 # designed to keep the generated image locally coherent
 def total_variation_loss(x):
     assert K.ndim(x) == 4
-    a = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, 1:, :img_height-1])
-    b = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, :img_width-1, 1:])
+    if K.image_dim_ordering() == 'th':
+        a = K.square(x[:, :, :img_nrows-1, :img_ncols-1] - x[:, :, 1:, :img_ncols-1])
+        b = K.square(x[:, :, :img_nrows-1, :img_ncols-1] - x[:, :, :img_nrows-1, 1:])
+    else:
+        a = K.square(x[:, :img_nrows-1, :img_ncols-1, :] - x[:, 1:, :img_ncols-1, :])
+        b = K.square(x[:, :img_nrows-1, :img_ncols-1, :] - x[:, :img_nrows-1, 1:, :])
     return K.sum(K.pow(a + b, 1.25))
 
 # combine these loss functions into a single scalar
 loss = K.variable(0.)
-layer_features = outputs_dict['conv4_2']
+layer_features = outputs_dict['block4_conv2']
 base_image_features = layer_features[0, :, :, :]
 combination_features = layer_features[2, :, :, :]
 loss += content_weight * content_loss(base_image_features,
                                       combination_features)
 
-feature_layers = ['conv1_1', 'conv2_1', 'conv3_1', 'conv4_1', 'conv5_1']
+feature_layers = ['block1_conv1', 'block2_conv1',
+                  'block3_conv1', 'block4_conv1',
+                  'block5_conv1']
 for layer_name in feature_layers:
     layer_features = outputs_dict[layer_name]
     style_reference_features = layer_features[1, :, :, :]
@@ -235,8 +198,12 @@ else:
     outputs.append(grads)
 
 f_outputs = K.function([combination_image], outputs)
+
 def eval_loss_and_grads(x):
-    x = x.reshape((1, 3, img_width, img_height))
+    if K.image_dim_ordering() == 'th':
+        x = x.reshape((1, 3, img_nrows, img_ncols))
+    else:
+        x = x.reshape((1, img_nrows, img_ncols, 3))
     outs = f_outputs([x])
     loss_value = outs[0]
     if len(outs[1:]) == 1:
@@ -274,7 +241,11 @@ evaluator = Evaluator()
 
 # run scipy-based optimization (L-BFGS) over the pixels of the generated image
 # so as to minimize the neural style loss
-x = np.random.uniform(0, 255, (1, 3, img_width, img_height))
+if K.image_dim_ordering() == 'th':
+    x = np.random.uniform(0, 255, (1, 3, img_nrows, img_ncols)) - 128.
+else:
+    x = np.random.uniform(0, 255, (1, img_nrows, img_ncols, 3)) - 128.
+
 for i in range(10):
     print('Start of iteration', i)
     start_time = time.time()
@@ -282,7 +253,7 @@ for i in range(10):
                                      fprime=evaluator.grads, maxfun=20)
     print('Current loss value:', min_val)
     # save current generated image
-    img = deprocess_image(x.reshape((3, img_width, img_height)))
+    img = deprocess_image(x.copy())
     fname = result_prefix + '_at_iteration_%d.png' % i
     imsave(fname, img)
     end_time = time.time()

examples/pretrained_word_embeddings.py

@@ -0,0 +1,144 @@
+'''This script loads pre-trained word embeddings (GloVe embeddings)
+into a frozen Keras Embedding layer, and uses it to
+train a text classification model on the 20 Newsgroup dataset
+(classication of newsgroup messages into 20 different categories).
+
+GloVe embedding data can be found at:
+http://nlp.stanford.edu/data/glove.6B.zip
+(source page: http://nlp.stanford.edu/projects/glove/)
+
+20 Newsgroup data can be found at:
+http://www.cs.cmu.edu/afs/cs.cmu.edu/project/theo-20/www/data/news20.html
+'''
+
+from __future__ import print_function
+import os
+import numpy as np
+np.random.seed(1337)
+
+from keras.preprocessing.text import Tokenizer
+from keras.preprocessing.sequence import pad_sequences
+from keras.utils.np_utils import to_categorical
+from keras.layers import Dense, Input, Flatten
+from keras.layers import Conv1D, MaxPooling1D, Embedding
+from keras.models import Model
+import sys
+
+BASE_DIR = ''
+GLOVE_DIR = BASE_DIR + '/glove.6B/'
+TEXT_DATA_DIR = BASE_DIR + '/20_newsgroup/'
+MAX_SEQUENCE_LENGTH = 1000
+MAX_NB_WORDS = 20000
+EMBEDDING_DIM = 100
+VALIDATION_SPLIT = 0.2
+
+# first, build index mapping words in the embeddings set
+# to their embedding vector
+
+print('Indexing word vectors.')
+
+embeddings_index = {}
+f = open(os.path.join(GLOVE_DIR, 'glove.6B.100d.txt'))
+for line in f:
+    values = line.split()
+    word = values[0]
+    coefs = np.asarray(values[1:], dtype='float32')
+    embeddings_index[word] = coefs
+f.close()
+
+print('Found %s word vectors.' % len(embeddings_index))
+
+# second, prepare text samples and their labels
+print('Processing text dataset')
+
+texts = []  # list of text samples
+labels_index = {}  # dictionary mapping label name to numeric id
+labels = []  # list of label ids
+for name in sorted(os.listdir(TEXT_DATA_DIR)):
+    path = os.path.join(TEXT_DATA_DIR, name)
+    if os.path.isdir(path):
+        label_id = len(labels_index)
+        labels_index[name] = label_id
+        for fname in sorted(os.listdir(path)):
+            if fname.isdigit():
+                fpath = os.path.join(path, fname)
+                if sys.version_info < (3,):
+                    f = open(fpath)
+                else:
+                    f = open(fpath, encoding='latin-1')
+                texts.append(f.read())
+                f.close()
+                labels.append(label_id)
+
+print('Found %s texts.' % len(texts))
+
+# finally, vectorize the text samples into a 2D integer tensor
+tokenizer = Tokenizer(nb_words=MAX_NB_WORDS)
+tokenizer.fit_on_texts(texts)
+sequences = tokenizer.texts_to_sequences(texts)
+
+word_index = tokenizer.word_index
+print('Found %s unique tokens.' % len(word_index))
+
+data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)
+
+labels = to_categorical(np.asarray(labels))
+print('Shape of data tensor:', data.shape)
+print('Shape of label tensor:', labels.shape)
+
+# split the data into a training set and a validation set
+indices = np.arange(data.shape[0])
+np.random.shuffle(indices)
+data = data[indices]
+labels = labels[indices]
+nb_validation_samples = int(VALIDATION_SPLIT * data.shape[0])
+
+x_train = data[:-nb_validation_samples]
+y_train = labels[:-nb_validation_samples]
+x_val = data[-nb_validation_samples:]
+y_val = labels[-nb_validation_samples:]
+
+print('Preparing embedding matrix.')
+
+# prepare embedding matrix
+nb_words = min(MAX_NB_WORDS, len(word_index))
+embedding_matrix = np.zeros((nb_words + 1, EMBEDDING_DIM))
+for word, i in word_index.items():
+    if i > MAX_NB_WORDS:
+        continue
+    embedding_vector = embeddings_index.get(word)
+    if embedding_vector is not None:
+        # words not found in embedding index will be all-zeros.
+        embedding_matrix[i] = embedding_vector
+
+# load pre-trained word embeddings into an Embedding layer
+# note that we set trainable = False so as to keep the embeddings fixed
+embedding_layer = Embedding(nb_words + 1,
+                            EMBEDDING_DIM,
+                            weights=[embedding_matrix],
+                            input_length=MAX_SEQUENCE_LENGTH,
+                            trainable=False)
+
+print('Training model.')
+
+# train a 1D convnet with global maxpooling
+sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
+embedded_sequences = embedding_layer(sequence_input)
+x = Conv1D(128, 5, activation='relu')(embedded_sequences)
+x = MaxPooling1D(5)(x)
+x = Conv1D(128, 5, activation='relu')(x)
+x = MaxPooling1D(5)(x)
+x = Conv1D(128, 5, activation='relu')(x)
+x = MaxPooling1D(35)(x)
+x = Flatten()(x)
+x = Dense(128, activation='relu')(x)
+preds = Dense(len(labels_index), activation='softmax')(x)
+
+model = Model(sequence_input, preds)
+model.compile(loss='categorical_crossentropy',
+              optimizer='rmsprop',
+              metrics=['acc'])
+
+# happy learning!
+model.fit(x_train, y_train, validation_data=(x_val, y_val),
+          nb_epoch=2, batch_size=128)

examples/stateful_lstm.py

@@ -16,7 +16,7 @@ epochs = 25
 lahead = 1
 
 
-def gen_cosine_amp(amp=100, period=25, x0=0, xn=50000, step=1, k=0.0001):
+def gen_cosine_amp(amp=100, period=1000, x0=0, xn=50000, step=1, k=0.0001):
     """Generates an absolute cosine time series with the amplitude
     exponentially decreasing
 
@@ -31,7 +31,7 @@ def gen_cosine_amp(amp=100, period=25, x0=0, xn=50000, step=1, k=0.0001):
     cos = np.zeros(((xn - x0) * step, 1, 1))
     for i in range(len(cos)):
         idx = x0 + i * step
-        cos[i, 0, 0] = amp * np.cos(idx / (2 * np.pi * period))
+        cos[i, 0, 0] = amp * np.cos(2 * np.pi * idx / period)
         cos[i, 0, 0] = cos[i, 0, 0] * np.exp(-k * idx)
     return cos
 
@@ -54,7 +54,6 @@ model.add(LSTM(50,
                return_sequences=True,
                stateful=True))
 model.add(LSTM(50,
-               batch_input_shape=(batch_size, tsteps, 1),
                return_sequences=False,
                stateful=True))
 model.add(Dense(1))

examples/variational_autoencoder.py

@@ -4,6 +4,7 @@ Reference: "Auto-Encoding Variational Bayes" https://arxiv.org/abs/1312.6114
 '''
 import numpy as np
 import matplotlib.pyplot as plt
+from scipy.stats import norm
 
 from keras.layers import Input, Dense, Lambda
 from keras.models import Model
@@ -11,27 +12,27 @@ from keras import backend as K
 from keras import objectives
 from keras.datasets import mnist
 
-batch_size = 16
+batch_size = 100
 original_dim = 784
 latent_dim = 2
-intermediate_dim = 128
-epsilon_std = 0.01
-nb_epoch = 40
+intermediate_dim = 256
+nb_epoch = 50
+epsilon_std = 1.0
 
 x = Input(batch_shape=(batch_size, original_dim))
 h = Dense(intermediate_dim, activation='relu')(x)
 z_mean = Dense(latent_dim)(h)
-z_log_std = Dense(latent_dim)(h)
+z_log_var = Dense(latent_dim)(h)
+
 
 def sampling(args):
-    z_mean, z_log_std = args
-    epsilon = K.random_normal(shape=(batch_size, latent_dim),
-                              mean=0., std=epsilon_std)
-    return z_mean + K.exp(z_log_std) * epsilon
+    z_mean, z_log_var = args
+    epsilon = K.random_normal(shape=(batch_size, latent_dim), mean=0.,
+                              std=epsilon_std)
+    return z_mean + K.exp(z_log_var / 2) * epsilon
 
 # note that "output_shape" isn't necessary with the TensorFlow backend
-# so you could write `Lambda(sampling)([z_mean, z_log_std])`
-z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_std])
+z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_var])
 
 # we instantiate these layers separately so as to reuse them later
 decoder_h = Dense(intermediate_dim, activation='relu')
@@ -39,9 +40,10 @@ decoder_mean = Dense(original_dim, activation='sigmoid')
 h_decoded = decoder_h(z)
 x_decoded_mean = decoder_mean(h_decoded)
 
+
 def vae_loss(x, x_decoded_mean):
-    xent_loss = objectives.binary_crossentropy(x, x_decoded_mean)
-    kl_loss = - 0.5 * K.mean(1 + z_log_std - K.square(z_mean) - K.exp(z_log_std), axis=-1)
+    xent_loss = original_dim * objectives.binary_crossentropy(x, x_decoded_mean)
+    kl_loss = - 0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
     return xent_loss + kl_loss
 
 vae = Model(x, x_decoded_mean)
@@ -81,18 +83,19 @@ generator = Model(decoder_input, _x_decoded_mean)
 n = 15  # figure with 15x15 digits
 digit_size = 28
 figure = np.zeros((digit_size * n, digit_size * n))
-# we will sample n points within [-15, 15] standard deviations
-grid_x = np.linspace(-15, 15, n)
-grid_y = np.linspace(-15, 15, n)
+# linearly spaced coordinates on the unit square were transformed through the inverse CDF (ppf) of the Gaussian
+# to produce values of the latent variables z, since the prior of the latent space is Gaussian
+grid_x = norm.ppf(np.linspace(0.05, 0.95, n))
+grid_y = norm.ppf(np.linspace(0.05, 0.95, n))
 
 for i, yi in enumerate(grid_x):
     for j, xi in enumerate(grid_y):
-        z_sample = np.array([[xi, yi]]) * epsilon_std
+        z_sample = np.array([[xi, yi]])
         x_decoded = generator.predict(z_sample)
         digit = x_decoded[0].reshape(digit_size, digit_size)
         figure[i * digit_size: (i + 1) * digit_size,
                j * digit_size: (j + 1) * digit_size] = digit
 
 plt.figure(figsize=(10, 10))
-plt.imshow(figure)
+plt.imshow(figure, cmap='Greys_r')
 plt.show()

examples/variational_autoencoder_deconv.py

@@ -0,0 +1,173 @@
+'''This script demonstrates how to build a variational autoencoder
+with Keras and deconvolution layers.
+
+Reference: "Auto-Encoding Variational Bayes" https://arxiv.org/abs/1312.6114
+'''
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.stats import norm
+
+from keras.layers import Input, Dense, Lambda, Flatten, Reshape
+from keras.layers import Convolution2D, Deconvolution2D
+from keras.models import Model
+from keras import backend as K
+from keras import objectives
+from keras.datasets import mnist
+
+# input image dimensions
+img_rows, img_cols, img_chns = 28, 28, 1
+# number of convolutional filters to use
+nb_filters = 64
+# convolution kernel size
+nb_conv = 3
+
+batch_size = 100
+if K.image_dim_ordering() == 'th':
+    original_img_size = (img_chns, img_rows, img_cols)
+else:
+    original_img_size = (img_rows, img_cols, img_chns)
+latent_dim = 2
+intermediate_dim = 128
+epsilon_std = 1.0
+nb_epoch = 5
+
+x = Input(batch_shape=(batch_size,) + original_img_size)
+conv_1 = Convolution2D(img_chns, 2, 2, border_mode='same', activation='relu')(x)
+conv_2 = Convolution2D(nb_filters, 2, 2,
+                       border_mode='same', activation='relu',
+                       subsample=(2, 2))(conv_1)
+conv_3 = Convolution2D(nb_filters, nb_conv, nb_conv,
+                       border_mode='same', activation='relu',
+                       subsample=(1, 1))(conv_2)
+conv_4 = Convolution2D(nb_filters, nb_conv, nb_conv,
+                       border_mode='same', activation='relu',
+                       subsample=(1, 1))(conv_3)
+flat = Flatten()(conv_4)
+hidden = Dense(intermediate_dim, activation='relu')(flat)
+
+z_mean = Dense(latent_dim)(hidden)
+z_log_var = Dense(latent_dim)(hidden)
+
+
+def sampling(args):
+    z_mean, z_log_var = args
+    epsilon = K.random_normal(shape=(batch_size, latent_dim),
+                              mean=0., std=epsilon_std)
+    return z_mean + K.exp(z_log_var) * epsilon
+
+# note that "output_shape" isn't necessary with the TensorFlow backend
+# so you could write `Lambda(sampling)([z_mean, z_log_var])`
+z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_var])
+
+# we instantiate these layers separately so as to reuse them later
+decoder_hid = Dense(intermediate_dim, activation='relu')
+decoder_upsample = Dense(nb_filters * 14 * 14, activation='relu')
+
+if K.image_dim_ordering() == 'th':
+    output_shape = (batch_size, nb_filters, 14, 14)
+else:
+    output_shape = (batch_size, 14, 14, nb_filters)
+
+decoder_reshape = Reshape(output_shape[1:])
+decoder_deconv_1 = Deconvolution2D(nb_filters, nb_conv, nb_conv,
+                                   output_shape,
+                                   border_mode='same',
+                                   subsample=(1, 1),
+                                   activation='relu')
+decoder_deconv_2 = Deconvolution2D(nb_filters, nb_conv, nb_conv,
+                                   output_shape,
+                                   border_mode='same',
+                                   subsample=(1, 1),
+                                   activation='relu')
+if K.image_dim_ordering() == 'th':
+    output_shape = (batch_size, nb_filters, 29, 29)
+else:
+    output_shape = (batch_size, 29, 29, nb_filters)
+decoder_deconv_3_upsamp = Deconvolution2D(nb_filters, 2, 2,
+                                          output_shape,
+                                          border_mode='valid',
+                                          subsample=(2, 2),
+                                          activation='relu')
+decoder_mean_squash = Convolution2D(img_chns, 2, 2,
+                                    border_mode='valid',
+                                    activation='sigmoid')
+
+hid_decoded = decoder_hid(z)
+up_decoded = decoder_upsample(hid_decoded)
+reshape_decoded = decoder_reshape(up_decoded)
+deconv_1_decoded = decoder_deconv_1(reshape_decoded)
+deconv_2_decoded = decoder_deconv_2(deconv_1_decoded)
+x_decoded_relu = decoder_deconv_3_upsamp(deconv_2_decoded)
+x_decoded_mean_squash = decoder_mean_squash(x_decoded_relu)
+
+def vae_loss(x, x_decoded_mean):
+    # NOTE: binary_crossentropy expects a batch_size by dim
+    # for x and x_decoded_mean, so we MUST flatten these!
+    x = K.flatten(x)
+    x_decoded_mean = K.flatten(x_decoded_mean)
+    xent_loss = img_rows * img_cols * objectives.binary_crossentropy(x, x_decoded_mean)
+    kl_loss = - 0.5 * K.mean(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
+    return xent_loss + kl_loss
+
+vae = Model(x, x_decoded_mean_squash)
+vae.compile(optimizer='rmsprop', loss=vae_loss)
+vae.summary()
+
+# train the VAE on MNIST digits
+(x_train, _), (x_test, y_test) = mnist.load_data()
+
+x_train = x_train.astype('float32') / 255.
+x_train = x_train.reshape((x_train.shape[0],) + original_img_size)
+x_test = x_test.astype('float32') / 255.
+x_test = x_test.reshape((x_test.shape[0],) + original_img_size)
+
+print('x_train.shape:', x_train.shape)
+
+vae.fit(x_train, x_train,
+        shuffle=True,
+        nb_epoch=nb_epoch,
+        batch_size=batch_size,
+        validation_data=(x_test, x_test))
+
+# build a model to project inputs on the latent space
+encoder = Model(x, z_mean)
+
+# display a 2D plot of the digit classes in the latent space
+x_test_encoded = encoder.predict(x_test, batch_size=batch_size)
+plt.figure(figsize=(6, 6))
+plt.scatter(x_test_encoded[:, 0], x_test_encoded[:, 1], c=y_test)
+plt.colorbar()
+plt.show()
+
+# build a digit generator that can sample from the learned distribution
+decoder_input = Input(shape=(latent_dim,))
+_hid_decoded = decoder_hid(decoder_input)
+_up_decoded = decoder_upsample(_hid_decoded)
+_reshape_decoded = decoder_reshape(_up_decoded)
+_deconv_1_decoded = decoder_deconv_1(_reshape_decoded)
+_deconv_2_decoded = decoder_deconv_2(_deconv_1_decoded)
+_x_decoded_relu = decoder_deconv_3_upsamp(_deconv_2_decoded)
+_x_decoded_mean_squash = decoder_mean_squash(_x_decoded_relu)
+generator = Model(decoder_input, _x_decoded_mean_squash)
+
+# display a 2D manifold of the digits
+n = 15  # figure with 15x15 digits
+digit_size = 28
+figure = np.zeros((digit_size * n, digit_size * n))
+# linearly spaced coordinates on the unit square were transformed through the inverse CDF (ppf) of the Gaussian
+# to produce values of the latent variables z, since the prior of the latent space is Gaussian
+grid_x = norm.ppf(np.linspace(0.05, 0.95, n))
+grid_y = norm.ppf(np.linspace(0.05, 0.95, n))
+
+for i, yi in enumerate(grid_x):
+    for j, xi in enumerate(grid_y):
+        z_sample = np.array([[xi, yi]])
+        z_sample = np.tile(z_sample, batch_size).reshape(batch_size, 2)
+        x_decoded = generator.predict(z_sample, batch_size=batch_size)
+        digit = x_decoded[0].reshape(digit_size, digit_size)
+        figure[i * digit_size: (i + 1) * digit_size,
+               j * digit_size: (j + 1) * digit_size] = digit
+
+plt.figure(figsize=(10, 10))
+plt.imshow(figure, cmap='Greys_r')
+plt.show()

keras/__init__.py

@@ -15,4 +15,4 @@ from . import objectives
 from . import optimizers
 from . import regularizers
 
-__version__ = '1.0.6'
+__version__ = '1.1.2'

keras/activations.py

@@ -1,5 +1,6 @@
 from __future__ import absolute_import
 from . import backend as K
+from .utils.generic_utils import get_from_module
 
 
 def softmax(x):
@@ -11,8 +12,13 @@ def softmax(x):
         s = K.sum(e, axis=-1, keepdims=True)
         return e / s
     else:
-        raise Exception('Cannot apply softmax to a tensor that is not 2D or 3D. ' +
-                        'Here, ndim=' + str(ndim))
+        raise ValueError('Cannot apply softmax to a tensor '
+                         'that is not 2D or 3D. '
+                         'Here, ndim=' + str(ndim))
+
+
+def elu(x, alpha=1.0):
+    return K.elu(x, alpha)
 
 
 def softplus(x):
@@ -40,13 +46,9 @@ def hard_sigmoid(x):
 
 
 def linear(x):
-    '''
-    The function returns the variable that is passed in, so all types work.
-    '''
     return x
 
 
-from .utils.generic_utils import get_from_module
 def get(identifier):
     if identifier is None:
         return linear

keras/applications/__init__.py

@@ -0,0 +1,5 @@
+from .vgg16 import VGG16
+from .vgg19 import VGG19
+from .resnet50 import ResNet50
+from .inception_v3 import InceptionV3
+from .xception import Xception

keras/applications/audio_conv_utils.py

@@ -0,0 +1,86 @@
+import numpy as np
+from .. import backend as K
+
+
+TAGS = ['rock', 'pop', 'alternative', 'indie', 'electronic',
+        'female vocalists', 'dance', '00s', 'alternative rock', 'jazz',
+        'beautiful', 'metal', 'chillout', 'male vocalists',
+        'classic rock', 'soul', 'indie rock', 'Mellow', 'electronica',
+        '80s', 'folk', '90s', 'chill', 'instrumental', 'punk',
+        'oldies', 'blues', 'hard rock', 'ambient', 'acoustic',
+        'experimental', 'female vocalist', 'guitar', 'Hip-Hop',
+        '70s', 'party', 'country', 'easy listening',
+        'sexy', 'catchy', 'funk', 'electro', 'heavy metal',
+        'Progressive rock', '60s', 'rnb', 'indie pop',
+        'sad', 'House', 'happy']
+
+
+def librosa_exists():
+    try:
+        __import__('librosa')
+    except ImportError:
+        return False
+    else:
+        return True
+
+
+def preprocess_input(audio_path, dim_ordering='default'):
+    '''Reads an audio file and outputs a Mel-spectrogram.
+    '''
+    if dim_ordering == 'default':
+        dim_ordering = K.image_dim_ordering()
+    assert dim_ordering in {'tf', 'th'}
+
+    if librosa_exists():
+        import librosa
+    else:
+        raise RuntimeError('Librosa is required to process audio files.\n' +
+                           'Install it via `pip install librosa` \nor visit ' +
+                           'http://librosa.github.io/librosa/ for details.')
+
+    # mel-spectrogram parameters
+    SR = 12000
+    N_FFT = 512
+    N_MELS = 96
+    HOP_LEN = 256
+    DURA = 29.12
+
+    src, sr = librosa.load(audio_path, sr=SR)
+    n_sample = src.shape[0]
+    n_sample_wanted = int(DURA * SR)
+
+    # trim the signal at the center
+    if n_sample < n_sample_wanted:  # if too short
+        src = np.hstack((src, np.zeros((int(DURA * SR) - n_sample,))))
+    elif n_sample > n_sample_wanted:  # if too long
+        src = src[(n_sample - n_sample_wanted) / 2:
+                  (n_sample + n_sample_wanted) / 2]
+
+    logam = librosa.logamplitude
+    melgram = librosa.feature.melspectrogram
+    x = logam(melgram(y=src, sr=SR, hop_length=HOP_LEN,
+                      n_fft=N_FFT, n_mels=N_MELS) ** 2,
+              ref_power=1.0)
+
+    if dim_ordering == 'th':
+        x = np.expand_dims(x, axis=0)
+    elif dim_ordering == 'tf':
+        x = np.expand_dims(x, axis=3)
+    return x
+
+
+def decode_predictions(preds, top_n=5):
+    '''Decode the output of a music tagger model.
+
+    # Arguments
+        preds: 2-dimensional numpy array
+        top_n: integer in [0, 50], number of items to show
+
+    '''
+    assert len(preds.shape) == 2 and preds.shape[1] == 50
+    results = []
+    for pred in preds:
+        result = zip(TAGS, pred)
+        result = sorted(result, key=lambda x: x[1], reverse=True)
+        results.append(result[:top_n])
+    return results

keras/applications/imagenet_utils.py

@@ -0,0 +1,51 @@
+import numpy as np
+import json
+
+from ..utils.data_utils import get_file
+from .. import backend as K
+
+CLASS_INDEX = None
+CLASS_INDEX_PATH = 'https://s3.amazonaws.com/deep-learning-models/image-models/imagenet_class_index.json'
+
+
+def preprocess_input(x, dim_ordering='default'):
+    if dim_ordering == 'default':
+        dim_ordering = K.image_dim_ordering()
+    assert dim_ordering in {'tf', 'th'}
+
+    if dim_ordering == 'th':
+        # 'RGB'->'BGR'
+        x = x[:, ::-1, :, :]
+        # Zero-center by mean pixel
+        x[:, 0, :, :] -= 103.939
+        x[:, 1, :, :] -= 116.779
+        x[:, 2, :, :] -= 123.68
+    else:
+        # 'RGB'->'BGR'
+        x = x[:, :, :, ::-1]
+        # Zero-center by mean pixel
+        x[:, :, :, 0] -= 103.939
+        x[:, :, :, 1] -= 116.779
+        x[:, :, :, 2] -= 123.68
+    return x
+
+
+def decode_predictions(preds, top=5):
+    global CLASS_INDEX
+    if len(preds.shape) != 2 or preds.shape[1] != 1000:
+        raise ValueError('`decode_predictions` expects '
+                         'a batch of predictions '
+                         '(i.e. a 2D array of shape (samples, 1000)). '
+                         'Found array with shape: ' + str(preds.shape))
+    if CLASS_INDEX is None:
+        fpath = get_file('imagenet_class_index.json',
+                         CLASS_INDEX_PATH,
+                         cache_subdir='models')
+        CLASS_INDEX = json.load(open(fpath))
+    results = []
+    for pred in preds:
+        top_indices = pred.argsort()[-top:][::-1]
+        result = [tuple(CLASS_INDEX[str(i)]) + (pred[i],) for i in top_indices]
+        result.sort(key=lambda x: x[2], reverse=True)
+        results.append(result)
+    return results

keras/applications/inception_v3.py

@@ -0,0 +1,312 @@
+# -*- coding: utf-8 -*-
+'''Inception V3 model for Keras.
+
+Note that the ImageNet weights provided are from a model that had not fully converged.
+Inception v3 should be able to reach 6.9% top-5 error, but our model
+only gets to 7.8% (same as a fully-converged ResNet 50).
+For comparison, VGG16 only gets to 9.9%, quite a bit worse.
+
+Also, do note that the input image format for this model is different than for
+the VGG16 and ResNet models (299x299 instead of 224x224), and that the input preprocessing function
+is also different (same as Xception).
+
+# Reference:
+
+- [Rethinking the Inception Architecture for Computer Vision](http://arxiv.org/abs/1512.00567)
+
+'''
+from __future__ import print_function
+from __future__ import absolute_import
+
+import warnings
+
+from ..models import Model
+from ..layers import Flatten, Dense, Input, BatchNormalization, merge
+from ..layers import Convolution2D, MaxPooling2D, AveragePooling2D
+from ..utils.layer_utils import convert_all_kernels_in_model
+from ..utils.data_utils import get_file
+from .. import backend as K
+from .imagenet_utils import decode_predictions
+
+
+TH_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.2/inception_v3_weights_th_dim_ordering_th_kernels.h5'
+TF_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.2/inception_v3_weights_tf_dim_ordering_tf_kernels.h5'
+TH_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.2/inception_v3_weights_th_dim_ordering_th_kernels_notop.h5'
+TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.2/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5'
+
+
+def conv2d_bn(x, nb_filter, nb_row, nb_col,
+              border_mode='same', subsample=(1, 1),
+              name=None):
+    '''Utility function to apply conv + BN.
+    '''
+    if name is not None:
+        bn_name = name + '_bn'
+        conv_name = name + '_conv'
+    else:
+        bn_name = None
+        conv_name = None
+    if K.image_dim_ordering() == 'th':
+        bn_axis = 1
+    else:
+        bn_axis = 3
+    x = Convolution2D(nb_filter, nb_row, nb_col,
+                      subsample=subsample,
+                      activation='relu',
+                      border_mode=border_mode,
+                      name=conv_name)(x)
+    x = BatchNormalization(axis=bn_axis, name=bn_name)(x)
+    return x
+
+
+def InceptionV3(include_top=True, weights='imagenet',
+                input_tensor=None):
+    '''Instantiate the Inception v3 architecture,
+    optionally loading weights pre-trained
+    on ImageNet. Note that when using TensorFlow,
+    for best performance you should set
+    `image_dim_ordering="tf"` in your Keras config
+    at ~/.keras/keras.json.
+
+    The model and the weights are compatible with both
+    TensorFlow and Theano. The dimension ordering
+    convention used by the model is the one
+    specified in your Keras config file.
+
+    Note that the default input image size for this model is 299x299.
+
+    # Arguments
+        include_top: whether to include the fully-connected
+            layer at the top of the network.
+        weights: one of `None` (random initialization)
+            or "imagenet" (pre-training on ImageNet).
+        input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
+            to use as image input for the model.
+
+    # Returns
+        A Keras model instance.
+    '''
+    if weights not in {'imagenet', None}:
+        raise ValueError('The `weights` argument should be either '
+                         '`None` (random initialization) or `imagenet` '
+                         '(pre-training on ImageNet).')
+    # Determine proper input shape
+    if K.image_dim_ordering() == 'th':
+        if include_top:
+            input_shape = (3, 299, 299)
+        else:
+            input_shape = (3, None, None)
+    else:
+        if include_top:
+            input_shape = (299, 299, 3)
+        else:
+            input_shape = (None, None, 3)
+
+    if input_tensor is None:
+        img_input = Input(shape=input_shape)
+    else:
+        if not K.is_keras_tensor(input_tensor):
+            img_input = Input(tensor=input_tensor, shape=input_shape)
+        else:
+            img_input = input_tensor
+
+    if K.image_dim_ordering() == 'th':
+        channel_axis = 1
+    else:
+        channel_axis = 3
+
+    x = conv2d_bn(img_input, 32, 3, 3, subsample=(2, 2), border_mode='valid')
+    x = conv2d_bn(x, 32, 3, 3, border_mode='valid')
+    x = conv2d_bn(x, 64, 3, 3)
+    x = MaxPooling2D((3, 3), strides=(2, 2))(x)
+
+    x = conv2d_bn(x, 80, 1, 1, border_mode='valid')
+    x = conv2d_bn(x, 192, 3, 3, border_mode='valid')
+    x = MaxPooling2D((3, 3), strides=(2, 2))(x)
+
+    # mixed 0, 1, 2: 35 x 35 x 256
+    for i in range(3):
+        branch1x1 = conv2d_bn(x, 64, 1, 1)
+
+        branch5x5 = conv2d_bn(x, 48, 1, 1)
+        branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
+
+        branch3x3dbl = conv2d_bn(x, 64, 1, 1)
+        branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
+        branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
+
+        branch_pool = AveragePooling2D(
+            (3, 3), strides=(1, 1), border_mode='same')(x)
+        branch_pool = conv2d_bn(branch_pool, 32, 1, 1)
+        x = merge([branch1x1, branch5x5, branch3x3dbl, branch_pool],
+                  mode='concat', concat_axis=channel_axis,
+                  name='mixed' + str(i))
+
+    # mixed 3: 17 x 17 x 768
+    branch3x3 = conv2d_bn(x, 384, 3, 3, subsample=(2, 2), border_mode='valid')
+
+    branch3x3dbl = conv2d_bn(x, 64, 1, 1)
+    branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
+    branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3,
+                             subsample=(2, 2), border_mode='valid')
+
+    branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x)
+    x = merge([branch3x3, branch3x3dbl, branch_pool],
+              mode='concat', concat_axis=channel_axis,
+              name='mixed3')
+
+    # mixed 4: 17 x 17 x 768
+    branch1x1 = conv2d_bn(x, 192, 1, 1)
+
+    branch7x7 = conv2d_bn(x, 128, 1, 1)
+    branch7x7 = conv2d_bn(branch7x7, 128, 1, 7)
+    branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
+
+    branch7x7dbl = conv2d_bn(x, 128, 1, 1)
+    branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
+    branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 1, 7)
+    branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
+    branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
+
+    branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same')(x)
+    branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
+    x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool],
+              mode='concat', concat_axis=channel_axis,
+              name='mixed4')
+
+    # mixed 5, 6: 17 x 17 x 768
+    for i in range(2):
+        branch1x1 = conv2d_bn(x, 192, 1, 1)
+
+        branch7x7 = conv2d_bn(x, 160, 1, 1)
+        branch7x7 = conv2d_bn(branch7x7, 160, 1, 7)
+        branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
+
+        branch7x7dbl = conv2d_bn(x, 160, 1, 1)
+        branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
+        branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 1, 7)
+        branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
+        branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
+
+        branch_pool = AveragePooling2D(
+            (3, 3), strides=(1, 1), border_mode='same')(x)
+        branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
+        x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool],
+                  mode='concat', concat_axis=channel_axis,
+                  name='mixed' + str(5 + i))
+
+    # mixed 7: 17 x 17 x 768
+    branch1x1 = conv2d_bn(x, 192, 1, 1)
+
+    branch7x7 = conv2d_bn(x, 192, 1, 1)
+    branch7x7 = conv2d_bn(branch7x7, 192, 1, 7)
+    branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
+
+    branch7x7dbl = conv2d_bn(x, 160, 1, 1)
+    branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
+    branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
+    branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
+    branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
+
+    branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same')(x)
+    branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
+    x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool],
+              mode='concat', concat_axis=channel_axis,
+              name='mixed7')
+
+    # mixed 8: 8 x 8 x 1280
+    branch3x3 = conv2d_bn(x, 192, 1, 1)
+    branch3x3 = conv2d_bn(branch3x3, 320, 3, 3,
+                          subsample=(2, 2), border_mode='valid')
+
+    branch7x7x3 = conv2d_bn(x, 192, 1, 1)
+    branch7x7x3 = conv2d_bn(branch7x7x3, 192, 1, 7)
+    branch7x7x3 = conv2d_bn(branch7x7x3, 192, 7, 1)
+    branch7x7x3 = conv2d_bn(branch7x7x3, 192, 3, 3,
+                            subsample=(2, 2), border_mode='valid')
+
+    branch_pool = AveragePooling2D((3, 3), strides=(2, 2))(x)
+    x = merge([branch3x3, branch7x7x3, branch_pool],
+              mode='concat', concat_axis=channel_axis,
+              name='mixed8')
+
+    # mixed 9: 8 x 8 x 2048
+    for i in range(2):
+        branch1x1 = conv2d_bn(x, 320, 1, 1)
+
+        branch3x3 = conv2d_bn(x, 384, 1, 1)
+        branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3)
+        branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1)
+        branch3x3 = merge([branch3x3_1, branch3x3_2],
+                          mode='concat', concat_axis=channel_axis,
+                          name='mixed9_' + str(i))
+
+        branch3x3dbl = conv2d_bn(x, 448, 1, 1)
+        branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3)
+        branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3)
+        branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1)
+        branch3x3dbl = merge([branch3x3dbl_1, branch3x3dbl_2],
+                             mode='concat', concat_axis=channel_axis)
+
+        branch_pool = AveragePooling2D(
+            (3, 3), strides=(1, 1), border_mode='same')(x)
+        branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
+        x = merge([branch1x1, branch3x3, branch3x3dbl, branch_pool],
+                  mode='concat', concat_axis=channel_axis,
+                  name='mixed' + str(9 + i))
+
+    if include_top:
+        # Classification block
+        x = AveragePooling2D((8, 8), strides=(8, 8), name='avg_pool')(x)
+        x = Flatten(name='flatten')(x)
+        x = Dense(1000, activation='softmax', name='predictions')(x)
+
+    # Create model
+    model = Model(img_input, x)
+
+    # load weights
+    if weights == 'imagenet':
+        if K.image_dim_ordering() == 'th':
+            if include_top:
+                weights_path = get_file('inception_v3_weights_th_dim_ordering_th_kernels.h5',
+                                        TH_WEIGHTS_PATH,
+                                        cache_subdir='models',
+                                        md5_hash='b3baf3070cc4bf476d43a2ea61b0ca5f')
+            else:
+                weights_path = get_file('inception_v3_weights_th_dim_ordering_th_kernels_notop.h5',
+                                        TH_WEIGHTS_PATH_NO_TOP,
+                                        cache_subdir='models',
+                                        md5_hash='79aaa90ab4372b4593ba3df64e142f05')
+            model.load_weights(weights_path)
+            if K.backend() == 'tensorflow':
+                warnings.warn('You are using the TensorFlow backend, yet you '
+                              'are using the Theano '
+                              'image dimension ordering convention '
+                              '(`image_dim_ordering="th"`). '
+                              'For best performance, set '
+                              '`image_dim_ordering="tf"` in '
+                              'your Keras config '
+                              'at ~/.keras/keras.json.')
+                convert_all_kernels_in_model(model)
+        else:
+            if include_top:
+                weights_path = get_file('inception_v3_weights_tf_dim_ordering_tf_kernels.h5',
+                                        TF_WEIGHTS_PATH,
+                                        cache_subdir='models',
+                                        md5_hash='fe114b3ff2ea4bf891e9353d1bbfb32f')
+            else:
+                weights_path = get_file('inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5',
+                                        TF_WEIGHTS_PATH_NO_TOP,
+                                        cache_subdir='models',
+                                        md5_hash='2f3609166de1d967d1a481094754f691')
+            model.load_weights(weights_path)
+            if K.backend() == 'theano':
+                convert_all_kernels_in_model(model)
+    return model
+
+
+def preprocess_input(x):
+    x /= 255.
+    x -= 0.5
+    x *= 2.
+    return x

keras/applications/music_tagger_crnn.py

@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*-
+'''MusicTaggerCRNN model for Keras.
+
+# Reference:
+
+- [Music-auto_tagging-keras](https://github.com/keunwoochoi/music-auto_tagging-keras)
+
+'''
+from __future__ import print_function
+from __future__ import absolute_import
+
+from .. import backend as K
+from ..layers import Input, Dense
+from ..models import Model
+from ..layers import Dense, Dropout, Reshape, Permute
+from ..layers.convolutional import Convolution2D
+from ..layers.convolutional import MaxPooling2D, ZeroPadding2D
+from ..layers.normalization import BatchNormalization
+from ..layers.advanced_activations import ELU
+from ..layers.recurrent import GRU
+from ..utils.data_utils import get_file
+from ..utils.layer_utils import convert_all_kernels_in_model
+from .audio_conv_utils import decode_predictions, preprocess_input
+
+TH_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.3/music_tagger_crnn_weights_tf_kernels_th_dim_ordering.h5'
+TF_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.3/music_tagger_crnn_weights_tf_kernels_tf_dim_ordering.h5'
+
+
+def MusicTaggerCRNN(weights='msd', input_tensor=None,
+                    include_top=True):
+    '''Instantiate the MusicTaggerCRNN architecture,
+    optionally loading weights pre-trained
+    on Million Song Dataset. Note that when using TensorFlow,
+    for best performance you should set
+    `image_dim_ordering="tf"` in your Keras config
+    at ~/.keras/keras.json.
+
+    The model and the weights are compatible with both
+    TensorFlow and Theano. The dimension ordering
+    convention used by the model is the one
+    specified in your Keras config file.
+
+    For preparing mel-spectrogram input, see
+    `audio_conv_utils.py` in [applications](https://github.com/fchollet/keras/tree/master/keras/applications).
+    You will need to install [Librosa](http://librosa.github.io/librosa/)
+    to use it.
+
+    # Arguments
+        weights: one of `None` (random initialization)
+            or "msd" (pre-training on ImageNet).
+        input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
+            to use as image input for the model.
+        include_top: whether to include the 1 fully-connected
+            layer (output layer) at the top of the network.
+            If False, the network outputs 32-dim features.
+
+
+    # Returns
+        A Keras model instance.
+    '''
+    if weights not in {'msd', None}:
+        raise ValueError('The `weights` argument should be either '
+                         '`None` (random initialization) or `msd` '
+                         '(pre-training on Million Song Dataset).')
+
+    # Determine proper input shape
+    if K.image_dim_ordering() == 'th':
+        input_shape = (1, 96, 1366)
+    else:
+        input_shape = (96, 1366, 1)
+
+    if input_tensor is None:
+        melgram_input = Input(shape=input_shape)
+    else:
+        if not K.is_keras_tensor(input_tensor):
+            melgram_input = Input(tensor=input_tensor, shape=input_shape)
+        else:
+            melgram_input = input_tensor
+
+    # Determine input axis
+    if K.image_dim_ordering() == 'th':
+        channel_axis = 1
+        freq_axis = 2
+        time_axis = 3
+    else:
+        channel_axis = 3
+        freq_axis = 1
+        time_axis = 2
+
+    # Input block
+    x = ZeroPadding2D(padding=(0, 37))(melgram_input)
+    x = BatchNormalization(axis=time_axis, name='bn_0_freq')(x)
+
+    # Conv block 1
+    x = Convolution2D(64, 3, 3, border_mode='same', name='conv1')(x)
+    x = BatchNormalization(axis=channel_axis, mode=0, name='bn1')(x)
+    x = ELU()(x)
+    x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool1')(x)
+
+    # Conv block 2
+    x = Convolution2D(128, 3, 3, border_mode='same', name='conv2')(x)
+    x = BatchNormalization(axis=channel_axis, mode=0, name='bn2')(x)
+    x = ELU()(x)
+    x = MaxPooling2D(pool_size=(3, 3), strides=(3, 3), name='pool2')(x)
+
+    # Conv block 3
+    x = Convolution2D(128, 3, 3, border_mode='same', name='conv3')(x)
+    x = BatchNormalization(axis=channel_axis, mode=0, name='bn3')(x)
+    x = ELU()(x)
+    x = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool3')(x)
+
+    # Conv block 4
+    x = Convolution2D(128, 3, 3, border_mode='same', name='conv4')(x)
+    x = BatchNormalization(axis=channel_axis, mode=0, name='bn4')(x)
+    x = ELU()(x)
+    x = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool4')(x)
+
+    # reshaping
+    if K.image_dim_ordering() == 'th':
+        x = Permute((3, 1, 2))(x)
+    x = Reshape((15, 128))(x)
+
+    # GRU block 1, 2, output
+    x = GRU(32, return_sequences=True, name='gru1')(x)
+    x = GRU(32, return_sequences=False, name='gru2')(x)
+
+    if include_top:
+        x = Dense(50, activation='sigmoid', name='output')(x)
+
+    # Create model
+    model = Model(melgram_input, x)
+    if weights is None:
+        return model
+    else:
+        # Load weights
+        if K.image_dim_ordering() == 'tf':
+            weights_path = get_file('music_tagger_crnn_weights_tf_kernels_tf_dim_ordering.h5',
+                                    TF_WEIGHTS_PATH,
+                                    cache_subdir='models')
+        else:
+            weights_path = get_file('music_tagger_crnn_weights_tf_kernels_th_dim_ordering.h5',
+                                    TH_WEIGHTS_PATH,
+                                    cache_subdir='models')
+        model.load_weights(weights_path, by_name=True)
+        if K.backend() == 'theano':
+            convert_all_kernels_in_model(model)
+        return model

keras/applications/resnet50.py

@@ -0,0 +1,235 @@
+# -*- coding: utf-8 -*-
+'''ResNet50 model for Keras.
+
+# Reference:
+
+- [Deep Residual Learning for Image Recognition](https://arxiv.org/abs/1512.03385)
+
+Adapted from code contributed by BigMoyan.
+'''
+from __future__ import print_function
+from __future__ import absolute_import
+
+import warnings
+
+from ..layers import merge, Input
+from ..layers import Dense, Activation, Flatten
+from ..layers import Convolution2D, MaxPooling2D, ZeroPadding2D, AveragePooling2D
+from ..layers import BatchNormalization
+from ..models import Model
+from .. import backend as K
+from ..utils.layer_utils import convert_all_kernels_in_model
+from ..utils.data_utils import get_file
+from .imagenet_utils import decode_predictions, preprocess_input
+
+
+TH_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.2/resnet50_weights_th_dim_ordering_th_kernels.h5'
+TF_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.2/resnet50_weights_tf_dim_ordering_tf_kernels.h5'
+TH_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.2/resnet50_weights_th_dim_ordering_th_kernels_notop.h5'
+TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.2/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5'
+
+
+def identity_block(input_tensor, kernel_size, filters, stage, block):
+    '''The identity_block is the block that has no conv layer at shortcut
+
+    # Arguments
+        input_tensor: input tensor
+        kernel_size: defualt 3, the kernel size of middle conv layer at main path
+        filters: list of integers, the nb_filters of 3 conv layer at main path
+        stage: integer, current stage label, used for generating layer names
+        block: 'a','b'..., current block label, used for generating layer names
+    '''
+    nb_filter1, nb_filter2, nb_filter3 = filters
+    if K.image_dim_ordering() == 'tf':
+        bn_axis = 3
+    else:
+        bn_axis = 1
+    conv_name_base = 'res' + str(stage) + block + '_branch'
+    bn_name_base = 'bn' + str(stage) + block + '_branch'
+
+    x = Convolution2D(nb_filter1, 1, 1, name=conv_name_base + '2a')(input_tensor)
+    x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
+    x = Activation('relu')(x)
+
+    x = Convolution2D(nb_filter2, kernel_size, kernel_size,
+                      border_mode='same', name=conv_name_base + '2b')(x)
+    x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)
+    x = Activation('relu')(x)
+
+    x = Convolution2D(nb_filter3, 1, 1, name=conv_name_base + '2c')(x)
+    x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)
+
+    x = merge([x, input_tensor], mode='sum')
+    x = Activation('relu')(x)
+    return x
+
+
+def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)):
+    '''conv_block is the block that has a conv layer at shortcut
+
+    # Arguments
+        input_tensor: input tensor
+        kernel_size: defualt 3, the kernel size of middle conv layer at main path
+        filters: list of integers, the nb_filters of 3 conv layer at main path
+        stage: integer, current stage label, used for generating layer names
+        block: 'a','b'..., current block label, used for generating layer names
+
+    Note that from stage 3, the first conv layer at main path is with subsample=(2,2)
+    And the shortcut should have subsample=(2,2) as well
+    '''
+    nb_filter1, nb_filter2, nb_filter3 = filters
+    if K.image_dim_ordering() == 'tf':
+        bn_axis = 3
+    else:
+        bn_axis = 1
+    conv_name_base = 'res' + str(stage) + block + '_branch'
+    bn_name_base = 'bn' + str(stage) + block + '_branch'
+
+    x = Convolution2D(nb_filter1, 1, 1, subsample=strides,
+                      name=conv_name_base + '2a')(input_tensor)
+    x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
+    x = Activation('relu')(x)
+
+    x = Convolution2D(nb_filter2, kernel_size, kernel_size, border_mode='same',
+                      name=conv_name_base + '2b')(x)
+    x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)
+    x = Activation('relu')(x)
+
+    x = Convolution2D(nb_filter3, 1, 1, name=conv_name_base + '2c')(x)
+    x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)
+
+    shortcut = Convolution2D(nb_filter3, 1, 1, subsample=strides,
+                             name=conv_name_base + '1')(input_tensor)
+    shortcut = BatchNormalization(axis=bn_axis, name=bn_name_base + '1')(shortcut)
+
+    x = merge([x, shortcut], mode='sum')
+    x = Activation('relu')(x)
+    return x
+
+
+def ResNet50(include_top=True, weights='imagenet',
+             input_tensor=None):
+    '''Instantiate the ResNet50 architecture,
+    optionally loading weights pre-trained
+    on ImageNet. Note that when using TensorFlow,
+    for best performance you should set
+    `image_dim_ordering="tf"` in your Keras config
+    at ~/.keras/keras.json.
+
+    The model and the weights are compatible with both
+    TensorFlow and Theano. The dimension ordering
+    convention used by the model is the one
+    specified in your Keras config file.
+
+    # Arguments
+        include_top: whether to include the 3 fully-connected
+            layers at the top of the network.
+        weights: one of `None` (random initialization)
+            or "imagenet" (pre-training on ImageNet).
+        input_tensor: optional Keras tensor (i.e. xput of `layers.Input()`)
+            to use as image input for the model.
+
+    # Returns
+        A Keras model instance.
+    '''
+    if weights not in {'imagenet', None}:
+        raise ValueError('The `weights` argument should be either '
+                         '`None` (random initialization) or `imagenet` '
+                         '(pre-training on ImageNet).')
+    # Determine proper input shape
+    if K.image_dim_ordering() == 'th':
+        if include_top:
+            input_shape = (3, 224, 224)
+        else:
+            input_shape = (3, None, None)
+    else:
+        if include_top:
+            input_shape = (224, 224, 3)
+        else:
+            input_shape = (None, None, 3)
+
+    if input_tensor is None:
+        img_input = Input(shape=input_shape)
+    else:
+        if not K.is_keras_tensor(input_tensor):
+            img_input = Input(tensor=input_tensor, shape=input_shape)
+        else:
+            img_input = input_tensor
+    if K.image_dim_ordering() == 'tf':
+        bn_axis = 3
+    else:
+        bn_axis = 1
+
+    x = ZeroPadding2D((3, 3))(img_input)
+    x = Convolution2D(64, 7, 7, subsample=(2, 2), name='conv1')(x)
+    x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
+    x = Activation('relu')(x)
+    x = MaxPooling2D((3, 3), strides=(2, 2))(x)
+
+    x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
+    x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
+    x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
+
+    x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
+    x = identity_block(x, 3, [128, 128, 512], stage=3, block='b')
+    x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')
+    x = identity_block(x, 3, [128, 128, 512], stage=3, block='d')
+
+    x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
+    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b')
+    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c')
+    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d')
+    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e')
+    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f')
+
+    x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
+    x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')
+    x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')
+
+    x = AveragePooling2D((7, 7), name='avg_pool')(x)
+
+    if include_top:
+        x = Flatten()(x)
+        x = Dense(1000, activation='softmax', name='fc1000')(x)
+
+    model = Model(img_input, x)
+
+    # load weights
+    if weights == 'imagenet':
+        if K.image_dim_ordering() == 'th':
+            if include_top:
+                weights_path = get_file('resnet50_weights_th_dim_ordering_th_kernels.h5',
+                                        TH_WEIGHTS_PATH,
+                                        cache_subdir='models',
+                                        md5_hash='1c1f8f5b0c8ee28fe9d950625a230e1c')
+            else:
+                weights_path = get_file('resnet50_weights_th_dim_ordering_th_kernels_notop.h5',
+                                        TH_WEIGHTS_PATH_NO_TOP,
+                                        cache_subdir='models',
+                                        md5_hash='f64f049c92468c9affcd44b0976cdafe')
+            model.load_weights(weights_path)
+            if K.backend() == 'tensorflow':
+                warnings.warn('You are using the TensorFlow backend, yet you '
+                              'are using the Theano '
+                              'image dimension ordering convention '
+                              '(`image_dim_ordering="th"`). '
+                              'For best performance, set '
+                              '`image_dim_ordering="tf"` in '
+                              'your Keras config '
+                              'at ~/.keras/keras.json.')
+                convert_all_kernels_in_model(model)
+        else:
+            if include_top:
+                weights_path = get_file('resnet50_weights_tf_dim_ordering_tf_kernels.h5',
+                                        TF_WEIGHTS_PATH,
+                                        cache_subdir='models',
+                                        md5_hash='a7b3fe01876f51b976af0dea6bc144eb')
+            else:
+                weights_path = get_file('resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5',
+                                        TF_WEIGHTS_PATH_NO_TOP,
+                                        cache_subdir='models',
+                                        md5_hash='a268eb855778b3df3c7506639542a6af')
+            model.load_weights(weights_path)
+            if K.backend() == 'theano':
+                convert_all_kernels_in_model(model)
+    return model

keras/applications/vgg16.py

@@ -0,0 +1,149 @@
+# -*- coding: utf-8 -*-
+'''VGG16 model for Keras.
+
+# Reference:
+
+- [Very Deep Convolutional Networks for Large-Scale Image Recognition](https://arxiv.org/abs/1409.1556)
+
+'''
+from __future__ import print_function
+from __future__ import absolute_import
+
+import warnings
+
+from ..models import Model
+from ..layers import Flatten, Dense, Input
+from ..layers import Convolution2D, MaxPooling2D
+from ..utils.layer_utils import convert_all_kernels_in_model
+from ..utils.data_utils import get_file
+from .. import backend as K
+from .imagenet_utils import decode_predictions, preprocess_input
+
+
+TH_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_th_dim_ordering_th_kernels.h5'
+TF_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_tf_dim_ordering_tf_kernels.h5'
+TH_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_th_dim_ordering_th_kernels_notop.h5'
+TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5'
+
+
+def VGG16(include_top=True, weights='imagenet',
+          input_tensor=None):
+    '''Instantiate the VGG16 architecture,
+    optionally loading weights pre-trained
+    on ImageNet. Note that when using TensorFlow,
+    for best performance you should set
+    `image_dim_ordering="tf"` in your Keras config
+    at ~/.keras/keras.json.
+
+    The model and the weights are compatible with both
+    TensorFlow and Theano. The dimension ordering
+    convention used by the model is the one
+    specified in your Keras config file.
+
+    # Arguments
+        include_top: whether to include the 3 fully-connected
+            layers at the top of the network.
+        weights: one of `None` (random initialization)
+            or "imagenet" (pre-training on ImageNet).
+        input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
+            to use as image input for the model.
+
+    # Returns
+        A Keras model instance.
+    '''
+    if weights not in {'imagenet', None}:
+        raise ValueError('The `weights` argument should be either '
+                         '`None` (random initialization) or `imagenet` '
+                         '(pre-training on ImageNet).')
+    # Determine proper input shape
+    if K.image_dim_ordering() == 'th':
+        if include_top:
+            input_shape = (3, 224, 224)
+        else:
+            input_shape = (3, None, None)
+    else:
+        if include_top:
+            input_shape = (224, 224, 3)
+        else:
+            input_shape = (None, None, 3)
+
+    if input_tensor is None:
+        img_input = Input(shape=input_shape)
+    else:
+        if not K.is_keras_tensor(input_tensor):
+            img_input = Input(tensor=input_tensor, shape=input_shape)
+        else:
+            img_input = input_tensor
+    # Block 1
+    x = Convolution2D(64, 3, 3, activation='relu', border_mode='same', name='block1_conv1')(img_input)
+    x = Convolution2D(64, 3, 3, activation='relu', border_mode='same', name='block1_conv2')(x)
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
+
+    # Block 2
+    x = Convolution2D(128, 3, 3, activation='relu', border_mode='same', name='block2_conv1')(x)
+    x = Convolution2D(128, 3, 3, activation='relu', border_mode='same', name='block2_conv2')(x)
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
+
+    # Block 3
+    x = Convolution2D(256, 3, 3, activation='relu', border_mode='same', name='block3_conv1')(x)
+    x = Convolution2D(256, 3, 3, activation='relu', border_mode='same', name='block3_conv2')(x)
+    x = Convolution2D(256, 3, 3, activation='relu', border_mode='same', name='block3_conv3')(x)
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
+
+    # Block 4
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block4_conv1')(x)
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block4_conv2')(x)
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block4_conv3')(x)
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
+
+    # Block 5
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block5_conv1')(x)
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block5_conv2')(x)
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block5_conv3')(x)
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)
+
+    if include_top:
+        # Classification block
+        x = Flatten(name='flatten')(x)
+        x = Dense(4096, activation='relu', name='fc1')(x)
+        x = Dense(4096, activation='relu', name='fc2')(x)
+        x = Dense(1000, activation='softmax', name='predictions')(x)
+
+    # Create model
+    model = Model(img_input, x)
+
+    # load weights
+    if weights == 'imagenet':
+        if K.image_dim_ordering() == 'th':
+            if include_top:
+                weights_path = get_file('vgg16_weights_th_dim_ordering_th_kernels.h5',
+                                        TH_WEIGHTS_PATH,
+                                        cache_subdir='models')
+            else:
+                weights_path = get_file('vgg16_weights_th_dim_ordering_th_kernels_notop.h5',
+                                        TH_WEIGHTS_PATH_NO_TOP,
+                                        cache_subdir='models')
+            model.load_weights(weights_path)
+            if K.backend() == 'tensorflow':
+                warnings.warn('You are using the TensorFlow backend, yet you '
+                              'are using the Theano '
+                              'image dimension ordering convention '
+                              '(`image_dim_ordering="th"`). '
+                              'For best performance, set '
+                              '`image_dim_ordering="tf"` in '
+                              'your Keras config '
+                              'at ~/.keras/keras.json.')
+                convert_all_kernels_in_model(model)
+        else:
+            if include_top:
+                weights_path = get_file('vgg16_weights_tf_dim_ordering_tf_kernels.h5',
+                                        TF_WEIGHTS_PATH,
+                                        cache_subdir='models')
+            else:
+                weights_path = get_file('vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5',
+                                        TF_WEIGHTS_PATH_NO_TOP,
+                                        cache_subdir='models')
+            model.load_weights(weights_path)
+            if K.backend() == 'theano':
+                convert_all_kernels_in_model(model)
+    return model

keras/applications/vgg19.py

@@ -0,0 +1,152 @@
+# -*- coding: utf-8 -*-
+'''VGG19 model for Keras.
+
+# Reference:
+
+- [Very Deep Convolutional Networks for Large-Scale Image Recognition](https://arxiv.org/abs/1409.1556)
+
+'''
+from __future__ import print_function
+from __future__ import absolute_import
+
+import warnings
+
+from ..models import Model
+from ..layers import Flatten, Dense, Input
+from ..layers import Convolution2D, MaxPooling2D
+from ..utils.layer_utils import convert_all_kernels_in_model
+from ..utils.data_utils import get_file
+from .. import backend as K
+from .imagenet_utils import decode_predictions, preprocess_input
+
+
+TH_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg19_weights_th_dim_ordering_th_kernels.h5'
+TF_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg19_weights_tf_dim_ordering_tf_kernels.h5'
+TH_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg19_weights_th_dim_ordering_th_kernels_notop.h5'
+TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5'
+
+
+def VGG19(include_top=True, weights='imagenet',
+          input_tensor=None):
+    '''Instantiate the VGG19 architecture,
+    optionally loading weights pre-trained
+    on ImageNet. Note that when using TensorFlow,
+    for best performance you should set
+    `image_dim_ordering="tf"` in your Keras config
+    at ~/.keras/keras.json.
+
+    The model and the weights are compatible with both
+    TensorFlow and Theano. The dimension ordering
+    convention used by the model is the one
+    specified in your Keras config file.
+
+    # Arguments
+        include_top: whether to include the 3 fully-connected
+            layers at the top of the network.
+        weights: one of `None` (random initialization)
+            or "imagenet" (pre-training on ImageNet).
+        input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
+            to use as image input for the model.
+
+    # Returns
+        A Keras model instance.
+    '''
+    if weights not in {'imagenet', None}:
+        raise ValueError('The `weights` argument should be either '
+                         '`None` (random initialization) or `imagenet` '
+                         '(pre-training on ImageNet).')
+    # Determine proper input shape
+    if K.image_dim_ordering() == 'th':
+        if include_top:
+            input_shape = (3, 224, 224)
+        else:
+            input_shape = (3, None, None)
+    else:
+        if include_top:
+            input_shape = (224, 224, 3)
+        else:
+            input_shape = (None, None, 3)
+
+    if input_tensor is None:
+        img_input = Input(shape=input_shape)
+    else:
+        if not K.is_keras_tensor(input_tensor):
+            img_input = Input(tensor=input_tensor, shape=input_shape)
+        else:
+            img_input = input_tensor
+    # Block 1
+    x = Convolution2D(64, 3, 3, activation='relu', border_mode='same', name='block1_conv1')(img_input)
+    x = Convolution2D(64, 3, 3, activation='relu', border_mode='same', name='block1_conv2')(x)
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
+
+    # Block 2
+    x = Convolution2D(128, 3, 3, activation='relu', border_mode='same', name='block2_conv1')(x)
+    x = Convolution2D(128, 3, 3, activation='relu', border_mode='same', name='block2_conv2')(x)
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
+
+    # Block 3
+    x = Convolution2D(256, 3, 3, activation='relu', border_mode='same', name='block3_conv1')(x)
+    x = Convolution2D(256, 3, 3, activation='relu', border_mode='same', name='block3_conv2')(x)
+    x = Convolution2D(256, 3, 3, activation='relu', border_mode='same', name='block3_conv3')(x)
+    x = Convolution2D(256, 3, 3, activation='relu', border_mode='same', name='block3_conv4')(x)
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
+
+    # Block 4
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block4_conv1')(x)
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block4_conv2')(x)
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block4_conv3')(x)
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block4_conv4')(x)
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
+
+    # Block 5
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block5_conv1')(x)
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block5_conv2')(x)
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block5_conv3')(x)
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block5_conv4')(x)
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)
+
+    if include_top:
+        # Classification block
+        x = Flatten(name='flatten')(x)
+        x = Dense(4096, activation='relu', name='fc1')(x)
+        x = Dense(4096, activation='relu', name='fc2')(x)
+        x = Dense(1000, activation='softmax', name='predictions')(x)
+
+    # Create model
+    model = Model(img_input, x)
+
+    # load weights
+    if weights == 'imagenet':
+        if K.image_dim_ordering() == 'th':
+            if include_top:
+                weights_path = get_file('vgg19_weights_th_dim_ordering_th_kernels.h5',
+                                        TH_WEIGHTS_PATH,
+                                        cache_subdir='models')
+            else:
+                weights_path = get_file('vgg19_weights_th_dim_ordering_th_kernels_notop.h5',
+                                        TH_WEIGHTS_PATH_NO_TOP,
+                                        cache_subdir='models')
+            model.load_weights(weights_path)
+            if K.backend() == 'tensorflow':
+                warnings.warn('You are using the TensorFlow backend, yet you '
+                              'are using the Theano '
+                              'image dimension ordering convention '
+                              '(`image_dim_ordering="th"`). '
+                              'For best performance, set '
+                              '`image_dim_ordering="tf"` in '
+                              'your Keras config '
+                              'at ~/.keras/keras.json.')
+                convert_all_kernels_in_model(model)
+        else:
+            if include_top:
+                weights_path = get_file('vgg19_weights_tf_dim_ordering_tf_kernels.h5',
+                                        TF_WEIGHTS_PATH,
+                                        cache_subdir='models')
+            else:
+                weights_path = get_file('vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5',
+                                        TF_WEIGHTS_PATH_NO_TOP,
+                                        cache_subdir='models')
+            model.load_weights(weights_path)
+            if K.backend() == 'theano':
+                convert_all_kernels_in_model(model)
+    return model

keras/applications/xception.py

@@ -0,0 +1,210 @@
+# -*- coding: utf-8 -*-
+'''Xception V1 model for Keras.
+
+On ImageNet, this model gets to a top-1 validation accuracy of 0.790
+and a top-5 validation accuracy of 0.945.
+
+Do note that the input image format for this model is different than for
+the VGG16 and ResNet models (299x299 instead of 224x224),
+and that the input preprocessing function
+is also different (same as Inception V3).
+
+Also do note that this model is only available for the TensorFlow backend,
+due to its reliance on `SeparableConvolution` layers.
+
+# Reference:
+
+- [Xception: Deep Learning with Depthwise Separable Convolutions](https://arxiv.org/abs/1610.02357)
+
+'''
+from __future__ import print_function
+from __future__ import absolute_import
+
+import warnings
+
+from ..models import Model
+from ..layers import Dense, Input, BatchNormalization, Activation, merge
+from ..layers import Conv2D, SeparableConv2D, MaxPooling2D, GlobalAveragePooling2D
+from ..utils.data_utils import get_file
+from .. import backend as K
+from .imagenet_utils import decode_predictions
+
+
+TF_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.4/xception_weights_tf_dim_ordering_tf_kernels.h5'
+TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.4/xception_weights_tf_dim_ordering_tf_kernels_notop.h5'
+
+
+def Xception(include_top=True, weights='imagenet',
+             input_tensor=None):
+    '''Instantiate the Xception architecture,
+    optionally loading weights pre-trained
+    on ImageNet. This model is available for TensorFlow only,
+    and can only be used with inputs following the TensorFlow
+    dimension ordering `(width, height, channels)`.
+    You should set `image_dim_ordering="tf"` in your Keras config
+    located at ~/.keras/keras.json.
+
+    Note that the default input image size for this model is 299x299.
+
+    # Arguments
+        include_top: whether to include the fully-connected
+            layer at the top of the network.
+        weights: one of `None` (random initialization)
+            or "imagenet" (pre-training on ImageNet).
+        input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
+            to use as image input for the model.
+
+    # Returns
+        A Keras model instance.
+    '''
+    if weights not in {'imagenet', None}:
+        raise ValueError('The `weights` argument should be either '
+                         '`None` (random initialization) or `imagenet` '
+                         '(pre-training on ImageNet).')
+    if K.backend() != 'tensorflow':
+        raise Exception('The Xception model is only available with '
+                        'the TensorFlow backend.')
+    if K.image_dim_ordering() != 'tf':
+        warnings.warn('The Xception model is only available for the '
+                      'input dimension ordering "tf" '
+                      '(width, height, channels). '
+                      'However your settings specify the default '
+                      'dimension ordering "th" (channels, width, height). '
+                      'You should set `image_dim_ordering="tf"` in your Keras '
+                      'config located at ~/.keras/keras.json. '
+                      'The model being returned right now will expect inputs '
+                      'to follow the "tf" dimension ordering.')
+        K.set_image_dim_ordering('tf')
+        old_dim_ordering = 'th'
+    else:
+        old_dim_ordering = None
+
+    # Determine proper input shape
+    if include_top:
+        input_shape = (299, 299, 3)
+    else:
+        input_shape = (None, None, 3)
+
+    if input_tensor is None:
+        img_input = Input(shape=input_shape)
+    else:
+        if not K.is_keras_tensor(input_tensor):
+            img_input = Input(tensor=input_tensor, shape=input_shape)
+        else:
+            img_input = input_tensor
+
+    x = Conv2D(32, 3, 3, subsample=(2, 2), bias=False, name='block1_conv1')(img_input)
+    x = BatchNormalization(name='block1_conv1_bn')(x)
+    x = Activation('relu', name='block1_conv1_act')(x)
+    x = Conv2D(64, 3, 3, bias=False, name='block1_conv2')(x)
+    x = BatchNormalization(name='block1_conv2_bn')(x)
+    x = Activation('relu', name='block1_conv2_act')(x)
+
+    residual = Conv2D(128, 1, 1, subsample=(2, 2),
+                      border_mode='same', bias=False)(x)
+    residual = BatchNormalization()(residual)
+
+    x = SeparableConv2D(128, 3, 3, border_mode='same', bias=False, name='block2_sepconv1')(x)
+    x = BatchNormalization(name='block2_sepconv1_bn')(x)
+    x = Activation('relu', name='block2_sepconv2_act')(x)
+    x = SeparableConv2D(128, 3, 3, border_mode='same', bias=False, name='block2_sepconv2')(x)
+    x = BatchNormalization(name='block2_sepconv2_bn')(x)
+
+    x = MaxPooling2D((3, 3), strides=(2, 2), border_mode='same', name='block2_pool')(x)
+    x = merge([x, residual], mode='sum')
+
+    residual = Conv2D(256, 1, 1, subsample=(2, 2),
+                      border_mode='same', bias=False)(x)
+    residual = BatchNormalization()(residual)
+
+    x = Activation('relu', name='block3_sepconv1_act')(x)
+    x = SeparableConv2D(256, 3, 3, border_mode='same', bias=False, name='block3_sepconv1')(x)
+    x = BatchNormalization(name='block3_sepconv1_bn')(x)
+    x = Activation('relu', name='block3_sepconv2_act')(x)
+    x = SeparableConv2D(256, 3, 3, border_mode='same', bias=False, name='block3_sepconv2')(x)
+    x = BatchNormalization(name='block3_sepconv2_bn')(x)
+
+    x = MaxPooling2D((3, 3), strides=(2, 2), border_mode='same', name='block3_pool')(x)
+    x = merge([x, residual], mode='sum')
+
+    residual = Conv2D(728, 1, 1, subsample=(2, 2),
+                      border_mode='same', bias=False)(x)
+    residual = BatchNormalization()(residual)
+
+    x = Activation('relu', name='block4_sepconv1_act')(x)
+    x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name='block4_sepconv1')(x)
+    x = BatchNormalization(name='block4_sepconv1_bn')(x)
+    x = Activation('relu', name='block4_sepconv2_act')(x)
+    x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name='block4_sepconv2')(x)
+    x = BatchNormalization(name='block4_sepconv2_bn')(x)
+
+    x = MaxPooling2D((3, 3), strides=(2, 2), border_mode='same', name='block4_pool')(x)
+    x = merge([x, residual], mode='sum')
+
+    for i in range(8):
+        residual = x
+        prefix = 'block' + str(i + 5)
+
+        x = Activation('relu', name=prefix + '_sepconv1_act')(x)
+        x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name=prefix + '_sepconv1')(x)
+        x = BatchNormalization(name=prefix + '_sepconv1_bn')(x)
+        x = Activation('relu', name=prefix + '_sepconv2_act')(x)
+        x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name=prefix + '_sepconv2')(x)
+        x = BatchNormalization(name=prefix + '_sepconv2_bn')(x)
+        x = Activation('relu', name=prefix + '_sepconv3_act')(x)
+        x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name=prefix + '_sepconv3')(x)
+        x = BatchNormalization(name=prefix + '_sepconv3_bn')(x)
+
+        x = merge([x, residual], mode='sum')
+
+    residual = Conv2D(1024, 1, 1, subsample=(2, 2),
+                      border_mode='same', bias=False)(x)
+    residual = BatchNormalization()(residual)
+
+    x = Activation('relu', name='block13_sepconv1_act')(x)
+    x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name='block13_sepconv1')(x)
+    x = BatchNormalization(name='block13_sepconv1_bn')(x)
+    x = Activation('relu', name='block13_sepconv2_act')(x)
+    x = SeparableConv2D(1024, 3, 3, border_mode='same', bias=False, name='block13_sepconv2')(x)
+    x = BatchNormalization(name='block13_sepconv2_bn')(x)
+
+    x = MaxPooling2D((3, 3), strides=(2, 2), border_mode='same', name='block13_pool')(x)
+    x = merge([x, residual], mode='sum')
+
+    x = SeparableConv2D(1536, 3, 3, border_mode='same', bias=False, name='block14_sepconv1')(x)
+    x = BatchNormalization(name='block14_sepconv1_bn')(x)
+    x = Activation('relu', name='block14_sepconv1_act')(x)
+
+    x = SeparableConv2D(2048, 3, 3, border_mode='same', bias=False, name='block14_sepconv2')(x)
+    x = BatchNormalization(name='block14_sepconv2_bn')(x)
+    x = Activation('relu', name='block14_sepconv2_act')(x)
+
+    if include_top:
+        x = GlobalAveragePooling2D(name='avg_pool')(x)
+        x = Dense(1000, activation='softmax', name='predictions')(x)
+
+    # Create model
+    model = Model(img_input, x)
+
+    # load weights
+    if weights == 'imagenet':
+        if include_top:
+            weights_path = get_file('xception_weights_tf_dim_ordering_tf_kernels.h5',
+                                    TF_WEIGHTS_PATH,
+                                    cache_subdir='models')
+        else:
+            weights_path = get_file('xception_weights_tf_dim_ordering_tf_kernels_notop.h5',
+                                    TF_WEIGHTS_PATH_NO_TOP,
+                                    cache_subdir='models')
+        model.load_weights(weights_path)
+
+    if old_dim_ordering:
+        K.set_image_dim_ordering(old_dim_ordering)
+    return model
+
+
+def preprocess_input(x):
+    x /= 255.
+    x -= 0.5
+    x *= 2.
+    return x

keras/backend/__init__.py

@@ -11,6 +11,9 @@ from .common import get_uid
 from .common import cast_to_floatx
 from .common import image_dim_ordering
 from .common import set_image_dim_ordering
+from .common import is_keras_tensor
+from .common import legacy_weight_ordering
+from .common import set_legacy_weight_ordering
 
 _keras_base_dir = os.path.expanduser('~')
 if not os.access(_keras_base_dir, os.W_OK):
@@ -20,7 +23,12 @@ _keras_dir = os.path.join(_keras_base_dir, '.keras')
 if not os.path.exists(_keras_dir):
     os.makedirs(_keras_dir)
 
-_BACKEND = 'theano'
+# Set theano as default backend for Windows users since tensorflow is not available for Windows yet.
+if os.name == 'nt':
+    _BACKEND = 'theano'
+else:
+    _BACKEND = 'tensorflow'
+
 _config_path = os.path.expanduser(os.path.join(_keras_dir, 'keras.json'))
 if os.path.exists(_config_path):
     _config = json.load(open(_config_path))
@@ -39,12 +47,13 @@ if os.path.exists(_config_path):
     _BACKEND = _backend
 
 # save config file
-_config = {'floatx': floatx(),
-           'epsilon': epsilon(),
-           'backend': _BACKEND,
-           'image_dim_ordering': image_dim_ordering()}
-with open(_config_path, 'w') as f:
-    f.write(json.dumps(_config, indent=4))
+if not os.path.exists(_config_path):
+    _config = {'floatx': floatx(),
+               'epsilon': epsilon(),
+               'backend': _BACKEND,
+               'image_dim_ordering': image_dim_ordering()}
+    with open(_config_path, 'w') as f:
+        f.write(json.dumps(_config, indent=4))
 
 if 'KERAS_BACKEND' in os.environ:
     _backend = os.environ['KERAS_BACKEND']
@@ -60,3 +69,10 @@ elif _BACKEND == 'tensorflow':
     from .tensorflow_backend import *
 else:
     raise Exception('Unknown backend: ' + str(_BACKEND))
+
+
+def backend():
+    '''Publicly accessible method
+    for determining the current backend.
+    '''
+    return _BACKEND

keras/backend/common.py

@@ -6,7 +6,8 @@ from collections import defaultdict
 _FLOATX = 'float32'
 _EPSILON = 10e-8
 _UID_PREFIXES = defaultdict(int)
-_IMAGE_DIM_ORDERING = 'th'
+_IMAGE_DIM_ORDERING = 'tf'
+_LEGACY_WEIGHT_ORDERING = False
 
 
 def epsilon():
@@ -64,3 +65,25 @@ def set_image_dim_ordering(dim_ordering):
 def get_uid(prefix=''):
     _UID_PREFIXES[prefix] += 1
     return _UID_PREFIXES[prefix]
+
+
+def reset_uids():
+    global _UID_PREFIXES
+    _UID_PREFIXES = defaultdict(int)
+
+
+def is_keras_tensor(x):
+    if hasattr(x, '_keras_shape'):
+        return True
+    else:
+        return False
+
+
+def set_legacy_weight_ordering(value):
+    global _LEGACY_WEIGHT_ORDERING
+    assert value in {True, False}
+    _LEGACY_WEIGHT_ORDERING = value
+
+
+def legacy_weight_ordering():
+    return _LEGACY_WEIGHT_ORDERING

keras/callbacks.py

@@ -1,14 +1,18 @@
 from __future__ import absolute_import
 from __future__ import print_function
 
+import os
+import csv
+
 import numpy as np
 import time
 import json
 import warnings
 
-from collections import deque
+from collections import deque, OrderedDict, Iterable
 from .utils.generic_utils import Progbar
 from keras import backend as K
+from pkg_resources import parse_version
 
 
 class CallbackList(object):
@@ -212,6 +216,7 @@ class History(Callback):
         for k, v in logs.items():
             self.history.setdefault(k, []).append(v)
 
+
 class ModelCheckpoint(Callback):
     '''Save the model after every epoch.
 
@@ -229,25 +234,29 @@ class ModelCheckpoint(Callback):
         verbose: verbosity mode, 0 or 1.
         save_best_only: if `save_best_only=True`,
             the latest best model according to
-            the validation loss will not be overwritten.
+            the quantity monitored will not be overwritten.
         mode: one of {auto, min, max}.
             If `save_best_only=True`, the decision
             to overwrite the current save file is made
             based on either the maximization or the
-            minization of the monitored. For `val_acc`,
+            minimization of the monitored quantity. For `val_acc`,
             this should be `max`, for `val_loss` this should
             be `min`, etc. In `auto` mode, the direction is
             automatically inferred from the name of the monitored quantity.
+        save_weights_only: if True, then only the model's weights will be
+            saved (`model.save_weights(filepath)`), else the full model
+            is saved (`model.save(filepath)`).
 
     '''
     def __init__(self, filepath, monitor='val_loss', verbose=0,
-                 save_best_only=False, mode='auto'):
-
+                 save_best_only=False, save_weights_only=False,
+                 mode='auto'):
         super(ModelCheckpoint, self).__init__()
         self.monitor = monitor
         self.verbose = verbose
         self.filepath = filepath
         self.save_best_only = save_best_only
+        self.save_weights_only = save_weights_only
 
         if mode not in ['auto', 'min', 'max']:
             warnings.warn('ModelCheckpoint mode %s is unknown, '
@@ -284,7 +293,10 @@ class ModelCheckpoint(Callback):
                               % (epoch, self.monitor, self.best,
                                  current, filepath))
                     self.best = current
-                    self.model.save_weights(filepath, overwrite=True)
+                    if self.save_weights_only:
+                        self.model.save_weights(filepath, overwrite=True)
+                    else:
+                        self.model.save(filepath, overwrite=True)
                 else:
                     if self.verbose > 0:
                         print('Epoch %05d: %s did not improve' %
@@ -292,7 +304,10 @@ class ModelCheckpoint(Callback):
         else:
             if self.verbose > 0:
                 print('Epoch %05d: saving model to %s' % (epoch, filepath))
-            self.model.save_weights(filepath, overwrite=True)
+            if self.save_weights_only:
+                self.model.save_weights(filepath, overwrite=True)
+            else:
+                self.model.save(filepath, overwrite=True)
 
 
 class EarlyStopping(Callback):
@@ -300,26 +315,35 @@ class EarlyStopping(Callback):
 
     # Arguments
         monitor: quantity to be monitored.
+        min_delta: minimum change in the monitored quantity
+            to qualify as an improvement, i.e. an absolute
+            change of less than min_delta, will count as no
+            improvement.
         patience: number of epochs with no improvement
             after which training will be stopped.
         verbose: verbosity mode.
-        mode: one of {auto, min, max}. In 'min' mode,
+        mode: one of {auto, min, max}. In `min` mode,
             training will stop when the quantity
-            monitored has stopped decreasing; in 'max'
+            monitored has stopped decreasing; in `max`
             mode it will stop when the quantity
-            monitored has stopped increasing.
+            monitored has stopped increasing; in `auto`
+            mode, the direction is automatically inferred
+            from the name of the monitored quantity.
     '''
-    def __init__(self, monitor='val_loss', patience=0, verbose=0, mode='auto'):
+    def __init__(self, monitor='val_loss', min_delta=0, patience=0, verbose=0, mode='auto'):
         super(EarlyStopping, self).__init__()
 
         self.monitor = monitor
         self.patience = patience
         self.verbose = verbose
+        self.min_delta = min_delta
         self.wait = 0
+        self.stopped_epoch = 0
 
         if mode not in ['auto', 'min', 'max']:
             warnings.warn('EarlyStopping mode %s is unknown, '
-                          'fallback to auto mode.' % (self.mode), RuntimeWarning)
+                          'fallback to auto mode.' % (self.mode),
+                          RuntimeWarning)
             mode = 'auto'
 
         if mode == 'min':
@@ -332,6 +356,11 @@ class EarlyStopping(Callback):
             else:
                 self.monitor_op = np.less
 
+        if self.monitor_op == np.greater:
+            self.min_delta *= 1
+        else:
+            self.min_delta *= -1
+
     def on_train_begin(self, logs={}):
         self.wait = 0       # Allow instances to be re-used
         self.best = np.Inf if self.monitor_op == np.less else -np.Inf
@@ -342,16 +371,19 @@ class EarlyStopping(Callback):
             warnings.warn('Early stopping requires %s available!' %
                           (self.monitor), RuntimeWarning)
 
-        if self.monitor_op(current, self.best):
+        if self.monitor_op(current - self.min_delta, self.best):
             self.best = current
             self.wait = 0
         else:
             if self.wait >= self.patience:
-                if self.verbose > 0:
-                    print('Epoch %05d: early stopping' % (epoch))
+                self.stopped_epoch = epoch
                 self.model.stop_training = True
             self.wait += 1
 
+    def on_train_end(self, logs={}):
+        if self.stopped_epoch > 0 and self.verbose > 0:
+            print('Epoch %05d: early stopping' % (self.stopped_epoch))
+
 
 class RemoteMonitor(Callback):
     '''Callback used to stream events to a server.
@@ -361,8 +393,8 @@ class RemoteMonitor(Callback):
     # Arguments
         root: root url to which the events will be sent (at the end
             of every epoch). Events are sent to
-            `root + '/publish/epoch/end/'` by default. Calls are 
-            HTTP POST, with a `data` argument which is a 
+            `root + '/publish/epoch/end/'` by default. Calls are
+            HTTP POST, with a `data` argument which is a
             JSON-encoded dictionary of event data.
     '''
 
@@ -405,7 +437,11 @@ class LearningRateScheduler(Callback):
         assert hasattr(self.model.optimizer, 'lr'), \
             'Optimizer must have a "lr" attribute.'
         lr = self.schedule(epoch)
-        assert type(lr) == float, 'The output of the "schedule" function should be float.'
+
+        if not isinstance(lr, (float, np.float32, np.float64)):
+            raise ValueError('The output of the "schedule" function '
+                             'should be float.')
+
         K.set_value(self.model.optimizer.lr, lr)
 
 
@@ -433,11 +469,12 @@ class TensorBoard(Callback):
         histogram_freq: frequency (in epochs) at which to compute activation
             histograms for the layers of the model. If set to 0,
             histograms won't be computed.
-        write_graph: whether to visualize the graph in Tensorboard. The log file can
-            become quite large when write_graph is set to True.
+        write_graph: whether to visualize the graph in Tensorboard.
+            The log file can become quite large when
+            write_graph is set to True.
     '''
 
-    def __init__(self, log_dir='./logs', histogram_freq=0, write_graph=True):
+    def __init__(self, log_dir='./logs', histogram_freq=0, write_graph=True, write_images=False):
         super(TensorBoard, self).__init__()
         if K._BACKEND != 'tensorflow':
             raise Exception('TensorBoard callback only works '
@@ -446,6 +483,7 @@ class TensorBoard(Callback):
         self.histogram_freq = histogram_freq
         self.merged = None
         self.write_graph = write_graph
+        self.write_images = write_images
 
     def _set_model(self, model):
         import tensorflow as tf
@@ -454,18 +492,31 @@ class TensorBoard(Callback):
         self.model = model
         self.sess = KTF.get_session()
         if self.histogram_freq and self.merged is None:
-            layers = self.model.layers
-            for layer in layers:
-                if hasattr(layer, 'W'):
-                    tf.histogram_summary('{}_W'.format(layer), layer.W)
-                if hasattr(layer, 'b'):
-                    tf.histogram_summary('{}_b'.format(layer), layer.b)
+            for layer in self.model.layers:
+
+                for weight in layer.weights:
+                    tf.histogram_summary(weight.name, weight)
+
+                    if self.write_images:
+                        w_img = tf.squeeze(weight)
+
+                        shape = w_img.get_shape()
+                        if len(shape) > 1 and shape[0] > shape[1]:
+                            w_img = tf.transpose(w_img)
+
+                        if len(shape) == 1:
+                            w_img = tf.expand_dims(w_img, 0)
+
+                        w_img = tf.expand_dims(tf.expand_dims(w_img, 0), -1)
+
+                        tf.image_summary(weight.name, w_img)
+
                 if hasattr(layer, 'output'):
-                    tf.histogram_summary('{}_out'.format(layer),
+                    tf.histogram_summary('{}_out'.format(layer.name),
                                          layer.output)
         self.merged = tf.merge_all_summaries()
         if self.write_graph:
-            if tf.__version__ >= '0.8.0':
+            if parse_version(tf.__version__) >= parse_version('0.8.0'):
                 self.writer = tf.train.SummaryWriter(self.log_dir,
                                                      self.sess.graph)
             else:
@@ -498,7 +549,226 @@ class TensorBoard(Callback):
                 continue
             summary = tf.Summary()
             summary_value = summary.value.add()
-            summary_value.simple_value = value
+            summary_value.simple_value = value.item()
             summary_value.tag = name
             self.writer.add_summary(summary, epoch)
         self.writer.flush()
+
+
+class ReduceLROnPlateau(Callback):
+    '''Reduce learning rate when a metric has stopped improving.
+
+    Models often benefit from reducing the learning rate by a factor
+    of 2-10 once learning stagnates. This callback monitors a
+    quantity and if no improvement is seen for a 'patience' number
+    of epochs, the learning rate is reduced.
+
+    # Example
+        ```python
+            reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
+                                          patience=5, min_lr=0.001)
+            model.fit(X_train, Y_train, callbacks=[reduce_lr])
+        ```
+
+    # Arguments
+        monitor: quantity to be monitored.
+        factor: factor by which the learning rate will
+            be reduced. new_lr = lr * factor
+        patience: number of epochs with no improvement
+            after which learning rate will be reduced.
+        verbose: int. 0: quiet, 1: update messages.
+        mode: one of {auto, min, max}. In `min` mode,
+            lr will be reduced when the quantity
+            monitored has stopped decreasing; in `max`
+            mode it will be reduced when the quantity
+            monitored has stopped increasing; in `auto`
+            mode, the direction is automatically inferred
+            from the name of the monitored quantity.
+        epsilon: threshold for measuring the new optimum,
+            to only focus on significant changes.
+        cooldown: number of epochs to wait before resuming
+            normal operation after lr has been reduced.
+        min_lr: lower bound on the learning rate.
+    '''
+
+    def __init__(self, monitor='val_loss', factor=0.1, patience=10,
+                 verbose=0, mode='auto', epsilon=1e-4, cooldown=0, min_lr=0):
+        super(Callback, self).__init__()
+
+        self.monitor = monitor
+        if factor >= 1.0:
+            raise ValueError('ReduceLROnPlateau does not support a factor >= 1.0.')
+        self.factor = factor
+        self.min_lr = min_lr
+        self.epsilon = epsilon
+        self.patience = patience
+        self.verbose = verbose
+        self.cooldown = cooldown
+        self.cooldown_counter = 0  # Cooldown counter.
+        self.wait = 0
+        self.best = 0
+        self.mode = mode
+        self.monitor_op = None
+        self.reset()
+
+    def reset(self):
+        if self.mode not in ['auto', 'min', 'max']:
+            warnings.warn('Learning Rate Plateau Reducing mode %s is unknown, '
+                          'fallback to auto mode.' % (self.mode), RuntimeWarning)
+            self.mode = 'auto'
+        if self.mode == 'min' or (self.mode == 'auto' and 'acc' not in self.monitor):
+            self.monitor_op = lambda a, b: np.less(a, b - self.epsilon)
+            self.best = np.Inf
+        else:
+            self.monitor_op = lambda a, b: np.greater(a, b + self.epsilon)
+            self.best = -np.Inf
+        self.cooldown_counter = 0
+        self.wait = 0
+        self.lr_epsilon = self.min_lr * 1e-4
+
+    def on_train_begin(self, logs={}):
+        self.reset()
+
+    def on_epoch_end(self, epoch, logs={}):
+        logs['lr'] = K.get_value(self.model.optimizer.lr)
+        current = logs.get(self.monitor)
+        if current is None:
+            warnings.warn('Learning Rate Plateau Reducing requires %s available!' %
+                          self.monitor, RuntimeWarning)
+        else:
+            if self.in_cooldown():
+                self.cooldown_counter -= 1
+                self.wait = 0
+
+            if self.monitor_op(current, self.best):
+                self.best = current
+                self.wait = 0
+            elif not self.in_cooldown():
+                if self.wait >= self.patience:
+                    old_lr = float(K.get_value(self.model.optimizer.lr))
+                    if old_lr > self.min_lr + self.lr_epsilon:
+                        new_lr = old_lr * self.factor
+                        new_lr = max(new_lr, self.min_lr)
+                        K.set_value(self.model.optimizer.lr, new_lr)
+                        if self.verbose > 0:
+                            print('\nEpoch %05d: reducing learning rate to %s.' % (epoch, new_lr))
+                        self.cooldown_counter = self.cooldown
+                        self.wait = 0
+                self.wait += 1
+
+    def in_cooldown(self):
+        return self.cooldown_counter > 0
+
+
+class CSVLogger(Callback):
+    '''Callback that streams epoch results to a csv file.
+    Supports all values that can be represented as a string,
+    including 1D iterables such as np.ndarray.
+
+    # Example
+        ```python
+            csv_logger = CSVLogger('training.log')
+            model.fit(X_train, Y_train, callbacks=[csv_logger])
+        ```
+
+    Arguments
+        filename: filename of the csv file, e.g. 'run/log.csv'.
+        separator: string used to separate elements in the csv file.
+        append: True: append if file exists (useful for continuing
+            training). False: overwrite existing file,
+    '''
+
+    def __init__(self, filename, separator=',', append=False):
+        self.sep = separator
+        self.filename = filename
+        self.append = append
+        self.writer = None
+        self.keys = None
+        self.append_header = True
+        super(CSVLogger, self).__init__()
+
+    def on_train_begin(self, logs={}):
+        if self.append:
+            if os.path.exists(self.filename):
+                with open(self.filename) as f:
+                    self.append_header = len(f.readline()) == 0
+            self.csv_file = open(self.filename, 'a')
+        else:
+            self.csv_file = open(self.filename, 'w')
+
+    def on_epoch_end(self, epoch, logs={}):
+        def handle_value(k):
+            is_zero_dim_ndarray = isinstance(k, np.ndarray) and k.ndim == 0
+            if isinstance(k, Iterable) and not is_zero_dim_ndarray:
+                return '"[%s]"' % (', '.join(map(lambda x: str(x), k)))
+            else:
+                return k
+
+        if not self.writer:
+            self.keys = sorted(logs.keys())
+            self.writer = csv.DictWriter(self.csv_file, fieldnames=['epoch'] + self.keys)
+            if self.append_header:
+                self.writer.writeheader()
+
+        row_dict = OrderedDict({'epoch': epoch})
+        row_dict.update((key, handle_value(logs[key])) for key in self.keys)
+        self.writer.writerow(row_dict)
+        self.csv_file.flush()
+
+    def on_train_end(self, logs={}):
+        self.csv_file.close()
+
+
+class LambdaCallback(Callback):
+    """Callback for creating simple, custom callbacks on-the-fly.
+
+    This callback is constructed with anonymous functions that will be called
+    at the appropiate time. Note that the callbacks expects positional
+    arguments, as:
+     - `on_epoch_begin` and `on_epoch_end` expect two positional arguments: `epoch`, `logs`
+     - `on_batch_begin` and `on_batch_end` expect two positional arguments: `batch`, `logs`
+     - `on_train_begin` and `on_train_end` expect one positional argument: `logs`
+
+    # Arguments
+        on_epoch_begin: called at the beginning of every epoch.
+        on_epoch_end: called at the end of every epoch.
+        on_batch_begin: called at the beginning of every batch.
+        on_batch_end: called at the end of every batch.
+        on_train_begin: called at the beginning of model training.
+        on_train_end: called at the end of model training.
+
+    # Example
+        ```python
+        # Print the batch number at the beginning of every batch.
+        batch_print_callback = LambdaCallback(on_batch_begin=lambda batch, logs: print(batch))
+
+        # Plot the loss after every epoch.
+        import numpy as np
+        import matplotlib.pyplot as plt
+        plot_loss_callback = LambdaCallback(on_epoch_end=lambda epoch, logs: plt.plot(np.arange(epoch), logs['loss']))
+
+        # Terminate some processes after having finished model training.
+        processes = ...
+        cleanup_callback = LambdaCallback(on_train_end=lambda logs: [p.terminate() for p in processes if p.is_alive()])
+
+        model.fit(..., callbacks=[batch_print_callback, plot_loss_callback, cleanup_callback])
+        ```
+
+    """
+
+    def __init__(self,
+                 on_epoch_begin=None,
+                 on_epoch_end=None,
+                 on_batch_begin=None,
+                 on_batch_end=None,
+                 on_train_begin=None,
+                 on_train_end=None,
+                 **kwargs):
+        super(Callback, self).__init__()
+        self.__dict__.update(kwargs)
+        self.on_epoch_begin = on_epoch_begin if on_epoch_begin else lambda epoch, logs: None
+        self.on_epoch_end = on_epoch_end if on_epoch_end else lambda epoch, logs: None
+        self.on_batch_begin = on_batch_begin if on_batch_begin else lambda batch, logs: None
+        self.on_batch_end = on_batch_end if on_batch_end else lambda batch, logs: None
+        self.on_train_begin = on_train_begin if on_train_begin else lambda logs: None
+        self.on_train_end = on_train_end if on_train_end else lambda logs: None

keras/datasets/cifar.py

@@ -11,9 +11,10 @@ def load_batch(fpath, label_key='labels'):
     else:
         d = cPickle.load(f, encoding="bytes")
         # decode utf8
+        d_decoded = {}
         for k, v in d.items():
-            del(d[k])
-            d[k.decode("utf8")] = v
+            d_decoded[k.decode("utf8")] = v
+        d = d_decoded
     f.close()
     data = d["data"]
     labels = d[label_key]

keras/datasets/cifar10.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 from .cifar import load_batch
 from ..utils.data_utils import get_file
+from .. import backend as K
 import numpy as np
 import os
 
@@ -18,8 +19,8 @@ def load_data():
     for i in range(1, 6):
         fpath = os.path.join(path, 'data_batch_' + str(i))
         data, labels = load_batch(fpath)
-        X_train[(i-1)*10000:i*10000, :, :, :] = data
-        y_train[(i-1)*10000:i*10000] = labels
+        X_train[(i - 1) * 10000: i * 10000, :, :, :] = data
+        y_train[(i - 1) * 10000: i * 10000] = labels
 
     fpath = os.path.join(path, 'test_batch')
     X_test, y_test = load_batch(fpath)
@@ -27,4 +28,8 @@ def load_data():
     y_train = np.reshape(y_train, (len(y_train), 1))
     y_test = np.reshape(y_test, (len(y_test), 1))
 
+    if K.image_dim_ordering() == 'tf':
+        X_train = X_train.transpose(0, 2, 3, 1)
+        X_test = X_test.transpose(0, 2, 3, 1)
+
     return (X_train, y_train), (X_test, y_test)

keras/datasets/cifar100.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 from .cifar import load_batch
 from ..utils.data_utils import get_file
+from .. import backend as K
 import numpy as np
 import os
 
@@ -13,9 +14,6 @@ def load_data(label_mode='fine'):
     origin = "http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz"
     path = get_file(dirname, origin=origin, untar=True)
 
-    nb_test_samples = 10000
-    nb_train_samples = 50000
-
     fpath = os.path.join(path, 'train')
     X_train, y_train = load_batch(fpath, label_key=label_mode+'_labels')
 
@@ -25,4 +23,8 @@ def load_data(label_mode='fine'):
     y_train = np.reshape(y_train, (len(y_train), 1))
     y_test = np.reshape(y_test, (len(y_test), 1))
 
+    if K.image_dim_ordering() == 'tf':
+        X_train = X_train.transpose(0, 2, 3, 1)
+        X_test = X_test.transpose(0, 2, 3, 1)
+
     return (X_train, y_train), (X_test, y_test)

keras/datasets/imdb.py

@@ -4,26 +4,58 @@ import gzip
 from ..utils.data_utils import get_file
 from six.moves import zip
 import numpy as np
+import sys
 
 
-def load_data(path="imdb.pkl", nb_words=None, skip_top=0,
-              maxlen=None, test_split=0.2, seed=113,
+def load_data(path='imdb_full.pkl', nb_words=None, skip_top=0,
+              maxlen=None, seed=113,
               start_char=1, oov_char=2, index_from=3):
+    '''
+    # Arguments
+        path: where to store the data (in `/.keras/dataset`)
+        nb_words: max number of words to include. Words are ranked
+            by how often they occur (in the training set) and only
+            the most frequent words are kept
+        skip_top: skip the top N most frequently occuring words
+            (which may not be informative).
+        maxlen: truncate sequences after this length.
+        seed: random seed for sample shuffling.
+        start_char: The start of a sequence will be marked with this character.
+            Set to 1 because 0 is usually the padding character.
+        oov_char: words that were cut out because of the `nb_words`
+            or `skip_top` limit will be replaced with this character.
+        index_from: index actual words with this index and higher.
 
-    path = get_file(path, origin="https://s3.amazonaws.com/text-datasets/imdb.pkl")
+    Note that the 'out of vocabulary' character is only used for
+    words that were present in the training set but are not included
+    because they're not making the `nb_words` cut here.
+    Words that were not seen in the trining set but are in the test set
+    have simply been skipped.
+    '''
+    path = get_file(path,
+                    origin='https://s3.amazonaws.com/text-datasets/imdb_full.pkl',
+                    md5_hash='d091312047c43cf9e4e38fef92437263')
 
-    if path.endswith(".gz"):
+    if path.endswith('.gz'):
         f = gzip.open(path, 'rb')
     else:
         f = open(path, 'rb')
 
-    X, labels = cPickle.load(f)
+    (x_train, labels_train), (x_test, labels_test) = cPickle.load(f)
     f.close()
 
     np.random.seed(seed)
-    np.random.shuffle(X)
+    np.random.shuffle(x_train)
     np.random.seed(seed)
-    np.random.shuffle(labels)
+    np.random.shuffle(labels_train)
+
+    np.random.seed(seed * 2)
+    np.random.shuffle(x_test)
+    np.random.seed(seed * 2)
+    np.random.shuffle(labels_test)
+
+    X = x_train + x_test
+    labels = labels_train + labels_test
 
     if start_char is not None:
         X = [[start_char] + [w + index_from for w in x] for x in X]
@@ -60,10 +92,25 @@ def load_data(path="imdb.pkl", nb_words=None, skip_top=0,
             nX.append(nx)
         X = nX
 
-    X_train = np.array(X[:int(len(X) * (1 - test_split))])
-    y_train = np.array(labels[:int(len(X) * (1 - test_split))])
+    X_train = np.array(X[:len(x_train)])
+    y_train = np.array(labels[:len(x_train)])
 
-    X_test = np.array(X[int(len(X) * (1 - test_split)):])
-    y_test = np.array(labels[int(len(X) * (1 - test_split)):])
+    X_test = np.array(X[len(x_train):])
+    y_test = np.array(labels[len(x_train):])
 
     return (X_train, y_train), (X_test, y_test)
+
+
+def get_word_index(path='imdb_word_index.pkl'):
+    path = get_file(path,
+                    origin='https://s3.amazonaws.com/text-datasets/imdb_word_index.pkl',
+                    md5_hash='72d94b01291be4ff843198d3b0e1e4d7')
+    f = open(path, 'rb')
+
+    if sys.version_info < (3,):
+        data = cPickle.load(f)
+    else:
+        data = cPickle.load(f, encoding='latin1')
+
+    f.close()
+    return data

keras/datasets/mnist.py

@@ -1,14 +1,13 @@
-# -*- coding: utf-8 -*-
 import gzip
 from ..utils.data_utils import get_file
 from six.moves import cPickle
 import sys
 
 
-def load_data(path="mnist.pkl.gz"):
-    path = get_file(path, origin="https://s3.amazonaws.com/img-datasets/mnist.pkl.gz")
+def load_data(path='mnist.pkl.gz'):
+    path = get_file(path, origin='https://s3.amazonaws.com/img-datasets/mnist.pkl.gz')
 
-    if path.endswith(".gz"):
+    if path.endswith('.gz'):
         f = gzip.open(path, 'rb')
     else:
         f = open(path, 'rb')
@@ -16,7 +15,7 @@ def load_data(path="mnist.pkl.gz"):
     if sys.version_info < (3,):
         data = cPickle.load(f)
     else:
-        data = cPickle.load(f, encoding="bytes")
+        data = cPickle.load(f, encoding='bytes')
 
     f.close()
     return data  # (X_train, y_train), (X_test, y_test)

keras/datasets/reuters.py

@@ -7,11 +7,35 @@ import numpy as np
 import sys
 
 
-def load_data(path="reuters.pkl", nb_words=None, skip_top=0,
+def load_data(path='reuters.pkl', nb_words=None, skip_top=0,
               maxlen=None, test_split=0.2, seed=113,
               start_char=1, oov_char=2, index_from=3):
+    '''Loads the Reuters newswire classification dataset.
 
-    path = get_file(path, origin="https://s3.amazonaws.com/text-datasets/reuters.pkl")
+    # Arguments
+        path: where to store the data (in `/.keras/dataset`)
+        nb_words: max number of words to include. Words are ranked
+            by how often they occur (in the training set) and only
+            the most frequent words are kept
+        skip_top: skip the top N most frequently occuring words
+            (which may not be informative).
+        maxlen: truncate sequences after this length.
+        test_split: Fraction of the dataset to be used as test data.
+        seed: random seed for sample shuffling.
+        start_char: The start of a sequence will be marked with this character.
+            Set to 1 because 0 is usually the padding character.
+        oov_char: words that were cut out because of the `nb_words`
+            or `skip_top` limit will be replaced with this character.
+        index_from: index actual words with this index and higher.
+
+    Note that the 'out of vocabulary' character is only used for
+    words that were present in the training set but are not included
+    because they're not making the `nb_words` cut here.
+    Words that were not seen in the trining set but are in the test set
+    have simply been skipped.
+    '''
+
+    path = get_file(path, origin='https://s3.amazonaws.com/text-datasets/reuters.pkl')
     f = open(path, 'rb')
     X, labels = cPickle.load(f)
     f.close()
@@ -62,14 +86,14 @@ def load_data(path="reuters.pkl", nb_words=None, skip_top=0,
     return (X_train, y_train), (X_test, y_test)
 
 
-def get_word_index(path="reuters_word_index.pkl"):
-    path = get_file(path, origin="https://s3.amazonaws.com/text-datasets/reuters_word_index.pkl")
+def get_word_index(path='reuters_word_index.pkl'):
+    path = get_file(path, origin='https://s3.amazonaws.com/text-datasets/reuters_word_index.pkl')
     f = open(path, 'rb')
 
     if sys.version_info < (3,):
         data = cPickle.load(f)
     else:
-        data = cPickle.load(f, encoding="latin1")
+        data = cPickle.load(f, encoding='latin1')
 
     f.close()
     return data

keras/engine/training.py

@@ -7,6 +7,9 @@ import time
 import numpy as np
 import multiprocessing
 import threading
+
+import six
+
 try:
     import queue
 except ImportError:
@@ -183,13 +186,12 @@ def check_array_lengths(X, Y, W):
 
 
 def check_loss_and_target_compatibility(targets, losses, output_shapes):
-    assert len(targets) == len(losses) == len(output_shapes)
     key_losses = {'mean_square_error',
                   'binary_crossentropy',
                   'categorical_crossentropy'}
     for y, loss, shape in zip(targets, losses, output_shapes):
         if loss.__name__ == 'categorical_crossentropy':
-            if y.shape[1] == 1:
+            if y.shape[-1] == 1:
                 raise Exception('You are passing a target array of shape ' + str(y.shape) +
                                 ' while using as loss `categorical_crossentropy`. '
                                 '`categorical_crossentropy` expects '
@@ -205,13 +207,15 @@ def check_loss_and_target_compatibility(targets, losses, output_shapes):
                                 'Alternatively, you can use the loss function '
                                 '`sparse_categorical_crossentropy` instead, '
                                 'which does expect integer targets.')
-        if loss.__name__ in key_losses and shape[1] is not None and y.shape[1] != shape[1]:
-            raise Exception('A target array with shape ' + str(y.shape) +
-                            ' was passed for an output of shape ' + str(shape) +
-                            ' while using as loss `' + loss.__name__ + '`. '
-                            'This loss expects '
-                            'targets to have the same shape '
-                            'as the output.')
+        if loss.__name__ in key_losses:
+            for target_dim, out_dim in zip(y.shape[1:], shape[1:]):
+                if out_dim is not None and target_dim != out_dim:
+                    raise Exception('A target array with shape ' + str(y.shape) +
+                                    ' was passed for an output of shape ' + str(shape) +
+                                    ' while using as loss `' + loss.__name__ + '`. '
+                                    'This loss expects '
+                                    'targets to have the same shape '
+                                    'as the output.')
 
 
 def collect_metrics(metrics, output_names):
@@ -234,31 +238,6 @@ def collect_metrics(metrics, output_names):
                         str(metrics))
 
 
-def collect_trainable_weights(layer):
-    '''Collects all `trainable_weights` attributes,
-    excluding any sublayers where `trainable` is set the `False`.
-    '''
-    trainable = getattr(layer, 'trainable', True)
-    if not trainable:
-        return []
-    weights = []
-    if layer.__class__.__name__ == 'Sequential':
-        for sublayer in layer.flattened_layers:
-            weights += collect_trainable_weights(sublayer)
-    elif layer.__class__.__name__ == 'Model':
-        for sublayer in layer.layers:
-            weights += collect_trainable_weights(sublayer)
-    elif layer.__class__.__name__ == 'Graph':
-        for sublayer in layer._graph_nodes.values():
-            weights += collect_trainable_weights(sublayer)
-    else:
-        weights += layer.trainable_weights
-    # dedupe weights
-    weights = list(set(weights))
-    weights.sort(key=lambda x: x.name)
-    return weights
-
-
 def batch_shuffle(index_array, batch_size):
     '''This shuffles an array in a batch-wise fashion.
     Useful for shuffling HDF5 arrays
@@ -418,15 +397,11 @@ def generator_queue(generator, max_q_size=10,
         _stop = threading.Event()
 
     try:
-
         def data_generator_task():
             while not _stop.is_set():
                 try:
-                    if q.qsize() < max_q_size:
-                        try:
-                            generator_output = next(generator)
-                        except ValueError:
-                            continue
+                    if pickle_safe or q.qsize() < max_q_size:
+                        generator_output = next(generator)
                         q.put(generator_output)
                     else:
                         time.sleep(wait_time)
@@ -444,7 +419,6 @@ def generator_queue(generator, max_q_size=10,
             generator_threads.append(thread)
             thread.daemon = True
             thread.start()
-
     except:
         _stop.set()
         if pickle_safe:
@@ -455,7 +429,7 @@ def generator_queue(generator, max_q_size=10,
             q.close()
         raise
 
-    return q, _stop
+    return q, _stop, generator_threads
 
 
 class Model(Container):
@@ -607,11 +581,15 @@ class Model(Container):
         for i in range(len(self.outputs)):
             shape = self.internal_output_shapes[i]
             name = self.output_names[i]
-            self.targets.append(K.placeholder(ndim=len(shape), name=name + '_target'))
+            self.targets.append(K.placeholder(ndim=len(shape),
+                                name=name + '_target',
+                                sparse=K.is_sparse(self.outputs[i]),
+                                dtype=K.dtype(self.outputs[i])))
 
         # prepare metrics
+        self.metrics = metrics
         self.metrics_names = ['loss']
-        self.metrics = []
+        self.metrics_tensors = []
 
         # compute total loss
         total_loss = None
@@ -625,7 +603,7 @@ class Model(Container):
             output_loss = weighted_loss(y_true, y_pred,
                                         sample_weight, mask)
             if len(self.outputs) > 1:
-                self.metrics.append(output_loss)
+                self.metrics_tensors.append(output_loss)
                 self.metrics_names.append(self.output_names[i] + '_loss')
             if total_loss is None:
                 total_loss = loss_weight * output_loss
@@ -639,6 +617,15 @@ class Model(Container):
         # list of same size as output_names.
         # contains tuples (metrics for output, names of metrics)
         nested_metrics = collect_metrics(metrics, self.output_names)
+
+        def append_metric(layer_num, metric_name, metric_tensor):
+            """Helper function, used in loop below"""
+            if len(self.output_names) > 1:
+                metric_name = self.output_layers[layer_num].name + '_' + metric_name
+
+            self.metrics_names.append(metric_name)
+            self.metrics_tensors.append(metric_tensor)
+
         for i in range(len(self.outputs)):
             y_true = self.targets[i]
             y_pred = self.outputs[i]
@@ -648,27 +635,28 @@ class Model(Container):
                 if metric == 'accuracy' or metric == 'acc':
                     # custom handling of accuracy (because of class mode duality)
                     output_shape = self.internal_output_shapes[i]
+                    acc_fn = None
                     if output_shape[-1] == 1 or self.loss_functions[i] == objectives.binary_crossentropy:
                         # case: binary accuracy
-                        self.metrics.append(metrics_module.binary_accuracy(y_true, y_pred))
+                        acc_fn = metrics_module.binary_accuracy
                     elif self.loss_functions[i] == objectives.sparse_categorical_crossentropy:
                         # case: categorical accuracy with sparse targets
-                        self.metrics.append(
-                            metrics_module.sparse_categorical_accuracy(y_true, y_pred))
+                        acc_fn = metrics_module.sparse_categorical_accuracy
                     else:
-                        # case: categorical accuracy with dense targets
-                        self.metrics.append(metrics_module.categorical_accuracy(y_true, y_pred))
-                    if len(self.output_names) == 1:
-                        self.metrics_names.append('acc')
-                    else:
-                        self.metrics_names.append(self.output_layers[i].name + '_acc')
+                        acc_fn = metrics_module.categorical_accuracy
+
+                    append_metric(i, 'acc', acc_fn(y_true, y_pred))
                 else:
                     metric_fn = metrics_module.get(metric)
-                    self.metrics.append(metric_fn(y_true, y_pred))
-                    if len(self.output_names) == 1:
-                        self.metrics_names.append(metric_fn.__name__)
-                    else:
-                        self.metrics_names.append(self.output_layers[i].name + '_' + metric_fn.__name__)
+                    metric_result = metric_fn(y_true, y_pred)
+
+                    if not isinstance(metric_result, dict):
+                        metric_result = {
+                            metric_fn.__name__: metric_result
+                        }
+
+                    for name, tensor in six.iteritems(metric_result):
+                        append_metric(i, name, tensor)
 
         # prepare gradient updates and state updates
         self.optimizer = optimizers.get(optimizer)
@@ -684,23 +672,33 @@ class Model(Container):
         self.test_function = None
         self.predict_function = None
 
+        # collected trainable weights and sort them deterministically.
+        trainable_weights = self.trainable_weights
+        # Sort weights by name
+        if trainable_weights:
+            if K.backend() == 'theano':
+                trainable_weights.sort(key=lambda x: x.name if x.name else x.auto_name)
+            else:
+                trainable_weights.sort(key=lambda x: x.name)
+        self._collected_trainable_weights = trainable_weights
+
     def _make_train_function(self):
         if not hasattr(self, 'train_function'):
             raise Exception('You must compile your model before using it.')
         if self.train_function is None:
-            if self.uses_learning_phase:
+            if self.uses_learning_phase and type(K.learning_phase()) is not int:
                 inputs = self.inputs + self.targets + self.sample_weights + [K.learning_phase()]
             else:
                 inputs = self.inputs + self.targets + self.sample_weights
 
-            # get trainable weights
-            trainable_weights = collect_trainable_weights(self)
-            training_updates = self.optimizer.get_updates(trainable_weights, self.constraints, self.total_loss)
+            training_updates = self.optimizer.get_updates(self._collected_trainable_weights,
+                                                          self.constraints,
+                                                          self.total_loss)
             updates = self.updates + training_updates
 
             # returns loss and metrics. Updates weights at each call.
             self.train_function = K.function(inputs,
-                                             [self.total_loss] + self.metrics,
+                                             [self.total_loss] + self.metrics_tensors,
                                              updates=updates,
                                              **self._function_kwargs)
 
@@ -708,14 +706,14 @@ class Model(Container):
         if not hasattr(self, 'test_function'):
             raise Exception('You must compile your model before using it.')
         if self.test_function is None:
-            if self.uses_learning_phase:
+            if self.uses_learning_phase and type(K.learning_phase()) is not int:
                 inputs = self.inputs + self.targets + self.sample_weights + [K.learning_phase()]
             else:
                 inputs = self.inputs + self.targets + self.sample_weights
             # return loss and metrics, no gradient updates.
             # Does update the network states.
             self.test_function = K.function(inputs,
-                                            [self.total_loss] + self.metrics,
+                                            [self.total_loss] + self.metrics_tensors,
                                             updates=self.state_updates,
                                             **self._function_kwargs)
 
@@ -723,7 +721,7 @@ class Model(Container):
         if not hasattr(self, 'predict_function'):
             self.predict_function = None
         if self.predict_function is None:
-            if self.uses_learning_phase:
+            if self.uses_learning_phase and type(K.learning_phase()) is not int:
                 inputs = self.inputs + [K.learning_phase()]
             else:
                 inputs = self.inputs
@@ -738,7 +736,7 @@ class Model(Container):
     def _fit_loop(self, f, ins, out_labels=[], batch_size=32,
                   nb_epoch=100, verbose=1, callbacks=[],
                   val_f=None, val_ins=None, shuffle=True,
-                  callback_metrics=[]):
+                  callback_metrics=[], initial_epoch=0):
         '''Abstract fit function for f(ins).
         Assume that f returns a list, labeled by out_labels.
 
@@ -758,6 +756,8 @@ class Model(Container):
                 passed to the callbacks. They should be the
                 concatenation of list the display names of the outputs of
                  `f` and the list of display names of the outputs of `f_val`.
+            initial_epoch: epoch at which to start training
+                (useful for resuming a previous training run)
 
         # Returns
             `History` object.
@@ -767,9 +767,9 @@ class Model(Container):
             do_validation = True
             if verbose:
                 print('Train on %d samples, validate on %d samples' %
-                      (len(ins[0]), len(val_ins[0])))
+                      (ins[0].shape[0], val_ins[0].shape[0]))
 
-        nb_train_sample = len(ins[0])
+        nb_train_sample = ins[0].shape[0]
         index_array = np.arange(nb_train_sample)
 
         self.history = cbks.History()
@@ -798,7 +798,7 @@ class Model(Container):
         callback_model.stop_training = False
         self.validation_data = val_ins
 
-        for epoch in range(nb_epoch):
+        for epoch in range(initial_epoch, nb_epoch):
             callbacks.on_epoch_begin(epoch)
             if shuffle == 'batch':
                 index_array = batch_shuffle(index_array, batch_size)
@@ -863,7 +863,7 @@ class Model(Container):
             or list of arrays of predictions
             (if the model has multiple outputs).
         '''
-        nb_sample = len(ins[0])
+        nb_sample = ins[0].shape[0]
         outs = []
         if verbose == 1:
             progbar = Progbar(target=nb_sample)
@@ -908,7 +908,7 @@ class Model(Container):
             and/or metrics). The attribute `model.metrics_names` will give you
             the display labels for the scalar outputs.
         '''
-        nb_sample = len(ins[0])
+        nb_sample = ins[0].shape[0]
         outs = []
         if verbose == 1:
             progbar = Progbar(target=nb_sample)
@@ -985,7 +985,7 @@ class Model(Container):
 
     def fit(self, x, y, batch_size=32, nb_epoch=10, verbose=1, callbacks=[],
             validation_split=0., validation_data=None, shuffle=True,
-            class_weight=None, sample_weight=None):
+            class_weight=None, sample_weight=None, initial_epoch=0):
         '''Trains the model for a fixed number of epochs (iterations on a dataset).
 
         # Arguments
@@ -1009,7 +1009,7 @@ class Model(Container):
                 on this data at the end of each epoch.
             validation_data: data on which to evaluate the loss and any model metrics
                 at the end of each epoch. The model will not be trained on this data.
-                This could be a tuple (x_val, y_val) or a tuple (val_x, val_y, val_sample_weights).
+                This could be a tuple (x_val, y_val) or a tuple (x_val, y_val, val_sample_weights).
             shuffle: boolean, whether to shuffle the training data before each epoch.
             class_weight: optional dictionary mapping class indices (integers) to
                 a weight (float) to apply to the model's loss for the samples
@@ -1022,6 +1022,8 @@ class Model(Container):
                 with shape (samples, sequence_length),
                 to apply a different weight to every timestep of every sample.
                 In this case you should make sure to specify sample_weight_mode="temporal" in compile().
+            initial_epoch: epoch at which to start training
+                (useful for resuming a previous training run)
 
 
         # Returns
@@ -1050,7 +1052,7 @@ class Model(Container):
                                                                            batch_size=batch_size)
             self._make_test_function()
             val_f = self.test_function
-            if self.uses_learning_phase:
+            if self.uses_learning_phase and type(K.learning_phase()) is not int:
                 val_ins = val_x + val_y + val_sample_weights + [0.]
             else:
                 val_ins = val_x + val_y + val_sample_weights
@@ -1064,7 +1066,7 @@ class Model(Container):
                 slice_X(sample_weights, 0, split_at), slice_X(sample_weights, split_at))
             self._make_test_function()
             val_f = self.test_function
-            if self.uses_learning_phase:
+            if self.uses_learning_phase and type(K.learning_phase()) is not int:
                 val_ins = val_x + val_y + val_sample_weights + [0.]
             else:
                 val_ins = val_x + val_y + val_sample_weights
@@ -1074,7 +1076,7 @@ class Model(Container):
             val_ins = None
 
         # prepare input arrays and training function
-        if self.uses_learning_phase:
+        if self.uses_learning_phase and type(K.learning_phase()) is not int:
             ins = x + y + sample_weights + [1.]
         else:
             ins = x + y + sample_weights
@@ -1105,7 +1107,8 @@ class Model(Container):
                               batch_size=batch_size, nb_epoch=nb_epoch,
                               verbose=verbose, callbacks=callbacks,
                               val_f=val_f, val_ins=val_ins, shuffle=shuffle,
-                              callback_metrics=callback_metrics)
+                              callback_metrics=callback_metrics,
+                              initial_epoch=initial_epoch)
 
     def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None):
         '''Returns the loss value and metrics values for the model
@@ -1134,7 +1137,7 @@ class Model(Container):
                                                            check_batch_dim=False,
                                                            batch_size=batch_size)
         # prepare inputs, delegate logic to _test_loop
-        if self.uses_learning_phase:
+        if self.uses_learning_phase and type(K.learning_phase()) is not int:
             ins = x + y + sample_weights + [0.]
         else:
             ins = x + y + sample_weights
@@ -1171,7 +1174,7 @@ class Model(Container):
                                 'Batch size: ' + str(batch_size) + '.')
 
         # prepare inputs, delegate logic to _predict_loop
-        if self.uses_learning_phase:
+        if self.uses_learning_phase and type(K.learning_phase()) is not int:
             ins = x + [0.]
         else:
             ins = x
@@ -1215,7 +1218,7 @@ class Model(Container):
                                                            sample_weight=sample_weight,
                                                            class_weight=class_weight,
                                                            check_batch_dim=True)
-        if self.uses_learning_phase:
+        if self.uses_learning_phase and type(K.learning_phase()) is not int:
             ins = x + y + sample_weights + [1.]
         else:
             ins = x + y + sample_weights
@@ -1253,7 +1256,7 @@ class Model(Container):
         x, y, sample_weights = self._standardize_user_data(x, y,
                                                            sample_weight=sample_weight,
                                                            check_batch_dim=True)
-        if self.uses_learning_phase:
+        if self.uses_learning_phase and type(K.learning_phase()) is not int:
             ins = x + y + sample_weights + [0.]
         else:
             ins = x + y + sample_weights
@@ -1268,7 +1271,7 @@ class Model(Container):
         '''
         x = standardize_input_data(x, self.input_names,
                                    self.internal_input_shapes)
-        if self.uses_learning_phase:
+        if self.uses_learning_phase and type(K.learning_phase()) is not int:
             ins = x + [0.]
         else:
             ins = x
@@ -1281,7 +1284,8 @@ class Model(Container):
     def fit_generator(self, generator, samples_per_epoch, nb_epoch,
                       verbose=1, callbacks=[],
                       validation_data=None, nb_val_samples=None,
-                      class_weight={}, max_q_size=10, nb_worker=1, pickle_safe=False):
+                      class_weight={}, max_q_size=10, nb_worker=1, pickle_safe=False,
+                      initial_epoch=0):
         '''Fits the model on data generated batch-by-batch by
         a Python generator.
         The generator is run in parallel to the model, for efficiency.
@@ -1314,9 +1318,11 @@ class Model(Container):
             max_q_size: maximum size for the generator queue
             nb_worker: maximum number of processes to spin up when using process based threading
             pickle_safe: if True, use process based threading. Note that because
-                this implementation relies on multiprocessing, you should not pass non
+                this implementation relies on multiprocessing, you should not pass
                 non picklable arguments to the generator as they can't be passed
                 easily to children processes.
+            initial_epoch: epoch at which to start training
+                (useful for resuming a previous training run)
 
         # Returns
             A `History` object.
@@ -1339,7 +1345,7 @@ class Model(Container):
         ```
         '''
         wait_time = 0.01  # in seconds
-        epoch = 0
+        epoch = initial_epoch
 
         do_validation = bool(validation_data)
         self._make_train_function()
@@ -1395,8 +1401,8 @@ class Model(Container):
             self.validation_data = None
 
         # start generator thread storing batches into a queue
-        data_gen_queue, _stop = generator_queue(generator, max_q_size=max_q_size, nb_worker=nb_worker,
-                                                pickle_safe=pickle_safe)
+        data_gen_queue, _stop, generator_threads = generator_queue(generator, max_q_size=max_q_size, nb_worker=nb_worker,
+                                                                   pickle_safe=pickle_safe)
 
         callback_model.stop_training = False
         while epoch < nb_epoch:
@@ -1430,11 +1436,11 @@ class Model(Container):
                 # build batch logs
                 batch_logs = {}
                 if type(x) is list:
-                    batch_size = len(x[0])
+                    batch_size = x[0].shape[0]
                 elif type(x) is dict:
-                    batch_size = len(list(x.values())[0])
+                    batch_size = list(x.values())[0].shape[0]
                 else:
-                    batch_size = len(x)
+                    batch_size = x.shape[0]
                 batch_logs['batch'] = batch_index
                 batch_logs['size'] = batch_size
                 callbacks.on_batch_begin(batch_index, batch_logs)
@@ -1470,11 +1476,14 @@ class Model(Container):
                     if val_gen:
                         val_outs = self.evaluate_generator(validation_data,
                                                            nb_val_samples,
-                                                           max_q_size=max_q_size)
+                                                           max_q_size=max_q_size,
+                                                           nb_worker=nb_worker,
+                                                           pickle_safe=pickle_safe)
                     else:
                         # no need for try/except because
                         # data has already been validated
                         val_outs = self.evaluate(val_x, val_y,
+                                                 batch_size=batch_size,
                                                  sample_weight=val_sample_weights,
                                                  verbose=0)
                     if type(val_outs) is not list:
@@ -1490,6 +1499,10 @@ class Model(Container):
 
         _stop.set()
         if pickle_safe:
+            # Terminate all daemon processes
+            for p in generator_threads:
+                if p.is_alive():
+                    p.terminate()
             data_gen_queue.close()
         callbacks.on_train_end()
         return self.history
@@ -1508,7 +1521,7 @@ class Model(Container):
             max_q_size: maximum size for the generator queue
             nb_worker: maximum number of processes to spin up when using process based threading
             pickle_safe: if True, use process based threading. Note that because
-                this implementation relies on multiprocessing, you should not pass non
+                this implementation relies on multiprocessing, you should not pass
                 non picklable arguments to the generator as they can't be passed
                 easily to children processes.
 
@@ -1524,8 +1537,8 @@ class Model(Container):
         wait_time = 0.01
         all_outs = []
         weights = []
-        data_gen_queue, _stop = generator_queue(generator, max_q_size=max_q_size, nb_worker=nb_worker,
-                                                pickle_safe=pickle_safe)
+        data_gen_queue, _stop, generator_threads = generator_queue(generator, max_q_size=max_q_size, nb_worker=nb_worker,
+                                                                   pickle_safe=pickle_safe)
 
         while processed_samples < val_samples:
             generator_output = None
@@ -1570,6 +1583,10 @@ class Model(Container):
 
         _stop.set()
         if pickle_safe:
+            # Terminate all daemon processes
+            for p in generator_threads:
+                if p.is_alive():
+                    p.terminate()
             data_gen_queue.close()
         if type(outs) is not list:
             return np.average(np.asarray(all_outs),
@@ -1593,7 +1610,7 @@ class Model(Container):
             max_q_size: maximum size for the generator queue
             nb_worker: maximum number of processes to spin up when using process based threading
             pickle_safe: if True, use process based threading. Note that because
-                this implementation relies on multiprocessing, you should not pass non
+                this implementation relies on multiprocessing, you should not pass
                 non picklable arguments to the generator as they can't be passed
                 easily to children processes.
 
@@ -1605,8 +1622,8 @@ class Model(Container):
         processed_samples = 0
         wait_time = 0.01
         all_outs = []
-        data_gen_queue, _stop = generator_queue(generator, max_q_size=max_q_size, nb_worker=nb_worker,
-                                                pickle_safe=pickle_safe)
+        data_gen_queue, _stop, generator_threads = generator_queue(generator, max_q_size=max_q_size, nb_worker=nb_worker,
+                                                                   pickle_safe=pickle_safe)
 
         while processed_samples < val_samples:
             generator_output = None
@@ -1659,6 +1676,10 @@ class Model(Container):
 
         _stop.set()
         if pickle_safe:
+            # Terminate all daemon processes
+            for p in generator_threads:
+                if p.is_alive():
+                    p.terminate()
             data_gen_queue.close()
         if len(all_outs) == 1:
             return all_outs[0]

keras/initializations.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 import numpy as np
 from . import backend as K
+from .utils.generic_utils import get_from_module
 
 
 def get_fans(shape, dim_ordering='th'):
@@ -20,7 +21,7 @@ def get_fans(shape, dim_ordering='th'):
             fan_in = shape[-2] * receptive_field_size
             fan_out = shape[-1] * receptive_field_size
         else:
-            raise Exception('Invalid dim_ordering: ' + dim_ordering)
+            raise ValueError('Invalid dim_ordering: ' + dim_ordering)
     else:
         # no specific assumptions
         fan_in = np.sqrt(np.prod(shape))
@@ -29,13 +30,11 @@ def get_fans(shape, dim_ordering='th'):
 
 
 def uniform(shape, scale=0.05, name=None):
-    return K.variable(np.random.uniform(low=-scale, high=scale, size=shape),
-                      name=name)
+    return K.random_uniform_variable(shape, -scale, scale, name=name)
 
 
 def normal(shape, scale=0.05, name=None):
-    return K.variable(np.random.normal(loc=0.0, scale=scale, size=shape),
-                      name=name)
+    return K.random_normal_variable(shape, 0.0, scale, name=name)
 
 
 def lecun_uniform(shape, name=None, dim_ordering='th'):
@@ -103,7 +102,6 @@ def one(shape, name=None):
     return K.ones(shape, name=name)
 
 
-from .utils.generic_utils import get_from_module
 def get(identifier, **kwargs):
     return get_from_module(identifier, globals(),
                            'initialization', kwargs=kwargs)

keras/layers/__init__.py

@@ -10,3 +10,4 @@ from .embeddings import *
 from .noise import *
 from .advanced_activations import *
 from .wrappers import *
+from .convolutional_recurrent import *

keras/layers/advanced_activations.py

@@ -107,12 +107,10 @@ class ELU(Layer):
         super(ELU, self).__init__(**kwargs)
 
     def call(self, x, mask=None):
-        pos = K.relu(x)
-        neg = (x - abs(x)) * 0.5
-        return pos + self.alpha * (K.exp(neg) - 1.)
+        return K.elu(x, self.alpha)
 
     def get_config(self):
-        config = {'alpha': self.alpha}
+        config = {'alpha': float(self.alpha)}
         base_config = super(ELU, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
@@ -161,8 +159,8 @@ class ParametricSoftplus(Layer):
         return K.softplus(self.betas * x) * self.alphas
 
     def get_config(self):
-        config = {'alpha_init': self.alpha_init,
-                  'beta_init': self.beta_init}
+        config = {'alpha_init': float(self.alpha_init),
+                  'beta_init': float(self.beta_init)}
         base_config = super(ParametricSoftplus, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
@@ -195,7 +193,7 @@ class ThresholdedReLU(Layer):
         return x * K.cast(x > self.theta, K.floatx())
 
     def get_config(self):
-        config = {'theta': self.theta}
+        config = {'theta': float(self.theta)}
         base_config = super(ThresholdedReLU, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 

keras/layers/convolutional_recurrent.py

@@ -0,0 +1,516 @@
+from .. import backend as K
+from .. import activations, initializations, regularizers
+
+import numpy as np
+from ..engine import Layer, InputSpec
+from ..utils.np_utils import conv_output_length
+import warnings
+
+
+class ConvRecurrent2D(Layer):
+    '''Abstract base class for convolutional recurrent layers.
+    Do not use in a model -- it's not a functional layer!
+
+    ConvLSTM2D
+    follow the specifications of this class and accept
+    the keyword arguments listed below.
+
+    # Input shape
+        5D tensor with shape `(nb_samples, timesteps, channels, rows, cols)`.
+
+    # Output shape
+        - if `return_sequences`: 5D tensor with shape
+            `(nb_samples, timesteps, channels, rows, cols)`.
+        - else, 4D tensor with shape `(nb_samples, channels, rows, cols)`.
+
+    # Arguments
+        weights: list of numpy arrays to set as initial weights.
+            The list should have 3 elements, of shapes:
+            `[(input_dim, nb_filter), (nb_filter, nb_filter), (nb_filter,)]`.
+        return_sequences: Boolean. Whether to return the last output
+            in the output sequence, or the full sequence.
+        go_backwards: Boolean (default False).
+            If True, rocess the input sequence backwards.
+        stateful: Boolean (default False). If True, the last state
+            for each sample at index i in a batch will be used as initial
+            state for the sample of index i in the following batch.
+        nb_filter: Number of convolution filters to use.
+        nb_row: Number of rows in the convolution kernel.
+        nb_col: Number of columns in the convolution kernel.
+            is required when using this layer as the first layer in a model.
+        input_shape: input_shape
+
+    # Masking
+        This layer supports masking for input data with a variable number
+        of timesteps. To introduce masks to your data,
+        use an [Embedding](embeddings.md) layer with the `mask_zero` parameter
+        set to `True`.
+        **Note:** for the time being, masking is only supported with Theano.
+
+    # TensorFlow warning
+        For the time being, when using the TensorFlow backend,
+        the number of timesteps used must be specified in your model.
+        Make sure to pass an `input_length` int argument to your
+        recurrent layer (if it comes first in your model),
+        or to pass a complete `input_shape` argument to the first layer
+        in your model otherwise.
+
+
+    # Note on using statefulness in RNNs
+        You can set RNN layers to be 'stateful', which means that the states
+        computed for the samples in one batch will be reused as initial states
+        for the samples in the next batch.
+        This assumes a one-to-one mapping between
+        samples in different successive batches.
+
+        To enable statefulness:
+            - specify `stateful=True` in the layer constructor.
+            - specify a fixed batch size for your model, by passing
+                a `batch_input_size=(...)` to the first layer in your model.
+                This is the expected shape of your inputs *including the batch
+                size*.
+                It should be a tuple of integers, e.g. `(32, 10, 100)`.
+
+        To reset the states of your model, call `.reset_states()` on either
+        a specific layer, or on your entire model.
+    '''
+
+    def __init__(self, weights=None, nb_row=None, nb_col=None, nb_filter=None,
+                 return_sequences=False, go_backwards=False, stateful=False,
+                 dim_ordering=None, **kwargs):
+        self.return_sequences = return_sequences
+        self.go_backwards = go_backwards
+        self.stateful = stateful
+        self.initial_weights = weights
+        self.nb_row = nb_row
+        self.nb_col = nb_col
+        self.nb_filter = nb_filter
+        self.dim_ordering = dim_ordering
+        self.input_spec = [InputSpec(ndim=5)]
+
+        super(ConvRecurrent2D, self).__init__(**kwargs)
+
+    def compute_mask(self, input, mask):
+        if self.return_sequences:
+            return mask
+        else:
+            return None
+
+    def get_output_shape_for(self, input_shape):
+
+        if self.dim_ordering == 'th':
+            rows = input_shape[3]
+            cols = input_shape[4]
+        elif self.dim_ordering == 'tf':
+            rows = input_shape[2]
+            cols = input_shape[3]
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+        rows = conv_output_length(rows, self.nb_row,
+                                  self.border_mode, self.subsample[0])
+        cols = conv_output_length(cols, self.nb_col,
+                                  self.border_mode, self.subsample[1])
+
+        if self.return_sequences:
+            if self.dim_ordering == 'th':
+                return (input_shape[0], input_shape[1],
+                        self.nb_filter, rows, cols)
+            elif self.dim_ordering == 'tf':
+                return (input_shape[0], input_shape[1],
+                        rows, cols, self.nb_filter)
+            else:
+                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+        else:
+            if self.dim_ordering == 'th':
+                return (input_shape[0], self.nb_filter, rows, cols)
+            elif self.dim_ordering == 'tf':
+                return (input_shape[0], rows, cols, self.nb_filter)
+            else:
+                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+    def step(self, x, states):
+        raise NotImplementedError
+
+    def get_constants(self, X, train=False):
+        return None
+
+    def get_initial_states(self, X):
+        # (samples, timesteps, row, col, filter)
+        initial_state = K.zeros_like(X)
+        # (samples,row, col, filter)
+        initial_state = K.sum(initial_state, axis=1)
+        initial_state = self.conv_step(initial_state, K.zeros(self.W_shape),
+                                       border_mode=self.border_mode)
+
+        initial_states = [initial_state for _ in range(2)]
+        return initial_states
+
+    def preprocess_input(self, x):
+        return x
+
+    def call(self, x, mask=None):
+        assert K.ndim(x) == 5
+        input_shape = self.input_spec[0].shape
+        unroll = False
+
+        if self.stateful:
+            initial_states = self.states
+        else:
+            initial_states = self.get_initial_states(x)
+
+        constants = self.get_constants(x)
+        preprocessed_input = self.preprocess_input(x)
+
+        last_output, outputs, states = K.rnn(self.step, preprocessed_input,
+                                             initial_states,
+                                             go_backwards=self.go_backwards,
+                                             mask=mask,
+                                             constants=constants,
+                                             unroll=unroll,
+                                             input_length=input_shape[1])
+        if self.stateful:
+            self.updates = []
+            for i in range(len(states)):
+                self.updates.append((self.states[i], states[i]))
+
+        if self.return_sequences:
+            return outputs
+        else:
+            return last_output
+
+    def get_config(self):
+        config = {'return_sequences': self.return_sequences,
+                  'go_backwards': self.go_backwards,
+                  'stateful': self.stateful}
+        if self.stateful:
+            config['batch_input_shape'] = self.input_spec[0].shape
+
+        base_config = super(ConvRecurrent2D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
+class ConvLSTM2D(ConvRecurrent2D):
+    '''Convolutional LSTM.
+
+    # Input shape
+        - if dim_ordering='th'
+            5D tensor with shape:
+            `(samples,time, channels, rows, cols)`
+        - if dim_ordering='tf'
+            5D tensor with shape:
+            `(samples,time, rows, cols, channels)`
+
+     # Output shape
+        - if `return_sequences`
+             - if dim_ordering='th'
+                5D tensor with shape:
+                `(samples, time, nb_filter, output_row, output_col)`
+             - if dim_ordering='tf'
+                5D tensor with shape:
+                `(samples, time, output_row, output_col, nb_filter)`
+        - else
+            - if dim_ordering ='th'
+                4D tensor with shape:
+                `(samples, nb_filter, output_row, output_col)`
+            - if dim_ordering='tf'
+                4D tensor with shape:
+                `(samples, output_row, output_col, nb_filter)`
+
+        where o_row and o_col depend on the shape of the filter and
+        the border_mode
+
+        # Arguments
+            nb_filter: Number of convolution filters to use.
+            nb_row: Number of rows in the convolution kernel.
+            nb_col: Number of columns in the convolution kernel.
+            border_mode: 'valid' or 'same'.
+            sub_sample: tuple of length 2. Factor by which to subsample output.
+                Also called strides elsewhere.
+            dim_ordering: 'tf' if the feature are at the last dimension or 'th'
+            stateful : Boolean (default False). If True, the last state
+                for each sample at index i in a batch will be used as initial
+                state for the sample of index i in the following batch.
+            init: weight initialization function.
+                Can be the name of an existing function (str),
+                or a Theano function
+                (see: [initializations](../initializations.md)).
+            inner_init: initialization function of the inner cells.
+            forget_bias_init: initialization function for the bias of the
+            forget gate.
+                [Jozefowicz et al.](http://www.jmlr.org/proceedings/papers/v37/jozefowicz15.pdf)
+                recommend initializing with ones.
+            activation: activation function.
+                Can be the name of an existing function (str),
+                or a Theano function (see: [activations](../activations.md)).
+            inner_activation: activation function for the inner cells.
+
+    # References
+        - [Convolutional LSTM Network: A Machine Learning Approach for
+        Precipitation Nowcasting](http://arxiv.org/pdf/1506.04214v1.pdf)
+        The current implementation does not include the feedback loop on the
+        cells output
+    '''
+    def __init__(self, nb_filter, nb_row, nb_col,
+                 init='glorot_uniform', inner_init='orthogonal',
+                 forget_bias_init='one', activation='tanh',
+                 inner_activation='hard_sigmoid',
+                 dim_ordering='default',
+                 border_mode='valid', subsample=(1, 1),
+                 W_regularizer=None, U_regularizer=None, b_regularizer=None,
+                 dropout_W=0., dropout_U=0., **kwargs):
+
+        if dim_ordering == 'default':
+            dim_ordering = K.image_dim_ordering()
+        if dim_ordering not in {'tf', 'th'}:
+            raise ValueError('dim_ordering must be in {tf,th}', dim_ordering)
+        self.nb_filter = nb_filter
+        self.nb_row = nb_row
+        self.nb_col = nb_col
+        self.init = initializations.get(init)
+        self.inner_init = initializations.get(inner_init)
+        self.forget_bias_init = initializations.get(forget_bias_init)
+        self.activation = activations.get(activation)
+        self.inner_activation = activations.get(inner_activation)
+        self.border_mode = border_mode
+        self.subsample = subsample
+
+        if dim_ordering == 'th':
+            warnings.warn('Be carefull if used with convolution3D layers:\n'
+                          'th in convolution 3D corresponds to '
+                          '(samples, channels, conv_dim1, conv_dim2,'
+                          'conv_dim3)\n'
+                          'while for this network it corresponds to: '
+                          '(samples, time, channels, rows, cols)')
+        self.dim_ordering = dim_ordering
+
+        kwargs['nb_filter'] = nb_filter
+        kwargs['nb_row'] = nb_row
+        kwargs['nb_col'] = nb_col
+        kwargs['dim_ordering'] = dim_ordering
+
+        self.W_regularizer = regularizers.get(W_regularizer)
+        self.U_regularizer = regularizers.get(U_regularizer)
+        self.b_regularizer = regularizers.get(b_regularizer)
+        self.dropout_W, self.dropout_U = dropout_W, dropout_U
+        if self.dropout_W or self.dropout_U:
+            self.uses_learning_phase = True
+
+        super(ConvLSTM2D, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+        self.input_spec = [InputSpec(shape=input_shape)]
+
+        if self.dim_ordering == 'th':
+            stack_size = input_shape[2]
+            self.W_shape = (self.nb_filter, stack_size,
+                            self.nb_row, self.nb_col)
+        elif self.dim_ordering == 'tf':
+            stack_size = input_shape[4]
+            self.W_shape = (self.nb_row, self.nb_col,
+                            stack_size, self.nb_filter)
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+        if self.dim_ordering == 'th':
+            self.W_shape1 = (self.nb_filter, self.nb_filter,
+                             self.nb_row, self.nb_col)
+        elif self.dim_ordering == 'tf':
+            self.W_shape1 = (self.nb_row, self.nb_col,
+                             self.nb_filter, self.nb_filter)
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+        if self.stateful:
+            self.reset_states()
+        else:
+            # initial states: 2 all-zero tensor of shape (nb_filter)
+            self.states = [None, None, None, None]
+
+        self.W_i = self.init(self.W_shape, name='{}_W_i'.format(self.name))
+        self.U_i = self.inner_init(self.W_shape1,
+                                   name='{}_U_i'.format(self.name))
+        self.b_i = K.zeros((self.nb_filter,), name='{}_b_i'.format(self.name))
+
+        self.W_f = self.init(self.W_shape, name='{}_W_f'.format(self.name))
+        self.U_f = self.inner_init(self.W_shape1,
+                                   name='{}_U_f'.format(self.name))
+        self.b_f = self.forget_bias_init((self.nb_filter,),
+                                         name='{}_b_f'.format(self.name))
+
+        self.W_c = self.init(self.W_shape, name='{}_W_c'.format(self.name))
+        self.U_c = self.inner_init(self.W_shape1,
+                                   name='{}_U_c'.format(self.name))
+        self.b_c = K.zeros((self.nb_filter,), name='{}_b_c'.format(self.name))
+
+        self.W_o = self.init(self.W_shape, name='{}_W_o'.format(self.name))
+        self.U_o = self.inner_init(self.W_shape1,
+                                   name='{}_U_o'.format(self.name))
+        self.b_o = K.zeros((self.nb_filter,), name='{}_b_o'.format(self.name))
+
+        self.trainable_weights = [self.W_i, self.U_i, self.b_i,
+                                  self.W_c, self.U_c, self.b_c,
+                                  self.W_f, self.U_f, self.b_f,
+                                  self.W_o, self.U_o, self.b_o]
+
+        self.W = K.concatenate([self.W_i, self.W_f, self.W_c, self.W_o])
+        self.U = K.concatenate([self.U_i, self.U_f, self.U_c, self.U_o])
+        self.b = K.concatenate([self.b_i, self.b_f, self.b_c, self.b_o])
+
+        self.regularizers = []
+        if self.W_regularizer:
+            self.W_regularizer.set_param(self.W)
+            self.regularizers.append(self.W_regularizer)
+        if self.U_regularizer:
+            self.U_regularizer.set_param(self.U)
+            self.regularizers.append(self.U_regularizer)
+        if self.b_regularizer:
+            self.b_regularizer.set_param(self.b)
+            self.regularizers.append(self.b_regularizer)
+
+        if self.initial_weights is not None:
+            self.set_weights(self.initial_weights)
+            del self.initial_weights
+        self.built = True
+
+    def reset_states(self):
+        assert self.stateful, 'Layer must be stateful.'
+        input_shape = self.input_spec[0].shape
+        output_shape = self.get_output_shape_for(input_shape)
+        if not input_shape[0]:
+            raise Exception('If a RNN is stateful, a complete ' +
+                            'input_shape must be provided ' +
+                            '(including batch size).')
+
+        if self.return_sequences:
+            out_row, out_col, out_filter = output_shape[2:]
+        else:
+            out_row, out_col, out_filter = output_shape[1:]
+
+        if hasattr(self, 'states'):
+            K.set_value(self.states[0],
+                        np.zeros((input_shape[0],
+                                  out_row, out_col, out_filter)))
+            K.set_value(self.states[1],
+                        np.zeros((input_shape[0],
+                                  out_row, out_col, out_filter)))
+        else:
+            self.states = [K.zeros((input_shape[0],
+                                    out_row, out_col, out_filter)),
+                           K.zeros((input_shape[0],
+                                    out_row, out_col, out_filter))]
+
+    def conv_step(self, x, W, b=None, border_mode='valid'):
+        input_shape = self.input_spec[0].shape
+
+        conv_out = K.conv2d(x, W, strides=self.subsample,
+                            border_mode=border_mode,
+                            dim_ordering=self.dim_ordering,
+                            image_shape=(input_shape[0],
+                                         input_shape[2],
+                                         input_shape[3],
+                                         input_shape[4]),
+                            filter_shape=self.W_shape)
+        if b:
+            if self.dim_ordering == 'th':
+                conv_out = conv_out + K.reshape(b, (1, self.nb_filter, 1, 1))
+            elif self.dim_ordering == 'tf':
+                conv_out = conv_out + K.reshape(b, (1, 1, 1, self.nb_filter))
+            else:
+                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+        return conv_out
+
+    def conv_step_hidden(self, x, W, border_mode='valid'):
+        # This new function was defined because the
+        # image shape must be hardcoded
+        input_shape = self.input_spec[0].shape
+        output_shape = self.get_output_shape_for(input_shape)
+        if self.return_sequences:
+            out_row, out_col, out_filter = output_shape[2:]
+        else:
+            out_row, out_col, out_filter = output_shape[1:]
+
+        conv_out = K.conv2d(x, W, strides=(1, 1),
+                            border_mode=border_mode,
+                            dim_ordering=self.dim_ordering,
+                            image_shape=(input_shape[0],
+                                         out_row, out_col,
+                                         out_filter),
+                            filter_shape=self.W_shape1)
+
+        return conv_out
+
+    def step(self, x, states):
+        assert len(states) == 4
+        h_tm1 = states[0]
+        c_tm1 = states[1]
+        B_U = states[2]
+        B_W = states[3]
+
+        x_i = self.conv_step(x * B_W[0], self.W_i, self.b_i,
+                             border_mode=self.border_mode)
+        x_f = self.conv_step(x * B_W[1], self.W_f, self.b_f,
+                             border_mode=self.border_mode)
+        x_c = self.conv_step(x * B_W[2], self.W_c, self.b_c,
+                             border_mode=self.border_mode)
+        x_o = self.conv_step(x * B_W[3], self.W_o, self.b_o,
+                             border_mode=self.border_mode)
+
+        # U : from nb_filter to nb_filter
+        # Same because must be stable in the output space
+        h_i = self.conv_step_hidden(h_tm1 * B_U[0], self.U_i,
+                                    border_mode='same')
+        h_f = self.conv_step_hidden(h_tm1 * B_U[1], self.U_f,
+                                    border_mode='same')
+        h_c = self.conv_step_hidden(h_tm1 * B_U[2], self.U_c,
+                                    border_mode='same')
+        h_o = self.conv_step_hidden(h_tm1 * B_U[3], self.U_o,
+                                    border_mode='same')
+
+        i = self.inner_activation(x_i + h_i)
+        f = self.inner_activation(x_f + h_f)
+        c = f * c_tm1 + i * self.activation(x_c + h_c)
+        o = self.inner_activation(x_o + h_o)
+        h = o * self.activation(c)
+
+        return h, [h, c]
+
+    def get_constants(self, x):
+        constants = []
+        if 0 < self.dropout_U < 1:
+            ones = K.zeros_like(x)
+            ones = K.sum(ones, axis=1)
+            ones = self.conv_step(ones, K.zeros(self.W_shape),
+                                  border_mode=self.border_mode)
+            ones = ones + 1
+            B_U = [K.in_train_phase(K.dropout(ones, self.dropout_U), ones)
+                   for _ in range(4)]
+            constants.append(B_U)
+        else:
+            constants.append([K.cast_to_floatx(1.) for _ in range(4)])
+
+        if 0 < self.dropout_W < 1:
+            ones = K.zeros_like(x)
+            ones = K.sum(ones, axis=1)
+            ones = ones + 1
+            B_W = [K.in_train_phase(K.dropout(ones, self.dropout_W), ones)
+                   for _ in range(4)]
+            constants.append(B_W)
+        else:
+            constants.append([K.cast_to_floatx(1.) for _ in range(4)])
+        return constants
+
+    def get_config(self):
+        config = {'nb_filter': self.nb_filter,
+                  'nb_row': self.nb_row,
+                  'nb_col': self.nb_col,
+                  'init': self.init.__name__,
+                  'inner_init': self.inner_init.__name__,
+                  'forget_bias_init': self.forget_bias_init.__name__,
+                  'activation': self.activation.__name__,
+                  'dim_ordering': self.dim_ordering,
+                  'border_mode': self.border_mode,
+                  'inner_activation': self.inner_activation.__name__}
+        base_config = super(ConvLSTM2D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))

keras/layers/core.py

@@ -7,14 +7,13 @@ import numpy as np
 import copy
 import inspect
 import types as python_types
-import marshal
-import sys
 import warnings
 
 from .. import backend as K
 from .. import activations, initializations, regularizers, constraints
 from ..engine import InputSpec, Layer, Merge
 from ..regularizers import ActivityRegularizer
+from ..utils.generic_utils import func_dump, func_load
 
 
 class Masking(Layer):
@@ -82,9 +81,13 @@ class Dropout(Layer):
         self.supports_masking = True
         super(Dropout, self).__init__(**kwargs)
 
+    def _get_noise_shape(self, x):
+        return None
+
     def call(self, x, mask=None):
         if 0. < self.p < 1.:
-            x = K.in_train_phase(K.dropout(x, level=self.p), x)
+            noise_shape = self._get_noise_shape(x)
+            x = K.in_train_phase(K.dropout(x, self.p, noise_shape), x)
         return x
 
     def get_config(self):
@@ -93,6 +96,132 @@ class Dropout(Layer):
         return dict(list(base_config.items()) + list(config.items()))
 
 
+class SpatialDropout1D(Dropout):
+    '''This version performs the same function as Dropout, however it drops
+    entire 1D feature maps instead of individual elements. If adjacent frames
+    within feature maps are strongly correlated (as is normally the case in
+    early convolution layers) then regular dropout will not regularize the
+    activations and will otherwise just result in an effective learning rate
+    decrease. In this case, SpatialDropout1D will help promote independence
+    between feature maps and should be used instead.
+
+    # Arguments
+        p: float between 0 and 1. Fraction of the input units to drop.
+
+    # Input shape
+        3D tensor with shape:
+        `(samples, timesteps, channels)`
+
+    # Output shape
+        Same as input
+
+    # References
+        - [Efficient Object Localization Using Convolutional Networks](https://arxiv.org/pdf/1411.4280.pdf)
+    '''
+    def __init__(self, p, **kwargs):
+        super(SpatialDropout1D, self).__init__(p, **kwargs)
+
+    def _get_noise_shape(self, x):
+        input_shape = K.shape(x)
+        noise_shape = (input_shape[0], 1, input_shape[2])
+        return noise_shape
+    
+    
+class SpatialDropout2D(Dropout):
+    '''This version performs the same function as Dropout, however it drops
+    entire 2D feature maps instead of individual elements. If adjacent pixels
+    within feature maps are strongly correlated (as is normally the case in
+    early convolution layers) then regular dropout will not regularize the
+    activations and will otherwise just result in an effective learning rate
+    decrease. In this case, SpatialDropout2D will help promote independence
+    between feature maps and should be used instead.
+
+    # Arguments
+        p: float between 0 and 1. Fraction of the input units to drop.
+        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
+            (the depth) is at index 1, in 'tf' mode is it at index 3.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "tf".
+
+    # Input shape
+        4D tensor with shape:
+        `(samples, channels, rows, cols)` if dim_ordering='th'
+        or 4D tensor with shape:
+        `(samples, rows, cols, channels)` if dim_ordering='tf'.
+
+    # Output shape
+        Same as input
+
+    # References
+        - [Efficient Object Localization Using Convolutional Networks](https://arxiv.org/pdf/1411.4280.pdf)
+    '''
+    def __init__(self, p, dim_ordering='default', **kwargs):
+        if dim_ordering == 'default':
+            dim_ordering = K.image_dim_ordering()
+        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
+        self.dim_ordering = dim_ordering
+        super(SpatialDropout2D, self).__init__(p, **kwargs)
+
+    def _get_noise_shape(self, x):
+        input_shape = K.shape(x)
+        if self.dim_ordering == 'th':
+            noise_shape = (input_shape[0], input_shape[1], 1, 1)
+        elif self.dim_ordering == 'tf':
+            noise_shape = (input_shape[0], 1, 1, input_shape[3])
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+        return noise_shape
+
+
+class SpatialDropout3D(Dropout):
+    '''This version performs the same function as Dropout, however it drops
+    entire 3D feature maps instead of individual elements. If adjacent voxels
+    within feature maps are strongly correlated (as is normally the case in
+    early convolution layers) then regular dropout will not regularize the
+    activations and will otherwise just result in an effective learning rate
+    decrease. In this case, SpatialDropout3D will help promote independence
+    between feature maps and should be used instead.
+
+    # Arguments
+        p: float between 0 and 1. Fraction of the input units to drop.
+        dim_ordering: 'th' or 'tf'.
+            In 'th' mode, the channels dimension (the depth)
+            is at index 1, in 'tf' mode is it at index 4.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "tf".
+
+    # Input shape
+        5D tensor with shape:
+        `(samples, channels, dim1, dim2, dim3)` if dim_ordering='th'
+        or 5D tensor with shape:
+        `(samples, dim1, dim2, dim3, channels)` if dim_ordering='tf'.
+
+    # Output shape
+        Same as input
+
+    # References
+        - [Efficient Object Localization Using Convolutional Networks](https://arxiv.org/pdf/1411.4280.pdf)
+    '''
+    def __init__(self, p, dim_ordering='default', **kwargs):
+        if dim_ordering == 'default':
+            dim_ordering = K.image_dim_ordering()
+        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
+        self.dim_ordering = dim_ordering
+        super(SpatialDropout3D, self).__init__(p, **kwargs)
+
+    def _get_noise_shape(self, x):
+        input_shape = K.shape(x)
+        if self.dim_ordering == 'th':
+            noise_shape = (input_shape[0], input_shape[1], 1, 1, 1)
+        elif self.dim_ordering == 'tf':
+            noise_shape = (input_shape[0], 1, 1, 1, input_shape[4])
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+        return noise_shape
+
+
 class Activation(Layer):
     '''Applies an activation function to an output.
 
@@ -385,16 +514,16 @@ class Lambda(Layer):
 
     # Arguments
         function: The function to be evaluated.
-            Takes one argument: the output of previous layer
+            Takes input tensor as first argument.
         output_shape: Expected output shape from function.
             Can be a tuple or function.
-            If a tuple, it only specifies the first dimension onward; 
+            If a tuple, it only specifies the first dimension onward;
                  sample dimension is assumed either the same as the input:
                  `output_shape = (input_shape[0], ) + output_shape`
                  or, the input is `None` and the sample dimension is also `None`:
                  `output_shape = (None, ) + output_shape`
-            If a function, it specifies the entire shape as a function of 
-                 the input shape: `output_shape = f(input_shape)`
+            If a function, it specifies the entire shape as a function of the
+            input shape: `output_shape = f(input_shape)`
         arguments: optional dictionary of keyword arguments to be passed
             to the function.
 
@@ -439,7 +568,10 @@ class Lambda(Layer):
             # otherwise, we default to the input shape
             return input_shape
         elif type(self._output_shape) in {tuple, list}:
-            nb_samples = input_shape[0] if input_shape else None
+            if type(input_shape) is list:
+                nb_samples = input_shape[0][0]
+            else:
+                nb_samples = input_shape[0] if input_shape else None
             return (nb_samples,) + tuple(self._output_shape)
         else:
             shape = self._output_shape(input_shape)
@@ -455,23 +587,15 @@ class Lambda(Layer):
         return self.function(x, **arguments)
 
     def get_config(self):
-        py3 = sys.version_info[0] == 3
-
         if isinstance(self.function, python_types.LambdaType):
-            if py3:
-                function = marshal.dumps(self.function.__code__).decode('raw_unicode_escape')
-            else:
-                function = marshal.dumps(self.function.func_code).decode('raw_unicode_escape')
+            function = func_dump(self.function)
             function_type = 'lambda'
         else:
             function = self.function.__name__
             function_type = 'function'
 
         if isinstance(self._output_shape, python_types.LambdaType):
-            if py3:
-                output_shape = marshal.dumps(self._output_shape.__code__).decode('raw_unicode_escape')
-            else:
-                output_shape = marshal.dumps(self._output_shape.func_code).decode('raw_unicode_escape')
+            output_shape = func_dump(self._output_shape)
             output_shape_type = 'lambda'
         elif callable(self._output_shape):
             output_shape = self._output_shape.__name__
@@ -494,8 +618,7 @@ class Lambda(Layer):
         if function_type == 'function':
             function = globals()[config['function']]
         elif function_type == 'lambda':
-            function = marshal.loads(config['function'].encode('raw_unicode_escape'))
-            function = python_types.FunctionType(function, globals())
+            function = func_load(config['function'], globs=globals())
         else:
             raise Exception('Unknown function type: ' + function_type)
 
@@ -503,8 +626,7 @@ class Lambda(Layer):
         if output_shape_type == 'function':
             output_shape = globals()[config['output_shape']]
         elif output_shape_type == 'lambda':
-            output_shape = marshal.loads(config['output_shape'].encode('raw_unicode_escape'))
-            output_shape = python_types.FunctionType(output_shape, globals())
+            output_shape = func_load(config['output_shape'], globs=globals())
         else:
             output_shape = config['output_shape']
 
@@ -570,7 +692,8 @@ class Dense(Layer):
     # Output shape
         2D tensor with shape: `(nb_samples, output_dim)`.
     '''
-    def __init__(self, output_dim, init='glorot_uniform', activation='linear', weights=None,
+    def __init__(self, output_dim, init='glorot_uniform',
+                 activation=None, weights=None,
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                  W_constraint=None, b_constraint=None,
                  bias=True, input_dim=None, **kwargs):
@@ -631,6 +754,7 @@ class Dense(Layer):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def call(self, x, mask=None):
         output = K.dot(x, self.W)
@@ -799,6 +923,7 @@ class MaxoutDense(Layer):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def get_output_shape_for(self, input_shape):
         assert input_shape and len(input_shape) == 2
@@ -871,7 +996,7 @@ class Highway(Layer):
         - [Highway Networks](http://arxiv.org/pdf/1505.00387v2.pdf)
     '''
     def __init__(self, init='glorot_uniform', transform_bias=-2,
-                 activation='linear', weights=None,
+                 activation=None, weights=None,
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                  W_constraint=None, b_constraint=None,
                  bias=True, input_dim=None, **kwargs):
@@ -936,6 +1061,7 @@ class Highway(Layer):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def call(self, x, mask=None):
         y = K.dot(x, self.W_carry)
@@ -1014,7 +1140,7 @@ class TimeDistributedDense(Layer):
     '''
 
     def __init__(self, output_dim,
-                 init='glorot_uniform', activation='linear', weights=None,
+                 init='glorot_uniform', activation=None, weights=None,
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                  W_constraint=None, b_constraint=None,
                  bias=True, input_dim=None, input_length=None, **kwargs):
@@ -1076,6 +1202,7 @@ class TimeDistributedDense(Layer):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def get_output_shape_for(self, input_shape):
         return (input_shape[0], input_shape[1], self.output_dim)

keras/layers/embeddings.py

@@ -110,6 +110,7 @@ class Embedding(Layer):
 
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
+        self.built = True
 
     def compute_mask(self, x, mask=None):
         if not self.mask_zero:
@@ -125,6 +126,8 @@ class Embedding(Layer):
         return (input_shape[0], input_length, self.output_dim)
 
     def call(self, x, mask=None):
+        if K.dtype(x) != 'int32':
+            x = K.cast(x, 'int32')
         if 0. < self.dropout < 1.:
             retain_p = 1. - self.dropout
             B = K.random_binomial((self.input_dim,), p=retain_p) * (1. / retain_p)

keras/layers/local.py

@@ -8,14 +8,17 @@ from ..utils.np_utils import conv_output_length
 
 
 class LocallyConnected1D(Layer):
-    '''LocallyConnected1D layer works almost the same as Convolution1D layer,
-    except that weights are unshared, that is, a different set of filters is
-    applied at each different patch of the input. When using this layer as the
-    first layer in a model, either provide the keyword argument `input_dim`
+    '''The `LocallyConnected1D` layer works similarly to
+    the `Convolution1D` layer, except that weights are unshared,
+    that is, a different set of filters is applied at each different patch
+    of the input.
+    When using this layer as the first layer in a model,
+    either provide the keyword argument `input_dim`
     (int, e.g. 128 for sequences of 128-dimensional vectors), or `input_shape`
-    (tuple of integers, e.g. (10, 128) for sequences of 10 vectors of
-    128-dimensional vectors). Also, you will need to fix shape of the previous
-    layer, since the weights can only be defined with determined output shape.
+    (tuple of integers, e.g. `input_shape=(10, 128)`
+    for sequences of 10 vectors of 128-dimensional vectors).
+    Also, note that this layer can only be used with
+    a fully-specified input shape (`None` dimensions not allowed).
 
     # Example
     ```python
@@ -28,6 +31,7 @@ class LocallyConnected1D(Layer):
         model.add(LocallyConnected1D(32, 3))
         # now model.output_shape == (None, 6, 32)
     ```
+
     # Arguments
         nb_filter: Dimensionality of the output.
         filter_length: The extension (spatial or temporal) of each filter.
@@ -62,20 +66,22 @@ class LocallyConnected1D(Layer):
             This argument is required if you are going to connect
             `Flatten` then `Dense` layers upstream
             (without it, the shape of the dense outputs cannot be computed).
+
     # Input shape
         3D tensor with shape: `(samples, steps, input_dim)`.
+
     # Output shape
         3D tensor with shape: `(samples, new_steps, nb_filter)`.
         `steps` value might have changed due to padding.
     '''
     def __init__(self, nb_filter, filter_length,
-                 init='uniform', activation='linear', weights=None,
+                 init='glorot_uniform', activation=None, weights=None,
                  border_mode='valid', subsample_length=1,
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                  W_constraint=None, b_constraint=None,
                  bias=True, input_dim=None, input_length=None, **kwargs):
         if border_mode != 'valid':
-            raise Exception('Invalid border mode for Convolution2D '
+            raise Exception('Invalid border mode for LocallyConnected1D '
                             '(only "valid" is supported):', border_mode)
         self.nb_filter = nb_filter
         self.filter_length = filter_length
@@ -133,6 +139,7 @@ class LocallyConnected1D(Layer):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def get_output_shape_for(self, input_shape):
         length = conv_output_length(input_shape[1],
@@ -180,14 +187,16 @@ class LocallyConnected1D(Layer):
 
 
 class LocallyConnected2D(Layer):
-    '''LocallyConnected2D layer works almost the same as Convolution2D layer,
-    except that weights are unshared, that is, a different set of filters is
-    applied at each different patch of the input. When using this layer as the
+    '''The `LocallyConnected2D` layer works similarly
+    to the `Convolution2D` layer, except that weights are unshared,
+    that is, a different set of filters is applied at each
+    different patch of the input.
+    When using this layer as the
     first layer in a model, provide the keyword argument `input_shape` (tuple
     of integers, does not include the sample axis), e.g.
-    `input_shape=(3, 128, 128)` for 128x128 RGB pictures. Also, you will need
-    to fix shape of the previous layer, since the weights can only be defined
-    with determined output shape.
+    `input_shape=(3, 128, 128)` for 128x128 RGB pictures.
+    Also, note that this layer can only be used with
+    a fully-specified input shape (`None` dimensions not allowed).
 
     # Examples
     ```python
@@ -249,14 +258,16 @@ class LocallyConnected2D(Layer):
         `rows` and `cols` values might have changed due to padding.
     '''
     def __init__(self, nb_filter, nb_row, nb_col,
-                 init='glorot_uniform', activation='linear', weights=None,
+                 init='glorot_uniform', activation=None, weights=None,
                  border_mode='valid', subsample=(1, 1),
-                 dim_ordering=K.image_dim_ordering(),
+                 dim_ordering='default',
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                  W_constraint=None, b_constraint=None,
                  bias=True, **kwargs):
+        if dim_ordering == 'default':
+            dim_ordering = K.image_dim_ordering()
         if border_mode != 'valid':
-            raise Exception('Invalid border mode for Convolution2D '
+            raise Exception('Invalid border mode for LocallyConnected2D '
                             '(only "valid" is supported):', border_mode)
         self.nb_filter = nb_filter
         self.nb_row = nb_row
@@ -323,6 +334,7 @@ class LocallyConnected2D(Layer):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def get_output_shape_for(self, input_shape):
         if self.dim_ordering == 'th':

keras/layers/noise.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 from ..engine import Layer
 from .. import backend as K
+import numpy as np
 
 
 class GaussianNoise(Layer):
@@ -71,7 +72,7 @@ class GaussianDropout(Layer):
     def call(self, x, mask=None):
         if 0 < self.p < 1:
             noise_x = x * K.random_normal(shape=K.shape(x), mean=1.0,
-                                          std=K.sqrt(self.p / (1.0 - self.p)))
+                                          std=np.sqrt(self.p / (1.0 - self.p)))
             return K.in_train_phase(noise_x, x)
         return x
 

keras/layers/normalization.py

@@ -1,5 +1,5 @@
 from ..engine import Layer, InputSpec
-from .. import initializations
+from .. import initializations, regularizers
 from .. import backend as K
 
 
@@ -35,6 +35,7 @@ class BatchNormalization(Layer):
         weights: Initialization weights.
             List of 2 Numpy arrays, with shapes:
             `[(input_shape,), (input_shape,)]`
+            Note that the order of this list is [gamma, beta, mean, std]
         beta_init: name of initialization function for shift parameter
             (see [initializations](../initializations.md)), or alternatively,
             Theano/TensorFlow function to use for weights initialization.
@@ -43,6 +44,10 @@ class BatchNormalization(Layer):
             [initializations](../initializations.md)), or alternatively,
             Theano/TensorFlow function to use for weights initialization.
             This parameter is only relevant if you don't pass a `weights` argument.
+        gamma_regularizer: instance of [WeightRegularizer](../regularizers.md)
+            (eg. L1 or L2 regularization), applied to the gamma vector.
+        beta_regularizer: instance of [WeightRegularizer](../regularizers.md),
+            applied to the beta vector.
 
     # Input shape
         Arbitrary. Use the keyword argument `input_shape`
@@ -53,10 +58,11 @@ class BatchNormalization(Layer):
         Same shape as input.
 
     # References
-        - [Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift](http://jmlr.org/proceedings/papers/v37/ioffe15.html)
+        - [Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift](http://jmlr.org/proceedings/papers/v37/ioffe15.pdf)
     '''
-    def __init__(self, epsilon=1e-6, mode=0, axis=-1, momentum=0.9,
-                 weights=None, beta_init='zero', gamma_init='one', **kwargs):
+    def __init__(self, epsilon=1e-5, mode=0, axis=-1, momentum=0.99,
+                 weights=None, beta_init='zero', gamma_init='one',
+                 gamma_regularizer=None, beta_regularizer=None, **kwargs):
         self.supports_masking = True
         self.beta_init = initializations.get(beta_init)
         self.gamma_init = initializations.get(gamma_init)
@@ -64,6 +70,8 @@ class BatchNormalization(Layer):
         self.mode = mode
         self.axis = axis
         self.momentum = momentum
+        self.gamma_regularizer = regularizers.get(gamma_regularizer)
+        self.beta_regularizer = regularizers.get(beta_regularizer)
         self.initial_weights = weights
         if self.mode == 0:
             self.uses_learning_phase = True
@@ -77,6 +85,15 @@ class BatchNormalization(Layer):
         self.beta = self.beta_init(shape, name='{}_beta'.format(self.name))
         self.trainable_weights = [self.gamma, self.beta]
 
+        self.regularizers = []
+        if self.gamma_regularizer:
+            self.gamma_regularizer.set_param(self.gamma)
+            self.regularizers.append(self.gamma_regularizer)
+
+        if self.beta_regularizer:
+            self.beta_regularizer.set_param(self.beta)
+            self.regularizers.append(self.beta_regularizer)
+
         self.running_mean = K.zeros(shape,
                                     name='{}_running_mean'.format(self.name))
         self.running_std = K.ones(shape,
@@ -87,7 +104,6 @@ class BatchNormalization(Layer):
             self.set_weights(self.initial_weights)
             del self.initial_weights
         self.built = True
-        self.called_with = None
 
     def call(self, x, mask=None):
         if self.mode == 0 or self.mode == 2:
@@ -99,53 +115,34 @@ class BatchNormalization(Layer):
             broadcast_shape = [1] * len(input_shape)
             broadcast_shape[self.axis] = input_shape[self.axis]
 
-            # # case: train mode (uses stats of the current batch)
-            # mean = K.mean(x, axis=reduction_axes)
-            # brodcast_mean = K.reshape(mean, broadcast_shape)
-            # std = K.mean(K.square(x - brodcast_mean) + self.epsilon, axis=reduction_axes)
-            # std = K.sqrt(std)
-            # brodcast_std = K.reshape(std, broadcast_shape)
-
             if self.mode == 2:
-                x_normed, mean, std = K.normalize_batch_in_training(x, self.gamma, self.beta, reduction_axes, epsilon=self.epsilon)
-                mean_update = self.momentum * self.running_mean + (1 - self.momentum) * mean
-                std_update = self.momentum * self.running_std + (1 - self.momentum) * std
+                x_normed, mean, std = K.normalize_batch_in_training(
+                    x, self.gamma, self.beta, reduction_axes,
+                    epsilon=self.epsilon)
             else:
                 # mode 0
-                if self.called_with not in {None, x}:
-                    raise Exception('You are attempting to share a '
-                                    'same `BatchNormalization` layer across '
-                                    'different data flows. '
-                                    'This is not possible. '
-                                    'You should use `mode=2` in '
-                                    '`BatchNormalization`, which has '
-                                    'a similar behavior but is shareable '
-                                    '(see docs for a description of '
-                                    'the behavior).')
-                self.called_with = x
-                x_normed, mean, std = K.normalize_batch_in_training(x, self.gamma, self.beta, reduction_axes, epsilon=self.epsilon)
-                mean_update = self.momentum * self.running_mean + (1 - self.momentum) * mean
-                std_update = self.momentum * self.running_std + (1 - self.momentum) * std
-                self.updates = [(self.running_mean, mean_update),
-                                (self.running_std, std_update)]
+                x_normed, mean, std = K.normalize_batch_in_training(
+                    x, self.gamma, self.beta, reduction_axes,
+                    epsilon=self.epsilon)
 
-                if sorted(reduction_axes) == range(K.ndim(x))[:-1]:
-                    x_normed_running = K.batch_normalization(x, self.running_mean,
-                                                             self.running_std,
-                                                             self.beta,
-                                                             self.gamma,
-                                                             epsilon=self.epsilon)
+                self.add_updates([K.moving_average_update(self.running_mean, mean, self.momentum),
+                                  K.moving_average_update(self.running_std, std, self.momentum)], x)
+
+                if K.backend() == 'tensorflow' and sorted(reduction_axes) == range(K.ndim(x))[:-1]:
+                    x_normed_running = K.batch_normalization(
+                        x, self.running_mean, self.running_std,
+                        self.beta, self.gamma,
+                        epsilon=self.epsilon)
                 else:
                     # need broadcasting
                     broadcast_running_mean = K.reshape(self.running_mean, broadcast_shape)
                     broadcast_running_std = K.reshape(self.running_std, broadcast_shape)
                     broadcast_beta = K.reshape(self.beta, broadcast_shape)
                     broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
-                    x_normed_running = K.batch_normalization(x, broadcast_running_mean,
-                                                             broadcast_running_std,
-                                                             broadcast_beta,
-                                                             broadcast_gamma,
-                                                             epsilon=self.epsilon)
+                    x_normed_running = K.batch_normalization(
+                        x, broadcast_running_mean, broadcast_running_std,
+                        broadcast_beta, broadcast_gamma,
+                        epsilon=self.epsilon)
 
                 # pick the normalized form of x corresponding to the training phase
                 x_normed = K.in_train_phase(x_normed, x_normed_running)
@@ -159,9 +156,11 @@ class BatchNormalization(Layer):
         return x_normed
 
     def get_config(self):
-        config = {"epsilon": self.epsilon,
-                  "mode": self.mode,
-                  "axis": self.axis,
-                  "momentum": self.momentum}
+        config = {'epsilon': self.epsilon,
+                  'mode': self.mode,
+                  'axis': self.axis,
+                  'gamma_regularizer': self.gamma_regularizer.get_config() if self.gamma_regularizer else None,
+                  'beta_regularizer': self.beta_regularizer.get_config() if self.beta_regularizer else None,
+                  'momentum': self.momentum}
         base_config = super(BatchNormalization, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))

keras/layers/pooling.py

@@ -34,14 +34,12 @@ class _Pooling1D(Layer):
         raise NotImplementedError
 
     def call(self, x, mask=None):
-        x = K.expand_dims(x, -1)   # add dummy last dimension
-        x = K.permute_dimensions(x, (0, 2, 1, 3))
+        x = K.expand_dims(x, 2)   # add dummy last dimension
         output = self._pooling_function(inputs=x, pool_size=self.pool_size,
                                         strides=self.st,
                                         border_mode=self.border_mode,
-                                        dim_ordering='th')
-        output = K.permute_dimensions(output, (0, 2, 1, 3))
-        return K.squeeze(output, 3)  # remove dummy last dimension
+                                        dim_ordering='tf')
+        return K.squeeze(output, 2)  # remove dummy last dimension
 
     def get_config(self):
         config = {'stride': self.stride,
@@ -61,11 +59,11 @@ class MaxPooling1D(_Pooling1D):
         3D tensor with shape: `(samples, downsampled_steps, features)`.
 
     # Arguments
-        pool_length: factor by which to downscale. 2 will halve the input.
-        stride: integer, or None. Stride value.
+        pool_length: size of the region to which max pooling is applied
+        stride: integer, or None. factor by which to downscale.
+            2 will halve the input.
             If None, it will default to `pool_length`.
         border_mode: 'valid' or 'same'.
-            Note: 'same' will only work with TensorFlow for the time being.
     '''
 
     def __init__(self, pool_length=2, stride=None,
@@ -88,7 +86,6 @@ class AveragePooling1D(_Pooling1D):
         stride: integer, or None. Stride value.
             If None, it will default to `pool_length`.
         border_mode: 'valid' or 'same'.
-            Note: 'same' will only work with TensorFlow for the time being.
 
     # Input shape
         3D tensor with shape: `(samples, steps, features)`.
@@ -114,8 +111,10 @@ class _Pooling2D(Layer):
     '''
 
     def __init__(self, pool_size=(2, 2), strides=None, border_mode='valid',
-                 dim_ordering=K.image_dim_ordering(), **kwargs):
+                 dim_ordering='default', **kwargs):
         super(_Pooling2D, self).__init__(**kwargs)
+        if dim_ordering == 'default':
+            dim_ordering = K.image_dim_ordering()
         self.pool_size = tuple(pool_size)
         if strides is None:
             strides = self.pool_size
@@ -178,12 +177,11 @@ class MaxPooling2D(_Pooling2D):
         strides: tuple of 2 integers, or None. Strides values.
             If None, it will default to `pool_size`.
         border_mode: 'valid' or 'same'.
-            Note: 'same' will only work with TensorFlow for the time being.
         dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
             (the depth) is at index 1, in 'tf' mode is it at index 3.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         4D tensor with shape:
@@ -199,7 +197,7 @@ class MaxPooling2D(_Pooling2D):
     '''
 
     def __init__(self, pool_size=(2, 2), strides=None, border_mode='valid',
-                 dim_ordering=K.image_dim_ordering(), **kwargs):
+                 dim_ordering='default', **kwargs):
         super(MaxPooling2D, self).__init__(pool_size, strides, border_mode,
                                            dim_ordering, **kwargs)
 
@@ -220,12 +218,11 @@ class AveragePooling2D(_Pooling2D):
         strides: tuple of 2 integers, or None. Strides values.
             If None, it will default to `pool_size`.
         border_mode: 'valid' or 'same'.
-            Note: 'same' will only work with TensorFlow for the time being.
         dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
             (the depth) is at index 1, in 'tf' mode is it at index 3.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         4D tensor with shape:
@@ -241,7 +238,7 @@ class AveragePooling2D(_Pooling2D):
     '''
 
     def __init__(self, pool_size=(2, 2), strides=None, border_mode='valid',
-                 dim_ordering=K.image_dim_ordering(), **kwargs):
+                 dim_ordering='default', **kwargs):
         super(AveragePooling2D, self).__init__(pool_size, strides, border_mode,
                                                dim_ordering, **kwargs)
 
@@ -257,8 +254,10 @@ class _Pooling3D(Layer):
     '''
 
     def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode='valid',
-                 dim_ordering=K.image_dim_ordering(), **kwargs):
+                 dim_ordering='default', **kwargs):
         super(_Pooling3D, self).__init__(**kwargs)
+        if dim_ordering == 'default':
+            dim_ordering = K.image_dim_ordering()
         self.pool_size = tuple(pool_size)
         if strides is None:
             strides = self.pool_size
@@ -328,7 +327,7 @@ class MaxPooling3D(_Pooling3D):
             (the depth) is at index 1, in 'tf' mode is it at index 4.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         5D tensor with shape:
@@ -344,7 +343,7 @@ class MaxPooling3D(_Pooling3D):
     '''
 
     def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode='valid',
-                 dim_ordering=K.image_dim_ordering(), **kwargs):
+                 dim_ordering='default', **kwargs):
         super(MaxPooling3D, self).__init__(pool_size, strides, border_mode,
                                            dim_ordering, **kwargs)
 
@@ -368,7 +367,7 @@ class AveragePooling3D(_Pooling3D):
             (the depth) is at index 1, in 'tf' mode is it at index 4.
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
+            If you never set it, then it will be "tf".
 
     # Input shape
         5D tensor with shape:
@@ -384,7 +383,7 @@ class AveragePooling3D(_Pooling3D):
     '''
 
     def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode='valid',
-                 dim_ordering=K.image_dim_ordering(), **kwargs):
+                 dim_ordering='default', **kwargs):
         super(AveragePooling3D, self).__init__(pool_size, strides, border_mode,
                                                dim_ordering, **kwargs)
 
@@ -393,3 +392,204 @@ class AveragePooling3D(_Pooling3D):
         output = K.pool3d(inputs, pool_size, strides,
                           border_mode, dim_ordering, pool_mode='avg')
         return output
+
+
+class _GlobalPooling1D(Layer):
+
+    def __init__(self, **kwargs):
+        super(_GlobalPooling1D, self).__init__(**kwargs)
+        self.input_spec = [InputSpec(ndim=3)]
+
+    def get_output_shape_for(self, input_shape):
+        return (input_shape[0], input_shape[2])
+
+    def call(self, x, mask=None):
+        raise NotImplementedError
+
+
+class GlobalAveragePooling1D(_GlobalPooling1D):
+    '''Global average pooling operation for temporal data.
+
+    # Input shape
+        3D tensor with shape: `(samples, steps, features)`.
+
+    # Output shape
+        2D tensor with shape: `(samples, features)`.
+    '''
+
+    def call(self, x, mask=None):
+        return K.mean(x, axis=1)
+
+
+class GlobalMaxPooling1D(_GlobalPooling1D):
+    '''Global max pooling operation for temporal data.
+
+    # Input shape
+        3D tensor with shape: `(samples, steps, features)`.
+
+    # Output shape
+        2D tensor with shape: `(samples, features)`.
+    '''
+
+    def call(self, x, mask=None):
+        return K.max(x, axis=1)
+
+
+class _GlobalPooling2D(Layer):
+
+    def __init__(self, dim_ordering='default', **kwargs):
+        super(_GlobalPooling2D, self).__init__(**kwargs)
+        if dim_ordering == 'default':
+            dim_ordering = K.image_dim_ordering()
+        self.dim_ordering = dim_ordering
+        self.input_spec = [InputSpec(ndim=4)]
+
+    def get_output_shape_for(self, input_shape):
+        if self.dim_ordering == 'tf':
+            return (input_shape[0], input_shape[3])
+        else:
+            return (input_shape[0], input_shape[1])
+
+    def call(self, x, mask=None):
+        raise NotImplementedError
+
+    def get_config(self):
+        config = {'dim_ordering': self.dim_ordering}
+        base_config = super(_GlobalPooling2D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
+class GlobalAveragePooling2D(_GlobalPooling2D):
+    '''Global average pooling operation for spatial data.
+
+    # Arguments
+        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
+            (the depth) is at index 1, in 'tf' mode is it at index 3.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "tf".
+
+    # Input shape
+        4D tensor with shape:
+        `(samples, channels, rows, cols)` if dim_ordering='th'
+        or 4D tensor with shape:
+        `(samples, rows, cols, channels)` if dim_ordering='tf'.
+
+    # Output shape
+        2D tensor with shape:
+        `(nb_samples, channels)`
+    '''
+
+    def call(self, x, mask=None):
+        if self.dim_ordering == 'tf':
+            return K.mean(x, axis=[1, 2])
+        else:
+            return K.mean(x, axis=[2, 3])
+
+
+class GlobalMaxPooling2D(_GlobalPooling2D):
+    '''Global max pooling operation for spatial data.
+
+    # Arguments
+        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
+            (the depth) is at index 1, in 'tf' mode is it at index 3.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "tf".
+
+    # Input shape
+        4D tensor with shape:
+        `(samples, channels, rows, cols)` if dim_ordering='th'
+        or 4D tensor with shape:
+        `(samples, rows, cols, channels)` if dim_ordering='tf'.
+
+    # Output shape
+        2D tensor with shape:
+        `(nb_samples, channels)`
+    '''
+
+    def call(self, x, mask=None):
+        if self.dim_ordering == 'tf':
+            return K.max(x, axis=[1, 2])
+        else:
+            return K.max(x, axis=[2, 3])
+
+
+class _GlobalPooling3D(Layer):
+
+    def __init__(self, dim_ordering='default', **kwargs):
+        super(_GlobalPooling3D, self).__init__(**kwargs)
+        if dim_ordering == 'default':
+            dim_ordering = K.image_dim_ordering()
+        self.dim_ordering = dim_ordering
+        self.input_spec = [InputSpec(ndim=5)]
+
+    def get_output_shape_for(self, input_shape):
+        if self.dim_ordering == 'tf':
+            return (input_shape[0], input_shape[4])
+        else:
+            return (input_shape[0], input_shape[1])
+
+    def call(self, x, mask=None):
+        raise NotImplementedError
+
+    def get_config(self):
+        config = {'dim_ordering': self.dim_ordering}
+        base_config = super(_GlobalPooling3D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
+class GlobalAveragePooling3D(_GlobalPooling3D):
+    '''Global Average pooling operation for 3D data.
+
+    # Arguments
+        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
+            (the depth) is at index 1, in 'tf' mode is it at index 4.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "tf".
+
+    # Input shape
+        5D tensor with shape:
+        `(samples, channels, len_pool_dim1, len_pool_dim2, len_pool_dim3)` if dim_ordering='th'
+        or 5D tensor with shape:
+        `(samples, len_pool_dim1, len_pool_dim2, len_pool_dim3, channels)` if dim_ordering='tf'.
+
+    # Output shape
+        2D tensor with shape:
+        `(nb_samples, channels)`
+    '''
+
+    def call(self, x, mask=None):
+        if self.dim_ordering == 'tf':
+            return K.mean(x, axis=[1, 2, 3])
+        else:
+            return K.mean(x, axis=[2, 3, 4])
+
+
+class GlobalMaxPooling3D(_GlobalPooling3D):
+    '''Global Max pooling operation for 3D data.
+
+    # Arguments
+        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
+            (the depth) is at index 1, in 'tf' mode is it at index 4.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "tf".
+
+    # Input shape
+        5D tensor with shape:
+        `(samples, channels, len_pool_dim1, len_pool_dim2, len_pool_dim3)` if dim_ordering='th'
+        or 5D tensor with shape:
+        `(samples, len_pool_dim1, len_pool_dim2, len_pool_dim3, channels)` if dim_ordering='tf'.
+
+    # Output shape
+        2D tensor with shape:
+        `(nb_samples, channels)`
+    '''
+
+    def call(self, x, mask=None):
+        if self.dim_ordering == 'tf':
+            return K.max(x, axis=[1, 2, 3])
+        else:
+            return K.max(x, axis=[2, 3, 4])

keras/layers/recurrent.py

@@ -12,13 +12,10 @@ def time_distributed_dense(x, w, b=None, dropout=None,
     '''Apply y.w + b for every temporal slice y of x.
     '''
     if not input_dim:
-        # won't work with TensorFlow
         input_dim = K.shape(x)[2]
     if not timesteps:
-        # won't work with TensorFlow
         timesteps = K.shape(x)[1]
     if not output_dim:
-        # won't work with TensorFlow
         output_dim = K.shape(w)[1]
 
     if dropout is not None and 0. < dropout < 1.:
@@ -30,12 +27,15 @@ def time_distributed_dense(x, w, b=None, dropout=None,
 
     # collapse time dimension and batch dimension together
     x = K.reshape(x, (-1, input_dim))
-
     x = K.dot(x, w)
     if b:
         x = x + b
     # reshape to 3D tensor
-    x = K.reshape(x, (-1, timesteps, output_dim))
+    if K.backend() == 'tensorflow':
+        x = K.reshape(x, K.pack([-1, timesteps, output_dim]))
+        x.set_shape([None, None, output_dim])
+    else:
+        x = K.reshape(x, (-1, timesteps, output_dim))
     return x
 
 
@@ -120,14 +120,10 @@ class Recurrent(Layer):
         use an [Embedding](embeddings.md) layer with the `mask_zero` parameter
         set to `True`.
 
-    # TensorFlow warning
-        For the time being, when using the TensorFlow backend,
-        the number of timesteps used must be specified in your model.
-        Make sure to pass an `input_length` int argument to your
-        recurrent layer (if it comes first in your model),
-        or to pass a complete `input_shape` argument to the first layer
-        in your model otherwise.
-
+    # Note on performance
+        You are likely to see better performance with RNNs in Theano compared
+        to TensorFlow. Additionally, when using TensorFlow, it is often
+        preferable to set `unroll=True` for better performance.
 
     # Note on using statefulness in RNNs
         You can set RNN layers to be 'stateful', which means that the states
@@ -139,16 +135,15 @@ class Recurrent(Layer):
         To enable statefulness:
             - specify `stateful=True` in the layer constructor.
             - specify a fixed batch size for your model, by passing
-                a `batch_input_shape=(...)` to the first layer in your model.
+                if sequential model:
+                  a `batch_input_shape=(...)` to the first layer in your model.
+                else for functional model with 1 or more Input layers:
+                  a `batch_shape=(...)` to all the first layers in your model.
                 This is the expected shape of your inputs *including the batch size*.
                 It should be a tuple of integers, e.g. `(32, 10, 100)`.
 
         To reset the states of your model, call `.reset_states()` on either
         a specific layer, or on your entire model.
-
-    # Note on using dropout with TensorFlow
-        When using the TensorFlow backend, specify a fixed batch size for your model
-        following the notes on statefulness RNNs.
     '''
     def __init__(self, weights=None,
                  return_sequences=False, go_backwards=False, stateful=False,
@@ -204,19 +199,18 @@ class Recurrent(Layer):
         # note that the .build() method of subclasses MUST define
         # self.input_spec with a complete input shape.
         input_shape = self.input_spec[0].shape
-        if K._BACKEND == 'tensorflow':
-            if not input_shape[1]:
-                raise Exception('When using TensorFlow, you should define '
-                                'explicitly the number of timesteps of '
-                                'your sequences.\n'
-                                'If your first layer is an Embedding, '
-                                'make sure to pass it an "input_length" '
-                                'argument. Otherwise, make sure '
-                                'the first layer has '
-                                'an "input_shape" or "batch_input_shape" '
-                                'argument, including the time axis. '
-                                'Found input shape at layer ' + self.name +
-                                ': ' + str(input_shape))
+        if self.unroll and input_shape[1] is None:
+            raise ValueError('Cannot unroll a RNN if the '
+                             'time dimension is undefined. \n'
+                             '- If using a Sequential model, '
+                             'specify the time dimension by passing '
+                             'an `input_shape` or `batch_input_shape` '
+                             'argument to your first layer. If your '
+                             'first layer is an Embedding, you can '
+                             'also use the `input_length` argument.\n'
+                             '- If using the functional API, specify '
+                             'the time dimension by passing a `shape` '
+                             'or `batch_shape` argument to your Input layer.')
         if self.stateful:
             initial_states = self.states
         else:
@@ -232,9 +226,10 @@ class Recurrent(Layer):
                                              unroll=self.unroll,
                                              input_length=input_shape[1])
         if self.stateful:
-            self.updates = []
+            updates = []
             for i in range(len(states)):
-                self.updates.append((self.states[i], states[i]))
+                updates.append((self.states[i], states[i]))
+            self.add_updates(updates, x)
 
         if self.return_sequences:
             return outputs
@@ -247,7 +242,7 @@ class Recurrent(Layer):
                   'stateful': self.stateful,
                   'unroll': self.unroll,
                   'consume_less': self.consume_less}
-        if self.stateful:
+        if self.stateful and self.input_spec[0].shape:
             config['batch_input_shape'] = self.input_spec[0].shape
         else:
             config['input_dim'] = self.input_dim
@@ -331,13 +326,22 @@ class SimpleRNN(Recurrent):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def reset_states(self):
         assert self.stateful, 'Layer must be stateful.'
         input_shape = self.input_spec[0].shape
         if not input_shape[0]:
-            raise Exception('If a RNN is stateful, a complete ' +
-                            'input_shape must be provided (including batch size).')
+            raise Exception('If a RNN is stateful, it needs to know '
+                            'its batch size. Specify the batch size '
+                            'of your input tensors: \n'
+                            '- If using a Sequential model, '
+                            'specify the batch size by passing '
+                            'a `batch_input_shape` '
+                            'argument to your first layer.\n'
+                            '- If using the functional API, specify '
+                            'the time dimension by passing a '
+                            '`batch_shape` argument to your Input layer.')
         if hasattr(self, 'states'):
             K.set_value(self.states[0],
                         np.zeros((input_shape[0], self.output_dim)))
@@ -372,7 +376,7 @@ class SimpleRNN(Recurrent):
         constants = []
         if 0 < self.dropout_U < 1:
             ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1)))
-            ones = K.concatenate([ones] * self.output_dim, 1)
+            ones = K.tile(ones, (1, self.output_dim))
             B_U = K.in_train_phase(K.dropout(ones, self.dropout_U), ones)
             constants.append(B_U)
         else:
@@ -381,7 +385,7 @@ class SimpleRNN(Recurrent):
             input_shape = self.input_spec[0].shape
             input_dim = input_shape[-1]
             ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1)))
-            ones = K.concatenate([ones] * input_dim, 1)
+            ones = K.tile(ones, (1, int(input_dim)))
             B_W = K.in_train_phase(K.dropout(ones, self.dropout_W), ones)
             constants.append(B_W)
         else:
@@ -513,6 +517,7 @@ class GRU(Recurrent):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def reset_states(self):
         assert self.stateful, 'Layer must be stateful.'
@@ -585,7 +590,7 @@ class GRU(Recurrent):
         constants = []
         if 0 < self.dropout_U < 1:
             ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1)))
-            ones = K.concatenate([ones] * self.output_dim, 1)
+            ones = K.tile(ones, (1, self.output_dim))
             B_U = [K.in_train_phase(K.dropout(ones, self.dropout_U), ones) for _ in range(3)]
             constants.append(B_U)
         else:
@@ -595,7 +600,7 @@ class GRU(Recurrent):
             input_shape = self.input_spec[0].shape
             input_dim = input_shape[-1]
             ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1)))
-            ones = K.concatenate([ones] * input_dim, 1)
+            ones = K.tile(ones, (1, int(input_dim)))
             B_W = [K.in_train_phase(K.dropout(ones, self.dropout_W), ones) for _ in range(3)]
             constants.append(B_W)
         else:
@@ -743,6 +748,7 @@ class LSTM(Recurrent):
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
             del self.initial_weights
+        self.built = True
 
     def reset_states(self):
         assert self.stateful, 'Layer must be stateful.'
@@ -825,7 +831,7 @@ class LSTM(Recurrent):
         constants = []
         if 0 < self.dropout_U < 1:
             ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1)))
-            ones = K.concatenate([ones] * self.output_dim, 1)
+            ones = K.tile(ones, (1, self.output_dim))
             B_U = [K.in_train_phase(K.dropout(ones, self.dropout_U), ones) for _ in range(4)]
             constants.append(B_U)
         else:
@@ -835,7 +841,7 @@ class LSTM(Recurrent):
             input_shape = self.input_spec[0].shape
             input_dim = input_shape[-1]
             ones = K.ones_like(K.reshape(x[:, 0, 0], (-1, 1)))
-            ones = K.concatenate([ones] * input_dim, 1)
+            ones = K.tile(ones, (1, int(input_dim)))
             B_W = [K.in_train_phase(K.dropout(ones, self.dropout_W), ones) for _ in range(4)]
             constants.append(B_W)
         else:

keras/layers/wrappers.py

@@ -20,6 +20,13 @@ class Wrapper(Layer):
         self.regularizers = getattr(self.layer, 'regularizers', [])
         self.constraints = getattr(self.layer, 'constraints', {})
 
+        # properly attribute the current layer to
+        # regularizers that need access to it
+        # (e.g. ActivityRegularizer).
+        for regularizer in self.regularizers:
+            if hasattr(regularizer, 'set_layer'):
+                regularizer.set_layer(self)
+
     def get_weights(self):
         weights = self.layer.get_weights()
         return weights
@@ -86,17 +93,6 @@ class TimeDistributed(Wrapper):
     def build(self, input_shape):
         assert len(input_shape) >= 3
         self.input_spec = [InputSpec(shape=input_shape)]
-        if K._BACKEND == 'tensorflow':
-            if not input_shape[1]:
-                raise Exception('When using TensorFlow, you should define '
-                                'explicitly the number of timesteps of '
-                                'your sequences.\n'
-                                'If your first layer is an Embedding, '
-                                'make sure to pass it an "input_length" '
-                                'argument. Otherwise, make sure '
-                                'the first layer has '
-                                'an "input_shape" or "batch_input_shape" '
-                                'argument, including the time axis.')
         child_input_shape = (input_shape[0],) + input_shape[2:]
         if not self.layer.built:
             self.layer.build(child_input_shape)
@@ -117,8 +113,10 @@ class TimeDistributed(Wrapper):
                 output = self.layer.call(x)
                 return output, []
 
-            last_output, outputs, states = K.rnn(step, X,
-                                                 initial_states=[])
+            _, outputs, _ = K.rnn(step, X,
+                                  initial_states=[],
+                                  input_length=input_shape[1],
+                                  unroll=False)
             y = outputs
         else:
             # no batch size specified, therefore the layer will be able
@@ -133,3 +131,135 @@ class TimeDistributed(Wrapper):
             output_shape = self.get_output_shape_for(input_shape)
             y = K.reshape(y, (-1, input_length) + output_shape[2:])
         return y
+
+
+class Bidirectional(Wrapper):
+    ''' Bidirectional wrapper for RNNs.
+
+    # Arguments:
+        layer: `Recurrent` instance.
+        merge_mode: Mode by which outputs of the
+            forward and backward RNNs will be combined.
+            One of {'sum', 'mul', 'concat', 'ave', None}.
+            If None, the outputs will not be combined,
+            they will be returned as a list.
+
+    # Examples:
+
+    ```python
+        model = Sequential()
+        model.add(Bidirectional(LSTM(10, return_sequences=True), input_shape=(5, 10)))
+        model.add(Bidirectional(LSTM(10)))
+        model.add(Dense(5))
+        model.add(Activation('softmax'))
+        model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
+    ```
+    '''
+    def __init__(self, layer, merge_mode='concat', weights=None, **kwargs):
+        if merge_mode not in ['sum', 'mul', 'ave', 'concat', None]:
+            raise ValueError('Invalid merge mode. '
+                             'Merge mode should be one of '
+                             '{"sum", "mul", "ave", "concat", None}')
+        self.forward_layer = layer
+        config = layer.get_config()
+        config['go_backwards'] = not config['go_backwards']
+        self.backward_layer = layer.__class__.from_config(config)
+        self.forward_layer.name = 'forward_' + self.forward_layer.name
+        self.backward_layer.name = 'backward_' + self.backward_layer.name
+        self.merge_mode = merge_mode
+        if weights:
+            nw = len(weights)
+            self.forward_layer.initial_weights = weights[:nw // 2]
+            self.backward_layer.initial_weights = weights[nw // 2:]
+        self.stateful = layer.stateful
+        self.return_sequences = layer.return_sequences
+        self.supports_masking = True
+        super(Bidirectional, self).__init__(layer, **kwargs)
+
+    def get_weights(self):
+        return self.forward_layer.get_weights() + self.backward_layer.get_weights()
+
+    def set_weights(self, weights):
+        nw = len(weights)
+        self.forward_layer.set_weights(weights[:nw // 2])
+        self.backward_layer.set_weights(weights[nw // 2:])
+
+    def get_output_shape_for(self, input_shape):
+        if self.merge_mode in ['sum', 'ave', 'mul']:
+            return self.forward_layer.get_output_shape_for(input_shape)
+        elif self.merge_mode == 'concat':
+            shape = list(self.forward_layer.get_output_shape_for(input_shape))
+            shape[-1] *= 2
+            return tuple(shape)
+        elif self.merge_mode is None:
+            return [self.forward_layer.get_output_shape_for(input_shape)] * 2
+
+    def call(self, X, mask=None):
+        Y = self.forward_layer.call(X, mask)
+        Y_rev = self.backward_layer.call(X, mask)
+        if self.return_sequences:
+            Y_rev = K.reverse(Y_rev, 1)
+        if self.merge_mode == 'concat':
+            return K.concatenate([Y, Y_rev])
+        elif self.merge_mode == 'sum':
+            return Y + Y_rev
+        elif self.merge_mode == 'ave':
+            return (Y + Y_rev) / 2
+        elif self.merge_mode == 'mul':
+            return Y * Y_rev
+        elif self.merge_mode is None:
+            return [Y, Y_rev]
+
+    def reset_states(self):
+        self.forward_layer.reset_states()
+        self.backward_layer.reset_states()
+
+    def build(self, input_shape):
+        self.forward_layer.build(input_shape)
+        self.backward_layer.build(input_shape)
+
+    def compute_mask(self, input, mask):
+        if self.return_sequences:
+            if not self.merge_mode:
+                return [mask, mask]
+            else:
+                return mask
+        else:
+            return None
+
+    @property
+    def trainable_weights(self):
+        if hasattr(self.forward_layer, 'trainable_weights'):
+            return self.forward_layer.trainable_weights + self.backward_layer.trainable_weights
+        return []
+
+    @property
+    def non_trainable_weights(self):
+        if hasattr(self.forward_layer, 'non_trainable_weights'):
+            return self.forward_layer.non_trainable_weights + self.backward_layer.non_trainable_weights
+        return []
+
+    @property
+    def updates(self):
+        if hasattr(self.forward_layer, 'updates'):
+            return self.forward_layer.updates + self.backward_layer.updates
+        return []
+
+    @property
+    def regularizers(self):
+        if hasattr(self.forward_layer, 'regularizers'):
+            return self.forward_layer.regularizers + self.backward_layer.regularizers
+        return []
+
+    @property
+    def constraints(self):
+        _constraints = {}
+        if hasattr(self.forward_layer, 'constraints'):
+            _constraints.update(self.forward_layer.constraints)
+            _constraints.update(self.backward_layer.constraints)
+        return _constraints
+
+    def get_config(self):
+        config = {"merge_mode": self.merge_mode}
+        base_config = super(Bidirectional, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))

keras/legacy/models.py

@@ -1,775 +0,0 @@
-from collections import OrderedDict
-import warnings
-import copy
-
-from .. import backend as K
-from ..layers import InputLayer, Layer, Merge
-from ..engine.training import Model
-
-
-class Graph(Model):
-    '''Arbitrary connection graph.
-
-    THIS IS A LEGACY MODEL AND SHOULD NOT BE USED
-    except for backwards compatibility support.
-
-    For multi-inputs/multi-outputs models, or
-    models using shared layers, use the functional API instead.
-    '''
-
-    def __init__(self, name=None):
-        # model attributes
-        self.inbound_nodes = []
-        self.outbound_nodes = []
-        self.built = False
-        self.supports_masking = False
-
-        # legacy attributes (we prefix them with _graph_)
-        self._graph_namespace = set()  # strings
-        self._graph_nodes = OrderedDict()  # layer-like
-        self._graph_inputs = OrderedDict()  # layer-like
-        self._graph_outputs = OrderedDict()  # layer-like
-        self._graph_input_config = []  # dicts
-        self._graph_output_config = []  # dicts
-        self._graph_node_config = []  # dicts
-        self._graph_shared_nodes_names = []
-
-        if not name:
-            prefix = 'graph_'
-            name = prefix + str(K.get_uid(prefix))
-        self.name = name
-
-    def __call__(self, x, mask=None):
-        self.build()
-        return super(Graph, self).__call__(x, mask)
-
-    def build(self, input_shape=None):
-        # this will crash if the input/output layers have multiple nodes
-        # no plans to support that case since Graph is deprecated
-        input_tensors = [layer.output for layer in self._graph_inputs.values()]
-        output_tensors = [layer.output for layer in self._graph_outputs.values()]
-        # actually create the model
-        super(Graph, self).__init__(input_tensors,
-                                    output_tensors,
-                                    name=self.name)
-        self.built = True
-
-    def compile(self, optimizer, loss,
-                metrics=[],
-                sample_weight_modes=None,
-                loss_weights=None,
-                **kwargs):
-        '''Configures the learning process.
-
-        # Arguments
-            optimizer: str (name of optimizer) or optimizer object.
-                See [optimizers](optimizers.md).
-            loss: dictionary mapping the name(s) of the output(s) to
-                a loss function (string name of objective function or
-                objective function. See [objectives](objectives.md)).
-            metrics: list of str (name of metrics) or
-                list of metrics functions. See [metrics](metrics.md).
-            sample_weight_modes: optional dictionary mapping certain
-                output names to a sample weight mode ("temporal" and None
-                are the only supported modes). If you need to do
-                timestep-wise loss weighting on one of your graph outputs,
-                you will need to set the sample weight mode for this output
-                to "temporal".
-            loss_weights: dictionary you can pass to specify a weight
-                coefficient for each loss function (in a multi-output model).
-                If no loss weight is specified for an output,
-                the weight for this output's loss will be considered to be 1.
-            kwargs: for Theano backend, these are passed into K.function.
-                Ignored for Tensorflow backend.
-        '''
-        # create the underlying Model
-        if not self.built:
-            self.build()
-        super(Graph, self).compile(optimizer, loss,
-                                   metrics=metrics,
-                                   sample_weight_mode=sample_weight_modes,
-                                   loss_weights=loss_weights,
-                                   **kwargs)
-
-    def add_input(self, name, input_shape=None,
-                  batch_input_shape=None, dtype='float'):
-        '''Adds an input to the graph.
-
-        # Arguments:
-            name: string. The name of the new input.
-                Must be unique in the graph.
-            input_shape: a tuple of integers,
-                the expected shape of the input samples.
-                Does not include the batch size.
-            batch_input_shape: a tuple of integers,
-                the expected shape of the whole input batch,
-                including the batch size.
-            dtype: 'float', or 'int'.
-        '''
-        if name in self._graph_namespace:
-            raise Exception('Duplicate node identifier: ' + name)
-        self._graph_namespace.add(name)
-        self.built = False
-
-        if dtype[:3] == 'int':
-            dtype = 'int32'
-        elif dtype[:5] == 'float':
-            dtype = K.floatx()
-        else:
-            raise Exception('Uknown dtype (should be "int" or "float"): ' +
-                            str(dtype))
-
-        # create input layer
-        input_layer = InputLayer(input_shape=input_shape,
-                                 batch_input_shape=batch_input_shape,
-                                 name=name, input_dtype=dtype)
-        self._graph_inputs[name] = input_layer
-
-        # append input config to self._graph_input_config
-        config = {'name': name, 'dtype': dtype}
-        if batch_input_shape:
-            config['batch_input_shape'] = batch_input_shape
-        else:
-            config['input_shape'] = input_shape
-        self._graph_input_config.append(config)
-
-    def add_node(self, layer, name, input=None, inputs=[],
-                 merge_mode='concat', concat_axis=-1, dot_axes=-1,
-                 create_output=False):
-        '''Adds a node in the graph. It can be connected to multiple
-        inputs, which will first be merged into one tensor
-        according to the mode specified.
-
-        # Arguments
-            layer: the layer at the node.
-            name: name for the node.
-            input: when connecting the layer to a single input,
-                this is the name of the incoming node.
-            inputs: when connecting the layer to multiple inputs,
-                this is a list of names of incoming nodes.
-            merge_mode: one of {concat, sum, dot, ave, mul}
-            concat_axis: when `merge_mode=='concat'`, this is the
-                input concatenation axis.
-            dot_axes: when `merge_mode='dot'`,
-                this is the contraction axes specification;
-                see the `Merge` layer for details.
-            create_output: boolean. Set this to `True` if you want the output
-                of your node to be an output of the graph.
-        '''
-        if name in self._graph_namespace:
-            raise Exception('Duplicate node identifier: ' + name)
-        self._graph_namespace.add(name)
-        layer.name = name
-        self.built = False
-
-        if input:
-            if input not in self._graph_namespace:
-                raise Exception('Unknown node/input identifier: ' + input)
-            if input in self._graph_nodes:
-                layer.add_inbound_node(self._graph_nodes[input])
-            elif input in self._graph_inputs:
-                layer.add_inbound_node(self._graph_inputs[input])
-        if inputs:
-            to_merge = []
-            for n in inputs:
-                if n in self._graph_nodes:
-                    to_merge.append(self._graph_nodes[n])
-                elif n in self._graph_inputs:
-                    to_merge.append(self._graph_inputs[n])
-                else:
-                    raise Exception('Unknown identifier: ' + n)
-            merge = Merge(to_merge, mode=merge_mode,
-                          concat_axis=concat_axis, dot_axes=dot_axes,
-                          name='merge_inputs_for_' + name)
-            layer.add_inbound_node(merge)
-        self._graph_nodes[name] = layer
-        self._graph_node_config.append({'name': name,
-                                        'input': input,
-                                        'inputs': inputs,
-                                        'merge_mode': merge_mode,
-                                        'concat_axis': concat_axis,
-                                        'dot_axes': dot_axes,
-                                        'create_output': create_output})
-        if create_output:
-            self.add_output(name, input=name)
-
-    def add_shared_node(self, layer, name, inputs=[], merge_mode=None,
-                        concat_axis=-1, dot_axes=-1, outputs=[],
-                        create_output=False):
-        '''Used to share a same layer across multiple nodes.
-
-        Supposed, for instance, that you want to apply one same `Dense` layer
-        after two different nodes ('node_a' and 'node_b').
-        You can then add the dense layer as a shared node by calling:
-
-        ```python
-        model.add_shared_node(my_dense, name='shared_dense', inputs=['node_a', 'node_b'], ...)
-        ```
-
-        If you want access to the output of dense(node_a) and dense(node_b) separately,
-        you can add these outputs to the Graph by passing an `outputs` argument:
-
-        ```python
-        model.add_shared_node(my_dense, name='shared_dense', inputs=['node_a', 'node_b'],
-                              outputs=['dense_output_a', 'dense_outputs_b'])
-        ```
-
-        Otherwise you can merge these different outputs via `merge_mode`.
-        In that case you can access the merged output
-        under the identifier `name`.
-
-        # Arguments
-            layer: The layer to be shared across multiple inputs
-            name: Name of the shared node
-            inputs: List of names of input nodes
-            merge_mode: Same meaning as `merge_mode` argument of `add_node()`
-            concat_axis: Same meaning as `concat_axis` argument of `add_node()`
-            dot_axes: Same meaning as `dot_axes` argument of `add_node()`
-            outputs: Used when `merge_mode=None`. Names for the output nodes.
-            create_output: Same meaning as `create_output` argument of `add_node()`.
-        '''
-        if name in self._graph_namespace:
-            raise Exception('Duplicate node identifier: ' + name)
-        self._graph_namespace.add(name)
-        self.built = False
-
-        for o in outputs:
-            if o in self._graph_namespace:
-                raise Exception('Duplicate node identifier: ' + o)
-        if merge_mode:
-            if merge_mode not in {'sum', 'ave', 'mul', 'dot', 'cos', 'concat'}:
-                raise Exception('Invalid merge mode:', merge_mode)
-        input_layers = []
-        for i in range(len(inputs)):
-            input = inputs[i]
-            if input in self._graph_nodes:
-                n = self._graph_nodes[input]
-                input_layers.append(n)
-            elif input in self._graph_inputs:
-                n = self._graph_inputs[input]
-                input_layers.append(n)
-            else:
-                raise Exception('Unknown identifier: ' + input)
-
-        created_node_indices = []
-        for input_layer in input_layers:
-            created_node_indices.append(len(layer.inbound_nodes))
-            layer.add_inbound_node(input_layer)
-
-        if merge_mode:
-            layer.name = 'input_for_' + name
-            # collect all output nodes of layer and merge them into a single output
-            merge = Merge([layer for _ in range(len(inputs))],
-                          mode=merge_mode,
-                          concat_axis=concat_axis, dot_axes=dot_axes,
-                          node_indices=created_node_indices,
-                          name=name)
-            self._graph_nodes[name] = merge
-            if create_output:
-                self.add_output(name, input=name)
-        else:
-            layer.name = name
-            # create one new layer per output node of layer,
-            # and add them to the Graph with their own identifiers
-            if len(outputs) != len(inputs):
-                raise Exception('When using merge_mode=None, '
-                                'you should provide a list of '
-                                'output names (`output` argument) '
-                                'the same size as `input`.')
-            for i in range(len(outputs)):
-                output_layer_name = outputs[i]
-                output_layer = Layer(name=output_layer_name)
-                output_layer.add_inbound_node(layer, created_node_indices[i])
-                self._graph_namespace.add(output_layer_name)
-                self._graph_nodes[output_layer_name] = output_layer
-                if create_output:
-                    self.add_output(output_layer_name, input=output_layer_name)
-
-        self._graph_node_config.append({'name': name,
-                                        'layer': {
-                                            'config': layer.get_config(),
-                                            'class_name': layer.__class__.__name__,
-                                        },
-                                        'inputs': inputs,
-                                        'merge_mode': merge_mode,
-                                        'concat_axis': concat_axis,
-                                        'dot_axes': dot_axes,
-                                        'outputs': outputs,
-                                        'create_output': create_output if merge_mode else False})
-        self._graph_shared_nodes_names.append(name)
-
-    def add_output(self, name, input=None, inputs=[],
-                   merge_mode='concat', concat_axis=-1, dot_axes=-1):
-        '''Adds an output to the graph.
-
-        This output can merge several node outputs into a single output.
-
-        # Arguments
-            name: name of the output.
-            input: when connecting the layer to a single input,
-                this is the name of the incoming node.
-            inputs: when connecting the layer to multiple inputs,
-                this is a list of names of incoming nodes.
-            merge_mode: one of {concat, sum, dot, ave, mul}
-            concat_axis: when `merge_mode=='concat'`, this is the
-                input concatenation axis.
-            dot_axes: when `merge_mode='dot'`,
-                this is the contraction axes specification;
-                see the `Merge layer for details.
-        '''
-        if name not in self._graph_namespace:
-            self._graph_namespace.add(name)
-        if name in self._graph_outputs:
-            raise Exception('Duplicate output identifier:', name)
-        self.built = False
-
-        if input:
-            if input in self._graph_nodes:
-                layer = self._graph_nodes[input]
-            elif input in self._graph_inputs:
-                layer = self._graph_inputs[input]
-            else:
-                raise Exception('Unknown node/input identifier: ' + input)
-            if layer.name == name:
-                self._graph_outputs[name] = layer
-            else:
-                layer.name = name
-                self._graph_outputs[name] = layer
-        if inputs:
-            to_merge = []
-            for n in inputs:
-                if n not in self._graph_nodes:
-                    raise Exception('Unknown identifier: ' + n)
-                to_merge.append(self._graph_nodes[n])
-            merge = Merge(to_merge, mode=merge_mode,
-                          concat_axis=concat_axis, dot_axes=dot_axes,
-                          name=name)
-            self._graph_outputs[name] = merge
-
-        self._graph_output_config.append({'name': name,
-                                          'input': input,
-                                          'inputs': inputs,
-                                          'merge_mode': merge_mode,
-                                          'concat_axis': concat_axis,
-                                          'dot_axes': dot_axes})
-
-    def _get_x(self, data):
-        x = []
-        for key in self._graph_inputs.keys():
-            if key not in data:
-                raise Exception('Expected to be provided an array '
-                                '(in dict argument `data`) for input "' +
-                                key + '".')
-            x.append(data[key])
-        return x
-
-    def _get_y(self, data):
-        y = []
-        for key in self._graph_outputs.keys():
-            if key not in data:
-                raise Exception('Expected to be provided an array '
-                                '(in dict argument `data`) for output "' +
-                                key + '".')
-            y.append(data[key])
-        return y
-
-    def fit(self, data, batch_size=32, nb_epoch=10, verbose=1, callbacks=[],
-            validation_split=0., validation_data=None, shuffle=True,
-            class_weight=None, sample_weight=None, **kwargs):
-        '''Trains the model for a fixed number of epochs.
-
-        Returns a history object. Its `history` attribute is a record of
-        training loss values at successive epochs,
-        as well as validation loss values (if applicable).
-
-        # Arguments
-            data: dictionary mapping input names and outputs names to
-                appropriate Numpy arrays. All arrays should contain
-                the same number of samples.
-            batch_size: int. Number of samples per gradient update.
-            nb_epoch: int.
-            verbose: 0 for no logging to stdout,
-                1 for progress bar logging, 2 for one log line per epoch.
-            callbacks: `keras.callbacks.Callback` list. List of callbacks
-                to apply during training. See [callbacks](callbacks.md).
-            validation_split: float (0. < x < 1). Fraction of the data to
-                use as held-out validation data.
-            validation_data: dictionary mapping input names and outputs names
-                to appropriate Numpy arrays to be used as
-                held-out validation data.
-                All arrays should contain the same number of samples.
-                Will override validation_split.
-            shuffle: boolean. Whether to shuffle the samples at each epoch.
-            class_weight: dictionary mapping output names to
-                class weight dictionaries.
-            sample_weight: dictionary mapping output names to
-                numpy arrays of sample weights.
-        '''
-        if 'show_accuracy' in kwargs:
-            kwargs.pop('show_accuracy')
-            warnings.warn('The "show_accuracy" argument is deprecated, '
-                          'instead you should pass the "accuracy" metric to '
-                          'the model at compile time:\n'
-                          '`model.compile(optimizer, loss, '
-                          'metrics=["accuracy"])`')
-        if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
-                            str(kwargs))
-        x = self._get_x(data)
-        y = self._get_y(data)
-
-        if type(validation_data) is tuple:
-            raise Exception('Cannot used sample_weight with '
-                            'validation data with legacy Graph model. '
-                            'validation_data should be a dictionary.')
-        if validation_data:
-            val_x = self._get_x(validation_data)
-            val_y = self._get_y(validation_data)
-            validation_data = (val_x, val_y)
-        return super(Graph, self).fit(x, y,
-                                      batch_size=batch_size,
-                                      nb_epoch=nb_epoch,
-                                      verbose=verbose,
-                                      callbacks=callbacks,
-                                      validation_split=validation_split,
-                                      validation_data=validation_data,
-                                      shuffle=shuffle,
-                                      class_weight=class_weight,
-                                      sample_weight=sample_weight)
-
-    def evaluate(self, data, batch_size=128,
-                 verbose=0, sample_weight={}, **kwargs):
-        '''Computes the loss on some input data, batch by batch.
-
-        Returns the scalar test loss over the data,
-        or a list of metrics values (starting with the test loss)
-        if applicable.
-
-        Arguments: see `fit` method.
-        '''
-        if 'show_accuracy' in kwargs:
-            kwargs.pop('show_accuracy')
-            warnings.warn('The "show_accuracy" argument is deprecated, '
-                          'instead you should pass the "accuracy" metric to '
-                          'the model at compile time:\n'
-                          '`model.compile(optimizer, loss, '
-                          'metrics=["accuracy"])`')
-        if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
-                            str(kwargs))
-        x = self._get_x(data)
-        y = self._get_y(data)
-        return super(Graph, self).evaluate(x, y,
-                                           batch_size=batch_size,
-                                           verbose=verbose,
-                                           sample_weight=sample_weight)
-
-    def predict(self, data, batch_size=128, verbose=0):
-        '''Generates output predictions for the input samples
-        batch by batch.
-
-        Arguments: see `fit` method.
-        '''
-        x = self._get_x(data)
-        output_list = super(Graph, self).predict(x, batch_size=batch_size,
-                                                 verbose=verbose)
-        if not isinstance(output_list, list):
-            output_list = [output_list]
-        return dict(zip(self._graph_outputs, output_list))
-
-    def train_on_batch(self, data,
-                       class_weight={},
-                       sample_weight={}, **kwargs):
-        '''Single gradient update on a batch of samples.
-
-        Returns the scalar train loss over the data,
-        or a list of metrics values (starting with the test loss)
-        if applicable.
-
-        Arguments: see `fit` method.
-        '''
-        if 'accuracy' in kwargs:
-            kwargs.pop('accuracy')
-            warnings.warn('The "accuracy" argument is deprecated, '
-                          'instead you should pass the "accuracy" metric to '
-                          'the model at compile time:\n'
-                          '`model.compile(optimizer, loss, '
-                          'metrics=["accuracy"])`')
-        if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
-                            str(kwargs))
-        x = self._get_x(data)
-        y = self._get_y(data)
-        return super(Graph, self).train_on_batch(x, y,
-                                                 sample_weight=sample_weight,
-                                                 class_weight=class_weight)
-
-    def test_on_batch(self, data, sample_weight={}, **kwargs):
-        '''Test the network on a single batch of samples.
-
-        Returns the scalar test loss over the data,
-        or a list of metrics values (starting with the test loss)
-        if applicable.
-
-        Arguments: see `fit` method.
-        '''
-        if 'accuracy' in kwargs:
-            kwargs.pop('accuracy')
-            warnings.warn('The "accuracy" argument is deprecated, '
-                          'instead you should pass the "accuracy" metric to '
-                          'the model at compile time:\n'
-                          '`model.compile(optimizer, loss, '
-                          'metrics=["accuracy"])`')
-        if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
-                            str(kwargs))
-        x = self._get_x(data)
-        y = self._get_y(data)
-        return super(Graph, self).test_on_batch(x, y,
-                                                sample_weight=sample_weight)
-
-    def predict_on_batch(self, data):
-        output_list = super(Graph, self).predict_on_batch(data)
-        if not isinstance(output_list, list):
-            output_list = [output_list]
-        return dict(zip(self._graph_outputs, output_list))
-
-    def fit_generator(self, generator, samples_per_epoch, nb_epoch,
-                      verbose=1, callbacks=[],
-                      validation_data=None, nb_val_samples=None,
-                      class_weight={},
-                      max_q_size=10, **kwargs):
-        '''Fits a model on data generated batch-by-batch by a Python generator.
-        The generator is run in parallel to the model, for efficiency.
-        For instance, this allows you to do real-time data augmentation
-        on images on CPU in parallel to training your model on GPU.
-
-        # Arguments
-            generator: a generator.
-                The output of the generator must be either a tuple
-                of dictionaries `(input_data, sample_weight)`
-                or a dictionary `input_data`
-                (mapping names of inputs and outputs to Numpy arrays).
-                All arrays should contain the same number of samples.
-                The generator is expected to loop over its data
-                indefinitely. An epoch finishes when `samples_per_epoch`
-                samples have been seen by the model.
-            samples_per_epoch: integer, number of samples to process before
-                going to the next epoch.
-            nb_epoch: integer, total number of iterations on the data.
-            verbose: verbosity mode, 0, 1, or 2.
-            callbacks: list of callbacks to be called during training.
-            validation_data: dictionary mapping input names and outputs names
-                to appropriate Numpy arrays to be used as
-                held-out validation data, or a generator yielding such
-                dictionaries. All arrays should contain the same number
-                of samples. If a generator, will be called until more than
-                `nb_val_samples` examples have been generated at the
-                end of every epoch. These examples will then be used
-                as the validation data.
-            nb_val_samples: number of samples to use from validation
-                generator at the end of every epoch.
-            class_weight: dictionary mapping class indices to a weight
-                for the class.
-
-        # Returns
-            A `History` object.
-
-        # Examples
-
-        ```python
-            def generate_arrays_from_file(path):
-                while 1:
-                    f = open(path)
-                    for line in f:
-                        # create Numpy arrays of input data
-                        # and labels, from each line in the file
-                        x1, x2, y = process_line(line)
-                        yield ({'input_1': x1, 'input_2': x2, 'output': y})
-                    f.close()
-
-            graph.fit_generator(generate_arrays_from_file('/my_file.txt'),
-                                samples_per_epoch=10000, nb_epoch=10)
-        ```
-        '''
-        if 'show_accuracy' in kwargs:
-            kwargs.pop('show_accuracy')
-            warnings.warn('The "show_accuracy" argument is deprecated, '
-                          'instead you should pass the "accuracy" metric to '
-                          'the model at compile time:\n'
-                          '`model.compile(optimizer, loss, '
-                          'metrics=["accuracy"])`')
-        if 'nb_worker' in kwargs:
-            kwargs.pop('nb_worker')
-            warnings.warn('The "nb_worker" argument is deprecated, '
-                          'please remove it from your code.')
-        if 'nb_val_worker' in kwargs:
-            kwargs.pop('nb_val_worker')
-            warnings.warn('The "nb_val_worker" argument is deprecated, '
-                          'please remove it from your code.')
-        if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
-                            str(kwargs))
-
-        self._train_on_batch = self.train_on_batch
-        self.train_on_batch = super(Graph, self).train_on_batch
-        self._evaluate = self.evaluate
-        self.evaluate = super(Graph, self).evaluate
-
-        if validation_data and type(validation_data) is tuple:
-            raise Exception('Cannot use sample_weight with '
-                            'validation_data in legacy Graph model.')
-        if validation_data and type(validation_data) is dict:
-            validation_data = (self._get_x(validation_data),
-                               self._get_y(validation_data))
-
-        original_generator = generator
-
-        def fixed_generator():
-            while 1:
-                data = next(original_generator)
-                if type(data) is tuple:
-                    data, sample_weight = data
-                    x = self._get_x(data)
-                    y = self._get_y(data)
-                    yield x, y, sample_weight
-                else:
-                    x = self._get_x(data)
-                    y = self._get_y(data)
-                    yield x, y
-
-        generator = fixed_generator()
-        history = super(Graph, self).fit_generator(generator,
-                                                   samples_per_epoch,
-                                                   nb_epoch,
-                                                   verbose=verbose,
-                                                   callbacks=callbacks,
-                                                   validation_data=validation_data,
-                                                   nb_val_samples=nb_val_samples,
-                                                   class_weight=class_weight,
-                                                   max_q_size=max_q_size)
-        self.train_on_batch = self._train_on_batch
-        self.evaluate = self._evaluate
-        return history
-
-    def evaluate_generator(self, generator, val_samples,
-                           verbose=1, max_q_size=10, **kwargs):
-        '''Evaluates the model on a generator. The generator should
-        return the same kind of data with every yield as accepted
-        by `evaluate`.
-
-        If `show_accuracy`, it returns a tuple `(loss, accuracy)`,
-        otherwise it returns the loss value.
-
-        Arguments:
-            generator:
-                generator yielding dictionaries of the kind accepted
-                by `evaluate`, or tuples of such dictionaries and
-                associated dictionaries of sample weights.
-            val_samples:
-                total number of samples to generate from `generator`
-                to use in validation.
-
-            Other arguments are the same as for `fit`.
-        '''
-        if 'show_accuracy' in kwargs:
-            kwargs.pop('show_accuracy')
-            warnings.warn('The "show_accuracy" argument is deprecated, '
-                          'instead you should pass the "accuracy" metric to '
-                          'the model at compile time:\n'
-                          '`model.compile(optimizer, loss, '
-                          'metrics=["accuracy"])`')
-        if 'verbose' in kwargs:
-            kwargs.pop('verbose')
-            warnings.warn('The "verbose" argument is deprecated.')
-        if kwargs:
-            raise Exception('Received unknown keyword arguments: ' +
-                            str(kwargs))
-
-        self._test_on_batch = self.test_on_batch
-        self.test_on_batch = super(Graph, self).test_on_batch
-
-        original_generator = generator
-
-        def fixed_generator():
-            while 1:
-                data = next(original_generator)
-                if type(data) is tuple:
-                    data, sample_weight = data
-                    x = self._get_x(data)
-                    y = self._get_y(data)
-                    yield x, y, sample_weight
-                else:
-                    x = self._get_x(data)
-                    y = self._get_y(data)
-                    yield x, y
-
-        generator = fixed_generator()
-        history = super(Graph, self).evaluate_generator(generator,
-                                                        val_samples,
-                                                        max_q_size=max_q_size)
-        self.test_on_batch = self._test_on_batch
-        return history
-
-    # get_weights, set_weights: inherited
-    def get_config(self):
-        config = {'input_config': self._graph_input_config,
-                  'node_config': self._graph_node_config,
-                  'output_config': self._graph_output_config}
-        nodes = {}
-        for name, node in self._graph_nodes.items():
-            nodes[name] = {'class_name': node.__class__.__name__,
-                           'config': node.get_config()}
-            if name in self._graph_shared_nodes_names:
-                nodes[name]['shared'] = True
-        config['nodes'] = nodes
-        return copy.deepcopy(config)
-
-    @classmethod
-    def from_config(cls, config):
-        # TODO: test legacy support
-        from keras.utils.layer_utils import layer_from_config
-
-        def normalize_legacy_config(conf):
-            if 'class_name' not in conf:
-                class_name = conf['name']
-                name = conf.get('custom_name')
-                conf['name'] = name
-                new_config = {
-                    'class_name': class_name,
-                    'config': conf,
-                }
-                return new_config
-            return conf
-
-        graph = cls()
-        inputs = config.get('input_config')
-        for input in inputs:
-            graph.add_input(**input)
-
-        nodes = config.get('node_config')
-        for node in nodes:
-            layer_config = config['nodes'][node['name']]
-            layer_config = normalize_legacy_config(layer_config)
-            if 'layer' in node:
-                # for add_shared_node
-                node['layer'] = layer_from_config(node['layer'])
-            else:
-                layer = layer_from_config(layer_config)
-                node['layer'] = layer
-
-            node['create_output'] = False  # outputs will be added below
-            if layer_config.get('shared'):
-                graph.add_shared_node(**node)
-            else:
-                graph.add_node(**node)
-
-        outputs = config.get('output_config')
-        for output in outputs:
-            graph.add_output(**output)
-        return graph
-
-    def load_weights(self, fname):
-        if not self.built:
-            self.build()
-        super(Graph, self).load_weights(fname)

keras/metrics.py

@@ -1,84 +1,216 @@
 import numpy as np
 from . import backend as K
+from .utils.generic_utils import get_from_module
 
 
 def binary_accuracy(y_true, y_pred):
+    '''Calculates the mean accuracy rate across all predictions for binary
+    classification problems.
+    '''
     return K.mean(K.equal(y_true, K.round(y_pred)))
 
 
 def categorical_accuracy(y_true, y_pred):
+    '''Calculates the mean accuracy rate across all predictions for
+    multiclass classification problems.
+    '''
     return K.mean(K.equal(K.argmax(y_true, axis=-1),
                   K.argmax(y_pred, axis=-1)))
 
 
 def sparse_categorical_accuracy(y_true, y_pred):
+    '''Same as categorical_accuracy, but useful when the predictions are for
+    sparse targets.
+    '''
     return K.mean(K.equal(K.max(y_true, axis=-1),
                           K.cast(K.argmax(y_pred, axis=-1), K.floatx())))
 
 
+def top_k_categorical_accuracy(y_true, y_pred, k=5):
+    '''Calculates the top-k categorical accuracy rate, i.e. success when the
+    target class is within the top-k predictions provided.
+    '''
+    return K.mean(K.in_top_k(y_pred, K.argmax(y_true, axis=-1), k))
+
+
 def mean_squared_error(y_true, y_pred):
+    '''Calculates the mean squared error (mse) rate
+    between predicted and target values.
+    '''
     return K.mean(K.square(y_pred - y_true))
 
 
 def mean_absolute_error(y_true, y_pred):
+    '''Calculates the mean absolute error (mae) rate
+    between predicted and target values.
+    '''
     return K.mean(K.abs(y_pred - y_true))
 
 
 def mean_absolute_percentage_error(y_true, y_pred):
+    '''Calculates the mean absolute percentage error (mape) rate
+    between predicted and target values.
+    '''
     diff = K.abs((y_true - y_pred) / K.clip(K.abs(y_true), K.epsilon(), np.inf))
     return 100. * K.mean(diff)
 
 
 def mean_squared_logarithmic_error(y_true, y_pred):
+    '''Calculates the mean squared logarithmic error (msle) rate
+    between predicted and target values.
+    '''
     first_log = K.log(K.clip(y_pred, K.epsilon(), np.inf) + 1.)
     second_log = K.log(K.clip(y_true, K.epsilon(), np.inf) + 1.)
     return K.mean(K.square(first_log - second_log))
 
 
-def squared_hinge(y_true, y_pred):
-    return K.mean(K.square(K.maximum(1. - y_true * y_pred, 0.)))
-
-
 def hinge(y_true, y_pred):
+    '''Calculates the hinge loss, which is defined as
+    `max(1 - y_true * y_pred, 0)`.
+    '''
     return K.mean(K.maximum(1. - y_true * y_pred, 0.))
 
 
+def squared_hinge(y_true, y_pred):
+    '''Calculates the squared value of the hinge loss.
+    '''
+    return K.mean(K.square(K.maximum(1. - y_true * y_pred, 0.)))
+
+
 def categorical_crossentropy(y_true, y_pred):
-    '''Expects a binary class matrix instead of a vector of scalar classes.
+    '''Calculates the cross-entropy value for multiclass classification
+    problems. Note: Expects a binary class matrix instead of a vector
+    of scalar classes.
     '''
     return K.mean(K.categorical_crossentropy(y_pred, y_true))
 
 
 def sparse_categorical_crossentropy(y_true, y_pred):
-    '''expects an array of integer classes.
-    Note: labels shape must have the same number of dimensions as output shape.
-    If you get a shape error, add a length-1 dimension to labels.
+    '''Calculates the cross-entropy value for multiclass classification
+    problems with sparse targets. Note: Expects an array of integer
+    classes. Labels shape must have the same number of dimensions as
+    output shape. If you get a shape error, add a length-1 dimension
+    to labels.
     '''
     return K.mean(K.sparse_categorical_crossentropy(y_pred, y_true))
 
 
 def binary_crossentropy(y_true, y_pred):
+    '''Calculates the cross-entropy value for binary classification
+    problems.
+    '''
     return K.mean(K.binary_crossentropy(y_pred, y_true))
 
 
+def kullback_leibler_divergence(y_true, y_pred):
+    '''Calculates the Kullback-Leibler (KL) divergence between prediction
+    and target values.
+    '''
+    y_true = K.clip(y_true, K.epsilon(), 1)
+    y_pred = K.clip(y_pred, K.epsilon(), 1)
+    return K.sum(y_true * K.log(y_true / y_pred), axis=-1)
+
+
 def poisson(y_true, y_pred):
+    '''Calculates the poisson function over prediction and target values.
+    '''
     return K.mean(y_pred - y_true * K.log(y_pred + K.epsilon()))
 
 
 def cosine_proximity(y_true, y_pred):
+    '''Calculates the cosine similarity between the prediction and target
+    values.
+    '''
     y_true = K.l2_normalize(y_true, axis=-1)
     y_pred = K.l2_normalize(y_pred, axis=-1)
     return -K.mean(y_true * y_pred)
 
 
+def matthews_correlation(y_true, y_pred):
+    '''Calculates the Matthews correlation coefficient measure for quality
+    of binary classification problems.
+    '''
+    y_pred_pos = K.round(K.clip(y_pred, 0, 1))
+    y_pred_neg = 1 - y_pred_pos
+
+    y_pos = K.round(K.clip(y_true, 0, 1))
+    y_neg = 1 - y_pos
+
+    tp = K.sum(y_pos * y_pred_pos)
+    tn = K.sum(y_neg * y_pred_neg)
+
+    fp = K.sum(y_neg * y_pred_pos)
+    fn = K.sum(y_pos * y_pred_neg)
+
+    numerator = (tp * tn - fp * fn)
+    denominator = K.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))
+
+    return numerator / (denominator + K.epsilon())
+
+
+def precision(y_true, y_pred):
+    '''Calculates the precision, a metric for multi-label classification of
+    how many selected items are relevant.
+    '''
+    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
+    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
+    precision = true_positives / (predicted_positives + K.epsilon())
+    return precision
+
+
+def recall(y_true, y_pred):
+    '''Calculates the recall, a metric for multi-label classification of
+    how many relevant items are selected.
+    '''
+    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
+    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
+    recall = true_positives / (possible_positives + K.epsilon())
+    return recall
+
+
+def fbeta_score(y_true, y_pred, beta=1):
+    '''Calculates the F score, the weighted harmonic mean of precision and recall.
+
+    This is useful for multi-label classification, where input samples can be
+    classified as sets of labels. By only using accuracy (precision) a model
+    would achieve a perfect score by simply assigning every class to every
+    input. In order to avoid this, a metric should penalize incorrect class
+    assignments as well (recall). The F-beta score (ranged from 0.0 to 1.0)
+    computes this, as a weighted mean of the proportion of correct class
+    assignments vs. the proportion of incorrect class assignments.
+
+    With beta = 1, this is equivalent to a F-measure. With beta < 1, assigning
+    correct classes becomes more important, and with beta > 1 the metric is
+    instead weighted towards penalizing incorrect class assignments.
+    '''
+    if beta < 0:
+        raise ValueError('The lowest choosable beta is zero (only precision).')
+        
+    # If there are no true positives, fix the F score at 0 like sklearn.
+    if K.sum(K.round(K.clip(y_true, 0, 1))) == 0:
+        return 0
+
+    p = precision(y_true, y_pred)
+    r = recall(y_true, y_pred)
+    bb = beta ** 2
+    fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon())
+    return fbeta_score
+
+
+def fmeasure(y_true, y_pred):
+    '''Calculates the f-measure, the harmonic mean of precision and recall.
+    '''
+    return fbeta_score(y_true, y_pred, beta=1)
+
+
 # aliases
 mse = MSE = mean_squared_error
 mae = MAE = mean_absolute_error
 mape = MAPE = mean_absolute_percentage_error
 msle = MSLE = mean_squared_logarithmic_error
 cosine = cosine_proximity
+fscore = f1score = fmeasure
 
 
-from .utils.generic_utils import get_from_module
 def get(identifier):
     return get_from_module(identifier, globals(), 'metric')

keras/models.py

@@ -1,11 +1,184 @@
 from __future__ import print_function
 import warnings
 import copy
+import json
+import os
+import numpy as np
 
 from . import backend as K
+from . import optimizers
+from .utils.io_utils import ask_to_proceed_with_overwrite
 from .engine.training import Model
-from .engine.topology import get_source_inputs, Node
-from .legacy.models import Graph
+from .engine.topology import get_source_inputs, Node, Layer, Merge
+from .optimizers import optimizer_from_config
+
+
+def save_model(model, filepath, overwrite=True):
+
+    def get_json_type(obj):
+        # if obj is a serializable Keras class instance
+        # e.g. optimizer, layer
+        if hasattr(obj, 'get_config'):
+            return {'class_name': obj.__class__.__name__,
+                    'config': obj.get_config()}
+
+        # if obj is any numpy type
+        if type(obj).__module__ == np.__name__:
+            return obj.item()
+
+        # misc functions (e.g. loss function)
+        if hasattr(obj, '__call__'):
+            return obj.__name__
+
+        # if obj is a python 'type'
+        if type(obj).__name__ == type.__name__:
+            return obj.__name__
+
+        raise TypeError('Not JSON Serializable:', obj)
+
+    import h5py
+    from keras import __version__ as keras_version
+
+    # if file exists and should not be overwritten
+    if not overwrite and os.path.isfile(filepath):
+        proceed = ask_to_proceed_with_overwrite(filepath)
+        if not proceed:
+            return
+
+    f = h5py.File(filepath, 'w')
+    f.attrs['keras_version'] = str(keras_version).encode('utf8')
+    f.attrs['model_config'] = json.dumps({
+        'class_name': model.__class__.__name__,
+        'config': model.get_config()
+    }, default=get_json_type).encode('utf8')
+
+    model_weights_group = f.create_group('model_weights')
+    model.save_weights_to_hdf5_group(model_weights_group)
+
+    if hasattr(model, 'optimizer'):
+        if isinstance(model.optimizer, optimizers.TFOptimizer):
+            warnings.warn(
+                'TensorFlow optimizers do not '
+                'make it possible to access '
+                'optimizer attributes or optimizer state '
+                'after instantiation. '
+                'As a result, we cannot save the optimizer '
+                'as part of the model save file.'
+                'You will have to compile your model again after loading it. '
+                'Prefer using a Keras optimizer instead '
+                '(see keras.io/optimizers).')
+        else:
+            f.attrs['training_config'] = json.dumps({
+                'optimizer_config': {
+                    'class_name': model.optimizer.__class__.__name__,
+                    'config': model.optimizer.get_config()
+                },
+                'loss': model.loss,
+                'metrics': model.metrics,
+                'sample_weight_mode': model.sample_weight_mode,
+                'loss_weights': model.loss_weights,
+            }, default=get_json_type).encode('utf8')
+
+            # save optimizer weights
+            symbolic_weights = getattr(model.optimizer, 'weights')
+            if symbolic_weights:
+                optimizer_weights_group = f.create_group('optimizer_weights')
+                weight_values = K.batch_get_value(symbolic_weights)
+                weight_names = []
+                for i, (w, val) in enumerate(zip(symbolic_weights, weight_values)):
+                    if hasattr(w, 'name') and w.name:
+                        name = str(w.name)
+                    else:
+                        name = 'param_' + str(i)
+                    weight_names.append(name.encode('utf8'))
+                optimizer_weights_group.attrs['weight_names'] = weight_names
+                for name, val in zip(weight_names, weight_values):
+                    param_dset = optimizer_weights_group.create_dataset(
+                        name,
+                        val.shape,
+                        dtype=val.dtype)
+                    if not val.shape:
+                        # scalar
+                        param_dset[()] = val
+                    else:
+                        param_dset[:] = val
+    f.flush()
+    f.close()
+
+
+def load_model(filepath, custom_objects={}):
+
+    def deserialize(obj):
+        if type(obj) is list:
+            deserialized = []
+            for value in obj:
+                if value in custom_objects:
+                    deserialized.append(custom_objects[value])
+                else:
+                    deserialized.append(value)
+            return deserialized
+        if type(obj) is dict:
+            deserialized = {}
+            for key, value in obj.items():
+                if value in custom_objects:
+                    deserialized[key] = custom_objects[value]
+                else:
+                    deserialized[key] = value
+            return deserialized
+        if obj in custom_objects:
+            return custom_objects[obj]
+        return obj
+
+    import h5py
+    f = h5py.File(filepath, mode='r')
+
+    # instantiate model
+    model_config = f.attrs.get('model_config')
+    if model_config is None:
+        raise ValueError('No model found in config file.')
+    model_config = json.loads(model_config.decode('utf-8'))
+    model = model_from_config(model_config, custom_objects=custom_objects)
+
+    # set weights
+    model.load_weights_from_hdf5_group(f['model_weights'])
+
+    # instantiate optimizer
+    training_config = f.attrs.get('training_config')
+    if training_config is None:
+        warnings.warn('No training configuration found in save file: '
+                      'the model was *not* compiled. Compile it manually.')
+        f.close()
+        return model
+    training_config = json.loads(training_config.decode('utf-8'))
+    optimizer_config = training_config['optimizer_config']
+    optimizer = optimizer_from_config(optimizer_config)
+
+    # recover loss functions and metrics
+    loss = deserialize(training_config['loss'])
+    metrics = deserialize(training_config['metrics'])
+    sample_weight_mode = training_config['sample_weight_mode']
+    loss_weights = training_config['loss_weights']
+
+    # compile model
+    model.compile(optimizer=optimizer,
+                  loss=loss,
+                  metrics=metrics,
+                  loss_weights=loss_weights,
+                  sample_weight_mode=sample_weight_mode)
+
+    # set optimizer weights
+    if 'optimizer_weights' in f:
+        # build train function (to get weight updates)
+        if isinstance(model, Sequential):
+            model.model._make_train_function()
+        else:
+            model._make_train_function()
+        optimizer_weights_group = f['optimizer_weights']
+        optimizer_weight_names = [n.decode('utf8') for n in optimizer_weights_group.attrs['weight_names']]
+        optimizer_weight_values = [optimizer_weights_group[n] for n in optimizer_weight_names]
+        model.optimizer.set_weights(optimizer_weight_values)
+    f.close()
+    return model
 
 
 def model_from_config(config, custom_objects={}):
@@ -77,6 +250,7 @@ class Sequential(Model):
         self.model = None  # internal Model instance
         self.inputs = []  # tensors
         self.outputs = []  # tensors (length 1)
+        self._trainable = True
 
         # model attributes
         self.inbound_nodes = []
@@ -98,6 +272,10 @@ class Sequential(Model):
         # Arguments
             layer: layer instance.
         '''
+        if not isinstance(layer, Layer):
+            raise ValueError('The added layer must be '
+                             'an instance of class Layer. '
+                             'Found: ' + str(layer))
         if not self.outputs:
             # first layer in model: check that it is an input layer
             if len(layer.inbound_nodes) == 0:
@@ -172,7 +350,27 @@ class Sequential(Model):
         else:
             self.layers[-1].outbound_nodes = []
             self.outputs = [self.layers[-1].output]
+            # update self.inbound_nodes
+            self.inbound_nodes[0].output_tensors = self.outputs
+            self.inbound_nodes[0].output_shapes = [self.outputs[0]._keras_shape]
         self.built = False
+        self._flattened_layers = None
+
+    def get_layer(self, name=None, index=None):
+        '''Returns a layer based on either its name (unique)
+        or its index in the graph. Indices are based on
+        order of horizontal graph traversal (bottom-up).
+
+        # Arguments
+            name: string, name of layer.
+            index: integer, index of layer.
+
+        # Returns
+            A layer instance.
+        '''
+        if not self.built:
+            self.build()
+        return self.model.get_layer(name, index)
 
     def call(self, x, mask=None):
         if not self.built:
@@ -185,6 +383,7 @@ class Sequential(Model):
                             ' Add some layers first.')
         # actually create the model
         self.model = Model(self.inputs, self.outputs[0], name=self.name + '_model')
+        self.model.trainable = self.trainable
 
         # mirror model attributes
         self.supports_masking = self.model.supports_masking
@@ -218,26 +417,27 @@ class Sequential(Model):
         if self._flattened_layers is not None:
             return self._flattened_layers
         layers = []
-        if self.layers[0].__class__.__name__ == 'Merge':
-            merge = self.layers[0]
-            for layer in merge.layers:
-                if hasattr(layer, 'flattened_layers'):
-                    for sublayer in layer.flattened_layers:
-                        if sublayer not in layers:
-                            layers.append(sublayer)
-                elif hasattr(layer, 'layers'):
-                    for sublayer in layer.layers:
-                        if sublayer not in layers:
-                            layers.append(sublayer)
-                else:
-                    if layer not in layers:
-                        layers.append(layer)
-        else:
-            if self.layers[0] not in layers:
-                layers.append(self.layers[0])
-        for layer in self.layers[1:]:
-            if layer not in layers:
-                layers.append(layer)
+        if self.layers:
+            if isinstance(self.layers[0], Merge):
+                merge = self.layers[0]
+                for layer in merge.layers:
+                    if hasattr(layer, 'flattened_layers'):
+                        for sublayer in layer.flattened_layers:
+                            if sublayer not in layers:
+                                layers.append(sublayer)
+                    elif hasattr(layer, 'layers'):
+                        for sublayer in layer.layers:
+                            if sublayer not in layers:
+                                layers.append(sublayer)
+                    else:
+                        if layer not in layers:
+                            layers.append(layer)
+            else:
+                if self.layers[0] not in layers:
+                    layers.append(self.layers[0])
+            for layer in self.layers[1:]:
+                if layer not in layers:
+                    layers.append(layer)
         self._flattened_layers = layers
         return layers
 
@@ -255,25 +455,43 @@ class Sequential(Model):
                              list(layer_dict.items()))
         return all_attrs
 
+    @property
+    def trainable(self):
+        return self._trainable
+
+    @trainable.setter
+    def trainable(self, value):
+        if self.model:
+            self.model.trainable = value
+        self._trainable = value
+
     @property
     def trainable_weights(self):
+        if not self.trainable:
+            return []
         # support for legacy behavior
         return self._gather_list_attr('trainable_weights')
 
     @property
     def non_trainable_weights(self):
         # support for legacy behavior
-        return self._gather_list_attr('non_trainable_weights')
+        weights = self._gather_list_attr('non_trainable_weights')
+        if not self.trainable:
+            trainable_weights = self._gather_list_attr('trainable_weights')
+            return trainable_weights + weights
+        return weights
 
     @property
     def updates(self):
-        # support for legacy behavior
-        return self._gather_list_attr('updates')
+        return self.model.updates
 
     @property
     def state_updates(self):
         # support for legacy behavior
-        return self._gather_list_attr('state_updates')
+        return self.model.state_updates
+
+    def get_updates_for(self, inputs):
+        return self.model.get_updates_for(inputs)
 
     @property
     def regularizers(self):
@@ -303,7 +521,7 @@ class Sequential(Model):
         '''
         # support for legacy behavior
         for layer in self.flattened_layers:
-            nb_param = len(layer.get_weights())
+            nb_param = len(layer.weights)
             layer.set_weights(weights[:nb_param])
             weights = weights[nb_param:]
 
@@ -329,6 +547,7 @@ class Sequential(Model):
             metrics: list of metrics to be evaluated by the model
                 during training and testing.
                 Typically you will use `metrics=['accuracy']`.
+                See [metrics](/metrics).
             sample_weight_mode: if you need to do timestep-wise
                 sample weighting (2D weights), set this to "temporal".
                 "None" defaults to sample-wise weights (1D).
@@ -359,6 +578,9 @@ class Sequential(Model):
                            **kwargs)
         self.optimizer = self.model.optimizer
         self.loss = self.model.loss
+        self.loss_weights = self.model.loss_weights
+        self.metrics = self.model.metrics
+        self.metrics_tensors = self.model.metrics_tensors
         self.metrics_names = self.model.metrics_names
         self.sample_weight_mode = self.model.sample_weight_mode
 
@@ -380,7 +602,8 @@ class Sequential(Model):
                 See [callbacks](/callbacks).
             validation_split: float (0. < x < 1).
                 Fraction of the data to use as held-out validation data.
-            validation_data: tuple (X, y) to be used as held-out
+            validation_data: tuple (x_val, y_val) or tuple
+                (x_val, y_val, val_sample_weights) to be used as held-out
                 validation data. Will override validation_split.
             shuffle: boolean or str (for 'batch').
                 Whether to shuffle the samples at each epoch.
@@ -594,7 +817,8 @@ class Sequential(Model):
     def fit_generator(self, generator, samples_per_epoch, nb_epoch,
                       verbose=1, callbacks=[],
                       validation_data=None, nb_val_samples=None,
-                      class_weight=None, max_q_size=10, nb_worker=1, pickle_safe=False, **kwargs):
+                      class_weight=None, max_q_size=10, nb_worker=1,
+                      pickle_safe=False, **kwargs):
         '''Fits the model on data generated batch-by-batch by
         a Python generator.
         The generator is run in parallel to the model, for efficiency.
@@ -627,7 +851,7 @@ class Sequential(Model):
             max_q_size: maximum size for the generator queue
             nb_worker: maximum number of processes to spin up
             pickle_safe: if True, use process based threading. Note that because
-                this implementation relies on multiprocessing, you should not pass non
+                this implementation relies on multiprocessing, you should not pass
                 non picklable arguments to the generator as they can't be passed
                 easily to children processes.
 
@@ -682,11 +906,13 @@ class Sequential(Model):
                                         nb_worker=nb_worker,
                                         pickle_safe=pickle_safe)
 
-    def evaluate_generator(self, generator, val_samples, max_q_size=10, nb_worker=1, pickle_safe=False, **kwargs):
+    def evaluate_generator(self, generator, val_samples,
+                           max_q_size=10, nb_worker=1,
+                           pickle_safe=False, **kwargs):
         '''Evaluates the model on a data generator. The generator should
         return the same kind of data as accepted by `test_on_batch`.
 
-        Arguments:
+        # Arguments
             generator:
                 generator yielding tuples (inputs, targets)
                 or (inputs, targets, sample_weights)
@@ -724,7 +950,8 @@ class Sequential(Model):
                                              nb_worker=nb_worker,
                                              pickle_safe=pickle_safe)
 
-    def predict_generator(self, generator, val_samples, max_q_size=10, nb_worker=1, pickle_safe=False):
+    def predict_generator(self, generator, val_samples,
+                          max_q_size=10, nb_worker=1, pickle_safe=False):
         '''Generates predictions for the input samples from a data generator.
         The generator should return the same kind of data as accepted by
         `predict_on_batch`.
@@ -758,7 +985,7 @@ class Sequential(Model):
         as a Python list.
         '''
         config = []
-        if self.layers[0].__class__.__name__ == 'Merge':
+        if isinstance(self.layers[0], Merge):
             assert hasattr(self.layers[0], 'layers')
             layers = []
             for layer in self.layers[0].layers:

keras/objectives.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 import numpy as np
 from . import backend as K
+from .utils.generic_utils import get_from_module
 
 
 def mean_squared_error(y_true, y_pred):
@@ -72,6 +73,6 @@ msle = MSLE = mean_squared_logarithmic_error
 kld = KLD = kullback_leibler_divergence
 cosine = cosine_proximity
 
-from .utils.generic_utils import get_from_module
+
 def get(identifier):
     return get_from_module(identifier, globals(), 'objective')

keras/optimizers.py

@@ -1,8 +1,8 @@
 from __future__ import absolute_import
 from . import backend as K
-import numpy as np
 from .utils.generic_utils import get_from_module
 from six.moves import zip
+import warnings
 
 
 def clip_norm(g, c, n):
@@ -11,8 +11,25 @@ def clip_norm(g, c, n):
     return g
 
 
-def kl_divergence(p, p_hat):
-    return p_hat - p + p * K.log(p / p_hat)
+def optimizer_from_config(config, custom_objects={}):
+    all_classes = {
+        'sgd': SGD,
+        'rmsprop': RMSprop,
+        'adagrad': Adagrad,
+        'adadelta': Adadelta,
+        'adam': Adam,
+        'adamax': Adamax,
+        'nadam': Nadam,
+        'tfoptimizer': TFOptimizer,
+    }
+    class_name = config['class_name']
+    if class_name in custom_objects:
+        cls = custom_objects[class_name]
+    else:
+        if class_name.lower() not in all_classes:
+            raise ValueError('Optimizer class not found:', class_name)
+        cls = all_classes[class_name.lower()]
+    return cls.from_config(config['config'])
 
 
 class Optimizer(object):
@@ -38,14 +55,6 @@ class Optimizer(object):
         self.updates = []
         self.weights = []
 
-    def get_state(self):
-        return [K.get_value(u[0]) for u in self.updates]
-
-    def set_state(self, value_list):
-        assert len(self.updates) == len(value_list)
-        for u, v in zip(self.updates, value_list):
-            K.set_value(u[0], v)
-
     def get_updates(self, params, constraints, loss):
         raise NotImplementedError
 
@@ -72,35 +81,35 @@ class Optimizer(object):
                 output of `get_weights`).
         '''
         params = self.weights
-        if len(params) != len(weights):
-            raise Exception('Provided weight array does not match  weights (' +
-                            str(len(params)) + ' optimizer params vs. ' +
-                            str(len(weights)) + ' provided weights)')
-        for p, w in zip(params, weights):
-            if K.get_value(p).shape != w.shape:
+        weight_value_tuples = []
+        param_values = K.batch_get_value(params)
+        for pv, p, w in zip(param_values, params, weights):
+            if pv.shape != w.shape:
                 raise Exception('Optimizer weight shape ' +
-                                str(K.get_value(p).shape) +
+                                str(pv.shape) +
                                 ' not compatible with '
                                 'provided weight shape ' + str(w.shape))
-            K.set_value(p, w)
+            weight_value_tuples.append((p, w))
+        K.batch_set_value(weight_value_tuples)
 
     def get_weights(self):
         '''Returns the current weights of the optimizer,
         as a list of numpy arrays.
         '''
-        weights = []
-        for p in self.weights:
-            weights.append(K.get_value(p))
-        return weights
+        return K.batch_get_value(self.weights)
 
     def get_config(self):
-        config = {'name': self.__class__.__name__}
+        config = {}
         if hasattr(self, 'clipnorm'):
             config['clipnorm'] = self.clipnorm
         if hasattr(self, 'clipvalue'):
             config['clipvalue'] = self.clipvalue
         return config
 
+    @classmethod
+    def from_config(cls, config):
+        return cls(**config)
+
 
 class SGD(Optimizer):
     '''Stochastic gradient descent, with support for momentum,
@@ -120,17 +129,24 @@ class SGD(Optimizer):
         self.lr = K.variable(lr)
         self.momentum = K.variable(momentum)
         self.decay = K.variable(decay)
+        self.inital_decay = decay
 
     def get_updates(self, params, constraints, loss):
         grads = self.get_gradients(loss, params)
-        lr = self.lr * (1. / (1. + self.decay * self.iterations))
-        self.updates = [(self.iterations, self.iterations + 1.)]
+        self.updates = []
+
+        lr = self.lr
+        if self.inital_decay > 0:
+            lr *= (1. / (1. + self.decay * self.iterations))
+            self.updates .append(K.update_add(self.iterations, 1))
 
         # momentum
-        self.weights = [K.variable(np.zeros(K.get_value(p).shape)) for p in params]
-        for p, g, m in zip(params, grads, self.weights):
+        shapes = [K.get_variable_shape(p) for p in params]
+        moments = [K.zeros(shape) for shape in shapes]
+        self.weights = [self.iterations] + moments
+        for p, g, m in zip(params, grads, moments):
             v = self.momentum * m - lr * g  # velocity
-            self.updates.append((m, v))
+            self.updates.append(K.update(m, v))
 
             if self.nesterov:
                 new_p = p + self.momentum * v - lr * g
@@ -141,7 +157,8 @@ class SGD(Optimizer):
             if p in constraints:
                 c = constraints[p]
                 new_p = c(new_p)
-            self.updates.append((p, new_p))
+
+            self.updates.append(K.update(p, new_p))
         return self.updates
 
     def get_config(self):
@@ -167,35 +184,47 @@ class RMSprop(Optimizer):
         lr: float >= 0. Learning rate.
         rho: float >= 0.
         epsilon: float >= 0. Fuzz factor.
+        decay: float >= 0. Learning rate decay over each update.
     '''
-    def __init__(self, lr=0.001, rho=0.9, epsilon=1e-8, **kwargs):
+    def __init__(self, lr=0.001, rho=0.9, epsilon=1e-8, decay=0.,
+                 **kwargs):
         super(RMSprop, self).__init__(**kwargs)
         self.__dict__.update(locals())
         self.lr = K.variable(lr)
         self.rho = K.variable(rho)
+        self.decay = K.variable(decay)
+        self.inital_decay = decay
+        self.iterations = K.variable(0.)
 
     def get_updates(self, params, constraints, loss):
         grads = self.get_gradients(loss, params)
-        # accumulators
-        self.weights = [K.variable(np.zeros(K.get_value(p).shape)) for p in params]
+        shapes = [K.get_variable_shape(p) for p in params]
+        accumulators = [K.zeros(shape) for shape in shapes]
+        self.weights = accumulators
         self.updates = []
 
-        for p, g, a in zip(params, grads, self.weights):
+        lr = self.lr
+        if self.inital_decay > 0:
+            lr *= (1. / (1. + self.decay * self.iterations))
+            self.updates.append(K.update_add(self.iterations, 1))
+
+        for p, g, a in zip(params, grads, accumulators):
             # update accumulator
             new_a = self.rho * a + (1. - self.rho) * K.square(g)
-            self.updates.append((a, new_a))
-            new_p = p - self.lr * g / (K.sqrt(new_a) + self.epsilon)
+            self.updates.append(K.update(a, new_a))
+            new_p = p - lr * g / (K.sqrt(new_a) + self.epsilon)
 
             # apply constraints
             if p in constraints:
                 c = constraints[p]
                 new_p = c(new_p)
-            self.updates.append((p, new_p))
+            self.updates.append(K.update(p, new_p))
         return self.updates
 
     def get_config(self):
         config = {'lr': float(K.get_value(self.lr)),
                   'rho': float(K.get_value(self.rho)),
+                  'decay': float(K.get_value(self.decay)),
                   'epsilon': self.epsilon}
         base_config = super(RMSprop, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
@@ -210,31 +239,44 @@ class Adagrad(Optimizer):
     # Arguments
         lr: float >= 0. Learning rate.
         epsilon: float >= 0.
+
+    # References
+        - [Adaptive Subgradient Methods for Online Learning and Stochastic Optimization](http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf)
     '''
-    def __init__(self, lr=0.01, epsilon=1e-8, **kwargs):
+    def __init__(self, lr=0.01, epsilon=1e-8, decay=0., **kwargs):
         super(Adagrad, self).__init__(**kwargs)
         self.__dict__.update(locals())
         self.lr = K.variable(lr)
+        self.decay = K.variable(decay)
+        self.inital_decay = decay
+        self.iterations = K.variable(0.)
 
     def get_updates(self, params, constraints, loss):
         grads = self.get_gradients(loss, params)
-        # accumulators
-        self.weights = [K.variable(np.zeros(K.get_value(p).shape)) for p in params]
+        shapes = [K.get_variable_shape(p) for p in params]
+        accumulators = [K.zeros(shape) for shape in shapes]
+        self.weights = accumulators
         self.updates = []
 
-        for p, g, a in zip(params, grads, self.weights):
+        lr = self.lr
+        if self.inital_decay > 0:
+            lr *= (1. / (1. + self.decay * self.iterations))
+            self.updates.append(K.update_add(self.iterations, 1))
+
+        for p, g, a in zip(params, grads, accumulators):
             new_a = a + K.square(g)  # update accumulator
-            self.updates.append((a, new_a))
-            new_p = p - self.lr * g / (K.sqrt(new_a) + self.epsilon)
+            self.updates.append(K.update(a, new_a))
+            new_p = p - lr * g / (K.sqrt(new_a) + self.epsilon)
             # apply constraints
             if p in constraints:
                 c = constraints[p]
                 new_p = c(new_p)
-            self.updates.append((p, new_p))
+            self.updates.append(K.update(p, new_p))
         return self.updates
 
     def get_config(self):
         config = {'lr': float(K.get_value(self.lr)),
+                  'decay': float(K.get_value(self.decay)),
                   'epsilon': self.epsilon}
         base_config = super(Adagrad, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
@@ -255,41 +297,52 @@ class Adadelta(Optimizer):
     # References
         - [Adadelta - an adaptive learning rate method](http://arxiv.org/abs/1212.5701)
     '''
-    def __init__(self, lr=1.0, rho=0.95, epsilon=1e-8, **kwargs):
+    def __init__(self, lr=1.0, rho=0.95, epsilon=1e-8, decay=0.,
+                 **kwargs):
         super(Adadelta, self).__init__(**kwargs)
         self.__dict__.update(locals())
         self.lr = K.variable(lr)
+        self.decay = K.variable(decay)
+        self.inital_decay = decay
+        self.iterations = K.variable(0.)
 
     def get_updates(self, params, constraints, loss):
         grads = self.get_gradients(loss, params)
-        accumulators = [K.variable(np.zeros(K.get_value(p).shape)) for p in params]
-        delta_accumulators = [K.variable(np.zeros(K.get_value(p).shape)) for p in params]
+        shapes = [K.get_variable_shape(p) for p in params]
+        accumulators = [K.zeros(shape) for shape in shapes]
+        delta_accumulators = [K.zeros(shape) for shape in shapes]
         self.weights = accumulators + delta_accumulators
         self.updates = []
 
+        lr = self.lr
+        if self.inital_decay > 0:
+            lr *= (1. / (1. + self.decay * self.iterations))
+            self.updates.append(K.update_add(self.iterations, 1))
+
         for p, g, a, d_a in zip(params, grads, accumulators, delta_accumulators):
             # update accumulator
             new_a = self.rho * a + (1. - self.rho) * K.square(g)
-            self.updates.append((a, new_a))
+            self.updates.append(K.update(a, new_a))
 
             # use the new accumulator and the *old* delta_accumulator
             update = g * K.sqrt(d_a + self.epsilon) / K.sqrt(new_a + self.epsilon)
 
-            new_p = p - self.lr * update
+            new_p = p - lr * update
             # apply constraints
             if p in constraints:
                 c = constraints[p]
                 new_p = c(new_p)
-            self.updates.append((p, new_p))
+            self.updates.append(K.update(p, new_p))
 
             # update delta_accumulator
             new_d_a = self.rho * d_a + (1 - self.rho) * K.square(update)
-            self.updates.append((d_a, new_d_a))
+            self.updates.append(K.update(d_a, new_d_a))
         return self.updates
 
     def get_config(self):
         config = {'lr': float(K.get_value(self.lr)),
                   'rho': self.rho,
+                  'decay': float(K.get_value(self.decay)),
                   'epsilon': self.epsilon}
         base_config = super(Adadelta, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
@@ -309,45 +362,53 @@ class Adam(Optimizer):
         - [Adam - A Method for Stochastic Optimization](http://arxiv.org/abs/1412.6980v8)
     '''
     def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
-                 epsilon=1e-8, **kwargs):
+                 epsilon=1e-8, decay=0., **kwargs):
         super(Adam, self).__init__(**kwargs)
         self.__dict__.update(locals())
         self.iterations = K.variable(0)
         self.lr = K.variable(lr)
         self.beta_1 = K.variable(beta_1)
         self.beta_2 = K.variable(beta_2)
+        self.decay = K.variable(decay)
+        self.inital_decay = decay
 
     def get_updates(self, params, constraints, loss):
         grads = self.get_gradients(loss, params)
-        self.updates = [(self.iterations, self.iterations + 1)]
+        self.updates = [K.update_add(self.iterations, 1)]
+
+        lr = self.lr
+        if self.inital_decay > 0:
+            lr *= (1. / (1. + self.decay * self.iterations))
 
         t = self.iterations + 1
-        lr_t = self.lr * K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t))
+        lr_t = lr * K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t))
 
-        ms = [K.variable(np.zeros(K.get_value(p).shape)) for p in params]
-        vs = [K.variable(np.zeros(K.get_value(p).shape)) for p in params]
-        self.weights = ms + vs
+        shapes = [K.get_variable_shape(p) for p in params]
+        ms = [K.zeros(shape) for shape in shapes]
+        vs = [K.zeros(shape) for shape in shapes]
+        self.weights = [self.iterations] + ms + vs
 
         for p, g, m, v in zip(params, grads, ms, vs):
             m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
             v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
             p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)
 
-            self.updates.append((m, m_t))
-            self.updates.append((v, v_t))
+            self.updates.append(K.update(m, m_t))
+            self.updates.append(K.update(v, v_t))
 
             new_p = p_t
             # apply constraints
             if p in constraints:
                 c = constraints[p]
                 new_p = c(new_p)
-            self.updates.append((p, new_p))
+            self.updates.append(K.update(p, new_p))
         return self.updates
 
     def get_config(self):
         config = {'lr': float(K.get_value(self.lr)),
                   'beta_1': float(K.get_value(self.beta_1)),
                   'beta_2': float(K.get_value(self.beta_2)),
+                  'decay': float(K.get_value(self.decay)),
                   'epsilon': self.epsilon}
         base_config = super(Adam, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
@@ -368,26 +429,33 @@ class Adamax(Optimizer):
         - [Adam - A Method for Stochastic Optimization](http://arxiv.org/abs/1412.6980v8)
     '''
     def __init__(self, lr=0.002, beta_1=0.9, beta_2=0.999,
-                 epsilon=1e-8, **kwargs):
+                 epsilon=1e-8, decay=0., **kwargs):
         super(Adamax, self).__init__(**kwargs)
         self.__dict__.update(locals())
         self.iterations = K.variable(0.)
         self.lr = K.variable(lr)
         self.beta_1 = K.variable(beta_1)
         self.beta_2 = K.variable(beta_2)
+        self.decay = K.variable(decay)
+        self.inital_decay = decay
 
     def get_updates(self, params, constraints, loss):
         grads = self.get_gradients(loss, params)
-        self.updates = [(self.iterations, self.iterations + 1)]
+        self.updates = [K.update_add(self.iterations, 1)]
+
+        lr = self.lr
+        if self.inital_decay > 0:
+            lr *= (1. / (1. + self.decay * self.iterations))
 
         t = self.iterations + 1
-        lr_t = self.lr / (1. - K.pow(self.beta_1, t))
+        lr_t = lr / (1. - K.pow(self.beta_1, t))
 
+        shapes = [K.get_variable_shape(p) for p in params]
         # zero init of 1st moment
-        ms = [K.variable(np.zeros(K.get_value(p).shape)) for p in params]
+        ms = [K.zeros(shape) for shape in shapes]
         # zero init of exponentially weighted infinity norm
-        us = [K.variable(np.zeros(K.get_value(p).shape)) for p in params]
-        self.weights = ms + us
+        us = [K.zeros(shape) for shape in shapes]
+        self.weights = [self.iterations] + ms + us
 
         for p, g, m, u in zip(params, grads, ms, us):
 
@@ -395,21 +463,22 @@ class Adamax(Optimizer):
             u_t = K.maximum(self.beta_2 * u, K.abs(g))
             p_t = p - lr_t * m_t / (u_t + self.epsilon)
 
-            self.updates.append((m, m_t))
-            self.updates.append((u, u_t))
+            self.updates.append(K.update(m, m_t))
+            self.updates.append(K.update(u, u_t))
 
             new_p = p_t
             # apply constraints
             if p in constraints:
                 c = constraints[p]
                 new_p = c(new_p)
-            self.updates.append((p, new_p))
+            self.updates.append(K.update(p, new_p))
         return self.updates
 
     def get_config(self):
         config = {'lr': float(K.get_value(self.lr)),
                   'beta_1': float(K.get_value(self.beta_1)),
                   'beta_2': float(K.get_value(self.beta_2)),
+                  'decay': float(K.get_value(self.decay)),
                   'epsilon': self.epsilon}
         base_config = super(Adamax, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
@@ -430,9 +499,8 @@ class Nadam(Optimizer):
         epsilon: float >= 0. Fuzz factor.
 
     # References
-        [1] Nadam report - http://cs229.stanford.edu/proj2015/054_report.pdf
-        [2] On the importance of initialization and momentum in deep learning -
-            http://www.cs.toronto.edu/~fritz/absps/momentum.pdf
+        - [Nadam report](http://cs229.stanford.edu/proj2015/054_report.pdf)
+        - [On the importance of initialization and momentum in deep learning](http://www.cs.toronto.edu/~fritz/absps/momentum.pdf)
     '''
     def __init__(self, lr=0.002, beta_1=0.9, beta_2=0.999,
                  epsilon=1e-8, schedule_decay=0.004, **kwargs):
@@ -447,7 +515,7 @@ class Nadam(Optimizer):
 
     def get_updates(self, params, constraints, loss):
         grads = self.get_gradients(loss, params)
-        self.updates = [(self.iterations, self.iterations + 1)]
+        self.updates = [K.update_add(self.iterations, 1)]
 
         t = self.iterations + 1
 
@@ -458,10 +526,11 @@ class Nadam(Optimizer):
         m_schedule_next = self.m_schedule * momentum_cache_t * momentum_cache_t_1
         self.updates.append((self.m_schedule, m_schedule_new))
 
-        ms = [K.variable(np.zeros(K.get_value(p).shape)) for p in params]
-        vs = [K.variable(np.zeros(K.get_value(p).shape)) for p in params]
+        shapes = [K.get_variable_shape(p) for p in params]
+        ms = [K.zeros(shape) for shape in shapes]
+        vs = [K.zeros(shape) for shape in shapes]
 
-        self.weights = ms + vs
+        self.weights = [self.iterations] + ms + vs
 
         for p, g, m, v in zip(params, grads, ms, vs):
             # the following equations given in [1]
@@ -472,8 +541,8 @@ class Nadam(Optimizer):
             v_t_prime = v_t / (1. - K.pow(self.beta_2, t))
             m_t_bar = (1. - momentum_cache_t) * g_prime + momentum_cache_t_1 * m_t_prime
 
-            self.updates.append((m, m_t))
-            self.updates.append((v, v_t))
+            self.updates.append(K.update(m, m_t))
+            self.updates.append(K.update(v, v_t))
 
             p_t = p - self.lr * m_t_bar / (K.sqrt(v_t_prime) + self.epsilon)
             new_p = p_t
@@ -482,7 +551,7 @@ class Nadam(Optimizer):
             if p in constraints:
                 c = constraints[p]
                 new_p = c(new_p)
-            self.updates.append((p, new_p))
+            self.updates.append(K.update(p, new_p))
         return self.updates
 
     def get_config(self):
@@ -495,6 +564,36 @@ class Nadam(Optimizer):
         return dict(list(base_config.items()) + list(config.items()))
 
 
+class TFOptimizer(Optimizer):
+
+    def __init__(self, optimizer):
+        self.optimizer = optimizer
+        self.iterations = K.variable(0.)
+        self.updates = []
+
+    def get_updates(self, params, constraints, loss):
+        if constraints:
+            raise ValueError('TF optimizers do not support '
+                             'weights constraints. Either remove '
+                             'all weights constraints in your model, '
+                             'or use a Keras optimizer.')
+        grads = self.optimizer.compute_gradients(loss, params)
+        opt_update = self.optimizer.apply_gradients(
+            grads, global_step=self.iterations)
+        self.updates.append(opt_update)
+        return self.updates
+
+    @property
+    def weights(self):
+        raise NotImplementedError
+
+    def get_config(self):
+        raise NotImplementedError
+
+    def from_config(self, config):
+        raise NotImplementedError
+
+
 # aliases
 sgd = SGD
 rmsprop = RMSprop
@@ -506,5 +605,11 @@ nadam = Nadam
 
 
 def get(identifier, kwargs=None):
+    if K.backend() == 'tensorflow':
+        # Wrap TF optimizer instances
+        import tensorflow as tf
+        if isinstance(identifier, tf.train.Optimizer):
+            return TFOptimizer(identifier)
+    # Instantiate a Keras optimizer
     return get_from_module(identifier, globals(), 'optimizer',
                            instantiate=True, kwargs=kwargs)

keras/preprocessing/image.py

@@ -118,13 +118,17 @@ def flip_axis(x, axis):
     return x
 
 
-def array_to_img(x, dim_ordering=K.image_dim_ordering(), scale=True):
+def array_to_img(x, dim_ordering='default', scale=True):
     from PIL import Image
+    if dim_ordering == 'default':
+        dim_ordering = K.image_dim_ordering()
     if dim_ordering == 'th':
         x = x.transpose(1, 2, 0)
     if scale:
         x += max(-np.min(x), 0)
-        x /= np.max(x)
+        x_max = np.max(x)
+        if x_max != 0:
+            x /= x_max
         x *= 255
     if x.shape[2] == 3:
         # RGB
@@ -136,7 +140,9 @@ def array_to_img(x, dim_ordering=K.image_dim_ordering(), scale=True):
         raise Exception('Unsupported channel number: ', x.shape[2])
 
 
-def img_to_array(img, dim_ordering=K.image_dim_ordering()):
+def img_to_array(img, dim_ordering='default'):
+    if dim_ordering == 'default':
+        dim_ordering = K.image_dim_ordering()
     if dim_ordering not in ['th', 'tf']:
         raise Exception('Unknown dim_ordering: ', dim_ordering)
     # image has dim_ordering (height, width, channel)
@@ -155,6 +161,14 @@ def img_to_array(img, dim_ordering=K.image_dim_ordering()):
 
 
 def load_img(path, grayscale=False, target_size=None):
+    '''Load an image into PIL format.
+
+    # Arguments
+        path: path to image file
+        grayscale: boolean
+        target_size: None (default to original size)
+            or (img_height, img_width)
+    '''
     from PIL import Image
     img = Image.open(path)
     if grayscale:
@@ -162,12 +176,12 @@ def load_img(path, grayscale=False, target_size=None):
     else:  # Ensure 3 channel even when loaded image is grayscale
         img = img.convert('RGB')
     if target_size:
-        img = img.resize(target_size)
+        img = img.resize((target_size[1], target_size[0]))
     return img
 
 
 def list_pictures(directory, ext='jpg|jpeg|bmp|png'):
-    return [os.path.join(directory, f) for f in os.listdir(directory)
+    return [os.path.join(directory, f) for f in sorted(os.listdir(directory))
             if os.path.isfile(os.path.join(directory, f)) and re.match('([\w]+\.(?:' + ext + '))', f)]
 
 
@@ -222,7 +236,9 @@ class ImageDataGenerator(object):
                  horizontal_flip=False,
                  vertical_flip=False,
                  rescale=None,
-                 dim_ordering=K.image_dim_ordering()):
+                 dim_ordering='default'):
+        if dim_ordering == 'default':
+            dim_ordering = K.image_dim_ordering()
         self.__dict__.update(locals())
         self.mean = None
         self.std = None
@@ -374,6 +390,9 @@ class ImageDataGenerator(object):
                 how many augmentation passes to do over the data
             seed: random seed.
         '''
+        if seed is not None:
+            np.random.seed(seed)
+
         X = np.copy(X)
         if augment:
             aX = np.zeros(tuple([rounds * X.shape[0]] + list(X.shape)[1:]))
@@ -392,7 +411,7 @@ class ImageDataGenerator(object):
 
         if self.zca_whitening:
             flatX = np.reshape(X, (X.shape[0], X.shape[1] * X.shape[2] * X.shape[3]))
-            sigma = np.dot(flatX.T, flatX) / flatX.shape[1]
+            sigma = np.dot(flatX.T, flatX) / flatX.shape[0]
             U, S, V = linalg.svd(sigma)
             self.principal_components = np.dot(np.dot(U, np.diag(1. / np.sqrt(S + 10e-7))), U.T)
 
@@ -415,11 +434,11 @@ class Iterator(object):
         # ensure self.batch_index is 0
         self.reset()
         while 1:
+            if seed is not None:
+                np.random.seed(seed + self.total_batches_seen)
             if self.batch_index == 0:
                 index_array = np.arange(N)
                 if shuffle:
-                    if seed is not None:
-                        np.random.seed(seed + self.total_batches_seen)
                     index_array = np.random.permutation(N)
 
             current_index = (self.batch_index * batch_size) % N
@@ -446,12 +465,14 @@ class NumpyArrayIterator(Iterator):
 
     def __init__(self, X, y, image_data_generator,
                  batch_size=32, shuffle=False, seed=None,
-                 dim_ordering=K.image_dim_ordering(),
+                 dim_ordering='default',
                  save_to_dir=None, save_prefix='', save_format='jpeg'):
         if y is not None and len(X) != len(y):
             raise Exception('X (images tensor) and y (labels) '
                             'should have the same length. '
                             'Found: X.shape = %s, y.shape = %s' % (np.asarray(X).shape, np.asarray(y).shape))
+        if dim_ordering == 'default':
+            dim_ordering = K.image_dim_ordering()
         self.X = X
         self.y = y
         self.image_data_generator = image_data_generator
@@ -493,10 +514,12 @@ class DirectoryIterator(Iterator):
 
     def __init__(self, directory, image_data_generator,
                  target_size=(256, 256), color_mode='rgb',
-                 dim_ordering=K.image_dim_ordering,
+                 dim_ordering='default',
                  classes=None, class_mode='categorical',
                  batch_size=32, shuffle=True, seed=None,
                  save_to_dir=None, save_prefix='', save_format='jpeg'):
+        if dim_ordering == 'default':
+            dim_ordering = K.image_dim_ordering()
         self.directory = directory
         self.image_data_generator = image_data_generator
         self.target_size = tuple(target_size)
@@ -540,7 +563,7 @@ class DirectoryIterator(Iterator):
 
         for subdir in classes:
             subpath = os.path.join(directory, subdir)
-            for fname in os.listdir(subpath):
+            for fname in sorted(os.listdir(subpath)):
                 is_valid = False
                 for extension in white_list_formats:
                     if fname.lower().endswith('.' + extension):
@@ -556,7 +579,7 @@ class DirectoryIterator(Iterator):
         i = 0
         for subdir in classes:
             subpath = os.path.join(directory, subdir)
-            for fname in os.listdir(subpath):
+            for fname in sorted(os.listdir(subpath)):
                 is_valid = False
                 for extension in white_list_formats:
                     if fname.lower().endswith('.' + extension):

keras/preprocessing/sequence.py

@@ -138,7 +138,7 @@ def skipgrams(sequence, vocabulary_size,
                     continue
                 couples.append([wi, wj])
                 if categorical:
-                    labels.append([0,1])
+                    labels.append([0, 1])
                 else:
                     labels.append(1)
 
@@ -149,12 +149,12 @@ def skipgrams(sequence, vocabulary_size,
 
         couples += [[words[i %len(words)], random.randint(1, vocabulary_size-1)] for i in range(nb_negative_samples)]
         if categorical:
-            labels += [[1,0]]*nb_negative_samples
+            labels += [[1, 0]]*nb_negative_samples
         else:
             labels += [0]*nb_negative_samples
 
     if shuffle:
-        seed = random.randint(0,10e6)
+        seed = random.randint(0, 10e6)
         random.seed(seed)
         random.shuffle(couples)
         random.seed(seed)

keras/preprocessing/text.py

@@ -99,6 +99,7 @@ class Tokenizer(object):
         wcounts = list(self.word_counts.items())
         wcounts.sort(key=lambda x: x[1], reverse=True)
         sorted_voc = [wc[0] for wc in wcounts]
+        # note that index 0 is reserved, never assigned to an existing word
         self.word_index = dict(list(zip(sorted_voc, list(range(1, len(sorted_voc) + 1)))))
 
         self.index_docs = {}

keras/regularizers.py

@@ -1,8 +1,10 @@
 from __future__ import absolute_import
 from . import backend as K
+from .utils.generic_utils import get_from_module
 
 
 class Regularizer(object):
+
     def set_param(self, p):
         self.p = p
 
@@ -29,6 +31,9 @@ class EigenvalueRegularizer(Regularizer):
         self.uses_learning_phase = True
 
     def set_param(self, p):
+        if hasattr(self, 'p'):
+            raise Exception('Regularizers cannot be reused. '
+                            'Instantiate one regularizer per layer.')
         self.p = p
 
     def __call__(self, loss):
@@ -43,30 +48,37 @@ class EigenvalueRegularizer(Regularizer):
 
         # power method for approximating the dominant eigenvector:
         o = K.ones([dim1, 1])  # initial values for the dominant eigenvector
-        domin_eigenvect = K.dot(WW, o)
+        main_eigenvect = K.dot(WW, o)
         for n in range(power - 1):
-            domin_eigenvect = K.dot(WW, domin_eigenvect)
+            main_eigenvect = K.dot(WW, main_eigenvect)
 
-        WWd = K.dot(WW, domin_eigenvect)
+        WWd = K.dot(WW, main_eigenvect)
 
         # the corresponding dominant eigenvalue:
-        domin_eigenval = K.dot(K.transpose(WWd), domin_eigenvect) / K.dot(K.transpose(domin_eigenvect), domin_eigenvect)
-        regularized_loss = loss + (domin_eigenval ** 0.5) * self.k  # multiplied by the given regularization gain
+        main_eigenval = (K.dot(K.transpose(WWd), main_eigenvect) /
+                         K.dot(K.transpose(main_eigenvect), main_eigenvect))
+        # multiplied by the given regularization gain
+        regularized_loss = loss + (main_eigenval ** 0.5) * self.k
 
         return K.in_train_phase(regularized_loss[0, 0], loss)
 
 
 class WeightRegularizer(Regularizer):
+
     def __init__(self, l1=0., l2=0.):
         self.l1 = K.cast_to_floatx(l1)
         self.l2 = K.cast_to_floatx(l2)
         self.uses_learning_phase = True
+        self.p = None
 
     def set_param(self, p):
+        if self.p is not None:
+            raise Exception('Regularizers cannot be reused. '
+                            'Instantiate one regularizer per layer.')
         self.p = p
 
     def __call__(self, loss):
-        if not hasattr(self, 'p'):
+        if self.p is None:
             raise Exception('Need to call `set_param` on '
                             'WeightRegularizer instance '
                             'before calling the instance. '
@@ -75,8 +87,11 @@ class WeightRegularizer(Regularizer):
                             'ActivityRegularizer '
                             '(i.e. activity_regularizer="l2" instead '
                             'of activity_regularizer="activity_l2".')
-        regularized_loss = loss + K.sum(K.abs(self.p)) * self.l1
-        regularized_loss += K.sum(K.square(self.p)) * self.l2
+        regularized_loss = loss
+        if self.l1:
+            regularized_loss += K.sum(self.l1 * K.abs(self.p))
+        if self.l2:
+            regularized_loss += K.sum(self.l2 * K.square(self.p))
         return K.in_train_phase(regularized_loss, loss)
 
     def get_config(self):
@@ -86,24 +101,30 @@ class WeightRegularizer(Regularizer):
 
 
 class ActivityRegularizer(Regularizer):
+
     def __init__(self, l1=0., l2=0.):
         self.l1 = K.cast_to_floatx(l1)
         self.l2 = K.cast_to_floatx(l2)
         self.uses_learning_phase = True
+        self.layer = None
 
     def set_layer(self, layer):
+        if self.layer is not None:
+            raise Exception('Regularizers cannot be reused')
         self.layer = layer
 
     def __call__(self, loss):
-        if not hasattr(self, 'layer'):
+        if self.layer is None:
             raise Exception('Need to call `set_layer` on '
                             'ActivityRegularizer instance '
                             'before calling the instance.')
         regularized_loss = loss
         for i in range(len(self.layer.inbound_nodes)):
             output = self.layer.get_output_at(i)
-            regularized_loss += self.l1 * K.sum(K.mean(K.abs(output), axis=0))
-            regularized_loss += self.l2 * K.sum(K.mean(K.square(output), axis=0))
+            if self.l1:
+                regularized_loss += K.sum(self.l1 * K.abs(output))
+            if self.l2:
+                regularized_loss += K.sum(self.l2 * K.square(output))
         return K.in_train_phase(regularized_loss, loss)
 
     def get_config(self):
@@ -136,7 +157,6 @@ def activity_l1l2(l1=0.01, l2=0.01):
     return ActivityRegularizer(l1=l1, l2=l2)
 
 
-from .utils.generic_utils import get_from_module
 def get(identifier, kwargs=None):
     return get_from_module(identifier, globals(), 'regularizer',
                            instantiate=True, kwargs=kwargs)

keras/utils/data_utils.py

@@ -5,6 +5,7 @@ import tarfile
 import os
 import sys
 import shutil
+import hashlib
 from six.moves.urllib.request import urlopen
 from six.moves.urllib.error import URLError, HTTPError
 
@@ -21,9 +22,10 @@ if sys.version_info[0] == 2:
             count = 0
             while 1:
                 chunk = response.read(chunk_size)
-                if not chunk:
-                    break
                 count += 1
+                if not chunk:
+                    reporthook(count, total_size, total_size)
+                    break
                 if reporthook:
                     reporthook(count, chunk_size, total_size)
                 yield chunk
@@ -36,11 +38,26 @@ else:
     from six.moves.urllib.request import urlretrieve
 
 
-def get_file(fname, origin, untar=False):
+def get_file(fname, origin, untar=False,
+             md5_hash=None, cache_subdir='datasets'):
+    '''Downloads a file from a URL if it not already in the cache.
+
+    Passing the MD5 hash will verify the file after download as well as if it is already present in the cache.
+
+    # Arguments
+        fname: name of the file
+        origin: original URL of the file
+        untar: boolean, whether the file should be decompressed
+        md5_hash: MD5 hash of the file for verification
+        cache_subdir: directory being used as the cache
+
+    # Returns
+        Path to the downloaded file
+    '''
     datadir_base = os.path.expanduser(os.path.join('~', '.keras'))
     if not os.access(datadir_base, os.W_OK):
         datadir_base = os.path.join('/tmp', '.keras')
-    datadir = os.path.join(datadir_base, 'datasets')
+    datadir = os.path.join(datadir_base, cache_subdir)
     if not os.path.exists(datadir):
         os.makedirs(datadir)
 
@@ -50,8 +67,19 @@ def get_file(fname, origin, untar=False):
     else:
         fpath = os.path.join(datadir, fname)
 
-    if not os.path.exists(fpath):
-        print('Downloading data from',  origin)
+    download = False
+    if os.path.exists(fpath):
+        # file found; verify integrity if a hash was provided
+        if md5_hash is not None:
+            if not validate_file(fpath, md5_hash):
+                print('A local file was found, but it seems to be '
+                      'incomplete or outdated.')
+                download = True
+    else:
+        download = True
+
+    if download:
+        print('Downloading data from', origin)
         global progbar
         progbar = None
 
@@ -60,7 +88,7 @@ def get_file(fname, origin, untar=False):
             if progbar is None:
                 progbar = Progbar(total_size)
             else:
-                progbar.update(count*block_size)
+                progbar.update(count * block_size)
 
         error_msg = 'URL fetch failure on {}: {} -- {}'
         try:
@@ -93,3 +121,23 @@ def get_file(fname, origin, untar=False):
         return untar_fpath
 
     return fpath
+
+
+def validate_file(fpath, md5_hash):
+    '''Validates a file against a MD5 hash
+
+    # Arguments
+        fpath: path to the file being validated
+        md5_hash: the MD5 hash being validated against
+
+    # Returns
+        Whether the file is valid
+    '''
+    hasher = hashlib.md5()
+    with open(fpath, 'rb') as f:
+        buf = f.read()
+        hasher.update(buf)
+    if str(hasher.hexdigest()) == str(md5_hash):
+        return True
+    else:
+        return False

keras/utils/generic_utils.py

@@ -3,6 +3,8 @@ import numpy as np
 import time
 import sys
 import six
+import marshal
+import types as python_types
 
 
 def get_from_module(identifier, module_params, module_name,
@@ -33,6 +35,43 @@ def make_tuple(*args):
     return args
 
 
+def func_dump(func):
+    '''Serialize user defined function.'''
+    code = marshal.dumps(func.__code__).decode('raw_unicode_escape')
+    defaults = func.__defaults__
+    if func.__closure__:
+        closure = tuple(c.cell_contents for c in func.__closure__)
+    else:
+        closure = None
+    return code, defaults, closure
+
+
+def func_load(code, defaults=None, closure=None, globs=None):
+    '''Deserialize user defined function.'''
+    if isinstance(code, (tuple, list)):  # unpack previous dump
+        code, defaults, closure = code
+    code = marshal.loads(code.encode('raw_unicode_escape'))
+    if closure is not None:
+        closure = func_reconstruct_closure(closure)
+    if globs is None:
+        globs = globals()
+    return python_types.FunctionType(code, globs, name=code.co_name, argdefs=defaults, closure=closure)
+
+
+def func_reconstruct_closure(values):
+    '''Deserialization helper that reconstructs a closure.'''
+    nums = range(len(values))
+    src = ["def func(arg):"]
+    src += ["  _%d = arg[%d]" % (n, n) for n in nums]
+    src += ["  return lambda:(%s)" % ','.join(["_%d" % n for n in nums]), ""]
+    src = '\n'.join(src)
+    try:
+        exec(src, globals())
+    except:
+        raise SyntaxError(src)
+    return func(values).__closure__
+
+
 class Progbar(object):
     def __init__(self, target, width=30, verbose=1, interval=0.01):
         '''

keras/utils/io_utils.py

@@ -1,21 +1,51 @@
 from __future__ import absolute_import
-import h5py
+from __future__ import print_function
 import numpy as np
+import sys
 from collections import defaultdict
 
 
 class HDF5Matrix():
+    '''Representation of HDF5 dataset which can be used instead of a
+    Numpy array.
+
+    # Example
+
+    ```python
+        X_data = HDF5Matrix('input/file.hdf5', 'data')
+        model.predict(X_data)
+    ```
+
+    Providing start and end allows use of a slice of the dataset.
+
+    Optionally, a normalizer function (or lambda) can be given. This will
+    be called on every slice of data retrieved.
+
+    # Arguments
+        datapath: string, path to a HDF5 file
+        dataset: string, name of the HDF5 dataset in the file specified
+            in datapath
+        start: int, start of desired slice of the specified dataset
+        end: int, end of desired slice of the specified dataset
+        normalizer: function to be called on data when retrieved
+
+    '''
     refs = defaultdict(int)
 
-    def __init__(self, datapath, dataset, start, end, normalizer=None):
+    def __init__(self, datapath, dataset, start=0, end=None, normalizer=None):
+        import h5py
+
         if datapath not in list(self.refs.keys()):
             f = h5py.File(datapath)
             self.refs[datapath] = f
         else:
             f = self.refs[datapath]
-        self.start = start
-        self.end = end
         self.data = f[dataset]
+        self.start = start
+        if end is None:
+            self.end = self.data.shape[0]
+        else:
+            self.end = end
         self.normalizer = normalizer
 
     def __len__(self):
@@ -29,7 +59,7 @@ class HDF5Matrix():
                 raise IndexError
         elif isinstance(key, int):
             if key + self.start < self.end:
-                idx = key+self.start
+                idx = key + self.start
             else:
                 raise IndexError
         elif isinstance(key, np.ndarray):
@@ -49,7 +79,7 @@ class HDF5Matrix():
 
     @property
     def shape(self):
-        return tuple([self.end - self.start, self.data.shape[1]])
+        return (self.end - self.start,) + self.data.shape[1:]
 
 
 def save_array(array, name):
@@ -69,3 +99,17 @@ def load_array(name):
     a[:] = array[:]
     f.close()
     return a
+
+
+def ask_to_proceed_with_overwrite(filepath):
+    get_input = input
+    if sys.version_info[:2] <= (2, 7):
+        get_input = raw_input
+    overwrite = get_input('[WARNING] %s already exists - overwrite? '
+                          '[y/n]' % (filepath))
+    while overwrite not in ['y', 'n']:
+        overwrite = get_input('Enter "y" (overwrite) or "n" (cancel).')
+    if overwrite == 'n':
+        return False
+    print('[TIP] Next time specify overwrite=True!')
+    return True

keras/utils/layer_utils.py

@@ -1,8 +1,9 @@
 from __future__ import print_function
 
 from .generic_utils import get_from_module
+from .np_utils import convert_kernel
 from ..layers import *
-from ..models import Model, Sequential, Graph
+from ..models import Model, Sequential
 from .. import backend as K
 
 
@@ -14,7 +15,7 @@ def layer_from_config(config, custom_objects={}):
             of custom (non-Keras) objects to class/functions
 
     # Returns
-        Layer instance (may be Model, Sequential, Graph, Layer...)
+        Layer instance (may be Model, Sequential, Layer...)
     '''
     # Insert custom layers into globals so they can
     # be accessed by `get_from_module`.
@@ -25,8 +26,6 @@ def layer_from_config(config, custom_objects={}):
 
     if class_name == 'Sequential':
         layer_class = Sequential
-    elif class_name == 'Graph':
-        layer_class = Graph
     elif class_name in ['Model', 'Container']:
         layer_class = Model
     else:
@@ -36,8 +35,14 @@ def layer_from_config(config, custom_objects={}):
 
 
 def print_summary(layers, relevant_nodes=None, line_length=100, positions=[.33, .55, .67, 1.]):
-    # line_length: total length of printed lines
-    # positions: relative or absolute positions of log elements in each line
+    '''Prints a summary of a layer
+
+    # Arguments
+        layers: list of layers to print summaries of
+        relevant_nodes: list of relevant nodes
+        line_length: total length of printed lines
+        positions: relative or absolute positions of log elements in each line
+    '''
     if positions[-1] <= 1:
         positions = [int(line_length * p) for p in positions]
     # header names for the different log elements
@@ -46,6 +51,8 @@ def print_summary(layers, relevant_nodes=None, line_length=100, positions=[.33,
     def print_row(fields, positions):
         line = ''
         for i in range(len(fields)):
+            if i > 0:
+                line = line[:-1] + ' '
             line += str(fields[i])
             line = line[:positions[i]]
             line += ' ' * (positions[i] - len(line))
@@ -86,14 +93,45 @@ def print_summary(layers, relevant_nodes=None, line_length=100, positions=[.33,
                 fields = ['', '', '', connections[i]]
                 print_row(fields, positions)
 
-    total_params = 0
     for i in range(len(layers)):
         print_layer_summary(layers[i])
         if i == len(layers) - 1:
             print('=' * line_length)
         else:
             print('_' * line_length)
-        total_params += layers[i].count_params()
 
-    print('Total params: %s' % total_params)
+    def count_total_params(layers, layer_set=None):
+        if layer_set is None:
+            layer_set = set()
+        total_params = 0
+        for layer in layers:
+            if layer in layer_set:
+                continue
+            layer_set.add(layer)
+            if type(layer) in (Model, Sequential):
+                total_params += count_total_params(layer.layers, layer_set)
+            else:
+                total_params += layer.count_params()
+        return total_params
+
+    print('Total params: %s' % count_total_params(layers))
     print('_' * line_length)
+
+
+def convert_all_kernels_in_model(model):
+    # Note: SeparableConvolution not included
+    # since only supported by TF.
+    conv_classes = {
+        'Convolution1D',
+        'Convolution2D',
+        'Convolution3D',
+        'AtrousConvolution2D',
+        'Deconvolution2D',
+    }
+    to_assign = []
+    for layer in model.layers:
+        if layer.__class__.__name__ in conv_classes:
+            original_w = K.get_value(layer.W)
+            converted_w = convert_kernel(original_w)
+            to_assign.append((layer.W, converted_w))
+    K.batch_set_value(to_assign)