Prepare PyPI release.

integer division for python3

.travis.yml

@@ -3,18 +3,18 @@ dist: trusty
 language: python
 matrix:
     include:
-        - python: 3.4
-          env: KERAS_BACKEND=theano
-        - python: 3.4
-          env: KERAS_BACKEND=tensorflow
         - python: 2.7
-          env: KERAS_BACKEND=theano
-        - python: 2.7
-          env: KERAS_BACKEND=tensorflow
+          env: KERAS_BACKEND=theano TEST_MODE=PEP8
         - python: 2.7
           env: KERAS_BACKEND=theano TEST_MODE=INTEGRATION_TESTS
         - python: 2.7
-          env: KERAS_BACKEND=theano TEST_MODE=PEP8
+          env: KERAS_BACKEND=tensorflow
+        - python: 3.5
+          env: KERAS_BACKEND=tensorflow
+        - python: 2.7
+          env: KERAS_BACKEND=theano
+        - python: 3.5
+          env: KERAS_BACKEND=theano
 install:
   # code below is taken from http://conda.pydata.org/docs/travis.html
   # We do this conditionally because it saves us some downloading if the
@@ -34,29 +34,29 @@ install:
 
   - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION numpy scipy matplotlib pandas pytest h5py
   - source activate test-environment
-  - pip install pytest-cov python-coveralls pytest-xdist coverage==3.7.1 #we need this version of coverage for coveralls.io to work
-  - pip install pep8 pytest-pep8
   - pip install git+git://github.com/Theano/Theano.git
 
   # install PIL for preprocessing tests
   - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
       conda install pil;
-    elif [[ "$TRAVIS_PYTHON_VERSION" == "3.4" ]]; then
+    elif [[ "$TRAVIS_PYTHON_VERSION" == "3.5" ]]; then
       conda install Pillow;
     fi
 
-  - python setup.py install
+  - pip install -e .[tests]
 
   # install TensorFlow
   - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
-      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.7.1-cp27-none-linux_x86_64.whl;
-    elif [[ "$TRAVIS_PYTHON_VERSION" == "3.4" ]]; then
-      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.7.1-cp34-none-linux_x86_64.whl;
+      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.12.1-cp27-none-linux_x86_64.whl;
+    elif [[ "$TRAVIS_PYTHON_VERSION" == "3.5" ]]; then
+      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.12.1-cp35-cp35m-linux_x86_64.whl;
     fi
 # command to run tests
 script:
   # run keras backend init to initialize backend config
   - python -c "import keras.backend"
+  # create dataset directory to avoid concurrent directory creation at runtime
+  - mkdir ~/.keras/datasets
   # set up keras backend
   - sed -i -e 's/"backend":[[:space:]]*"[^"]*/"backend":\ "'$KERAS_BACKEND'/g' ~/.keras/keras.json;
   - echo -e "Running tests with the following config:\n$(cat ~/.keras/keras.json)"

CONTRIBUTING.md

@@ -43,7 +43,7 @@ We love pull requests. Here's a quick guide:
 4. Write tests. Your code should have full unit test coverage. If you want to see your PR merged promptly, this is crucial.
 
 5. Run our test suite locally. It's easy: from the Keras folder, simply run: `py.test tests/`.
-  - You will need to install `pytest`, `coveralls`, `pytest-cov`, `pytest-xdist`: `pip install pytest pytest-cov python-coveralls pytest-xdist pep8 pytest-pep8`
+  - You will need to install the test requirements as well: `pip install -e .[tests]`.
 
 6. Make sure all tests are passing:
   - with the Theano backend, on Python 2.7 and Python 3.5

ISSUE_TEMPLATE.md

@@ -1,9 +1,13 @@
-Please make sure that the boxes below are checked before you submit your issue. Thank you!
+Please make sure that the boxes below are checked before you submit your issue. If your issue is an implementation question, please ask your question on [StackOverflow](http://stackoverflow.com/questions/tagged/keras) or [join the Keras Slack channel](https://keras-slack-autojoin.herokuapp.com/) and ask there instead of filing a GitHub issue.
+
+Thank you!
 
 - [ ] Check that you are up-to-date with the master branch of Keras. You can update with:
 pip install git+git://github.com/fchollet/keras.git --upgrade --no-deps
 
+- [ ] If running on TensorFlow, check that you are up-to-date with the latest version. The installation instructions can be found [here](https://www.tensorflow.org/get_started/os_setup).
+
 - [ ] If running on Theano, check that you are up-to-date with the master branch of Theano. You can update with:
 pip install git+git://github.com/Theano/Theano.git --upgrade --no-deps
 
-- [ ] Provide a link to a GitHub Gist of a Python script that can reproduce your issue (or just copy the script here if it is short).
+- [ ] Provide a link to a GitHub Gist of a Python script that can reproduce your issue (or just copy the script here if it is short).

README.md

@@ -1,18 +1,19 @@
-# Keras: Deep Learning library for Theano and TensorFlow
+# Keras: Deep Learning library for TensorFlow and Theano
 
 [![Build Status](https://travis-ci.org/fchollet/keras.svg?branch=master)](https://travis-ci.org/fchollet/keras)
-[![PyPI version](https://badge.fury.io/py/keras.svg)](https://badge.fury.io/py/keras)
+[![license](https://img.shields.io/github/license/mashape/apistatus.svg?maxAge=2592000)](https://github.com/fchollet/keras/blob/master/LICENSE)
+
 
 ## You have just found Keras.
 
-Keras is a minimalist, highly modular neural networks library, written in Python and capable of running on top of either [TensorFlow](https://github.com/tensorflow/tensorflow) or [Theano](https://github.com/Theano/Theano). It was developed with a focus on enabling fast experimentation. Being able to go from idea to result with the least possible delay is key to doing good research.
+Keras is a high-level neural networks library, written in Python and capable of running on top of either [TensorFlow](https://github.com/tensorflow/tensorflow) or [Theano](https://github.com/Theano/Theano). It was developed with a focus on enabling fast experimentation. *Being able to go from idea to result with the least possible delay is key to doing good research.*
 
 Use Keras if you need a deep learning library that:
 
-- allows for easy and fast prototyping (through total modularity, minimalism, and extensibility).
-- supports both convolutional networks and recurrent networks, as well as combinations of the two.
-- supports arbitrary connectivity schemes (including multi-input and multi-output training).
-- runs seamlessly on CPU and GPU.
+- Allows for easy and fast prototyping (through total modularity, minimalism, and extensibility).
+- Supports both convolutional networks and recurrent networks, as well as combinations of the two.
+- Supports arbitrary connectivity schemes (including multi-input and multi-output training).
+- Runs seamlessly on CPU and GPU.
 
 Read the documentation at [Keras.io](http://keras.io).
 
@@ -38,7 +39,7 @@ Keras is compatible with: __Python 2.7-3.5__.
 
 ## Getting started: 30 seconds to Keras
 
-The core data structure of Keras is a __model__, a way to organize layers. The main type of model is the [`Sequential`](http://keras.io/getting-started/sequential-model-guide) model, a linear stack of layers. For more complex architectures, you should use the [Keras function API](http://keras.io/getting-started/functional-api-guide).
+The core data structure of Keras is a __model__, a way to organize layers. The main type of model is the [`Sequential`](http://keras.io/getting-started/sequential-model-guide) model, a linear stack of layers. For more complex architectures, you should use the [Keras functional API](http://keras.io/getting-started/functional-api-guide).
 
 Here's the `Sequential` model:
 
@@ -51,12 +52,12 @@ model = Sequential()
 Stacking layers is as easy as `.add()`:
 
 ```python
-from keras.layers.core import Dense, Activation
+from keras.layers import Dense, Activation
 
 model.add(Dense(output_dim=64, input_dim=100))
-model.add(Activation("relu"))
+model.add(Activation('relu'))
 model.add(Dense(output_dim=10))
-model.add(Activation("softmax"))
+model.add(Activation('softmax'))
 ```
 
 Once your model looks good, configure its learning process with `.compile()`:
@@ -113,41 +114,45 @@ Keras uses the following dependencies:
 - HDF5 and h5py (optional, required if you use model saving/loading functions)
 - Optional but recommended if you use CNNs: cuDNN.
 
+
+*When using the TensorFlow backend:*
+
+- TensorFlow
+    - [See installation instructions](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/g3doc/get_started/os_setup.md)
+
 *When using the Theano backend:*
 
 - Theano
     - [See installation instructions](http://deeplearning.net/software/theano/install.html#install).
 
-*When using the TensorFlow backend:*
-
-- TensorFlow
-    - [See installation instructions](https://github.com/tensorflow/tensorflow#download-and-setup).
-
 To install Keras, `cd` to the Keras folder and run the install command:
-```
+```sh
 sudo python setup.py install
 ```
 
 You can also install Keras from PyPI:
-```
+```sh
 sudo pip install keras
 ```
 
 ------------------
 
 
-## Switching from Theano to TensorFlow
+## Switching from TensorFlow to Theano
 
-By default, Keras will use Theano as its tensor manipulation library. [Follow these instructions](http://keras.io/backend/) to configure the Keras backend.
+By default, Keras will use TensorFlow as its tensor manipulation library. [Follow these instructions](http://keras.io/backend/) to configure the Keras backend.
 
 ------------------
 
 
 ## Support
 
-You can ask questions and join the development discussion on the [Keras Google group](https://groups.google.com/forum/#!forum/keras-users).
+You can ask questions and join the development discussion:
 
-You can also post bug reports and feature requests in [Github issues](https://github.com/fchollet/keras/issues). Make sure to read [our guidelines](https://github.com/fchollet/keras/blob/master/CONTRIBUTING.md) first.
+- On the [Keras Google group](https://groups.google.com/forum/#!forum/keras-users).
+- On the [Keras Slack channel](https://kerasteam.slack.com). Use [this link](https://keras-slack-autojoin.herokuapp.com/) to request an invitation to the channel.
+
+You can also post **bug reports and feature requests** (only) in [Github issues](https://github.com/fchollet/keras/issues). Make sure to read [our guidelines](https://github.com/fchollet/keras/blob/master/CONTRIBUTING.md) first.
 
 
 ------------------

docker/Dockerfile

@@ -0,0 +1,46 @@
+FROM nvidia/cuda:8.0-cudnn5-devel
+
+ENV CONDA_DIR /opt/conda
+ENV PATH $CONDA_DIR/bin:$PATH
+
+RUN mkdir -p $CONDA_DIR && \
+    echo export PATH=$CONDA_DIR/bin:'$PATH' > /etc/profile.d/conda.sh && \
+    apt-get update && \
+    apt-get install -y wget git libhdf5-dev g++ graphviz && \
+    wget --quiet https://repo.continuum.io/miniconda/Miniconda3-3.9.1-Linux-x86_64.sh && \
+    echo "6c6b44acdd0bc4229377ee10d52c8ac6160c336d9cdd669db7371aa9344e1ac3 *Miniconda3-3.9.1-Linux-x86_64.sh" | sha256sum -c - && \
+    /bin/bash /Miniconda3-3.9.1-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
+    rm Miniconda3-3.9.1-Linux-x86_64.sh
+
+ENV NB_USER keras
+ENV NB_UID 1000
+
+RUN useradd -m -s /bin/bash -N -u $NB_UID $NB_USER && \
+    mkdir -p $CONDA_DIR && \
+    chown keras $CONDA_DIR -R && \
+    mkdir -p /src && \
+    chown keras /src
+
+USER keras
+
+# Python
+ARG python_version=3.5.2
+ARG tensorflow_version=0.12.0rc0-cp35-cp35m
+RUN conda install -y python=${python_version} && \
+    pip install https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-${tensorflow_version}-linux_x86_64.whl && \
+    pip install git+git://github.com/Theano/Theano.git && \
+    pip install ipdb pytest pytest-cov python-coveralls coverage==3.7.1 pytest-xdist pep8 pytest-pep8 pydot_ng && \
+    conda install Pillow scikit-learn notebook pandas matplotlib nose pyyaml six h5py && \
+    pip install git+git://github.com/fchollet/keras.git && \
+    conda clean -yt
+
+ADD theanorc /home/keras/.theanorc
+
+ENV PYTHONPATH='/src/:$PYTHONPATH'
+
+WORKDIR /src
+
+EXPOSE 8888
+
+CMD jupyter notebook --port=8888 --ip=0.0.0.0
+

docker/Makefile

@@ -0,0 +1,26 @@
+help:
+	@cat Makefile
+
+DATA?="${HOME}/Data"
+GPU?=0
+DOCKER_FILE=Dockerfile
+DOCKER=GPU=$(GPU) nvidia-docker
+BACKEND=tensorflow
+TEST=tests/
+SRC=$(shell dirname `pwd`)
+
+build:
+	docker build -t keras --build-arg python_version=3.5 -f $(DOCKER_FILE) .
+
+bash: build
+	$(DOCKER) run -it -v $(SRC):/src -v $(DATA):/data --env KERAS_BACKEND=$(BACKEND) keras bash
+
+ipython: build
+	$(DOCKER) run -it -v $(SRC):/src -v $(DATA):/data --env KERAS_BACKEND=$(BACKEND) keras ipython
+
+notebook: build
+	$(DOCKER) run -it -v $(SRC):/src -v $(DATA):/data --net=host --env KERAS_BACKEND=$(BACKEND) keras
+
+test: build
+	$(DOCKER) run -it -v $(SRC):/src -v $(DATA):/data --env KERAS_BACKEND=$(BACKEND) keras py.test $(TEST)
+

docker/README.md

@@ -0,0 +1,58 @@
+# Using Keras via Docker
+
+This directory contains `Dockerfile` to make it easy to get up and running with
+Keras via [Docker](http://www.docker.com/).
+
+## Installing Docker
+
+General installation instructions are
+[on the Docker site](https://docs.docker.com/installation/), but we give some
+quick links here:
+
+* [OSX](https://docs.docker.com/installation/mac/): [docker toolbox](https://www.docker.com/toolbox)
+* [ubuntu](https://docs.docker.com/installation/ubuntulinux/)
+
+## Running the container
+
+We are using `Makefile` to simplify docker commands within make commands.
+
+Build the container and start a jupyter notebook
+
+    $ make notebook
+
+Build the container and start an iPython shell
+
+    $ make ipython
+
+Build the container and start a bash
+
+    $ make bash
+
+For GPU support install NVidia drivers (ideally latest) and
+[nvidia-docker](https://github.com/NVIDIA/nvidia-docker). Run using
+
+    $ make notebook GPU=0 # or [ipython, bash]
+
+Switch between Theano and TensorFlow
+
+    $ make notebook BACKEND=theano
+    $ make notebook BACKEND=tensorflow
+
+Mount a volume for external data sets
+
+    $ make DATA=~/mydata
+
+Prints all make tasks
+
+    $ make help
+
+You can change Theano parameters by editing `/docker/theanorc`.
+
+
+Note: If you would have a problem running nvidia-docker you may try the old way
+we have used. But it is not recommended. If you find a bug in the nvidia-docker report
+it there please and try using the nvidia-docker as described above.
+
+    $ export CUDA_SO=$(\ls /usr/lib/x86_64-linux-gnu/libcuda.* | xargs -I{} echo '-v {}:{}')
+    $ export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}')
+    $ docker run -it -p 8888:8888 $CUDA_SO $DEVICES gcr.io/tensorflow/tensorflow:latest-gpu

docker/theanorc

@@ -0,0 +1,5 @@
+[global]
+floatX = float32
+optimizer=None
+device = gpu
+

docs/autogen.py

@@ -40,6 +40,7 @@ Index
     Sequence preprocessing
 
 Objectives
+Metrics
 Optimizers
 Activations
 Callbacks
@@ -53,12 +54,20 @@ Scikit-learn API
 
 '''
 from __future__ import print_function
+from __future__ import unicode_literals
+
 import re
 import inspect
 import os
 import shutil
+import sys
+if sys.version[0] == '2':
+    reload(sys)
+    sys.setdefaultencoding('utf8')
 
 from keras.layers import convolutional
+from keras.layers import pooling
+from keras.layers import local
 from keras.layers import recurrent
 from keras.layers import core
 from keras.layers import noise
@@ -71,16 +80,24 @@ from keras import callbacks
 from keras import models
 from keras.engine import topology
 from keras import objectives
+from keras import metrics
 from keras import backend
 from keras import constraints
 from keras import activations
 from keras import regularizers
+from keras.utils import data_utils
+from keras.utils import io_utils
+from keras.utils import layer_utils
+from keras.utils import np_utils
+from keras.utils import generic_utils
+
 
 EXCLUDE = {
     'Optimizer',
     'Wrapper',
     'get_session',
     'set_session',
+    'CallbackList',
 }
 
 PAGES = [
@@ -98,6 +115,7 @@ PAGES = [
             models.Sequential.predict_on_batch,
             models.Sequential.fit_generator,
             models.Sequential.evaluate_generator,
+            models.Sequential.predict_generator,
         ],
     },
     {
@@ -112,6 +130,7 @@ PAGES = [
             models.Model.predict_on_batch,
             models.Model.fit_generator,
             models.Model.evaluate_generator,
+            models.Model.predict_generator,
             models.Model.get_layer,
         ]
     },
@@ -131,21 +150,21 @@ PAGES = [
             core.Masking,
             core.Highway,
             core.MaxoutDense,
-            core.TimeDistributedDense,
         ],
     },
     {
         'page': 'layers/convolutional.md',
         'classes': [
             convolutional.Convolution1D,
+            convolutional.AtrousConvolution1D,
             convolutional.Convolution2D,
+            convolutional.AtrousConvolution2D,
+            convolutional.SeparableConvolution2D,
+            convolutional.Deconvolution2D,
             convolutional.Convolution3D,
-            convolutional.MaxPooling1D,
-            convolutional.MaxPooling2D,
-            convolutional.MaxPooling3D,
-            convolutional.AveragePooling1D,
-            convolutional.AveragePooling2D,
-            convolutional.AveragePooling3D,
+            convolutional.Cropping1D,
+            convolutional.Cropping2D,
+            convolutional.Cropping3D,
             convolutional.UpSampling1D,
             convolutional.UpSampling2D,
             convolutional.UpSampling3D,
@@ -154,6 +173,28 @@ PAGES = [
             convolutional.ZeroPadding3D,
         ],
     },
+    {
+        'page': 'layers/pooling.md',
+        'classes': [
+            pooling.MaxPooling1D,
+            pooling.MaxPooling2D,
+            pooling.MaxPooling3D,
+            pooling.AveragePooling1D,
+            pooling.AveragePooling2D,
+            pooling.AveragePooling3D,
+            pooling.GlobalMaxPooling1D,
+            pooling.GlobalAveragePooling1D,
+            pooling.GlobalMaxPooling2D,
+            pooling.GlobalAveragePooling2D,
+        ],
+    },
+    {
+        'page': 'layers/local.md',
+        'classes': [
+            local.LocallyConnected1D,
+            local.LocallyConnected2D,
+        ],
+    },
     {
         'page': 'layers/recurrent.md',
         'classes': [
@@ -187,8 +228,10 @@ PAGES = [
         'page': 'layers/wrappers.md',
         'all_module_classes': [wrappers],
     },
-
-
+    {
+        'page': 'metrics.md',
+        'all_module_functions': [metrics],
+    },
     {
         'page': 'optimizers.md',
         'all_module_classes': [optimizers],
@@ -201,6 +244,33 @@ PAGES = [
         'page': 'backend.md',
         'all_module_functions': [backend],
     },
+    {
+        'page': 'utils/data_utils.md',
+        'functions': [
+            data_utils.get_file,
+        ]
+    },
+    {
+        'page': 'utils/io_utils.md',
+        'classes': [
+            io_utils.HDF5Matrix
+        ],
+    },
+    {
+        'page': 'utils/layer_utils.md',
+        'functions': [
+            layer_utils.layer_from_config,
+        ]
+    },
+    {
+        'page': 'utils/np_utils.md',
+        'all_module_functions': [np_utils]
+    },
+    {
+        'page': 'utils/generic_utils.md',
+        'all_module_functions': [generic_utils],
+        'classes': [generic_utils.CustomObjectScope]
+    },
 ]
 
 ROOT = 'http://keras.io/'
@@ -248,10 +318,8 @@ def get_function_signature(function, method=True):
     for a in args:
         st += str(a) + ', '
     for a, v in kwargs:
-        if type(v) == str:
+        if isinstance(v, str):
             v = '\'' + v + '\''
-        elif type(v) == unicode:
-            v = 'u\'' + v + '\''
         st += str(a) + '=' + str(v) + ', '
     if kwargs or args:
         return st[:-2] + ')'
@@ -330,6 +398,7 @@ def process_function_docstring(docstring):
 print('Cleaning up existing sources directory.')
 if os.path.exists('sources'):
     shutil.rmtree('sources')
+
 print('Populating sources directory with templates.')
 for subdir, dirs, fnames in os.walk('templates'):
     for fname in fnames:
@@ -414,103 +483,3 @@ for page_data in PAGES:
     if not os.path.exists(subdir):
         os.makedirs(subdir)
     open(path, 'w').write(mkdown)
-
-
-# covered_so_far = set()
-# for module, module_name in MODULES:
-#     class_pages = []
-#     for name in dir(module):
-#         if name in SKIP:
-#             continue
-#         if name[0] == '_':
-#             continue
-#         module_member = getattr(module, name)
-#         if module_member in covered_so_far:
-#             continue
-#         if inspect.isclass(module_member):
-#             cls = module_member
-#             if cls.__module__ == module_name:
-
-#                 try:
-#                     class_signature = get_function_signature(cls.__init__)
-#                     class_signature = class_signature.replace('__init__', cls.__name__)
-#                 except:
-#                     # in case the class inherits from object and does not
-#                     # define __init__
-#                     class_signature = module_name + '.' + cls.__name__ + '()'
-
-#                 functions = []
-#                 functions_not_defined_here = []
-#                 for name in dir(cls):
-#                     if name in SKIP:
-#                         continue
-#                     if name[0] == '_':
-#                         continue
-#                     cls_member = getattr(cls, name)
-#                     if inspect.isfunction(cls_member):
-#                         function = cls_member
-#                         signature = inspect.getargspec(function)
-#                         defaults = signature.defaults
-#                         args = signature.args[1:]
-#                         if defaults:
-#                             kwargs = zip(args[-len(defaults):], defaults)
-#                             args = args[:-len(defaults)]
-#                         else:
-#                             kwargs = []
-
-#                         defined_by = get_earliest_class_that_defined_member(function.__name__, cls)
-#                         if cls == defined_by:
-#                             functions.append(function)
-#                         else:
-#                             functions_not_defined_here.append((function, defined_by))
-
-#                 blocks = []
-#                 blocks.append('<span style="float:right;">' + class_to_source_link(cls) + '</span>')
-#                 blocks.append('# ' + cls.__name__ + '\n')
-#                 blocks.append(code_snippet(class_signature))
-#                 docstring = cls.__doc__
-#                 if docstring:
-#                     blocks.append(process_class_docstring(docstring))
-
-#                 if cls.__name__ in INCLUDE_functionS_FOR:
-#                     if functions or functions_not_defined_here:
-#                         blocks.append('### functions\n')
-#                         for function in functions:
-#                             signature = get_function_signature(function)
-#                             signature = signature.replace(module_name + '.', '')
-#                             blocks.append(code_snippet(signature))
-#                             docstring = function.__doc__
-#                             if docstring:
-#                                 blocks.append(process_function_docstring(docstring))
-#                         for function, defined_by in functions_not_defined_here:
-#                             signature = get_function_signature(function)
-#                             function_module_name = function.__module__
-#                             signature = signature.replace(function_module_name + '.', '')
-#                             link = '[' + defined_by.__name__ + '](' + class_to_docs_link(defined_by) + ')'
-#                             blocks.append(code_snippet(signature))
-#                             blocks.append('Defined by ' + link + '.\n')
-
-#                 mkdown = '\n'.join(blocks)
-#                 class_pages.append((id(cls), mkdown))
-#                 covered_so_far.add(module_member)
-
-#     class_pages.sort(key=lambda x: x[0])
-#     class_pages = [x[1] for x in class_pages]
-#     module_page = '\n----\n\n'.join(class_pages)
-
-#     # save module page.
-#     # Either insert content into existing page,
-#     # or create page otherwise
-#     path = 'sources/' + module_name.replace('.', '/')[6:] + '.md'
-#     if os.path.exists(path):
-#         template = open(path).read()
-#         assert '{{autogenerated}}' in template, ('Template found for ' + path +
-#                                                  ' but missing {{autogenerated}} tag.')
-#         module_page = template.replace('{{autogenerated}}', module_page)
-#         print('...inserting autogenerated content into template:', path)
-#     else:
-#         print('...creating new page with autogenerated content:', path)
-#     subdir = os.path.dirname(path)
-#     if not os.path.exists(subdir):
-#         os.makedirs(subdir)
-#     open(path, 'w').write(module_page)

docs/mkdocs.yml

@@ -24,6 +24,8 @@ pages:
   - About Keras layers: layers/about-keras-layers.md
   - Core Layers: layers/core.md
   - Convolutional Layers: layers/convolutional.md
+  - Pooling Layers: layers/pooling.md
+  - Locally-connected Layers: layers/local.md
   - Recurrent Layers: layers/recurrent.md
   - Embedding Layers: layers/embeddings.md
   - Advanced Activations Layers: layers/advanced-activations.md
@@ -36,17 +38,24 @@ pages:
   - Text Preprocessing: preprocessing/text.md
   - Image Preprocessing: preprocessing/image.md
 - Objectives: objectives.md
+- Metrics: metrics.md
 - Optimizers: optimizers.md
 - Activations: activations.md
 - Callbacks: callbacks.md
 - Datasets: datasets.md
+- Applications: applications.md
 - Backend: backend.md
 - Initializations: initializations.md
 - Regularizers: regularizers.md
 - Constraints: constraints.md
 - Visualization: visualization.md
 - Scikit-learn API: scikit-learn-api.md
-
+- Utils:
+  - Data Utils: utils/data_utils.md
+  - I/O Utils: utils/io_utils.md
+  - Layer Utils: utils/layer_utils.md
+  - Numpy Utils: utils/np_utils.md
+  - Generic Utils: utils/generic_utils.md
 
 
 

docs/templates/activations.md

@@ -30,6 +30,7 @@ model.add(Activation(tanh))
 
 - __softmax__: Softmax applied across inputs last dimension. Expects shape either `(nb_samples, nb_timesteps, nb_dims)` or `(nb_samples, nb_dims)`.
 - __softplus__
+- __softsign__
 - __relu__
 - __tanh__
 - __sigmoid__

docs/templates/applications.md

@@ -0,0 +1,469 @@
+# Applications
+
+Keras Applications are deep learning models that are made available alongside pre-trained weights.
+These models can be used for prediction, feature extraction, and fine-tuning.
+
+Weights are downloaded automatically when instantiating a model. They are stored at `~/.keras/models/`.
+
+## Available models
+
+### Models for image classification with weights trained on ImageNet:
+
+- [Xception](#xception)
+- [VGG16](#vgg16)
+- [VGG19](#vgg19)
+- [ResNet50](#resnet50)
+- [InceptionV3](#inceptionv3)
+
+All of these architectures (except Xception) are compatible with both TensorFlow and Theano, and upon instantiation the models will be built according to the image dimension ordering set in your Keras configuration file at `~/.keras/keras.json`. For instance, if you have set `image_dim_ordering=tf`, then any model loaded from this repository will get built according to the TensorFlow dimension ordering convention, "Width-Height-Depth".
+
+The Xception model is only available for TensorFlow, due to its reliance on `SeparableConvolution` layers.
+
+### Model for music audio file auto-tagging (taking as input Mel-spectrograms):
+
+- [MusicTaggerCRNN](#musictaggercrnn)
+
+-----
+
+## Usage examples for image classification models
+
+### Classify ImageNet classes with ResNet50
+
+```python
+from keras.applications.resnet50 import ResNet50
+from keras.preprocessing import image
+from keras.applications.resnet50 import preprocess_input, decode_predictions
+import numpy as np
+
+model = ResNet50(weights='imagenet')
+
+img_path = 'elephant.jpg'
+img = image.load_img(img_path, target_size=(224, 224))
+x = image.img_to_array(img)
+x = np.expand_dims(x, axis=0)
+x = preprocess_input(x)
+
+preds = model.predict(x)
+# decode the results into a list of tuples (class, description, probability)
+# (one such list for each sample in the batch)
+print('Predicted:', decode_predictions(preds, top=3)[0])
+# Predicted: [(u'n02504013', u'Indian_elephant', 0.82658225), (u'n01871265', u'tusker', 0.1122357), (u'n02504458', u'African_elephant', 0.061040461)]
+```
+
+### Extract features with VGG16
+
+```python
+from keras.applications.vgg16 import VGG16
+from keras.preprocessing import image
+from keras.applications.vgg16 import preprocess_input
+import numpy as np
+
+model = VGG16(weights='imagenet', include_top=False)
+
+img_path = 'elephant.jpg'
+img = image.load_img(img_path, target_size=(224, 224))
+x = image.img_to_array(img)
+x = np.expand_dims(x, axis=0)
+x = preprocess_input(x)
+
+features = model.predict(x)
+```
+
+### Extract features from an arbitrary intermediate layer with VGG19
+
+```python
+from keras.applications.vgg19 import VGG19
+from keras.preprocessing import image
+from keras.applications.vgg19 import preprocess_input
+from keras.models import Model
+import numpy as np
+
+base_model = VGG19(weights='imagenet')
+model = Model(input=base_model.input, output=base_model.get_layer('block4_pool').output)
+
+img_path = 'elephant.jpg'
+img = image.load_img(img_path, target_size=(224, 224))
+x = image.img_to_array(img)
+x = np.expand_dims(x, axis=0)
+x = preprocess_input(x)
+
+block4_pool_features = model.predict(x)
+```
+
+### Fine-tune InceptionV3 on a new set of classes
+
+```python
+from keras.applications.inception_v3 import InceptionV3
+from keras.preprocessing import image
+from keras.models import Model
+from keras.layers import Dense, GlobalAveragePooling2D
+from keras import backend as K
+
+# create the base pre-trained model
+base_model = InceptionV3(weights='imagenet', include_top=False)
+
+# add a global spatial average pooling layer
+x = base_model.output
+x = GlobalAveragePooling2D()(x)
+# let's add a fully-connected layer
+x = Dense(1024, activation='relu')(x)
+# and a logistic layer -- let's say we have 200 classes
+predictions = Dense(200, activation='softmax')(x)
+
+# this is the model we will train
+model = Model(input=base_model.input, output=predictions)
+
+# first: train only the top layers (which were randomly initialized)
+# i.e. freeze all convolutional InceptionV3 layers
+for layer in base_model.layers:
+    layer.trainable = False
+
+# compile the model (should be done *after* setting layers to non-trainable)
+model.compile(optimizer='rmsprop', loss='categorical_crossentropy')
+
+# train the model on the new data for a few epochs
+model.fit_generator(...)
+
+# at this point, the top layers are well trained and we can start fine-tuning
+# convolutional layers from inception V3. We will freeze the bottom N layers
+# and train the remaining top layers.
+
+# let's visualize layer names and layer indices to see how many layers
+# we should freeze:
+for i, layer in enumerate(base_model.layers):
+   print(i, layer.name)
+
+# we chose to train the top 2 inception blocks, i.e. we will freeze
+# the first 172 layers and unfreeze the rest:
+for layer in model.layers[:172]:
+   layer.trainable = False
+for layer in model.layers[172:]:
+   layer.trainable = True
+
+# we need to recompile the model for these modifications to take effect
+# we use SGD with a low learning rate
+from keras.optimizers import SGD
+model.compile(optimizer=SGD(lr=0.0001, momentum=0.9), loss='categorical_crossentropy')
+
+# we train our model again (this time fine-tuning the top 2 inception blocks
+# alongside the top Dense layers
+model.fit_generator(...)
+```
+
+
+### Build InceptionV3 over a custom input tensor
+
+```python
+from keras.applications.inception_v3 import InceptionV3
+from keras.layers import Input
+
+# this could also be the output a different Keras model or layer
+input_tensor = Input(shape=(224, 224, 3))  # this assumes K.image_dim_ordering() == 'tf'
+
+model = InceptionV3(input_tensor=input_tensor, weights='imagenet', include_top=True)
+```
+
+-----
+
+# Documentation for individual models
+
+- [Xception](#xception)
+- [VGG16](#vgg16)
+- [VGG19](#vgg19)
+- [ResNet50](#resnet50)
+- [InceptionV3](#inceptionv3)
+- [MusicTaggerCRNN](#musictaggercrnn)
+
+-----
+
+
+## Xception
+
+
+```python
+keras.applications.xception.Xception(include_top=True, weights='imagenet', input_tensor=None, input_shape=None)
+```
+
+Xception V1 model, with weights pre-trained on ImageNet.
+
+On ImageNet, this model gets to a top-1 validation accuracy of 0.790
+and a top-5 validation accuracy of 0.945.
+
+Note that this model is only available for the TensorFlow backend,
+due to its reliance on `SeparableConvolution` layers. Additionally it only supports
+the dimension ordering "tf" (width, height, channels).
+
+The default input size for this model is 299x299.
+
+### Arguments
+
+- include_top: whether to include the fully-connected layer at the top of the network.
+- weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+- input_shape: optional shape tuple, only to be specified
+    if `include_top` is False (otherwise the input shape
+    has to be `(299, 299, 3)`.
+    It should have exactly 3 inputs channels,
+    and width and height should be no smaller than 71.
+    E.g. `(150, 150, 3)` would be one valid value.
+- classes: optional number of classes to classify images 
+    into, only to be specified if `include_top` is True, and 
+    if no `weights` argument is specified.
+
+### Returns
+
+A Keras model instance.
+
+### References
+
+- [Xception: Deep Learning with Depthwise Separable Convolutions](https://arxiv.org/abs/1610.02357)
+
+### License
+
+These weights are trained by ourselves and are released under the MIT license.
+
+
+-----
+
+
+## VGG16
+
+```python
+keras.applications.vgg16.VGG16(include_top=True, weights='imagenet', input_tensor=None, input_shape=None)
+```
+
+VGG16 model, with weights pre-trained on ImageNet.
+
+This model is available for both the Theano and TensorFlow backend, and can be built both
+with "th" dim ordering (channels, width, height) or "tf" dim ordering (width, height, channels).
+
+The default input size for this model is 224x224.
+
+### Arguments
+
+- include_top: whether to include the 3 fully-connected layers at the top of the network.
+- weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+- input_shape: optional shape tuple, only to be specified
+    if `include_top` is False (otherwise the input shape
+    has to be `(224, 224, 3)` (with `tf` dim ordering)
+    or `(3, 224, 244)` (with `th` dim ordering).
+    It should have exactly 3 inputs channels,
+    and width and height should be no smaller than 48.
+    E.g. `(200, 200, 3)` would be one valid value.
+- classes: optional number of classes to classify images 
+    into, only to be specified if `include_top` is True, and 
+    if no `weights` argument is specified.
+    
+### Returns
+
+A Keras model instance.
+
+### References
+
+- [Very Deep Convolutional Networks for Large-Scale Image Recognition](https://arxiv.org/abs/1409.1556): please cite this paper if you use the VGG models in your work.
+
+### License
+
+These weights are ported from the ones [released by VGG at Oxford](http://www.robots.ox.ac.uk/~vgg/research/very_deep/) under the [Creative Commons Attribution License](https://creativecommons.org/licenses/by/4.0/).
+
+-----
+
+## VGG19
+
+
+```python
+keras.applications.vgg19.VGG19(include_top=True, weights='imagenet', input_tensor=None, input_shape=None)
+```
+
+
+VGG19 model, with weights pre-trained on ImageNet.
+
+This model is available for both the Theano and TensorFlow backend, and can be built both
+with "th" dim ordering (channels, width, height) or "tf" dim ordering (width, height, channels).
+
+The default input size for this model is 224x224.
+
+### Arguments
+
+- include_top: whether to include the 3 fully-connected layers at the top of the network.
+- weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+- input_shape: optional shape tuple, only to be specified
+    if `include_top` is False (otherwise the input shape
+    has to be `(224, 224, 3)` (with `tf` dim ordering)
+    or `(3, 224, 244)` (with `th` dim ordering).
+    It should have exactly 3 inputs channels,
+    and width and height should be no smaller than 48.
+    E.g. `(200, 200, 3)` would be one valid value.
+- classes: optional number of classes to classify images 
+    into, only to be specified if `include_top` is True, and 
+    if no `weights` argument is specified.
+    
+### Returns
+
+A Keras model instance.
+
+
+### References
+
+- [Very Deep Convolutional Networks for Large-Scale Image Recognition](https://arxiv.org/abs/1409.1556)
+
+### License
+
+These weights are ported from the ones [released by VGG at Oxford](http://www.robots.ox.ac.uk/~vgg/research/very_deep/) under the [Creative Commons Attribution License](https://creativecommons.org/licenses/by/4.0/).
+
+-----
+
+## ResNet50
+
+
+```python
+keras.applications.resnet50.ResNet50(include_top=True, weights='imagenet', input_tensor=None, input_shape=None)
+```
+
+
+ResNet50 model, with weights pre-trained on ImageNet.
+
+This model is available for both the Theano and TensorFlow backend, and can be built both
+with "th" dim ordering (channels, width, height) or "tf" dim ordering (width, height, channels).
+
+The default input size for this model is 224x224.
+
+
+### Arguments
+
+- include_top: whether to include the fully-connected layer at the top of the network.
+- weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+- input_shape: optional shape tuple, only to be specified
+    if `include_top` is False (otherwise the input shape
+    has to be `(224, 224, 3)` (with `tf` dim ordering)
+    or `(3, 224, 244)` (with `th` dim ordering).
+    It should have exactly 3 inputs channels,
+    and width and height should be no smaller than 197.
+    E.g. `(200, 200, 3)` would be one valid value.
+- classes: optional number of classes to classify images 
+    into, only to be specified if `include_top` is True, and 
+    if no `weights` argument is specified.
+    
+### Returns
+
+A Keras model instance.
+
+### References
+
+- [Deep Residual Learning for Image Recognition](https://arxiv.org/abs/1512.03385)
+
+### License
+
+These weights are ported from the ones [released by Kaiming He](https://github.com/KaimingHe/deep-residual-networks) under the [MIT license](https://github.com/KaimingHe/deep-residual-networks/blob/master/LICENSE).
+
+-----
+
+## InceptionV3
+
+
+```python
+keras.applications.inception_v3.InceptionV3(include_top=True, weights='imagenet', input_tensor=None, input_shape=None)
+```
+
+Inception V3 model, with weights pre-trained on ImageNet.
+
+This model is available for both the Theano and TensorFlow backend, and can be built both
+with "th" dim ordering (channels, width, height) or "tf" dim ordering (width, height, channels).
+
+The default input size for this model is 299x299.
+
+
+### Arguments
+
+- include_top: whether to include the fully-connected layer at the top of the network.
+- weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+- input_shape: optional shape tuple, only to be specified
+    if `include_top` is False (otherwise the input shape
+    has to be `(299, 299, 3)` (with `tf` dim ordering)
+    or `(3, 299, 299)` (with `th` dim ordering).
+    It should have exactly 3 inputs channels,
+    and width and height should be no smaller than 139.
+    E.g. `(150, 150, 3)` would be one valid value.
+- classes: optional number of classes to classify images 
+    into, only to be specified if `include_top` is True, and 
+    if no `weights` argument is specified.
+    
+### Returns
+
+A Keras model instance.
+
+### References
+
+- [Rethinking the Inception Architecture for Computer Vision](http://arxiv.org/abs/1512.00567)
+
+### License
+
+These weights are trained by ourselves and are released under the MIT license.
+
+-----
+
+## MusicTaggerCRNN
+
+
+```python
+keras.applications.music_tagger_crnn.MusicTaggerCRNN(weights='msd', input_tensor=None, include_top=True)
+```
+
+A convolutional-recurrent model taking as input a vectorized representation of the MelSpectrogram of a music track and capable of outputting the musical genre of the track. You can use `keras.applications.music_tagger_crnn.preprocess_input` to convert a sound file to a vectorized spectrogram. This requires to have installed the [Librosa](http://librosa.github.io/librosa/) library. See [the usage example](#music-tagging-and-feature-extraction-with-musictaggercrnn).
+
+### Arguments
+
+- weights: one of `None` (random initialization) or "msd" (pre-training on [Million Song Dataset](http://labrosa.ee.columbia.edu/millionsong/)).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+- include_top: whether to include the 1 fully-connected layer (output layer) at the top of the network. If False, the network outputs 32-dim features.
+- classes: optional number of classes to classify images 
+    into, only to be specified if `include_top` is True, and 
+    if no `weights` argument is specified.
+    
+### Returns
+
+A Keras model instance.
+
+### References
+
+- [Convolutional Recurrent Neural Networks for Music Classification](https://arxiv.org/abs/1609.04243)
+
+### License
+
+These weights are ported from the ones [released by Keunwoo Choi](https://github.com/keunwoochoi/music-auto_tagging-keras) under the [MIT license](https://github.com/keunwoochoi/music-auto_tagging-keras/blob/master/LICENSE.md).
+
+### Examples: music tagging and audio feature extraction
+
+```python
+from keras.applications.music_tagger_crnn import MusicTaggerCRNN
+from keras.applications.music_tagger_crnn import preprocess_input, decode_predictions
+import numpy as np
+
+# 1. Tagging
+model = MusicTaggerCRNN(weights='msd')
+
+audio_path = 'audio_file.mp3'
+melgram = preprocess_input(audio_path)
+melgrams = np.expand_dims(melgram, axis=0)
+
+preds = model.predict(melgrams)
+print('Predicted:')
+print(decode_predictions(preds))
+# print: ('Predicted:', [[('rock', 0.097071797), ('pop', 0.042456303), ('alternative', 0.032439161), ('indie', 0.024491295), ('female vocalists', 0.016455274)]])
+
+#. 2. Feature extraction
+model = MusicTaggerCRNN(weights='msd', include_top=False)
+
+audio_path = 'audio_file.mp3'
+melgram = preprocess_input(audio_path)
+melgrams = np.expand_dims(melgram, axis=0)
+
+feats = model.predict(melgrams)
+print('Features:')
+print(feats[0, :10])
+# print: ('Features:', [-0.19160545 0.94259131 -0.9991011 0.47644514 -0.19089699 0.99033844 0.1103896 -0.00340496 0.14823607 0.59856361])
+```

docs/templates/backend.md

@@ -4,10 +4,12 @@
 
 Keras is a model-level library, providing high-level building blocks for developing deep learning models. It does not handle itself low-level operations such as tensor products, convolutions and so on. Instead, it relies on a specialized, well-optimized tensor manipulation library to do so, serving as the "backend engine" of Keras. Rather than picking one single tensor library and making the implementation of Keras tied to that library, Keras handles the problem in a modular way, and several different backend engines can be plugged seamlessly into Keras.
 
-At this time, Keras has two backend implementations available: the **Theano** backend and the **TensorFlow** backend.
+At this time, Keras has two backend implementations available: the **TensorFlow** backend and the **Theano** backend.
 
-- [Theano](http://deeplearning.net/software/theano/) is an open-source symbolic tensor manipulation framework developed by LISA/MILA Lab at Université de Montréal.
 - [TensorFlow](http://www.tensorflow.org/) is an open-source symbolic tensor manipulation framework developed by Google, Inc.
+- [Theano](http://deeplearning.net/software/theano/) is an open-source symbolic tensor manipulation framework developed by LISA/MILA Lab at Université de Montréal.
+
+In the future, we are likely to add more backend options. If you are interested in developing a new backend, get in touch!
 
 ----
 
@@ -19,9 +21,16 @@ If you have run Keras at least once, you will find the Keras configuration file
 
 If it isn't there, you can create it.
 
-It probably looks like this:
+The default configuration file looks like this:
 
-`{"epsilon": 1e-07, "floatx": "float32", "backend": "theano"}`
+```
+{
+    "image_dim_ordering": "tf",
+    "epsilon": 1e-07,
+    "floatx": "float32",
+    "backend": "tensorflow"
+}
+```
 
 Simply change the field `backend` to either `"theano"` or `"tensorflow"`, and Keras will use the new configuration next time you run any Keras code.
 
@@ -29,13 +38,35 @@ You can also define the environment variable ``KERAS_BACKEND`` and this will
 override what is defined in your config file :
 
 ```bash
-KERAS_BACKEND=tensorflow python -c "from keras import backend; print backend._BACKEND"
+KERAS_BACKEND=tensorflow python -c "from keras import backend"
 Using TensorFlow backend.
-tensorflow
 ```
 
 ----
 
+## keras.json details
+
+
+```
+{
+    "image_dim_ordering": "tf",
+    "epsilon": 1e-07,
+    "floatx": "float32",
+    "backend": "tensorflow"
+}
+```
+
+You can change these settings by editing `~/.keras/keras.json`. 
+
+* `image_dim_ordering`: string, either `"tf"` or `"th"`. It specifies which dimension ordering convention Keras will follow. (`keras.backend.image_dim_ordering()` returns it.)
+  - For 2D data (e.g. image), `"tf"` assumes `(rows, cols, channels)` while `"th"` assumes `(channels, rows, cols)`. 
+  - For 3D data, `"tf"` assumes `(conv_dim1, conv_dim2, conv_dim3, channels)` while `"th"` assumes `(channels, conv_dim1, conv_dim2, conv_dim3)`.
+* `epsilon`: float, a numeric fuzzing constant used to avoid dividing by zero in some operations.
+* `floatx`: string, `"float16"`, `"float32"`, or `"float64"`. Default float precision.
+* `backend`: string, `"tensorflow"` or `"theano"`.
+
+----
+
 ## Using the abstract Keras backend to write new code
 
 If you want the Keras modules you write to be compatible with both Theano and TensorFlow, you have to write them via the abstract Keras backend API. Here's an intro.

docs/templates/constraints.md

@@ -2,7 +2,7 @@
 
 Functions from the `constraints` module allow setting constraints (eg. non-negativity) on network parameters during optimization.
 
-The penalties are applied on a per-layer basis. The exact API will depend on the layer, but the layers `Dense`, `TimeDistributedDense`, `MaxoutDense`, `Convolution1D` and `Convolution2D` have a unified API.
+The penalties are applied on a per-layer basis. The exact API will depend on the layer, but the layers `Dense`, `TimeDistributedDense`, `MaxoutDense`, `Convolution1D`, `Convolution2D` and `Convolution3D` have a unified API.
 
 These layers expose 2 keyword arguments:
 

docs/templates/datasets.md

@@ -53,11 +53,14 @@ As a convention, "0" does not stand for a specific word, but instead is used to
 ```python
 from keras.datasets import imdb
 
-(X_train, y_train), (X_test, y_test) = imdb.load_data(path="imdb.pkl",
+(X_train, y_train), (X_test, y_test) = imdb.load_data(path="imdb_full.pkl",
                                                       nb_words=None,
                                                       skip_top=0,
                                                       maxlen=None,
-                                                      test_split=0.1)
+                                                      seed=113,
+                                                      start_char=1,
+                                                      oov_char=2,
+                                                      index_from=3)
 ```
 - __Return:__
     - 2 tuples:
@@ -70,8 +73,12 @@ from keras.datasets import imdb
     - __nb_words__: integer or None. Top most frequent words to consider. Any less frequent word will appear as 0 in the sequence data.
     - __skip_top__: integer. Top most frequent words to ignore (they will appear as 0s in the sequence data).
     - __maxlen__: int. Maximum sequence length. Any longer sequence will be truncated.
-    - __test_split__: float. Fraction of the dataset to be used as test data.
     - __seed__: int. Seed for reproducible data shuffling.
+    - __start_char__: char. The start of a sequence will be marked with this character.
+        Set to 1 because 0 is usually the padding character.
+    - __oov_char__: char. words that were cut out because of the `nb_words`
+        or `skip_top` limit will be replaced with this character.
+    - __index_from__: int. Index actual words with this index and higher.
 
 ---
 
@@ -88,10 +95,16 @@ from keras.datasets import reuters
                                                          nb_words=None,
                                                          skip_top=0,
                                                          maxlen=None,
-                                                         test_split=0.1)
+                                                         test_split=0.2,
+                                                         seed=113,
+                                                         start_char=1,
+                                                         oov_char=2,
+                                                         index_from=3)
 ```
 
-The specifications are the same as that of the IMDB dataset.
+The specifications are the same as that of the IMDB dataset, with the addition of:
+
+    - __test_split__: float. Fraction of the dataset to be used as test data.
 
 This dataset also makes available the word index used for encoding the sequences:
 

docs/templates/getting-started/faq.md

@@ -2,15 +2,19 @@
 
 - [How should I cite Keras?](#how-should-i-cite-keras)
 - [How can I run Keras on GPU?](#how-can-i-run-keras-on-gpu)
+- [What does \["sample", "batch", "epoch"\] mean?](#what-does-sample-batch-epoch-mean)
 - [How can I save a Keras model?](#how-can-i-save-a-keras-model)
 - [Why is the training loss much higher than the testing loss?](#why-is-the-training-loss-much-higher-than-the-testing-loss)
-- [How can I visualize the output of an intermediate layer?](#how-can-i-visualize-the-output-of-an-intermediate-layer)
+- [How can I obtain the output of an intermediate layer?](#how-can-i-obtain-the-output-of-an-intermediate-layer)
 - [How can I use Keras with datasets that don't fit in memory?](#how-can-i-use-keras-with-datasets-that-dont-fit-in-memory)
 - [How can I interrupt training when the validation loss isn't decreasing anymore?](#how-can-i-interrupt-training-when-the-validation-loss-isnt-decreasing-anymore)
 - [How is the validation split computed?](#how-is-the-validation-split-computed)
 - [Is the data shuffled during training?](#is-the-data-shuffled-during-training)
 - [How can I record the training / validation loss / accuracy at each epoch?](#how-can-i-record-the-training-validation-loss-accuracy-at-each-epoch)
+- [How can I "freeze" layers?](#how-can-i-freeze-keras-layers)
 - [How can I use stateful RNNs?](#how-can-i-use-stateful-rnns)
+- [How can I remove a layer from a Sequential model?](#how-can-i-remove-a-layer-from-a-sequential-model)
+- [How can I use pre-trained models in Keras?](#how-can-i-use-pre-trained-models-in-keras)
 
 ---
 
@@ -20,15 +24,16 @@ Please cite Keras in your publications if it helps your research. Here is an exa
 
 ```
 @misc{chollet2015keras,
-  author = {Chollet, François},
-  title = {Keras},
-  year = {2015},
-  publisher = {GitHub},
-  journal = {GitHub repository},
-  howpublished = {\url{https://github.com/fchollet/keras}}
+  title={Keras},
+  author={Chollet, Fran\c{c}ois},
+  year={2015},
+  publisher={GitHub},
+  howpublished={\url{https://github.com/fchollet/keras}},
 }
 ```
 
+---
+
 ### How can I run Keras on GPU?
 
 If you are running on the TensorFlow backend, your code will automatically run on GPU if any available GPU is detected.
@@ -52,11 +57,50 @@ theano.config.floatX = 'float32'
 
 ---
 
+### What does \["sample", "batch", "epoch"\] mean?
+
+Below are some common definitions that are necessary to know and understand to correctly utilize Keras:
+
+- **Sample**: one element of a dataset.
+  - *Example:* one image is a **sample** in a convolutional network
+  - *Example:* one audio file is a **sample** for a speech recognition model
+- **Batch**: a set of *N* samples. The samples in a **batch** are processed independently, in parallel. If training, a batch results in only one update to the model.
+  - A **batch** generally approximates the distribution of the input data better than a single input. The larger the batch, the better the approximation; however, it is also true that the batch will take longer to processes and will still result in only one update. For inference (evaluate/predict), it is recommended to pick a batch size that is as large as you can afford without going out of memory (since larger batches will usually result in faster evaluating/prediction).
+- **Epoch**: an arbitrary cutoff, generally defined as "one pass over the entire dataset", used to separate training into distinct phases, which is useful for logging and periodic evaluation.
+  - When using `evaluation_data` or `evaluation_split` with the `fit` method of Keras models, evaluation will be run at the end of every **epoch**.
+  - Within Keras, there is the ability to add [callbacks](https://keras.io/callbacks/) specifically designed to be run at the end of an **epoch**. Examples of these are learning rate changes and model checkpointing (saving).
+
+---
+
 ### How can I save a Keras model?
 
 *It is not recommended to use pickle or cPickle to save a Keras model.*
 
-If you only need to save the architecture of a model, and not its weights, you can do:
+You can use `model.save(filepath)` to save a Keras model into a single HDF5 file which will contain:
+
+- the architecture of the model, allowing to re-create the model
+- the weights of the model
+- the training configuration (loss, optimizer)
+- the state of the optimizer, allowing to resume training exactly where you left off.
+
+You can then use `keras.models.load_model(filepath)` to reinstantiate your model.
+`load_model` will also take care of compiling the model using the saved training configuration
+(unless the model was never compiled in the first place).
+
+Example:
+
+```python
+from keras.models import load_model
+
+model.save('my_model.h5')  # creates a HDF5 file 'my_model.h5'
+del model  # deletes the existing model
+
+# returns a compiled model
+# identical to the previous one
+model = load_model('my_model.h5')
+```
+
+If you only need to save the **architecture of a model**, and not its weights or its training configuration, you can do:
 
 ```python
 # save as JSON
@@ -66,6 +110,8 @@ json_string = model.to_json()
 yaml_string = model.to_yaml()
 ```
 
+The generated JSON / YAML files are human-readable and can be manually edited if needed.
+
 You can then build a fresh model from this data:
 
 ```python
@@ -77,7 +123,7 @@ model = model_from_json(json_string)
 model = model_from_yaml(yaml_string)
 ```
 
-If you need to save the weights of a model, you can do so in HDF5 with the code below.
+If you need to save the **weights of a model**, you can do so in HDF5 with the code below.
 
 Note that you will first need to install HDF5 and the Python library h5py, which do not come bundled with Keras.
 
@@ -85,21 +131,37 @@ Note that you will first need to install HDF5 and the Python library h5py, which
 model.save_weights('my_model_weights.h5')
 ```
 
-Assuming you have code for instantiating your model, you can then load the weights you saved into a model with the same architecture:
+Assuming you have code for instantiating your model, you can then load the weights you saved into a model with the *same* architecture:
 
 ```python
 model.load_weights('my_model_weights.h5')
 ```
 
-This leads us to a way to save and reconstruct models from only serialized data:
-```python
-json_string = model.to_json()
-open('my_model_architecture.json', 'w').write(json_string)
-model.save_weights('my_model_weights.h5')
+If you need to load weights into a *different* architecture (with some layers in common), for instance for fine-tuning or transfer-learning, you can load weights by *layer name*:
 
-# elsewhere...
-model = model_from_json(open('my_model_architecture.json').read())
-model.load_weights('my_model_weights.h5')
+```python
+model.load_weights('my_model_weights.h5', by_name=True)
+```
+
+For example:
+
+```python
+"""
+Assume original model looks like this:
+    model = Sequential()
+    model.add(Dense(2, input_dim=3, name="dense_1"))
+    model.add(Dense(3, name="dense_2"))
+    ...
+    model.save_weights(fname)
+"""
+
+# new model
+model = Sequential()
+model.add(Dense(2, input_dim=3, name="dense_1"))  # will be loaded
+model.add(Dense(10, name="new_dense"))  # will not be loaded
+
+# load weights from first model; will only affect the first layer, dense_1.
+model.load_weights(fname, by_name=True)
 ```
 
 ---
@@ -112,9 +174,22 @@ Besides, the training loss is the average of the losses over each batch of train
 
 ---
 
-### How can I visualize the output of an intermediate layer?
+### How can I obtain the output of an intermediate layer?
 
-You can build a Keras function that will return the output of a certain layer given a certain input, for example:
+One simple way is to create a new `Model` that will output the layers that you are interested in:
+
+```python
+from keras.models import Model
+
+model = ...  # create the original model
+
+layer_name = 'my_layer'
+intermediate_layer_model = Model(input=model.input,
+                                 output=model.get_layer(layer_name).output)
+intermediate_output = intermediate_layer_model.predict(data)
+```
+
+Alternatively, you can build a Keras function that will return the output of a certain layer given a certain input, for example:
 
 ```python
 from keras import backend as K
@@ -134,10 +209,10 @@ to pass the learning phase flag to your function:
 get_3rd_layer_output = K.function([model.layers[0].input, K.learning_phase()],
                                   [model.layers[3].output])
 
-# output in train mode
-layer_output = get_3rd_layer_output([X, 1])[0]
+# output in test mode = 0
+layer_output = get_3rd_layer_output([X, 0])[0]
 
-# output in test mode
+# output in train mode = 1
 layer_output = get_3rd_layer_output([X, 1])[0]
 ```
 
@@ -169,8 +244,9 @@ Find out more in the [callbacks documentation](/callbacks).
 
 ### How is the validation split computed?
 
-If you set the `validation_split` argument in `model.fit` to e.g. 0.1, then the validation data used will be the *last 10%* of the data. If you set it to 0.25, it will be the last 25% of the data, etc.
+If you set the `validation_split` argument in `model.fit` to e.g. 0.1, then the validation data used will be the *last 10%* of the data. If you set it to 0.25, it will be the last 25% of the data, etc. Note that the data isn't shuffled before extracting the validation split, so the validation is literally just the *last* x% of samples in the input you passed.
 
+The same validation set is used for all epochs (within a same call to `fit`).
 
 ---
 
@@ -194,6 +270,40 @@ print(hist.history)
 
 ---
 
+### How can I "freeze" Keras layers?
+
+To "freeze" a layer means to exclude it from training, i.e. its weights will never be updated. This is useful in the context of fine-tuning a model, or using fixed embeddings for a text input.
+
+You can pass a `trainable` argument (boolean) to a layer constructor to set a layer to be non-trainable:
+
+```python
+frozen_layer = Dense(32, trainable=False)
+```
+
+Additionally, you can set the `trainable` property of a layer to `True` or `False` after instantiation. For this to take effect, you will need to call `compile()` on your model after modifying the `trainable` property. Here's an example:
+
+```python
+x = Input(shape=(32,))
+layer = Dense(32)
+layer.trainable = False
+y = layer(x)
+
+frozen_model = Model(x, y)
+# in the model below, the weights of `layer` will not be updated during training
+frozen_model.compile(optimizer='rmsprop', loss='mse')
+
+layer.trainable = True
+trainable_model = Model(x, y)
+# with this model the weights of the layer will be updated during training
+# (which will also affect the above model since it uses the same layer instance)
+trainable_model.compile(optimizer='rmsprop', loss='mse')
+
+frozen_model.fit(data, labels)  # this does NOT update the weights of `layer`
+trainable_model.fit(data, labels)  # this updates the weights of `layer`
+```
+
+---
+
 ### How can I use stateful RNNs?
 
 Making a RNN stateful means that the states for the samples of each batch will be reused as initial states for the samples in the next batch.
@@ -241,3 +351,53 @@ model.layers[0].reset_states()
 
 Notes that the methods `predict`, `fit`, `train_on_batch`, `predict_classes`, etc. will *all* update the states of the stateful layers in a model. This allows you to do not only stateful training, but also stateful prediction.
 
+---
+
+### How can I remove a layer from a Sequential model?
+
+You can remove the last added layer in a Sequential model by calling `.pop()`:
+
+```python
+model = Sequential()
+model.add(Dense(32, activation='relu', input_dim=784))
+model.add(Dense(32, activation='relu'))
+
+print(len(model.layers))  # "2"
+
+model.pop()
+print(len(model.layers))  # "1"
+```
+
+---
+
+### How can I use pre-trained models in Keras?
+
+Code and pre-trained weights are available for the following image classification models:
+
+- Xception
+- VGG16
+- VGG19
+- ResNet50
+- Inception v3
+
+They can be imported from the module `keras.applications`:
+
+```python
+from keras.applications.xception import Xception
+from keras.applications.vgg16 import VGG16
+from keras.applications.vgg19 import VGG19
+from keras.applications.resnet50 import ResNet50
+from keras.applications.inception_v3 import InceptionV3
+
+model = VGG16(weights='imagenet', include_top=True)
+```
+
+For a few simple usage examples, see [the documentation for the Applications module](/applications).
+
+For a detailed example of how to use such a pre-trained model for feature extraction or for fine-tuning, see [this blog post](http://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html).
+
+The VGG16 model is also the basis for several Keras example scripts:
+
+- [Style transfer](https://github.com/fchollet/keras/blob/master/examples/neural_style_transfer.py)
+- [Feature visualization](https://github.com/fchollet/keras/blob/master/examples/conv_filter_visualization.py)
+- [Deep dream](https://github.com/fchollet/keras/blob/master/examples/deep_dream.py)

docs/templates/getting-started/functional-api-guide.md

@@ -45,7 +45,7 @@ With the functional API, it is easy to re-use trained models: you can treat any
 
 ```python
 x = Input(shape=(784,))
-# this works, and returns the 10-way softmax we defined above. 
+# this works, and returns the 10-way softmax we defined above.
 y = model(x)
 ```
 
@@ -75,7 +75,7 @@ The model will also be supervised via two loss functions. Using the main loss fu
 
 Here's what our model looks like:
 
-<img src="http://s3.amazonaws.com/keras.io/img/multi-input-multi-output-graph.png" alt="multi-input-multi-output-graph" style="width: 400px;"/>
+<img src="https://s3.amazonaws.com/keras.io/img/multi-input-multi-output-graph.png" alt="multi-input-multi-output-graph" style="width: 400px;"/>
 
 Let's implement it with the functional API.
 
@@ -102,7 +102,7 @@ lstm_out = LSTM(32)(x)
 Here we insert the auxiliary loss, allowing the LSTM and Embedding layer to be trained smoothly even though the main loss will be much higher in the model.
 
 ```python
-auxiliary_loss = Dense(1, activation='sigmoid', name='aux_output')(lstm_out)
+auxiliary_output = Dense(1, activation='sigmoid', name='aux_output')(lstm_out)
 ```
 
 At this point, we feed into the model our auxiliary input data by concatenating it with the LSTM output:
@@ -117,22 +117,22 @@ x = Dense(64, activation='relu')(x)
 x = Dense(64, activation='relu')(x)
 
 # and finally we add the main logistic regression layer
-main_loss = Dense(1, activation='sigmoid', name='main_output')(x)
+main_output = Dense(1, activation='sigmoid', name='main_output')(x)
 ```
 
 This defines a model with two inputs and two outputs:
 
 ```python
-model = Model(input=[main_input, auxiliary_input], output=[main_loss, auxiliary_loss])
+model = Model(input=[main_input, auxiliary_input], output=[main_output, auxiliary_output])
 ```
 
 We compile the model and assign a weight of 0.2 to the auxiliary loss.
-To specify different `loss_weight` or `loss` for each different output, you can use a list or a dictionary.
+To specify different `loss_weights` or `loss` for each different output, you can use a list or a dictionary.
 Here we pass a single loss as the `loss` argument, so the same loss will be used on all outputs.
 
 ```python
 model.compile(optimizer='rmsprop', loss='binary_crossentropy',
-              loss_weight=[1., 0.2])
+              loss_weights=[1., 0.2])
 ```
 
 We can train the model by passing it lists of input arrays and target arrays:
@@ -148,7 +148,7 @@ We could also have compiled the model via:
 ```python
 model.compile(optimizer='rmsprop',
               loss={'main_output': 'binary_crossentropy', 'aux_output': 'binary_crossentropy'},
-              loss_weight={'main_output': 1., 'aux_output': 0.2})
+              loss_weights={'main_output': 1., 'aux_output': 0.2})
 
 # and trained it via:
 model.fit({'main_input': headline_data, 'aux_input': additional_data},
@@ -166,7 +166,7 @@ Let's consider a dataset of tweets. We want to build a model that can tell wheth
 
 One way to achieve this is to build a model that encodes two tweets into two vectors, concatenates the vectors and adds a logistic regression of top, outputting a probability that the two tweets share the same author. The model would then be trained on positive tweet pairs and negative tweet pairs.
 
-Because the problem is symetric, the mechanism that encodes the first tweet should be reused (weights and all) to encode the second tweet. Here we use a shared LSTM layer to encode the tweets.
+Because the problem is symmetric, the mechanism that encodes the first tweet should be reused (weights and all) to encode the second tweet. Here we use a shared LSTM layer to encode the tweets.
 
 Let's build this with the functional API. We will take as input for a tweet a binary matrix of shape `(140, 256)`, i.e. a sequence of 140 vectors of size 256, where each dimension in the 256-dimensional vector encodes the presence/absence of a character (out of an alphabet of 256 frequent characters).
 
@@ -196,7 +196,7 @@ encoded_b = shared_lstm(tweet_b)
 merged_vector = merge([encoded_a, encoded_b], mode='concat', concat_axis=-1)
 
 # and add a logistic regression on top
-predictions = Dense(1, activation='sigmoid')(merged_vector) 
+predictions = Dense(1, activation='sigmoid')(merged_vector)
 
 # we define a trainable model linking the
 # tweet inputs to the predictions
@@ -309,8 +309,8 @@ from keras.layers import merge, Convolution2D, Input
 
 # input tensor for a 3-channel 256x256 image
 x = Input(shape=(3, 256, 256))
-# 3x3 conv with 16 output channels
-y = Convolution2D(16, 3, 3, border_mode='same')
+# 3x3 conv with 3 output channels (same as input channels)
+y = Convolution2D(3, 3, 3, border_mode='same')(x)
 # this returns x + y.
 z = merge([x, y], mode='sum')
 ```

docs/templates/getting-started/sequential-model-guide.md

@@ -6,6 +6,7 @@ You can create a `Sequential` model by passing a list of layer instances to the
 
 ```python
 from keras.models import Sequential
+from keras.layers import Dense, Activation
 
 model = Sequential([
     Dense(32, input_dim=784),
@@ -85,7 +86,14 @@ final_model.add(merged)
 final_model.add(Dense(10, activation='softmax'))
 ```
 
-<img src="http://s3.amazonaws.com/keras.io/img/two_branches_sequential_model.png" alt="two branch Sequential" style="width: 400px;"/>
+<img src="https://s3.amazonaws.com/keras.io/img/two_branches_sequential_model.png" alt="two branch Sequential" style="width: 400px;"/>
+
+Such a two-branch model can then be trained via e.g.:
+
+```python
+final_model.compile(optimizer='rmsprop', loss='categorical_crossentropy')
+final_model.fit([input_data_1, input_data_2], targets)  # we pass one data array per model input
+```
 
 The `Merge` layer supports a number of pre-defined modes:
 
@@ -99,7 +107,7 @@ The `Merge` layer supports a number of pre-defined modes:
 You can also pass a function as the `mode` argument, allowing for arbitrary transformations:
 
 ```python
-merged = Merge([left_branch, right_branch], mode=lambda x, y: x - y)
+merged = Merge([left_branch, right_branch], mode=lambda x: x[0] - x[1])
 ```
 
 Now you know enough to be able to define *almost* any model with Keras. For complex models that cannot be expressed via `Sequential` and `Merge`, you can use [the functional API](/getting-started/functional-api-guide).
@@ -112,8 +120,8 @@ Now you know enough to be able to define *almost* any model with Keras. For comp
 Before training a model, you need to configure the learning process, which is done via the `compile` method. It receives three arguments:
 
 - an optimizer. This could be the string identifier of an existing optimizer (such as `rmsprop` or `adagrad`), or an instance of the `Optimizer` class. See: [optimizers](/optimizers).
-- a loss function. This is the objective that the model will try to minimize. If can be the string identifier of an existing loss function (such as `categorical_crossentropy` or `mse`), or it can be an objective function. See: [objectives](/objectives).
-- a list of metrics. For any classification problem you will want to set this to `metrics=['accuracy']`. A metric could be the string identifier of an existing metric (only `accuracy` is supported at this point), or a custom metric function.
+- a loss function. This is the objective that the model will try to minimize. It can be the string identifier of an existing loss function (such as `categorical_crossentropy` or `mse`), or it can be an objective function. See: [objectives](/objectives).
+- a list of metrics. For any classification problem you will want to set this to `metrics=['accuracy']`. A metric could be the string identifier of an existing metric or a custom metric function.  Custom metric function should return either a single tensor value or a dict `metric_name -> metric_value`. See: [metrics](/metrics).
 
 ```python
 # for a multi-class classification problem
@@ -129,6 +137,24 @@ model.compile(optimizer='rmsprop',
 # for a mean squared error regression problem
 model.compile(optimizer='rmsprop',
               loss='mse')
+
+# for custom metrics
+import keras.backend as K
+
+def mean_pred(y_true, y_pred):
+    return K.mean(y_pred)
+
+def false_rates(y_true, y_pred):
+    false_neg = ...
+    false_pos = ...
+    return {
+        'false_neg': false_neg,
+        'false_pos': false_pos,
+    }
+
+model.compile(optimizer='rmsprop',
+              loss='binary_crossentropy',
+              metrics=['accuracy', mean_pred, false_rates])
 ```
 
 ----
@@ -141,7 +167,7 @@ Keras models are trained on Numpy arrays of input data and labels. For training
 # for a single-input model with 2 classes (binary):
 
 model = Sequential()
-model.add(Dense(1, input_dim=784, activation='softmax'))
+model.add(Dense(1, input_dim=784, activation='sigmoid'))
 model.compile(optimizer='rmsprop',
               loss='binary_crossentropy',
               metrics=['accuracy'])
@@ -373,7 +399,7 @@ image_model.load_weights('weight_file.h5')
 language_model = Sequential()
 language_model.add(Embedding(vocab_size, 256, input_length=max_caption_len))
 language_model.add(GRU(output_dim=128, return_sequences=True))
-language_model.add(TimeDistributedDense(128))
+language_model.add(TimeDistributed(Dense(128)))
 
 # let's repeat the image vector to turn it into a sequence.
 image_model.add(RepeatVector(max_caption_len))
@@ -410,7 +436,7 @@ The first two LSTMs return their full output sequences, but the last one only re
 the last step in its output sequence, thus dropping the temporal dimension
 (i.e. converting the input sequence into a single vector).
 
-<img src="http://keras.io/img/regular_stacked_lstm.png" alt="stacked LSTM" style="width: 300px;"/>
+<img src="https://keras.io/img/regular_stacked_lstm.png" alt="stacked LSTM" style="width: 300px;"/>
 
 ```python
 from keras.models import Sequential
@@ -499,7 +525,7 @@ In this model, two input sequences are encoded into vectors by two separate LSTM
 
 These two vectors are then concatenated, and a fully connected network is trained on top of the concatenated representations.
 
-<img src="http://keras.io/img/dual_lstm.png" alt="Dual LSTM" style="width: 600px;"/>
+<img src="https://keras.io/img/dual_lstm.png" alt="Dual LSTM" style="width: 600px;"/>
 
 ```python
 from keras.models import Sequential
@@ -538,4 +564,4 @@ y_val = np.random.random((100, nb_classes))
 decoder.fit([x_train_a, x_train_b], y_train,
             batch_size=64, nb_epoch=5,
             validation_data=([x_val_a, x_val_b], y_val))
-```
+```

docs/templates/index.md

@@ -2,14 +2,14 @@
 
 ## You have just found Keras.
 
-Keras is a minimalist, highly modular neural networks library, written in Python and capable of running on top of either [TensorFlow](https://github.com/tensorflow/tensorflow) or [Theano](https://github.com/Theano/Theano). It was developed with a focus on enabling fast experimentation. Being able to go from idea to result with the least possible delay is key to doing good research.
+Keras is a high-level neural networks library, written in Python and capable of running on top of either [TensorFlow](https://github.com/tensorflow/tensorflow) or [Theano](https://github.com/Theano/Theano). It was developed with a focus on enabling fast experimentation. *Being able to go from idea to result with the least possible delay is key to doing good research.*
 
 Use Keras if you need a deep learning library that:
 
-- allows for easy and fast prototyping (through total modularity, minimalism, and extensibility).
-- supports both convolutional networks and recurrent networks, as well as combinations of the two.
-- supports arbitrary connectivity schemes (including multi-input and multi-output training).
-- runs seamlessly on CPU and GPU.
+- Allows for easy and fast prototyping (through total modularity, minimalism, and extensibility).
+- Supports both convolutional networks and recurrent networks, as well as combinations of the two.
+- Supports arbitrary connectivity schemes (including multi-input and multi-output training).
+- Runs seamlessly on CPU and GPU.
 
 Read the documentation at [Keras.io](http://keras.io).
 
@@ -33,10 +33,9 @@ Keras is compatible with: __Python 2.7-3.5__.
 ------------------
 
 
-
 ## Getting started: 30 seconds to Keras
 
-The core data structure of Keras is a __model__, a way to organize layers. The main type of model is the [`Sequential`](http://keras.io/getting-started/sequential-model-guide) model, a linear stack of layers. For more complex architectures, you should use the [Keras function API](http://keras.io/getting-started/functional-api-guide).
+The core data structure of Keras is a __model__, a way to organize layers. The main type of model is the [`Sequential`](http://keras.io/getting-started/sequential-model-guide) model, a linear stack of layers. For more complex architectures, you should use the [Keras functional API](http://keras.io/getting-started/functional-api-guide).
 
 Here's the `Sequential` model:
 
@@ -49,12 +48,12 @@ model = Sequential()
 Stacking layers is as easy as `.add()`:
 
 ```python
-from keras.layers.core import Dense, Activation
+from keras.layers import Dense, Activation
 
 model.add(Dense(output_dim=64, input_dim=100))
-model.add(Activation("relu"))
+model.add(Activation('relu'))
 model.add(Dense(output_dim=10))
-model.add(Activation("softmax"))
+model.add(Activation('softmax'))
 ```
 
 Once your model looks good, configure its learning process with `.compile()`:
@@ -98,6 +97,7 @@ For a more in-depth tutorial about Keras, you can check out:
 
 In the [examples folder](https://github.com/fchollet/keras/tree/master/examples) of the repository, you will find more advanced models: question-answering with memory networks, text generation with stacked LSTMs, etc.
 
+
 ------------------
 
 
@@ -110,41 +110,45 @@ Keras uses the following dependencies:
 - HDF5 and h5py (optional, required if you use model saving/loading functions)
 - Optional but recommended if you use CNNs: cuDNN.
 
-*When using the Theano backend:*
-
-- Theano
-    - [See installation instructions](http://deeplearning.net/software/theano/install.html#install).
 
 *When using the TensorFlow backend:*
 
 - TensorFlow
     - [See installation instructions](https://github.com/tensorflow/tensorflow#download-and-setup).
 
+*When using the Theano backend:*
+
+- Theano
+    - [See installation instructions](http://deeplearning.net/software/theano/install.html#install).
+
 To install Keras, `cd` to the Keras folder and run the install command:
-```
+```sh
 sudo python setup.py install
 ```
 
 You can also install Keras from PyPI:
-```
+```sh
 sudo pip install keras
 ```
 
 ------------------
 
 
-## Switching from Theano to TensorFlow
+## Switching from TensorFlow to Theano
 
-By default, Keras will use Theano as its tensor manipulation library. [Follow these instructions](http://keras.io/backend/) to configure the Keras backend.
+By default, Keras will use TensorFlow as its tensor manipulation library. [Follow these instructions](http://keras.io/backend/) to configure the Keras backend.
 
 ------------------
 
 
 ## Support
 
-You can ask questions and join the development discussion on the [Keras Google group](https://groups.google.com/forum/#!forum/keras-users).
+You can ask questions and join the development discussion:
 
-You can also post bug reports and feature requests in [Github issues](https://github.com/fchollet/keras/issues). Make sure to read [our guidelines](https://github.com/fchollet/keras/blob/master/CONTRIBUTING.md) first.
+- On the [Keras Google group](https://groups.google.com/forum/#!forum/keras-users).
+- On the [Keras Slack channel](https://kerasteam.slack.com). Use [this link](https://keras-slack-autojoin.herokuapp.com/) to request an invitation to the channel.
+
+You can also post **bug reports and feature requests** (only) in [Github issues](https://github.com/fchollet/keras/issues). Make sure to read [our guidelines](https://github.com/fchollet/keras/blob/master/CONTRIBUTING.md) first.
 
 
 ------------------

docs/templates/initializations.md

@@ -1,7 +1,7 @@
 
 ## Usage of initializations
 
-Initializations define the probability distribution used to set the initial random weights of Keras layers.
+Initializations define the way to set the initial random weights of Keras layers.
 
 The keyword arguments used for passing initializations to layers will depend on the layer. Usually it is simply `init`:
 
@@ -17,7 +17,35 @@ model.add(Dense(64, init='uniform'))
 - __identity__: Use with square 2D layers (`shape[0] == shape[1]`).
 - __orthogonal__: Use with square 2D layers (`shape[0] == shape[1]`).
 - __zero__
+- __one__
 - __glorot_normal__: Gaussian initialization scaled by fan_in + fan_out (Glorot 2010)
 - __glorot_uniform__
 - __he_normal__: Gaussian initialization scaled by fan_in (He et al., 2014)
 - __he_uniform__
+
+
+An initialization may be passed as a string (must match one of the available initializations above), or as a callable.
+If a callable, then it must take two arguments: `shape` (shape of the variable to initialize) and `name` (name of the variable),
+and it must return a variable (e.g. output of `K.variable()`):
+
+```python
+from keras import backend as K
+import numpy as np
+
+def my_init(shape, name=None):
+    value = np.random.random(shape)
+    return K.variable(value, name=name)
+
+model.add(Dense(64, init=my_init))
+```
+
+You could also use functions from `keras.initializations` in this way:
+
+```python
+from keras import initializations
+
+def my_init(shape, name=None):
+    return initializations.normal(shape, scale=0.01, name=name)
+
+model.add(Dense(64, init=my_init))
+```

docs/templates/layers/about-keras-layers.md

@@ -5,11 +5,21 @@ All Keras layers have a number of methods in common:
 - `layer.get_weights()`: returns the weights of the layer as a list of Numpy arrays.
 - `layer.set_weights(weights)`: sets the weights of the layer from a list of Numpy arrays (with the same shapes as the output of `get_weights`).
 - `layer.get_config()`: returns a dictionary containing the configuration of the layer. The layer can be reinstantiated from its config via:
+
+```python
+layer = Dense(32)
+config = layer.get_config()
+reconstructed_layer = Dense.from_config(config)
+```
+
+Or:
+
 ```python
 from keras.utils.layer_utils import layer_from_config
 
 config = layer.get_config()
-layer = layer_from_config(config)
+layer = layer_from_config({'class_name': layer.__class__.__name__,
+                           'config': config})
 ```
 
 If a layer has a single node (i.e. if it isn't a shared layer), you can get its input tensor, output tensor, input shape and output shape via:

docs/templates/layers/writing-your-own-keras-layers.md

@@ -0,0 +1,35 @@
+# Writing your own Keras layers
+
+For simple, stateless custom operations, you are probably better off using `layers.core.Lambda` layers. But for any custom operation that has trainable weights, you should implement your own layer.
+
+Here is the skeleton of a Keras layer, **as of Keras 1.2.0** (if you have an older version, please upgrade). There are only three methods you need to implement:
+
+- `build(input_shape)`: this is where you will define your weights. This method must set `self.built = True`, which can be done by calling `super([Layer], self).build()`.
+- `call(x)`: this is where the layer's logic lives. Unless you want your layer to support masking, you only have to care about the first argument passed to `call`: the input tensor.
+- `get_output_shape_for(input_shape)`: in case your layer modifies the shape of its input, you should specify here the shape transformation logic. This allows Keras to do automatic shape inference.
+
+```python
+from keras import backend as K
+from keras.engine.topology import Layer
+import numpy as np
+
+class MyLayer(Layer):
+    def __init__(self, output_dim, **kwargs):
+        self.output_dim = output_dim
+        super(MyLayer, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+        # Create a trainable weight variable for this layer.
+        self.W = self.add_weight(shape=(input_shape[1], self.output_dim),
+                                 initializer='uniform',
+                                 trainable=True)
+        super(MyLayer, self).build(input_shape)  # Be sure to call this somewhere!
+
+    def call(self, x, mask=None):
+        return K.dot(x, self.W)
+
+    def get_output_shape_for(self, input_shape):
+        return (input_shape[0], self.output_dim)
+```
+
+The existing Keras layers provide examples of how to implement almost anything. Never hesitate to read the source code!

docs/templates/metrics.md

@@ -0,0 +1,51 @@
+
+## Usage of metrics
+
+A metric is a function that is used to judge the performance of your model. Metric functions are to be supplied in the `metrics` parameter when a model is compiled.
+
+A metric function is similar to an [objective function](/objectives), except that the results from evaluating a metric are not used when training the model.
+
+You can either pass the name of an existing metric, or pass a Theano/TensorFlow symbolic function (see [Custom metrics](#custom-metrics)).
+
+#### Arguments
+  - __y_true__: True labels. Theano/TensorFlow tensor.
+  - __y_pred__: Predictions. Theano/TensorFlow tensor of the same shape as y_true.
+
+#### Returns
+  Single tensor value representing the mean of the output array across all
+  datapoints.
+
+----
+
+## Available metrics
+
+
+{{autogenerated}}
+
+----
+
+## Custom metrics
+
+Custom metrics can be defined and passed via the compilation step. The
+function would need to take `(y_true, y_pred)` as arguments and return
+either a single tensor value or a dict `metric_name -> metric_value`.
+
+```python
+# for custom metrics
+import keras.backend as K
+
+def mean_pred(y_true, y_pred):
+    return K.mean(y_pred)
+
+def false_rates(y_true, y_pred):
+    false_neg = ...
+    false_pos = ...
+    return {
+        'false_neg': false_neg,
+        'false_pos': false_pos,
+    }
+
+model.compile(optimizer='rmsprop',
+              loss='binary_crossentropy',
+              metrics=['accuracy', mean_pred, false_rates])
+```

docs/templates/models/about-keras-models.md

@@ -30,4 +30,4 @@ yaml_string = model.to_yaml()
 model = model_from_yaml(yaml_string)
 ```
 - `model.save_weights(filepath)`: saves the weights of the model as a HDF5 file.
-- `model.load_weights(filepath)`: loads the weights of the model from a HDF5 file (created by `save_weights`).
+- `model.load_weights(filepath, by_name=False)`: loads the weights of the model from a HDF5 file (created by `save_weights`). By default, the architecture is expected to be unchanged. To load weights into a different architecture (with some layers in common), use `by_name=True` to load only those layers with the same name.

docs/templates/models/model.md

@@ -11,7 +11,7 @@ b = Dense(32)(a)
 model = Model(input=a, output=b)
 ```
 
-This model will include all layers required in the computation of `a` given `b`.
+This model will include all layers required in the computation of `b` given `a`.
 
 In the case of multi-input or multi-output models, you can use lists as well:
 
@@ -29,4 +29,4 @@ For a detailed introduction of what `Model` can do, read [this guide to the Kera
 
 ## Methods
 
-{{autogenerated}}
+{{autogenerated}}

docs/templates/objectives.md

@@ -26,5 +26,15 @@ For a few examples of such functions, check out the [objectives source](https://
 - __hinge__
 - __binary_crossentropy__: Also known as logloss. 
 - __categorical_crossentropy__: Also known as multiclass logloss. __Note__: using this objective requires that your labels are binary arrays of shape `(nb_samples, nb_classes)`.
-- __poisson__: mean of `(predictions - targets * log(predictions))`
-- __cosine_proximity__: the opposite (negative) of the mean cosine proximity between predictions and targets.
+- __sparse_categorical_crossentropy__: As above but accepts sparse labels. __Note__: this objective still requires that your labels have the same number of dimensions as your outputs; you may need to add a length-1 dimension to the shape of your labels, e.g with `np.expand_dims(y, -1)`.
+- __kullback_leibler_divergence__ / __kld__: Information gain from a predicted probability distribution Q to a true probability distribution P. Gives a measure of difference between both distributions.
+- __poisson__: Mean of `(predictions - targets * log(predictions))`
+- __cosine_proximity__: The opposite (negative) of the mean cosine proximity between predictions and targets.
+
+**Note**: when using the `categorical_crossentropy` objective, your targets should be in categorical format (e.g. if you have 10 classes, the target for each sample should be a 10-dimensional vector that is all-zeros expect for a 1 at the index corresponding to the class of the sample). In order to convert *integer targets* into *categorical targets*, you can use the Keras utility `to_categorical`:
+
+```python
+from keras.utils.np_utils import to_categorical
+
+categorical_labels = to_categorical(int_labels, nb_classes=None)
+```

docs/templates/optimizers.md

@@ -9,7 +9,7 @@ model.add(Dense(64, init='uniform', input_dim=10))
 model.add(Activation('tanh'))
 model.add(Activation('softmax'))
 
-sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
+sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
 model.compile(loss='mean_squared_error', optimizer=sgd)
 ```
 
@@ -22,4 +22,23 @@ model.compile(loss='mean_squared_error', optimizer='sgd')
 
 ---
 
+## Parameters common to all Keras optimizers
+
+The parameters `clipnorm` and `clipvalue` can be used with all optimizers to control gradient clipping:
+
+```python
+# all parameter gradients will be clipped to
+# a maximum norm of 1.
+sgd = SGD(lr=0.01, clipnorm=1.)
+```
+
+```python
+# all parameter gradients will be clipped to
+# a maximum value of 0.5 and
+# a minimum value of -0.5.
+sgd = SGD(lr=0.01, clipvalue=0.5)
+```
+
+---
+
 {{autogenerated}}

docs/templates/preprocessing/image.md

@@ -2,53 +2,103 @@
 ## ImageDataGenerator
 
 ```python
-keras.preprocessing.image.ImageDataGenerator(featurewise_center=True,
+keras.preprocessing.image.ImageDataGenerator(featurewise_center=False,
     samplewise_center=False,
-    featurewise_std_normalization=True,
+    featurewise_std_normalization=False,
     samplewise_std_normalization=False,
     zca_whitening=False,
     rotation_range=0.,
     width_shift_range=0.,
     height_shift_range=0.,
     shear_range=0.,
+    zoom_range=0.,
+    channel_shift_range=0.,
+    fill_mode='nearest',
+    cval=0.,
     horizontal_flip=False,
-    vertical_flip=False)
+    vertical_flip=False,
+    rescale=None,
+    dim_ordering=K.image_dim_ordering())
 ```
 
 Generate batches of tensor image data with real-time data augmentation. The data will be looped over (in batches) indefinitely.
 
 - __Arguments__:
-    - __featurewise_center__: Boolean. Set input mean to 0 over the dataset.
+    - __featurewise_center__: Boolean. Set input mean to 0 over the dataset, feature-wise.
     - __samplewise_center__: Boolean. Set each sample mean to 0.
-    - __featurewise_std_normalization__: Boolean. Divide inputs by std of the dataset.
+    - __featurewise_std_normalization__: Boolean. Divide inputs by std of the dataset, feature-wise.
     - __samplewise_std_normalization__: Boolean. Divide each input by its std.
     - __zca_whitening__: Boolean. Apply ZCA whitening.
     - __rotation_range__: Int. Degree range for random rotations.
     - __width_shift_range__: Float (fraction of total width). Range for random horizontal shifts.
     - __height_shift_range__: Float (fraction of total height). Range for random vertical shifts.
     - __shear_range__: Float. Shear Intensity (Shear angle in counter-clockwise direction as radians)
+    - __zoom_range__: Float or [lower, upper]. Range for random zoom. If a float, `[lower, upper] = [1-zoom_range, 1+zoom_range]`.
+    - __channel_shift_range__: Float. Range for random channel shifts.
+    - __fill_mode__: One of {"constant", "nearest", "reflect" or "wrap"}.  Points outside the boundaries of the input are filled according to the given mode.
+    - __cval__: Float or Int. Value used for points outside the boundaries when `fill_mode = "constant"`.
     - __horizontal_flip__: Boolean. Randomly flip inputs horizontally.
     - __vertical_flip__: Boolean. Randomly flip inputs vertically.
+    - __rescale__: rescaling factor. Defaults to None. If None or 0, no rescaling is applied,
+            otherwise we multiply the data by the value provided (before applying
+            any other transformation).
+    - __dim_ordering__: One of {"th", "tf"}.
+        "tf" mode means that the images should have shape `(samples, height, width, channels)`,
+        "th" mode means that the images should have shape `(samples, channels, height, width)`.
+        It defaults to the `image_dim_ordering` value found in your
+        Keras config file at `~/.keras/keras.json`.
+        If you never set it, then it will be "tf".
 
 - __Methods__:
-    - __fit(X)__: Required if featurewise_center or featurewise_std_normalization or zca_whitening. Compute necessary quantities on some sample data.
+    - __fit(X)__: Compute the internal data stats related to the data-dependent transformations, based on an array of sample data.
+        Only required if featurewise_center or featurewise_std_normalization or zca_whitening.
         - __Arguments__:
-            - __X__: sample data.
+            - __X__: sample data. Should have rank 4.
+                In case of grayscale data,
+                the channels axis should have value 1, and in case
+                of RGB data, it should have value 3.
             - __augment__: Boolean (default: False). Whether to fit on randomly augmented samples.
             - __rounds__: int (default: 1). If augment, how many augmentation passes over the data to use.
-    - __flow(X, y)__:
+            - __seed__: int (default: None). Random seed.
+    - __flow(X, y)__: Takes numpy data & label arrays, and generates batches of augmented/normalized data. Yields batches indefinitely, in an infinite loop.
         - __Arguments__:
-            - __X__: data.
+            - __X__: data. Should have rank 4.
+                In case of grayscale data,
+                the channels axis should have value 1, and in case
+                of RGB data, it should have value 3.
             - __y__: labels.
             - __batch_size__: int (default: 32).
-            - __shuffle__: boolean (defaut: False).
-            - __save_to_dir__: None or str. This allows you to optimally specify a directory to which to save the augmented pictures being generated (useful for visualizing what you are doing).
-            - __save_prefix__: str. Prefix to use for filenames of saved pictures.
-            - __save_format__: one of "png", jpeg".
+            - __shuffle__: boolean (defaut: True).
+            - __seed__: int (default: None).
+            - __save_to_dir__: None or str (default: None). This allows you to optimally specify a directory to which to save the augmented pictures being generated (useful for visualizing what you are doing).
+            - __save_prefix__: str (default: `''`). Prefix to use for filenames of saved pictures (only relevant if `save_to_dir` is set).
+            - __save_format__: one of "png", "jpeg" (only relevant if `save_to_dir` is set). Default: "jpeg".
+        - __yields__: Tuples of `(x, y)` where `x` is a numpy array of image data and `y` is a numpy array of corresponding labels.
+            The generator loops indefinitely.
+    - __flow_from_directory(directory)__: Takes the path to a directory, and generates batches of augmented/normalized data. Yields batches indefinitely, in an infinite loop.
+        - __Arguments__:
+            - __directory__: path to the target directory. It should contain one subdirectory per class.
+                Any PNG, JPG or BMP images inside each of the subdirectories directory tree will be included in the generator.
+                See [this script](https://gist.github.com/fchollet/0830affa1f7f19fd47b06d4cf89ed44d) for more details.
+            - __target_size__: tuple of integers, default: `(256, 256)`. The dimensions to which all images found will be resized.
+            - __color_mode__: one of "grayscale", "rbg". Default: "rgb". Whether the images will be converted to have 1 or 3 color channels.
+            - __classes__: optional list of class subdirectories (e.g. `['dogs', 'cats']`). Default: None. If not provided, the list of classes will be automatically inferred (and the order of the classes, which will map to the label indices, will be alphanumeric).
+            - __class_mode__: one of "categorical", "binary", "sparse" or None. Default: "categorical". Determines the type of label arrays that are returned: "categorical" will be 2D one-hot encoded labels, "binary" will be 1D binary labels, "sparse" will be 1D integer labels. If None, no labels are returned (the generator will only yield batches of image data, which is useful to use `model.predict_generator()`, `model.evaluate_generator()`, etc.).
+            - __batch_size__: size of the batches of data (default: 32).
+            - __shuffle__: whether to shuffle the data (default: True)
+            - __seed__: optional random seed for shuffling and transformations.
+            - __save_to_dir__: None or str (default: None). This allows you to optimally specify a directory to which to save the augmented pictures being generated (useful for visualizing what you are doing).
+            - __save_prefix__: str. Prefix to use for filenames of saved pictures (only relevant if `save_to_dir` is set).
+            - __save_format__: one of "png", "jpeg" (only relevant if `save_to_dir` is set). Default: "jpeg".
+            - __follow_links__: whether to follow symlinks inside class subdirectories (default: False).
+
+
+- __Examples__:
+
+Example of using `.flow(X, y)`:
 
-- __Example__:
 ```python
-(X_train, y_train), (X_test, y_test) = cifar10.load_data(test_split=0.1)
+(X_train, y_train), (X_test, y_test) = cifar10.load_data()
 Y_train = np_utils.to_categorical(y_train, nb_classes)
 Y_test = np_utils.to_categorical(y_test, nb_classes)
 
@@ -80,3 +130,71 @@ for e in range(nb_epoch):
             # the generator loops indefinitely
             break
 ```
+
+Example of using `.flow_from_directory(directory)`:
+
+```python
+train_datagen = ImageDataGenerator(
+        rescale=1./255,
+        shear_range=0.2,
+        zoom_range=0.2,
+        horizontal_flip=True)
+
+test_datagen = ImageDataGenerator(rescale=1./255)
+
+train_generator = train_datagen.flow_from_directory(
+        'data/train',
+        target_size=(150, 150),
+        batch_size=32,
+        class_mode='binary')
+
+validation_generator = test_datagen.flow_from_directory(
+        'data/validation',
+        target_size=(150, 150),
+        batch_size=32,
+        class_mode='binary')
+
+model.fit_generator(
+        train_generator,
+        samples_per_epoch=2000,
+        nb_epoch=50,
+        validation_data=validation_generator,
+        nb_val_samples=800)
+```
+
+Example of transforming images and masks together.
+
+```python
+# we create two instances with the same arguments
+data_gen_args = dict(featurewise_center=True,
+                     featurewise_std_normalization=True,
+                     rotation_range=90.,
+                     width_shift_range=0.1,
+                     height_shift_range=0.1,
+                     zoom_range=0.2)
+image_datagen = ImageDataGenerator(**data_gen_args)
+mask_datagen = ImageDataGenerator(**data_gen_args)
+
+# Provide the same seed and keyword arguments to the fit and flow methods
+seed = 1
+image_datagen.fit(images, augment=True, seed=seed)
+mask_datagen.fit(masks, augment=True, seed=seed)
+
+image_generator = image_datagen.flow_from_directory(
+    'data/images',
+    class_mode=None,
+    seed=seed)
+
+mask_generator = mask_datagen.flow_from_directory(
+    'data/masks',
+    class_mode=None,
+    seed=seed)
+
+# combine generators into one which yields image and masks
+train_generator = zip(image_generator, mask_generator)
+
+model.fit_generator(
+    train_generator,
+    samples_per_epoch=2000,
+    nb_epoch=50)
+```

docs/templates/preprocessing/sequence.md

@@ -1,17 +1,18 @@
 ## pad_sequences
 
 ```python
-keras.preprocessing.sequence.pad_sequences(sequences, maxlen=None, dtype='int32')
+keras.preprocessing.sequence.pad_sequences(sequences, maxlen=None, dtype='int32',
+    padding='pre', truncating='pre', value=0.)
 ```
 
-Transform a list of `nb_samples sequences` (lists of scalars) into a 2D numpy array of shape `(nb_samples, nb_timesteps)`. `nb_timesteps` is either the `maxlen` argument if provided, or the length of the longest sequence otherwise. Sequences that are shorter than `nb_timesteps` are padded with zeros at the end.
+Transform a list of `nb_samples` sequences (lists of scalars) into a 2D Numpy array of shape `(nb_samples, nb_timesteps)`. `nb_timesteps` is either the `maxlen` argument if provided, or the length of the longest sequence otherwise. Sequences that are shorter than `nb_timesteps` are padded with `value` at the end. Sequences longer than `nb_timesteps` are truncated so that it fits the desired length. Position where padding or truncation happens is determined by `padding` or `truncating`, respectively.
 
-- __Return__: 2D numpy array of shape `(nb_samples, nb_timesteps)`.
+- __Return__: 2D Numpy array of shape `(nb_samples, nb_timesteps)`.
 
 - __Arguments__:
     - __sequences__: List of lists of int or float.
     - __maxlen__: None or int. Maximum sequence length, longer sequences are truncated and shorter sequences are padded with zeros at the end.
-    - __dtype__: datatype of the numpy array returned.
+    - __dtype__: datatype of the Numpy array returned.
     - __padding__: 'pre' or 'post', pad either before or after each sequence.
     - __truncating__: 'pre' or 'post', remove values from sequences larger than maxlen either in the beginning or in the end of the sequence
     - __value__: float, value to pad the sequences to the desired value.
@@ -21,12 +22,12 @@ Transform a list of `nb_samples sequences` (lists of scalars) into a 2D numpy ar
 ## skipgrams
 
 ```python
-keras.preprocessing.sequence.skipgrams(sequence, vocabulary_size, 
-    window_size=4, negative_samples=1., shuffle=True, 
+keras.preprocessing.sequence.skipgrams(sequence, vocabulary_size,
+    window_size=4, negative_samples=1., shuffle=True,
     categorical=False, sampling_table=None)
 ```
 
-Transforms a sequence of word indexes (list of int) into couples of the form: 
+Transforms a sequence of word indexes (list of int) into couples of the form:
 
 - (word, word in the same window), with label 1 (positive samples).
 - (word, random word from the vocabulary), with label 0 (negative samples).
@@ -34,8 +35,8 @@ Transforms a sequence of word indexes (list of int) into couples of the form:
 Read more about Skipgram in this gnomic paper by Mikolov et al.: [Efficient Estimation of Word Representations in
 Vector Space](http://arxiv.org/pdf/1301.3781v3.pdf)
 
-- __Return__: tuple `(couples, labels)`. 
-    - `couples` is a list of 2-elements lists of int: `[word_index, other_word_index]`. 
+- __Return__: tuple `(couples, labels)`.
+    - `couples` is a list of 2-elements lists of int: `[word_index, other_word_index]`.
     - `labels` is a list of 0 and 1, where 1 indicates that `other_word_index` was found in the same window as `word_index`, and 0 indicates that `other_word_index` was random.
     - if categorical is set to True, the labels are categorical, ie. 1 becomes [0,1], and 0 becomes [1, 0].
 
@@ -46,7 +47,7 @@ Vector Space](http://arxiv.org/pdf/1301.3781v3.pdf)
     - __negative_samples__: float >= 0. 0 for no negative (=random) samples. 1 for same number as positive samples. etc.
     - __shuffle__: boolean. Whether to shuffle the samples.
     - __categorical__: boolean. Whether to make the returned labels categorical.
-    - __sampling_table__: numpy array of shape `(vocabulary_size,)` where `sampling_table[i]` is the probability of sampling the word with index i (assumed to be i-th most common word in the dataset).
+    - __sampling_table__: Numpy array of shape `(vocabulary_size,)` where `sampling_table[i]` is the probability of sampling the word with index i (assumed to be i-th most common word in the dataset).
 
 
 ---
@@ -59,7 +60,7 @@ keras.preprocessing.sequence.make_sampling_table(size, sampling_factor=1e-5)
 
 Used for generating the `sampling_table` argument for `skipgrams`. `sampling_table[i]` is the probability of sampling the word i-th most common word in a dataset (more common words should be sampled less frequently, for balance).
 
-- __Return__: numpy array of shape `(size,)`.
+- __Return__: Numpy array of shape `(size,)`.
 
 - __Arguments__:
     - __size__: size of the vocabulary considered.

docs/templates/regularizers.md

@@ -2,7 +2,7 @@
 
 Regularizers allow to apply penalties on layer parameters or layer activity during optimization. These penalties are incorporated in the loss function that the network optimizes.
 
-The penalties are applied on a per-layer basis. The exact API will depend on the layer, but the layers `Dense`, `TimeDistributedDense`, `MaxoutDense`, `Convolution1D` and `Convolution2D` have a unified API.
+The penalties are applied on a per-layer basis. The exact API will depend on the layer, but the layers `Dense`, `TimeDistributedDense`, `MaxoutDense`, `Convolution1D`, `Convolution2D` and `Convolution3D` have a unified API.
 
 These layers expose 3 keyword arguments:
 

docs/templates/scikit-learn-api.md

@@ -1,12 +1,12 @@
-# Wrappers for the Sciki-Learn API
+# Wrappers for the Scikit-Learn API
 
-You can use `Sequential` Keras models (single-input only) as part of your Scikit-Learn workflow via the wrappers found at `keras.wrappers.sklearn.py`.
+You can use `Sequential` Keras models (single-input only) as part of your Scikit-Learn workflow via the wrappers found at `keras.wrappers.scikit_learn.py`.
 
 There are two wrappers available:
 
-`keras.wrappers.sklearn.KerasClassifier(build_fn=None, **sk_params)`, which implements the sklearn classifier interface,
+`keras.wrappers.scikit_learn.KerasClassifier(build_fn=None, **sk_params)`, which implements the Scikit-Learn classifier interface,
 
-`keras.wrappers.sklearn.KerasRegressor(build_fn=None, **sk_params)`, which implements the sklearn regressor interface.
+`keras.wrappers.scikit_learn.KerasRegressor(build_fn=None, **sk_params)`, which implements the Scikit-Learn regressor interface.
 
 ### Arguments
 
@@ -25,7 +25,7 @@ present class will then be treated as the default build_fn.
 
 `sk_params` takes both model parameters and fitting parameters. Legal model
 parameters are the arguments of `build_fn`. Note that like all other
-estimators in scikit-learn, 'build_fn' should provide defalult values for
+estimators in scikit-learn, 'build_fn' should provide default values for
 its arguments, so that you could create the estimator without passing any
 values to `sk_params`.
 
@@ -42,4 +42,4 @@ fitting (predicting) parameters are selected in the following order:
 When using scikit-learn's `grid_search` API, legal tunable parameters are
 those you could pass to `sk_params`, including fitting parameters.
 In other words, you could use `grid_search` to search for the best
-`batch_size` or `nb_epoch` as well as the model parameters.
+`batch_size` or `nb_epoch` as well as the model parameters.

docs/templates/visualization.md

@@ -10,9 +10,10 @@ from keras.utils.visualize_util import plot
 plot(model, to_file='model.png')
 ```
 
-`plot` takes one optional arguments:
+`plot` takes two optional arguments:
 
 - `show_shapes` (defaults to False) controls whether output shapes are shown in the graph.
+- `show_layer_names` (defaults to True) controls whether layer names are shown in the graph.
 
 You can also directly obtain the `pydot.Graph` object and render it yourself,
 for example to show it in an ipython notebook :

examples/README.md

@@ -0,0 +1,100 @@
+# Keras examples directory
+
+[addition_rnn.py](addition_rnn.py)
+Implementation of sequence to sequence learning for performing addition of two numbers (as strings).
+
+[antirectifier.py](antirectifier.py)
+Demonstrates how to write custom layers for Keras.
+
+[babi_memnn.py](babi_memnn.py)
+Trains a memory network on the bAbI dataset for reading comprehension.
+
+[babi_rnn.py](babi_rnn.py)
+Trains a two-branch recurrent network on the bAbI dataset for reading comprehension.
+
+[cifar10_cnn.py](cifar10_cnn.py)
+Trains a simple deep CNN on the CIFAR10 small images dataset.
+
+[conv_filter_visualization.py](conv_filter_visualization.py)
+Visualization of the filters of VGG16, via gradient ascent in input space.
+
+[conv_lstm.py](conv_lstm.py)
+Demonstrates the use of a convolutional LSTM network.
+
+[deep_dream.py](deep_dream.py)
+Deep Dreams in Keras.
+
+[image_ocr.py](image_ocr.py)
+Trains a convolutional stack followed by a recurrent stack and a CTC logloss function to perform optical character recognition (OCR).
+
+[imdb_bidirectional_lstm.py](imdb_bidirectional_lstm.py)
+Trains a Bidirectional LSTM on the IMDB sentiment classification task.
+
+[imdb_cnn.py](imdb_cnn.py)
+Demonstrates the use of Convolution1D for text classification.
+
+[imdb_cnn_lstm.py](imdb_cnn_lstm.py)
+Trains a convolutional stack followed by a recurrent stack network on the IMDB sentiment classification task.
+
+[imdb_fasttext.py](imdb_fasttext.py)
+Trains a FastText model on the IMDB sentiment classification task.
+
+[imdb_lstm.py](imdb_lstm.py)
+Trains a LSTM on the IMDB sentiment classification task.
+
+[lstm_benchmark.py](lstm_benchmark.py)
+Compares different LSTM implementations on the IMDB sentiment classification task.
+
+[lstm_text_generation.py](lstm_text_generation.py)
+Generates text from Nietzsche's writings.
+
+[mnist_acgan.py](mnist_acgan.py)
+Implementation of AC-GAN ( Auxiliary Classifier GAN ) on the MNIST dataset
+
+[mnist_cnn.py](mnist_cnn.py)
+Trains a simple convnet on the MNIST dataset.
+
+[mnist_hierarchical_rnn.py](mnist_hierarchical_rnn.py)
+Trains a Hierarchical RNN (HRNN) to classify MNIST digits.
+
+[mnist_irnn.py](mnist_irnn.py)
+Reproduction of the IRNN experiment with pixel-by-pixel sequential MNIST in "A Simple Way to Initialize Recurrent Networks of Rectified Linear Units" by Le et al.
+
+[mnist_mlp.py](mnist_mlp.py)
+Trains a simple deep multi-layer perceptron on the MNIST dataset.
+
+[mnist_net2net.py](mnist_net2net.py)
+Reproduction of the Net2Net experiment with MNIST in "Net2Net: Accelerating Learning via Knowledge Transfer".
+
+[mnist_siamese_graph.py](mnist_siamese_graph.py)
+Trains a Siamese multi-layer perceptron on pairs of digits from the MNIST dataset.
+
+[mnist_sklearn_wrapper.py](mnist_sklearn_wrapper.py)
+Demonstrates how to use the sklearn wrapper.
+
+[mnist_swwae.py](mnist_swwae.py)
+Trains a Stacked What-Where AutoEncoder built on residual blocks on the MNIST dataset.
+
+[mnist_transfer_cnn.py](mnist_transfer_cnn.py)
+Transfer learning toy example.
+
+[neural_doodle.py](neural_doodle.py)
+Neural doodle.
+
+[neural_style_transfer.py](neural_style_transfer.py)
+Neural style transfer.
+
+[pretrained_word_embeddings.py](pretrained_word_embeddings.py)
+Loads pre-trained word embeddings (GloVe embeddings) into a frozen Keras Embedding layer, and uses it to train a text classification model on the 20 Newsgroup dataset.
+
+[reuters_mlp.py](reuters_mlp.py)
+Trains and evaluate a simple MLP on the Reuters newswire topic classification task.
+
+[stateful_lstm.py](stateful_lstm.py)
+Demonstrates how to use stateful RNNs to model long sequences efficiently.
+
+[variational_autoencoder.py](variational_autoencoder.py)
+Demonstrates how to build a variational autoencoder.
+
+[variational_autoencoder_deconv.py](variational_autoencoder_deconv.py)
+Demonstrates how to build a variational autoencoder with Keras using deconvolution layers.

examples/addition_rnn.py

@@ -29,28 +29,35 @@ Five digits inverted:
 from __future__ import print_function
 from keras.models import Sequential
 from keras.engine.training import slice_X
-from keras.layers.core import Activation, TimeDistributedDense, RepeatVector
-from keras.layers import recurrent
+from keras.layers import Activation, TimeDistributed, Dense, RepeatVector, recurrent
 import numpy as np
 from six.moves import range
 
 
 class CharacterTable(object):
-    '''
-    Given a set of characters:
+    """Given a set of characters:
     + Encode them to a one hot integer representation
     + Decode the one hot integer representation to their character output
-    + Decode a vector of probabilties to their character output
-    '''
-    def __init__(self, chars, maxlen):
+    + Decode a vector of probabilities to their character output
+    """
+    def __init__(self, chars):
+        """Initialize character table.
+
+        # Arguments
+            chars: Characters that can appear in the input.
+        """
         self.chars = sorted(set(chars))
         self.char_indices = dict((c, i) for i, c in enumerate(self.chars))
         self.indices_char = dict((i, c) for i, c in enumerate(self.chars))
-        self.maxlen = maxlen
 
-    def encode(self, C, maxlen=None):
-        maxlen = maxlen if maxlen else self.maxlen
-        X = np.zeros((maxlen, len(self.chars)))
+    def encode(self, C, num_rows):
+        """One hot encode given string C.
+
+        # Arguments
+            num_rows: Number of rows in the returned one hot encoding. This is
+                used to keep the # of rows for each data the same.
+        """
+        X = np.zeros((num_rows, len(self.chars)))
         for i, c in enumerate(C):
             X[i, self.char_indices[c]] = 1
         return X
@@ -66,40 +73,42 @@ class colors:
     fail = '\033[91m'
     close = '\033[0m'
 
-# Parameters for the model and dataset
+# Parameters for the model and dataset.
 TRAINING_SIZE = 50000
 DIGITS = 3
 INVERT = True
-# Try replacing GRU, or SimpleRNN
-RNN = recurrent.LSTM
-HIDDEN_SIZE = 128
-BATCH_SIZE = 128
-LAYERS = 1
+
+# Maximum length of input is 'int + int' (e.g., '345+678'). Maximum length of
+# int is DIGITS.
 MAXLEN = DIGITS + 1 + DIGITS
 
+# All the numbers, plus sign and space for padding.
 chars = '0123456789+ '
-ctable = CharacterTable(chars, MAXLEN)
+ctable = CharacterTable(chars)
 
 questions = []
 expected = []
 seen = set()
 print('Generating data...')
 while len(questions) < TRAINING_SIZE:
-    f = lambda: int(''.join(np.random.choice(list('0123456789')) for i in range(np.random.randint(1, DIGITS + 1))))
+    f = lambda: int(''.join(np.random.choice(list('0123456789'))
+                    for i in range(np.random.randint(1, DIGITS + 1))))
     a, b = f(), f()
     # Skip any addition questions we've already seen
-    # Also skip any such that X+Y == Y+X (hence the sorting)
+    # Also skip any such that X+Y == Y+X (hence the sorting).
     key = tuple(sorted((a, b)))
     if key in seen:
         continue
     seen.add(key)
-    # Pad the data with spaces such that it is always MAXLEN
+    # Pad the data with spaces such that it is always MAXLEN.
     q = '{}+{}'.format(a, b)
     query = q + ' ' * (MAXLEN - len(q))
     ans = str(a + b)
-    # Answers can be of maximum size DIGITS + 1
+    # Answers can be of maximum size DIGITS + 1.
     ans += ' ' * (DIGITS + 1 - len(ans))
     if INVERT:
+        # Reverse the query, e.g., '12+345  ' becomes '  543+21'. (Note the
+        # space used for padding.)
         query = query[::-1]
     questions.append(query)
     expected.append(ans)
@@ -109,53 +118,73 @@ print('Vectorization...')
 X = np.zeros((len(questions), MAXLEN, len(chars)), dtype=np.bool)
 y = np.zeros((len(questions), DIGITS + 1, len(chars)), dtype=np.bool)
 for i, sentence in enumerate(questions):
-    X[i] = ctable.encode(sentence, maxlen=MAXLEN)
+    X[i] = ctable.encode(sentence, MAXLEN)
 for i, sentence in enumerate(expected):
-    y[i] = ctable.encode(sentence, maxlen=DIGITS + 1)
+    y[i] = ctable.encode(sentence, DIGITS + 1)
 
-# Shuffle (X, y) in unison as the later parts of X will almost all be larger digits
+# Shuffle (X, y) in unison as the later parts of X will almost all be larger
+# digits.
 indices = np.arange(len(y))
 np.random.shuffle(indices)
 X = X[indices]
 y = y[indices]
 
-# Explicitly set apart 10% for validation data that we never train over
-split_at = len(X) - len(X) / 10
+# Explicitly set apart 10% for validation data that we never train over.
+split_at = len(X) - len(X) // 10
 (X_train, X_val) = (slice_X(X, 0, split_at), slice_X(X, split_at))
 (y_train, y_val) = (y[:split_at], y[split_at:])
 
+print('Training Data:')
 print(X_train.shape)
 print(y_train.shape)
 
+print('Validation Data:')
+print(X_val.shape)
+print(y_val.shape)
+
+# Try replacing GRU, or SimpleRNN.
+RNN = recurrent.LSTM
+HIDDEN_SIZE = 128
+BATCH_SIZE = 128
+LAYERS = 1
+
 print('Build model...')
 model = Sequential()
-# "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE
-# note: in a situation where your input sequences have a variable length,
+# "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE.
+# Note: In a situation where your input sequences have a variable length,
 # use input_shape=(None, nb_feature).
 model.add(RNN(HIDDEN_SIZE, input_shape=(MAXLEN, len(chars))))
-# For the decoder's input, we repeat the encoded input for each time step
+# As the decoder RNN's input, repeatedly provide with the last hidden state of
+# RNN for each time step. Repeat 'DIGITS + 1' times as that's the maximum
+# length of output, e.g., when DIGITS=3, max output is 999+999=1998.
 model.add(RepeatVector(DIGITS + 1))
-# The decoder RNN could be multiple layers stacked or a single layer
+# The decoder RNN could be multiple layers stacked or a single layer.
 for _ in range(LAYERS):
+    # By setting return_sequences to True, return not only the last output but
+    # all the outputs so far in the form of (nb_samples, timesteps,
+    # output_dim). This is necessary as TimeDistributed in the below expects
+    # the first dimension to be the timesteps.
     model.add(RNN(HIDDEN_SIZE, return_sequences=True))
 
-# For each of step of the output sequence, decide which character should be chosen
-model.add(TimeDistributedDense(len(chars)))
+# Apply a dense layer to the every temporal slice of an input. For each of step
+# of the output sequence, decide which character should be chosen.
+model.add(TimeDistributed(Dense(len(chars))))
 model.add(Activation('softmax'))
-
 model.compile(loss='categorical_crossentropy',
               optimizer='adam',
               metrics=['accuracy'])
+model.summary()
 
-# Train the model each generation and show predictions against the validation dataset
+# Train the model each generation and show predictions against the validation
+# dataset.
 for iteration in range(1, 200):
     print()
     print('-' * 50)
     print('Iteration', iteration)
     model.fit(X_train, y_train, batch_size=BATCH_SIZE, nb_epoch=1,
               validation_data=(X_val, y_val))
-    ###
-    # Select 10 samples from the validation set at random so we can visualize errors
+    # Select 10 samples from the validation set at random so we can visualize
+    # errors.
     for i in range(10):
         ind = np.random.randint(0, len(X_val))
         rowX, rowy = X_val[np.array([ind])], y_val[np.array([ind])]
@@ -165,5 +194,9 @@ for iteration in range(1, 200):
         guess = ctable.decode(preds[0], calc_argmax=False)
         print('Q', q[::-1] if INVERT else q)
         print('T', correct)
-        print(colors.ok + '☑' + colors.close if correct == guess else colors.fail + '☒' + colors.close, guess)
+        if correct == guess:
+            print(colors.ok + '☑' + colors.close, end=" ")
+        else:
+            print(colors.fail + '☒' + colors.close, end=" ")
+        print(guess)
         print('---')

examples/antirectifier.py

@@ -2,7 +2,7 @@
 
 We build a custom activation layer called 'Antirectifier',
 which modifies the shape of the tensor that passes through it.
-We need to specify two methods: `output_shape` and `get_output`.
+We need to specify two methods: `get_output_shape_for` and `call`.
 
 Note that the same result can also be achieved via a Lambda layer.
 
@@ -12,7 +12,7 @@ backend (`K`), our code can run both on TensorFlow and Theano.
 
 from __future__ import print_function
 from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Layer, Activation
+from keras.layers import Dense, Dropout, Layer, Activation
 from keras.datasets import mnist
 from keras import backend as K
 from keras.utils import np_utils
@@ -45,6 +45,7 @@ class Antirectifier(Layer):
         with twice less parameters yet with comparable
         classification accuracy as an equivalent ReLU-based network.
     '''
+
     def get_output_shape_for(self, input_shape):
         shape = list(input_shape)
         assert len(shape) == 2  # only valid for 2D tensors

examples/babi_memnn.py

@@ -16,8 +16,8 @@ Time per epoch: 3s on CPU (core i7).
 from __future__ import print_function
 from keras.models import Sequential
 from keras.layers.embeddings import Embedding
-from keras.layers.core import Activation, Dense, Merge, Permute, Dropout
-from keras.layers.recurrent import LSTM
+from keras.layers import Activation, Dense, Merge, Permute, Dropout
+from keras.layers import LSTM
 from keras.utils.data_utils import get_file
 from keras.preprocessing.sequence import pad_sequences
 from functools import reduce
@@ -94,8 +94,13 @@ def vectorize_stories(data, word_idx, story_maxlen, query_maxlen):
             pad_sequences(Xq, maxlen=query_maxlen), np.array(Y))
 
 
-path = get_file('babi-tasks-v1-2.tar.gz',
-                origin='http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz')
+try:
+    path = get_file('babi-tasks-v1-2.tar.gz', origin='https://s3.amazonaws.com/text-datasets/babi_tasks_1-20_v1-2.tar.gz')
+except:
+    print('Error downloading dataset, please download it manually:\n'
+          '$ wget http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz\n'
+          '$ mv tasks_1-20_v1-2.tar.gz ~/.keras/datasets/babi-tasks-v1-2.tar.gz')
+    raise
 tar = tarfile.open(path)
 
 challenges = {
@@ -168,6 +173,7 @@ match = Sequential()
 match.add(Merge([input_encoder_m, question_encoder],
                 mode='dot',
                 dot_axes=[2, 2]))
+match.add(Activation('softmax'))
 # output: (samples, story_maxlen, query_maxlen)
 # embed the input into a single vector with size = story_maxlen:
 input_encoder_c = Sequential()

examples/babi_rnn.py

@@ -66,7 +66,7 @@ np.random.seed(1337)  # for reproducibility
 
 from keras.utils.data_utils import get_file
 from keras.layers.embeddings import Embedding
-from keras.layers.core import Dense, Merge, Dropout, RepeatVector
+from keras.layers import Dense, Merge, Dropout, RepeatVector
 from keras.layers import recurrent
 from keras.models import Sequential
 from keras.preprocessing.sequence import pad_sequences
@@ -146,7 +146,13 @@ BATCH_SIZE = 32
 EPOCHS = 40
 print('RNN / Embed / Sent / Query = {}, {}, {}, {}'.format(RNN, EMBED_HIDDEN_SIZE, SENT_HIDDEN_SIZE, QUERY_HIDDEN_SIZE))
 
-path = get_file('babi-tasks-v1-2.tar.gz', origin='http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz')
+try:
+    path = get_file('babi-tasks-v1-2.tar.gz', origin='https://s3.amazonaws.com/text-datasets/babi_tasks_1-20_v1-2.tar.gz')
+except:
+    print('Error downloading dataset, please download it manually:\n'
+          '$ wget http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz\n'
+          '$ mv tasks_1-20_v1-2.tar.gz ~/.keras/datasets/babi-tasks-v1-2.tar.gz')
+    raise
 tar = tarfile.open(path)
 # Default QA1 with 1000 samples
 # challenge = 'tasks_1-20_v1-2/en/qa1_single-supporting-fact_{}.txt'

examples/cifar10_cnn.py

@@ -1,23 +1,18 @@
 '''Train a simple deep CNN on the CIFAR10 small images dataset.
 
-GPU run command:
+GPU run command with Theano backend (with TensorFlow, the GPU is automatically used):
     THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python cifar10_cnn.py
 
 It gets down to 0.65 test logloss in 25 epochs, and down to 0.55 after 50 epochs.
 (it's still underfitting at that point, though).
-
-Note: the data was pickled with Python 2, and some encoding issues might prevent you
-from loading it in Python 3. You might have to load it in Python 2,
-save it in a different format, load it in Python 3 and repickle it.
 '''
 
 from __future__ import print_function
 from keras.datasets import cifar10
 from keras.preprocessing.image import ImageDataGenerator
 from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation, Flatten
-from keras.layers.convolutional import Convolution2D, MaxPooling2D
-from keras.optimizers import SGD
+from keras.layers import Dense, Dropout, Activation, Flatten
+from keras.layers import Convolution2D, MaxPooling2D
 from keras.utils import np_utils
 
 batch_size = 32
@@ -27,23 +22,23 @@ data_augmentation = True
 
 # input image dimensions
 img_rows, img_cols = 32, 32
-# the CIFAR10 images are RGB
+# The CIFAR10 images are RGB.
 img_channels = 3
 
-# the data, shuffled and split between train and test sets
+# The data, shuffled and split between train and test sets:
 (X_train, y_train), (X_test, y_test) = cifar10.load_data()
 print('X_train shape:', X_train.shape)
 print(X_train.shape[0], 'train samples')
 print(X_test.shape[0], 'test samples')
 
-# convert class vectors to binary class matrices
+# Convert class vectors to binary class matrices.
 Y_train = np_utils.to_categorical(y_train, nb_classes)
 Y_test = np_utils.to_categorical(y_test, nb_classes)
 
 model = Sequential()
 
 model.add(Convolution2D(32, 3, 3, border_mode='same',
-                        input_shape=(img_channels, img_rows, img_cols)))
+                        input_shape=X_train.shape[1:]))
 model.add(Activation('relu'))
 model.add(Convolution2D(32, 3, 3))
 model.add(Activation('relu'))
@@ -64,10 +59,9 @@ model.add(Dropout(0.5))
 model.add(Dense(nb_classes))
 model.add(Activation('softmax'))
 
-# let's train the model using SGD + momentum (how original).
-sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
+# Let's train the model using RMSprop
 model.compile(loss='categorical_crossentropy',
-              optimizer=sgd,
+              optimizer='rmsprop',
               metrics=['accuracy'])
 
 X_train = X_train.astype('float32')
@@ -84,8 +78,7 @@ if not data_augmentation:
               shuffle=True)
 else:
     print('Using real-time data augmentation.')
-
-    # this will do preprocessing and realtime data augmentation
+    # This will do preprocessing and realtime data augmentation:
     datagen = ImageDataGenerator(
         featurewise_center=False,  # set input mean to 0 over the dataset
         samplewise_center=False,  # set each sample mean to 0
@@ -98,13 +91,13 @@ else:
         horizontal_flip=True,  # randomly flip images
         vertical_flip=False)  # randomly flip images
 
-    # compute quantities required for featurewise normalization
-    # (std, mean, and principal components if ZCA whitening is applied)
+    # Compute quantities required for featurewise normalization
+    # (std, mean, and principal components if ZCA whitening is applied).
     datagen.fit(X_train)
 
-    # fit the model on the batches generated by datagen.flow()
+    # Fit the model on the batches generated by datagen.flow().
     model.fit_generator(datagen.flow(X_train, Y_train,
-                        batch_size=batch_size),
+                                     batch_size=batch_size),
                         samples_per_epoch=X_train.shape[0],
                         nb_epoch=nb_epoch,
                         validation_data=(X_test, Y_test))

examples/conv_filter_visualization.py

@@ -3,34 +3,25 @@
 This script can run on CPU in a few minutes (with the TensorFlow backend).
 
 Results example: http://i.imgur.com/4nj4KjN.jpg
-
-Before running this script, download the weights for the VGG16 model at:
-https://drive.google.com/file/d/0Bz7KyqmuGsilT0J5dmRCM0ROVHc/view?usp=sharing
-(source: https://gist.github.com/baraldilorenzo/07d7802847aaad0a35d3)
-and make sure the variable `weights_path` in this script matches the location of the file.
 '''
 from __future__ import print_function
 from scipy.misc import imsave
 import numpy as np
 import time
-import os
-import h5py
-
-from keras.models import Sequential
-from keras.layers import Convolution2D, ZeroPadding2D, MaxPooling2D
+from keras.applications import vgg16
 from keras import backend as K
 
 # dimensions of the generated pictures for each filter.
 img_width = 128
 img_height = 128
 
-# path to the model weights file.
-weights_path = 'vgg16_weights.h5'
-
-# the name of the layer we want to visualize (see model definition below)
-layer_name = 'conv5_1'
+# the name of the layer we want to visualize
+# (see model definition at keras/applications/vgg16.py)
+layer_name = 'block5_conv1'
 
 # util function to convert a tensor into a valid image
+
+
 def deprocess_image(x):
     # normalize tensor: center on 0., ensure std is 0.1
     x -= x.mean()
@@ -43,70 +34,22 @@ def deprocess_image(x):
 
     # convert to RGB array
     x *= 255
-    x = x.transpose((1, 2, 0))
+    if K.image_dim_ordering() == 'th':
+        x = x.transpose((1, 2, 0))
     x = np.clip(x, 0, 255).astype('uint8')
     return x
 
-# build the VGG16 network
-model = Sequential()
-model.add(ZeroPadding2D((1, 1), batch_input_shape=(1, 3, img_width, img_height)))
-first_layer = model.layers[-1]
-# this is a placeholder tensor that will contain our generated images
-input_img = first_layer.input
-
-model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_2'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_2'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_2'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_3'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_3'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_2'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_3'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-# load the weights of the VGG16 networks
-# (trained on ImageNet, won the ILSVRC competition in 2014)
-# note: when there is a complete match between your model definition
-# and your weight savefile, you can simply call model.load_weights(filename)
-assert os.path.exists(weights_path), 'Model weights not found (see "weights_path" variable in script).'
-f = h5py.File(weights_path)
-for k in range(f.attrs['nb_layers']):
-    if k >= len(model.layers):
-        # we don't look at the last (fully-connected) layers in the savefile
-        break
-    g = f['layer_{}'.format(k)]
-    weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]
-    model.layers[k].set_weights(weights)
-f.close()
+# build the VGG16 network with ImageNet weights
+model = vgg16.VGG16(weights='imagenet', include_top=False)
 print('Model loaded.')
 
+model.summary()
+
+# this is the placeholder for the input images
+input_img = model.input
+
 # get the symbolic outputs of each "key" layer (we gave them unique names).
-layer_dict = dict([(layer.name, layer) for layer in model.layers])
+layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
 
 
 def normalize(x):
@@ -124,7 +67,10 @@ for filter_index in range(0, 200):
     # we build a loss function that maximizes the activation
     # of the nth filter of the layer considered
     layer_output = layer_dict[layer_name].output
-    loss = K.mean(layer_output[:, filter_index, :, :])
+    if K.image_dim_ordering() == 'th':
+        loss = K.mean(layer_output[:, filter_index, :, :])
+    else:
+        loss = K.mean(layer_output[:, :, :, filter_index])
 
     # we compute the gradient of the input picture wrt this loss
     grads = K.gradients(loss, input_img)[0]
@@ -139,7 +85,11 @@ for filter_index in range(0, 200):
     step = 1.
 
     # we start from a gray image with some random noise
-    input_img_data = np.random.random((1, 3, img_width, img_height)) * 20 + 128.
+    if K.image_dim_ordering() == 'th':
+        input_img_data = np.random.random((1, 3, img_width, img_height))
+    else:
+        input_img_data = np.random.random((1, img_width, img_height, 3))
+    input_img_data = (input_img_data - 0.5) * 20 + 128
 
     # we run gradient ascent for 20 steps
     for i in range(20):

examples/conv_lstm.py

@@ -0,0 +1,142 @@
+""" This script demonstrates the use of a convolutional LSTM network.
+This network is used to predict the next frame of an artificially
+generated movie which contains moving squares.
+"""
+from keras.models import Sequential
+from keras.layers.convolutional import Convolution3D
+from keras.layers.convolutional_recurrent import ConvLSTM2D
+from keras.layers.normalization import BatchNormalization
+import numpy as np
+import pylab as plt
+
+# We create a layer which take as input movies of shape
+# (n_frames, width, height, channels) and returns a movie
+# of identical shape.
+
+seq = Sequential()
+seq.add(ConvLSTM2D(nb_filter=40, nb_row=3, nb_col=3,
+                   input_shape=(None, 40, 40, 1),
+                   border_mode='same', return_sequences=True))
+seq.add(BatchNormalization())
+
+seq.add(ConvLSTM2D(nb_filter=40, nb_row=3, nb_col=3,
+                   border_mode='same', return_sequences=True))
+seq.add(BatchNormalization())
+
+seq.add(ConvLSTM2D(nb_filter=40, nb_row=3, nb_col=3,
+                   border_mode='same', return_sequences=True))
+seq.add(BatchNormalization())
+
+seq.add(ConvLSTM2D(nb_filter=40, nb_row=3, nb_col=3,
+                   border_mode='same', return_sequences=True))
+seq.add(BatchNormalization())
+
+seq.add(Convolution3D(nb_filter=1, kernel_dim1=1, kernel_dim2=3,
+                      kernel_dim3=3, activation='sigmoid',
+                      border_mode='same', dim_ordering='tf'))
+
+seq.compile(loss='binary_crossentropy', optimizer='adadelta')
+
+
+# Artificial data generation:
+# Generate movies with 3 to 7 moving squares inside.
+# The squares are of shape 1x1 or 2x2 pixels,
+# which move linearly over time.
+# For convenience we first create movies with bigger width and height (80x80)
+# and at the end we select a 40x40 window.
+
+def generate_movies(n_samples=1200, n_frames=15):
+    row = 80
+    col = 80
+    noisy_movies = np.zeros((n_samples, n_frames, row, col, 1), dtype=np.float)
+    shifted_movies = np.zeros((n_samples, n_frames, row, col, 1),
+                              dtype=np.float)
+
+    for i in range(n_samples):
+        # Add 3 to 7 moving squares
+        n = np.random.randint(3, 8)
+
+        for j in range(n):
+            # Initial position
+            xstart = np.random.randint(20, 60)
+            ystart = np.random.randint(20, 60)
+            # Direction of motion
+            directionx = np.random.randint(0, 3) - 1
+            directiony = np.random.randint(0, 3) - 1
+
+            # Size of the square
+            w = np.random.randint(2, 4)
+
+            for t in range(n_frames):
+                x_shift = xstart + directionx * t
+                y_shift = ystart + directiony * t
+                noisy_movies[i, t, x_shift - w: x_shift + w,
+                             y_shift - w: y_shift + w, 0] += 1
+
+                # Make it more robust by adding noise.
+                # The idea is that if during inference,
+                # the value of the pixel is not exactly one,
+                # we need to train the network to be robust and still
+                # consider it as a pixel belonging to a square.
+                if np.random.randint(0, 2):
+                    noise_f = (-1)**np.random.randint(0, 2)
+                    noisy_movies[i, t,
+                                 x_shift - w - 1: x_shift + w + 1,
+                                 y_shift - w - 1: y_shift + w + 1,
+                                 0] += noise_f * 0.1
+
+                # Shift the ground truth by 1
+                x_shift = xstart + directionx * (t + 1)
+                y_shift = ystart + directiony * (t + 1)
+                shifted_movies[i, t, x_shift - w: x_shift + w,
+                               y_shift - w: y_shift + w, 0] += 1
+
+    # Cut to a 40x40 window
+    noisy_movies = noisy_movies[::, ::, 20:60, 20:60, ::]
+    shifted_movies = shifted_movies[::, ::, 20:60, 20:60, ::]
+    noisy_movies[noisy_movies >= 1] = 1
+    shifted_movies[shifted_movies >= 1] = 1
+    return noisy_movies, shifted_movies
+
+# Train the network
+noisy_movies, shifted_movies = generate_movies(n_samples=1200)
+seq.fit(noisy_movies[:1000], shifted_movies[:1000], batch_size=10,
+        nb_epoch=300, validation_split=0.05)
+
+# Testing the network on one movie
+# feed it with the first 7 positions and then
+# predict the new positions
+which = 1004
+track = noisy_movies[which][:7, ::, ::, ::]
+
+for j in range(16):
+    new_pos = seq.predict(track[np.newaxis, ::, ::, ::, ::])
+    new = new_pos[::, -1, ::, ::, ::]
+    track = np.concatenate((track, new), axis=0)
+
+
+# And then compare the predictions
+# to the ground truth
+track2 = noisy_movies[which][::, ::, ::, ::]
+for i in range(15):
+    fig = plt.figure(figsize=(10, 5))
+
+    ax = fig.add_subplot(121)
+
+    if i >= 7:
+        ax.text(1, 3, 'Predictions !', fontsize=20, color='w')
+    else:
+        ax.text(1, 3, 'Inital trajectory', fontsize=20)
+
+    toplot = track[i, ::, ::, 0]
+
+    plt.imshow(toplot)
+    ax = fig.add_subplot(122)
+    plt.text(1, 3, 'Ground truth', fontsize=20)
+
+    toplot = track2[i, ::, ::, 0]
+    if i >= 2:
+        toplot = shifted_movies[which][i - 1, ::, ::, 0]
+
+    plt.imshow(toplot)
+    plt.savefig('%i_animate.png' % (i + 1))

examples/deep_dream.py

@@ -9,23 +9,22 @@ e.g.:
 python deep_dream.py img/mypic.jpg results/dream
 ```
 
-It is preferrable to run this script on GPU, for speed.
+It is preferable to run this script on GPU, for speed.
 If running on CPU, prefer the TensorFlow backend (much faster).
 
 Example results: http://i.imgur.com/FX6ROg9.jpg
 '''
 from __future__ import print_function
-from scipy.misc import imread, imresize, imsave
+from keras.preprocessing.image import load_img, img_to_array
 import numpy as np
+from scipy.misc import imsave
 from scipy.optimize import fmin_l_bfgs_b
 import time
 import argparse
-import h5py
-import os
 
-from keras.models import Sequential
-from keras.layers.convolutional import Convolution2D, ZeroPadding2D, MaxPooling2D
+from keras.applications import vgg16
 from keras import backend as K
+from keras.layers import Input
 
 parser = argparse.ArgumentParser(description='Deep Dreams with Keras.')
 parser.add_argument('base_image_path', metavar='base', type=str,
@@ -38,22 +37,22 @@ base_image_path = args.base_image_path
 result_prefix = args.result_prefix
 
 # dimensions of the generated picture.
-img_width = 600
 img_height = 600
+img_width = 600
 
 # path to the model weights file.
 weights_path = 'vgg16_weights.h5'
 
 # some settings we found interesting
 saved_settings = {
-    'bad_trip': {'features': {'conv4_1': 0.05,
-                              'conv4_2': 0.01,
-                              'conv4_3': 0.01},
+    'bad_trip': {'features': {'block4_conv1': 0.05,
+                              'block4_conv2': 0.01,
+                              'block4_conv3': 0.01},
                  'continuity': 0.1,
                  'dream_l2': 0.8,
                  'jitter': 5},
-    'dreamy': {'features': {'conv5_1': 0.05,
-                            'conv5_2': 0.02},
+    'dreamy': {'features': {'block5_conv1': 0.05,
+                            'block5_conv2': 0.02},
                'continuity': 0.1,
                'dream_l2': 0.02,
                'jitter': 0},
@@ -62,84 +61,64 @@ saved_settings = {
 settings = saved_settings['dreamy']
 
 # util function to open, resize and format pictures into appropriate tensors
+
+
 def preprocess_image(image_path):
-    img = imresize(imread(image_path), (img_width, img_height))
-    img = img.transpose((2, 0, 1)).astype('float64')
+    img = load_img(image_path, target_size=(img_height, img_width))
+    img = img_to_array(img)
     img = np.expand_dims(img, axis=0)
+    img = vgg16.preprocess_input(img)
     return img
 
 # util function to convert a tensor into a valid image
+
+
 def deprocess_image(x):
-    x = x.transpose((1, 2, 0))
+    if K.image_dim_ordering() == 'th':
+        x = x.reshape((3, img_height, img_width))
+        x = x.transpose((1, 2, 0))
+    else:
+        x = x.reshape((img_height, img_width, 3))
+    # Remove zero-center by mean pixel
+    x[:, :, 0] += 103.939
+    x[:, :, 1] += 116.779
+    x[:, :, 2] += 123.68
+    # 'BGR'->'RGB'
+    x = x[:, :, ::-1]
     x = np.clip(x, 0, 255).astype('uint8')
     return x
 
-# build the VGG16 network
-model = Sequential()
-model.add(ZeroPadding2D((1, 1), batch_input_shape=(1, 3, img_width, img_height)))
-first_layer = model.layers[-1]
-# this is a placeholder tensor that will contain our generated images
-dream = first_layer.input
+if K.image_dim_ordering() == 'th':
+    img_size = (3, img_height, img_width)
+else:
+    img_size = (img_height, img_width, 3)
+# this will contain our generated image
+dream = Input(batch_shape=(1,) + img_size)
 
-model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_2'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_2'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_2'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_3'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_3'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_2'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_3'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-# load the weights of the VGG16 networks
-# (trained on ImageNet, won the ILSVRC competition in 2014)
-# note: when there is a complete match between your model definition
-# and your weight savefile, you can simply call model.load_weights(filename)
-assert os.path.exists(weights_path), 'Model weights not found (see "weights_path" variable in script).'
-f = h5py.File(weights_path)
-for k in range(f.attrs['nb_layers']):
-    if k >= len(model.layers):
-        # we don't look at the last (fully-connected) layers in the savefile
-        break
-    g = f['layer_{}'.format(k)]
-    weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]
-    model.layers[k].set_weights(weights)
-f.close()
+# build the VGG16 network with our placeholder
+# the model will be loaded with pre-trained ImageNet weights
+model = vgg16.VGG16(input_tensor=dream,
+                    weights='imagenet', include_top=False)
 print('Model loaded.')
 
 # get the symbolic outputs of each "key" layer (we gave them unique names).
 layer_dict = dict([(layer.name, layer) for layer in model.layers])
 
 # continuity loss util function
+
+
 def continuity_loss(x):
     assert K.ndim(x) == 4
-    a = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, 1:, :img_height-1])
-    b = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, :img_width-1, 1:])
+    if K.image_dim_ordering() == 'th':
+        a = K.square(x[:, :, :img_height - 1, :img_width - 1] -
+                     x[:, :, 1:, :img_width - 1])
+        b = K.square(x[:, :, :img_height - 1, :img_width - 1] -
+                     x[:, :, :img_height - 1, 1:])
+    else:
+        a = K.square(x[:, :img_height - 1, :img_width - 1, :] -
+                     x[:, 1:, :img_width - 1, :])
+        b = K.square(x[:, :img_height - 1, :img_width - 1, :] -
+                     x[:, :img_height - 1, 1:, :])
     return K.sum(K.pow(a + b, 1.25))
 
 # define the loss
@@ -151,12 +130,15 @@ for layer_name in settings['features']:
     x = layer_dict[layer_name].output
     shape = layer_dict[layer_name].output_shape
     # we avoid border artifacts by only involving non-border pixels in the loss
-    loss -= coeff * K.sum(K.square(x[:, :, 2: shape[2]-2, 2: shape[3]-2])) / np.prod(shape[1:])
+    if K.image_dim_ordering() == 'th':
+        loss -= coeff * K.sum(K.square(x[:, :, 2: shape[2] - 2, 2: shape[3] - 2])) / np.prod(shape[1:])
+    else:
+        loss -= coeff * K.sum(K.square(x[:, 2: shape[1] - 2, 2: shape[2] - 2, :])) / np.prod(shape[1:])
 
 # add continuity loss (gives image local coherence, can result in an artful blur)
-loss += settings['continuity'] * continuity_loss(dream) / (3 * img_width * img_height)
+loss += settings['continuity'] * continuity_loss(dream) / np.prod(img_size)
 # add image L2 norm to loss (prevents pixels from taking very high values, makes image darker)
-loss += settings['dream_l2'] * K.sum(K.square(dream)) / (3 * img_width * img_height)
+loss += settings['dream_l2'] * K.sum(K.square(dream)) / np.prod(img_size)
 
 # feel free to further modify the loss as you see fit, to achieve new effects...
 
@@ -164,14 +146,16 @@ loss += settings['dream_l2'] * K.sum(K.square(dream)) / (3 * img_width * img_hei
 grads = K.gradients(loss, dream)
 
 outputs = [loss]
-if type(grads) in {list, tuple}:
+if isinstance(grads, (list, tuple)):
     outputs += grads
 else:
     outputs.append(grads)
 
 f_outputs = K.function([dream], outputs)
+
+
 def eval_loss_and_grads(x):
-    x = x.reshape((1, 3, img_width, img_height))
+    x = x.reshape((1,) + img_size)
     outs = f_outputs([x])
     loss_value = outs[0]
     if len(outs[1:]) == 1:
@@ -186,10 +170,13 @@ def eval_loss_and_grads(x):
 # "loss" and "grads". This is done because scipy.optimize
 # requires separate functions for loss and gradients,
 # but computing them separately would be inefficient.
+
+
 class Evaluator(object):
+
     def __init__(self):
         self.loss_value = None
-        self.grads_values = None
+        self.grad_values = None
 
     def loss(self, x):
         assert self.loss_value is None
@@ -215,7 +202,7 @@ for i in range(5):
     start_time = time.time()
 
     # add a random jitter to the initial image. This will be reverted at decoding time
-    random_jitter = (settings['jitter'] * 2) * (np.random.random((3, img_width, img_height)) - 0.5)
+    random_jitter = (settings['jitter'] * 2) * (np.random.random(img_size) - 0.5)
     x += random_jitter
 
     # run L-BFGS for 7 steps
@@ -223,9 +210,9 @@ for i in range(5):
                                      fprime=evaluator.grads, maxfun=7)
     print('Current loss value:', min_val)
     # decode the dream and save it
-    x = x.reshape((3, img_width, img_height))
+    x = x.reshape(img_size)
     x -= random_jitter
-    img = deprocess_image(x)
+    img = deprocess_image(np.copy(x))
     fname = result_prefix + '_at_iteration_%d.png' % i
     imsave(fname, img)
     end_time = time.time()

examples/image_ocr.py

@@ -0,0 +1,488 @@
+'''This example uses a convolutional stack followed by a recurrent stack
+and a CTC logloss function to perform optical character recognition
+of generated text images. I have no evidence of whether it actually
+learns general shapes of text, or just is able to recognize all
+the different fonts thrown at it...the purpose is more to demonstrate CTC
+inside of Keras.  Note that the font list may need to be updated
+for the particular OS in use.
+
+This starts off with 4 letter words.  For the first 12 epochs, the
+difficulty is gradually increased using the TextImageGenerator class
+which is both a generator class for test/train data and a Keras
+callback class. After 20 epochs, longer sequences are thrown at it
+by recompiling the model to handle a wider image and rebuilding
+the word list to include two words separated by a space.
+
+The table below shows normalized edit distance values. Theano uses
+a slightly different CTC implementation, hence the different results.
+
+            Norm. ED
+Epoch |   TF   |   TH
+------------------------
+    10   0.027   0.064
+    15   0.038   0.035
+    20   0.043   0.045
+    25   0.014   0.019
+
+This requires cairo and editdistance packages:
+pip install cairocffi
+pip install editdistance
+
+Created by Mike Henry
+https://github.com/mbhenry/
+'''
+import os
+import itertools
+import re
+import datetime
+import cairocffi as cairo
+import editdistance
+import numpy as np
+from scipy import ndimage
+import pylab
+from keras import backend as K
+from keras.layers.convolutional import Convolution2D, MaxPooling2D
+from keras.layers import Input, Dense, Activation
+from keras.layers import Reshape, Lambda, merge
+from keras.models import Model
+from keras.layers.recurrent import GRU
+from keras.optimizers import SGD
+from keras.utils.data_utils import get_file
+from keras.preprocessing import image
+import keras.callbacks
+
+
+OUTPUT_DIR = 'image_ocr'
+
+np.random.seed(55)
+
+
+# this creates larger "blotches" of noise which look
+# more realistic than just adding gaussian noise
+# assumes greyscale with pixels ranging from 0 to 1
+
+def speckle(img):
+    severity = np.random.uniform(0, 0.6)
+    blur = ndimage.gaussian_filter(np.random.randn(*img.shape) * severity, 1)
+    img_speck = (img + blur)
+    img_speck[img_speck > 1] = 1
+    img_speck[img_speck <= 0] = 0
+    return img_speck
+
+
+# paints the string in a random location the bounding box
+# also uses a random font, a slight random rotation,
+# and a random amount of speckle noise
+
+def paint_text(text, w, h, rotate=False, ud=False, multi_fonts=False):
+    surface = cairo.ImageSurface(cairo.FORMAT_RGB24, w, h)
+    with cairo.Context(surface) as context:
+        context.set_source_rgb(1, 1, 1)  # White
+        context.paint()
+        # this font list works in Centos 7
+        if multi_fonts:
+            fonts = ['Century Schoolbook', 'Courier', 'STIX', 'URW Chancery L', 'FreeMono']
+            context.select_font_face(np.random.choice(fonts), cairo.FONT_SLANT_NORMAL,
+                                     np.random.choice([cairo.FONT_WEIGHT_BOLD, cairo.FONT_WEIGHT_NORMAL]))
+        else:
+            context.select_font_face('Courier', cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_BOLD)
+        context.set_font_size(25)
+        box = context.text_extents(text)
+        border_w_h = (4, 4)
+        if box[2] > (w - 2 * border_w_h[1]) or box[3] > (h - 2 * border_w_h[0]):
+            raise IOError('Could not fit string into image. Max char count is too large for given image width.')
+
+        # teach the RNN translational invariance by
+        # fitting text box randomly on canvas, with some room to rotate
+        max_shift_x = w - box[2] - border_w_h[0]
+        max_shift_y = h - box[3] - border_w_h[1]
+        top_left_x = np.random.randint(0, int(max_shift_x))
+        if ud:
+            top_left_y = np.random.randint(0, int(max_shift_y))
+        else:
+            top_left_y = h // 2
+        context.move_to(top_left_x - int(box[0]), top_left_y - int(box[1]))
+        context.set_source_rgb(0, 0, 0)
+        context.show_text(text)
+
+    buf = surface.get_data()
+    a = np.frombuffer(buf, np.uint8)
+    a.shape = (h, w, 4)
+    a = a[:, :, 0]  # grab single channel
+    a = a.astype(np.float32) / 255
+    a = np.expand_dims(a, 0)
+    if rotate:
+        a = image.random_rotation(a, 3 * (w - top_left_x) / w + 1)
+    a = speckle(a)
+
+    return a
+
+
+def shuffle_mats_or_lists(matrix_list, stop_ind=None):
+    ret = []
+    assert all([len(i) == len(matrix_list[0]) for i in matrix_list])
+    len_val = len(matrix_list[0])
+    if stop_ind is None:
+        stop_ind = len_val
+    assert stop_ind <= len_val
+
+    a = range(stop_ind)
+    np.random.shuffle(a)
+    a += range(stop_ind, len_val)
+    for mat in matrix_list:
+        if isinstance(mat, np.ndarray):
+            ret.append(mat[a])
+        elif isinstance(mat, list):
+            ret.append([mat[i] for i in a])
+        else:
+            raise TypeError('shuffle_mats_or_lists only supports '
+                            'numpy.array and list objects')
+    return ret
+
+
+def text_to_labels(text, num_classes):
+    ret = []
+    for char in text:
+        if char >= 'a' and char <= 'z':
+            ret.append(ord(char) - ord('a'))
+        elif char == ' ':
+            ret.append(26)
+    return ret
+
+
+# only a-z and space..probably not to difficult
+# to expand to uppercase and symbols
+
+def is_valid_str(in_str):
+    search = re.compile(r'[^a-z\ ]').search
+    return not bool(search(in_str))
+
+
+# Uses generator functions to supply train/test with
+# data. Image renderings are text are created on the fly
+# each time with random perturbations
+
+class TextImageGenerator(keras.callbacks.Callback):
+
+    def __init__(self, monogram_file, bigram_file, minibatch_size,
+                 img_w, img_h, downsample_factor, val_split,
+                 absolute_max_string_len=16):
+
+        self.minibatch_size = minibatch_size
+        self.img_w = img_w
+        self.img_h = img_h
+        self.monogram_file = monogram_file
+        self.bigram_file = bigram_file
+        self.downsample_factor = downsample_factor
+        self.val_split = val_split
+        self.blank_label = self.get_output_size() - 1
+        self.absolute_max_string_len = absolute_max_string_len
+
+    def get_output_size(self):
+        return 28
+
+    # num_words can be independent of the epoch size due to the use of generators
+    # as max_string_len grows, num_words can grow
+    def build_word_list(self, num_words, max_string_len=None, mono_fraction=0.5):
+        assert max_string_len <= self.absolute_max_string_len
+        assert num_words % self.minibatch_size == 0
+        assert (self.val_split * num_words) % self.minibatch_size == 0
+        self.num_words = num_words
+        self.string_list = [''] * self.num_words
+        tmp_string_list = []
+        self.max_string_len = max_string_len
+        self.Y_data = np.ones([self.num_words, self.absolute_max_string_len]) * -1
+        self.X_text = []
+        self.Y_len = [0] * self.num_words
+
+        # monogram file is sorted by frequency in english speech
+        with open(self.monogram_file, 'rt') as f:
+            for line in f:
+                if len(tmp_string_list) == int(self.num_words * mono_fraction):
+                    break
+                word = line.rstrip()
+                if max_string_len == -1 or max_string_len is None or len(word) <= max_string_len:
+                    tmp_string_list.append(word)
+
+        # bigram file contains common word pairings in english speech
+        with open(self.bigram_file, 'rt') as f:
+            lines = f.readlines()
+            for line in lines:
+                if len(tmp_string_list) == self.num_words:
+                    break
+                columns = line.lower().split()
+                word = columns[0] + ' ' + columns[1]
+                if is_valid_str(word) and \
+                        (max_string_len == -1 or max_string_len is None or len(word) <= max_string_len):
+                    tmp_string_list.append(word)
+        if len(tmp_string_list) != self.num_words:
+            raise IOError('Could not pull enough words from supplied monogram and bigram files. ')
+        # interlace to mix up the easy and hard words
+        self.string_list[::2] = tmp_string_list[:self.num_words // 2]
+        self.string_list[1::2] = tmp_string_list[self.num_words // 2:]
+
+        for i, word in enumerate(self.string_list):
+            self.Y_len[i] = len(word)
+            self.Y_data[i, 0:len(word)] = text_to_labels(word, self.get_output_size())
+            self.X_text.append(word)
+        self.Y_len = np.expand_dims(np.array(self.Y_len), 1)
+
+        self.cur_val_index = self.val_split
+        self.cur_train_index = 0
+
+    # each time an image is requested from train/val/test, a new random
+    # painting of the text is performed
+    def get_batch(self, index, size, train):
+        # width and height are backwards from typical Keras convention
+        # because width is the time dimension when it gets fed into the RNN
+        if K.image_dim_ordering() == 'th':
+            X_data = np.ones([size, 1, self.img_w, self.img_h])
+        else:
+            X_data = np.ones([size, self.img_w, self.img_h, 1])
+
+        labels = np.ones([size, self.absolute_max_string_len])
+        input_length = np.zeros([size, 1])
+        label_length = np.zeros([size, 1])
+        source_str = []
+        for i in range(0, size):
+            # Mix in some blank inputs.  This seems to be important for
+            # achieving translational invariance
+            if train and i > size - 4:
+                if K.image_dim_ordering() == 'th':
+                    X_data[i, 0, 0:self.img_w, :] = self.paint_func('')[0, :, :].T
+                else:
+                    X_data[i, 0:self.img_w, :, 0] = self.paint_func('',)[0, :, :].T
+                labels[i, 0] = self.blank_label
+                input_length[i] = self.img_w // self.downsample_factor - 2
+                label_length[i] = 1
+                source_str.append('')
+            else:
+                if K.image_dim_ordering() == 'th':
+                    X_data[i, 0, 0:self.img_w, :] = self.paint_func(self.X_text[index + i])[0, :, :].T
+                else:
+                    X_data[i, 0:self.img_w, :, 0] = self.paint_func(self.X_text[index + i])[0, :, :].T
+                labels[i, :] = self.Y_data[index + i]
+                input_length[i] = self.img_w // self.downsample_factor - 2
+                label_length[i] = self.Y_len[index + i]
+                source_str.append(self.X_text[index + i])
+        inputs = {'the_input': X_data,
+                  'the_labels': labels,
+                  'input_length': input_length,
+                  'label_length': label_length,
+                  'source_str': source_str  # used for visualization only
+                  }
+        outputs = {'ctc': np.zeros([size])}  # dummy data for dummy loss function
+        return (inputs, outputs)
+
+    def next_train(self):
+        while 1:
+            ret = self.get_batch(self.cur_train_index, self.minibatch_size, train=True)
+            self.cur_train_index += self.minibatch_size
+            if self.cur_train_index >= self.val_split:
+                self.cur_train_index = self.cur_train_index % 32
+                (self.X_text, self.Y_data, self.Y_len) = shuffle_mats_or_lists(
+                    [self.X_text, self.Y_data, self.Y_len], self.val_split)
+            yield ret
+
+    def next_val(self):
+        while 1:
+            ret = self.get_batch(self.cur_val_index, self.minibatch_size, train=False)
+            self.cur_val_index += self.minibatch_size
+            if self.cur_val_index >= self.num_words:
+                self.cur_val_index = self.val_split + self.cur_val_index % 32
+            yield ret
+
+    def on_train_begin(self, logs={}):
+        self.build_word_list(16000, 4, 1)
+        self.paint_func = lambda text: paint_text(text, self.img_w, self.img_h,
+                                                  rotate=False, ud=False, multi_fonts=False)
+
+    def on_epoch_begin(self, epoch, logs={}):
+        # rebind the paint function to implement curriculum learning
+        if epoch >= 3 and epoch < 6:
+            self.paint_func = lambda text: paint_text(text, self.img_w, self.img_h,
+                                                      rotate=False, ud=True, multi_fonts=False)
+        elif epoch >= 6 and epoch < 9:
+            self.paint_func = lambda text: paint_text(text, self.img_w, self.img_h,
+                                                      rotate=False, ud=True, multi_fonts=True)
+        elif epoch >= 9:
+            self.paint_func = lambda text: paint_text(text, self.img_w, self.img_h,
+                                                      rotate=True, ud=True, multi_fonts=True)
+        if epoch >= 21 and self.max_string_len < 12:
+            self.build_word_list(32000, 12, 0.5)
+
+
+# the actual loss calc occurs here despite it not being
+# an internal Keras loss function
+
+def ctc_lambda_func(args):
+    y_pred, labels, input_length, label_length = args
+    # the 2 is critical here since the first couple outputs of the RNN
+    # tend to be garbage:
+    y_pred = y_pred[:, 2:, :]
+    return K.ctc_batch_cost(labels, y_pred, input_length, label_length)
+
+
+# For a real OCR application, this should be beam search with a dictionary
+# and language model.  For this example, best path is sufficient.
+
+def decode_batch(test_func, word_batch):
+    out = test_func([word_batch])[0]
+    ret = []
+    for j in range(out.shape[0]):
+        out_best = list(np.argmax(out[j, 2:], 1))
+        out_best = [k for k, g in itertools.groupby(out_best)]
+        # 26 is space, 27 is CTC blank char
+        outstr = ''
+        for c in out_best:
+            if c >= 0 and c < 26:
+                outstr += chr(c + ord('a'))
+            elif c == 26:
+                outstr += ' '
+        ret.append(outstr)
+    return ret
+
+
+class VizCallback(keras.callbacks.Callback):
+
+    def __init__(self, run_name, test_func, text_img_gen, num_display_words=6):
+        self.test_func = test_func
+        self.output_dir = os.path.join(
+            OUTPUT_DIR, run_name)
+        self.text_img_gen = text_img_gen
+        self.num_display_words = num_display_words
+        if not os.path.exists(self.output_dir):
+            os.makedirs(self.output_dir)
+
+    def show_edit_distance(self, num):
+        num_left = num
+        mean_norm_ed = 0.0
+        mean_ed = 0.0
+        while num_left > 0:
+            word_batch = next(self.text_img_gen)[0]
+            num_proc = min(word_batch['the_input'].shape[0], num_left)
+            decoded_res = decode_batch(self.test_func, word_batch['the_input'][0:num_proc])
+            for j in range(0, num_proc):
+                edit_dist = editdistance.eval(decoded_res[j], word_batch['source_str'][j])
+                mean_ed += float(edit_dist)
+                mean_norm_ed += float(edit_dist) / len(word_batch['source_str'][j])
+            num_left -= num_proc
+        mean_norm_ed = mean_norm_ed / num
+        mean_ed = mean_ed / num
+        print('\nOut of %d samples:  Mean edit distance: %.3f Mean normalized edit distance: %0.3f'
+              % (num, mean_ed, mean_norm_ed))
+
+    def on_epoch_end(self, epoch, logs={}):
+        self.model.save_weights(os.path.join(self.output_dir, 'weights%02d.h5' % (epoch)))
+        self.show_edit_distance(256)
+        word_batch = next(self.text_img_gen)[0]
+        res = decode_batch(self.test_func, word_batch['the_input'][0:self.num_display_words])
+        if word_batch['the_input'][0].shape[0] < 256:
+            cols = 2
+        else:
+            cols = 1
+        for i in range(self.num_display_words):
+            pylab.subplot(self.num_display_words // cols, cols, i + 1)
+            if K.image_dim_ordering() == 'th':
+                the_input = word_batch['the_input'][i, 0, :, :]
+            else:
+                the_input = word_batch['the_input'][i, :, :, 0]
+            pylab.imshow(the_input.T, cmap='Greys_r')
+            pylab.xlabel('Truth = \'%s\'\nDecoded = \'%s\'' % (word_batch['source_str'][i], res[i]))
+        fig = pylab.gcf()
+        fig.set_size_inches(10, 13)
+        pylab.savefig(os.path.join(self.output_dir, 'e%02d.png' % (epoch)))
+        pylab.close()
+
+
+def train(run_name, start_epoch, stop_epoch, img_w):
+    # Input Parameters
+    img_h = 64
+    words_per_epoch = 16000
+    val_split = 0.2
+    val_words = int(words_per_epoch * (val_split))
+
+    # Network parameters
+    conv_num_filters = 16
+    filter_size = 3
+    pool_size = 2
+    time_dense_size = 32
+    rnn_size = 512
+
+    if K.image_dim_ordering() == 'th':
+        input_shape = (1, img_w, img_h)
+    else:
+        input_shape = (img_w, img_h, 1)
+
+    fdir = os.path.dirname(get_file('wordlists.tgz',
+                                    origin='http://www.isosemi.com/datasets/wordlists.tgz', untar=True))
+
+    img_gen = TextImageGenerator(monogram_file=os.path.join(fdir, 'wordlist_mono_clean.txt'),
+                                 bigram_file=os.path.join(fdir, 'wordlist_bi_clean.txt'),
+                                 minibatch_size=32,
+                                 img_w=img_w,
+                                 img_h=img_h,
+                                 downsample_factor=(pool_size ** 2),
+                                 val_split=words_per_epoch - val_words
+                                 )
+    act = 'relu'
+    input_data = Input(name='the_input', shape=input_shape, dtype='float32')
+    inner = Convolution2D(conv_num_filters, filter_size, filter_size, border_mode='same',
+                          activation=act, init='he_normal', name='conv1')(input_data)
+    inner = MaxPooling2D(pool_size=(pool_size, pool_size), name='max1')(inner)
+    inner = Convolution2D(conv_num_filters, filter_size, filter_size, border_mode='same',
+                          activation=act, init='he_normal', name='conv2')(inner)
+    inner = MaxPooling2D(pool_size=(pool_size, pool_size), name='max2')(inner)
+
+    conv_to_rnn_dims = (img_w // (pool_size ** 2), (img_h // (pool_size ** 2)) * conv_num_filters)
+    inner = Reshape(target_shape=conv_to_rnn_dims, name='reshape')(inner)
+
+    # cuts down input size going into RNN:
+    inner = Dense(time_dense_size, activation=act, name='dense1')(inner)
+
+    # Two layers of bidirecitonal GRUs
+    # GRU seems to work as well, if not better than LSTM:
+    gru_1 = GRU(rnn_size, return_sequences=True, init='he_normal', name='gru1')(inner)
+    gru_1b = GRU(rnn_size, return_sequences=True, go_backwards=True, init='he_normal', name='gru1_b')(inner)
+    gru1_merged = merge([gru_1, gru_1b], mode='sum')
+    gru_2 = GRU(rnn_size, return_sequences=True, init='he_normal', name='gru2')(gru1_merged)
+    gru_2b = GRU(rnn_size, return_sequences=True, go_backwards=True, init='he_normal', name='gru2_b')(gru1_merged)
+
+    # transforms RNN output to character activations:
+    inner = Dense(img_gen.get_output_size(), init='he_normal',
+                  name='dense2')(merge([gru_2, gru_2b], mode='concat'))
+    y_pred = Activation('softmax', name='softmax')(inner)
+    Model(input=[input_data], output=y_pred).summary()
+
+    labels = Input(name='the_labels', shape=[img_gen.absolute_max_string_len], dtype='float32')
+    input_length = Input(name='input_length', shape=[1], dtype='int64')
+    label_length = Input(name='label_length', shape=[1], dtype='int64')
+    # Keras doesn't currently support loss funcs with extra parameters
+    # so CTC loss is implemented in a lambda layer
+    loss_out = Lambda(ctc_lambda_func, output_shape=(1,), name='ctc')([y_pred, labels, input_length, label_length])
+
+    # clipnorm seems to speeds up convergence
+    sgd = SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5)
+
+    model = Model(input=[input_data, labels, input_length, label_length], output=[loss_out])
+
+    # the loss calc occurs elsewhere, so use a dummy lambda func for the loss
+    model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd)
+    if start_epoch > 0:
+        weight_file = os.path.join(OUTPUT_DIR, os.path.join(run_name, 'weights%02d.h5' % (start_epoch - 1)))
+        model.load_weights(weight_file)
+    # captures output of softmax so we can decode the output during visualization
+    test_func = K.function([input_data], [y_pred])
+
+    viz_cb = VizCallback(run_name, test_func, img_gen.next_val())
+
+    model.fit_generator(generator=img_gen.next_train(), samples_per_epoch=(words_per_epoch - val_words),
+                        nb_epoch=stop_epoch, validation_data=img_gen.next_val(), nb_val_samples=val_words,
+                        callbacks=[viz_cb, img_gen], initial_epoch=start_epoch)
+
+
+if __name__ == '__main__':
+    run_name = datetime.datetime.now().strftime('%Y:%m:%d:%H:%M:%S')
+    train(run_name, 0, 20, 128)
+    # increase to wider images and start at epoch 20. The learned weights are reloaded
+    train(run_name, 20, 25, 512)

examples/imdb_bidirectional_lstm.py

@@ -9,8 +9,8 @@ import numpy as np
 np.random.seed(1337)  # for reproducibility
 
 from keras.preprocessing import sequence
-from keras.models import Model
-from keras.layers import Dense, Dropout, Embedding, LSTM, Input, merge
+from keras.models import Sequential
+from keras.layers import Dense, Dropout, Embedding, LSTM, Bidirectional
 from keras.datasets import imdb
 
 
@@ -19,8 +19,7 @@ maxlen = 100  # cut texts after this number of words (among top max_features mos
 batch_size = 32
 
 print('Loading data...')
-(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features,
-                                                      test_split=0.2)
+(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features)
 print(len(X_train), 'train sequences')
 print(len(X_test), 'test sequences')
 
@@ -32,24 +31,11 @@ print('X_test shape:', X_test.shape)
 y_train = np.array(y_train)
 y_test = np.array(y_test)
 
-
-# this is the placeholder tensor for the input sequences
-sequence = Input(shape=(maxlen,), dtype='int32')
-# this embedding layer will transform the sequences of integers
-# into vectors of size 128
-embedded = Embedding(max_features, 128, input_length=maxlen)(sequence)
-
-# apply forwards LSTM
-forwards = LSTM(64)(embedded)
-# apply backwards LSTM
-backwards = LSTM(64, go_backwards=True)(embedded)
-
-# concatenate the outputs of the 2 LSTMs
-merged = merge([forwards, backwards], mode='concat', concat_axis=-1)
-after_dp = Dropout(0.5)(merged)
-output = Dense(1, activation='sigmoid')(after_dp)
-
-model = Model(input=sequence, output=output)
+model = Sequential()
+model.add(Embedding(max_features, 128, input_length=maxlen))
+model.add(Bidirectional(LSTM(64)))
+model.add(Dropout(0.5))
+model.add(Dense(1, activation='sigmoid'))
 
 # try using different optimizers and different optimizer configs
 model.compile('adam', 'binary_crossentropy', metrics=['accuracy'])

examples/imdb_cnn.py

@@ -1,6 +1,9 @@
 '''This example demonstrates the use of Convolution1D for text classification.
 
-Gets to 0.835 test accuracy after 2 epochs. 100s/epoch on K520 GPU.
+Gets to 0.89 test accuracy after 2 epochs.
+90s/epoch on Intel i5 2.4Ghz CPU.
+10s/epoch on Tesla K40 GPU.
+
 '''
 
 from __future__ import print_function
@@ -9,25 +12,24 @@ np.random.seed(1337)  # for reproducibility
 
 from keras.preprocessing import sequence
 from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation, Flatten
-from keras.layers.embeddings import Embedding
-from keras.layers.convolutional import Convolution1D, MaxPooling1D
+from keras.layers import Dense, Dropout, Activation
+from keras.layers import Embedding
+from keras.layers import Convolution1D, GlobalMaxPooling1D
 from keras.datasets import imdb
 
 
 # set parameters:
 max_features = 5000
-maxlen = 100
+maxlen = 400
 batch_size = 32
-embedding_dims = 100
+embedding_dims = 50
 nb_filter = 250
 filter_length = 3
 hidden_dims = 250
 nb_epoch = 2
 
 print('Loading data...')
-(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features,
-                                                      test_split=0.2)
+(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features)
 print(len(X_train), 'train sequences')
 print(len(X_test), 'test sequences')
 
@@ -42,8 +44,10 @@ model = Sequential()
 
 # we start off with an efficient embedding layer which maps
 # our vocab indices into embedding_dims dimensions
-model.add(Embedding(max_features, embedding_dims, input_length=maxlen))
-model.add(Dropout(0.25))
+model.add(Embedding(max_features,
+                    embedding_dims,
+                    input_length=maxlen,
+                    dropout=0.2))
 
 # we add a Convolution1D, which will learn nb_filter
 # word group filters of size filter_length:
@@ -52,16 +56,12 @@ model.add(Convolution1D(nb_filter=nb_filter,
                         border_mode='valid',
                         activation='relu',
                         subsample_length=1))
-# we use standard max pooling (halving the output of the previous layer):
-model.add(MaxPooling1D(pool_length=2))
-
-# We flatten the output of the conv layer,
-# so that we can add a vanilla dense layer:
-model.add(Flatten())
+# we use max pooling:
+model.add(GlobalMaxPooling1D())
 
 # We add a vanilla hidden layer:
 model.add(Dense(hidden_dims))
-model.add(Dropout(0.25))
+model.add(Dropout(0.2))
 model.add(Activation('relu'))
 
 # We project onto a single unit output layer, and squash it with a sigmoid:
@@ -69,7 +69,7 @@ model.add(Dense(1))
 model.add(Activation('sigmoid'))
 
 model.compile(loss='binary_crossentropy',
-              optimizer='rmsprop',
+              optimizer='adam',
               metrics=['accuracy'])
 model.fit(X_train, y_train,
           batch_size=batch_size,

examples/imdb_cnn_lstm.py

@@ -9,10 +9,10 @@ np.random.seed(1337)  # for reproducibility
 
 from keras.preprocessing import sequence
 from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation
-from keras.layers.embeddings import Embedding
-from keras.layers.recurrent import LSTM, GRU, SimpleRNN
-from keras.layers.convolutional import Convolution1D, MaxPooling1D
+from keras.layers import Dense, Dropout, Activation
+from keras.layers import Embedding
+from keras.layers import LSTM
+from keras.layers import Convolution1D, MaxPooling1D
 from keras.datasets import imdb
 
 
@@ -22,9 +22,9 @@ maxlen = 100
 embedding_size = 128
 
 # Convolution
-filter_length = 3
+filter_length = 5
 nb_filter = 64
-pool_length = 2
+pool_length = 4
 
 # LSTM
 lstm_output_size = 70
@@ -40,7 +40,7 @@ Only 2 epochs are needed as the dataset is very small.
 '''
 
 print('Loading data...')
-(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features, test_split=0.2)
+(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features)
 print(len(X_train), 'train sequences')
 print(len(X_test), 'test sequences')
 

examples/imdb_fasttext.py

@@ -0,0 +1,136 @@
+'''This example demonstrates the use of fasttext for text classification
+
+Based on Joulin et al's paper:
+
+Bags of Tricks for Efficient Text Classification
+https://arxiv.org/abs/1607.01759
+
+Results on IMDB datasets with uni and bi-gram embeddings:
+    Uni-gram: 0.8813 test accuracy after 5 epochs. 8s/epoch on i7 cpu.
+    Bi-gram : 0.9056 test accuracy after 5 epochs. 2s/epoch on GTX 980M gpu.
+'''
+
+from __future__ import print_function
+import numpy as np
+np.random.seed(1337)  # for reproducibility
+
+from keras.preprocessing import sequence
+from keras.models import Sequential
+from keras.layers import Dense
+from keras.layers import Embedding
+from keras.layers import GlobalAveragePooling1D
+from keras.datasets import imdb
+
+
+def create_ngram_set(input_list, ngram_value=2):
+    """
+    Extract a set of n-grams from a list of integers.
+
+    >>> create_ngram_set([1, 4, 9, 4, 1, 4], ngram_value=2)
+    {(4, 9), (4, 1), (1, 4), (9, 4)}
+
+    >>> create_ngram_set([1, 4, 9, 4, 1, 4], ngram_value=3)
+    [(1, 4, 9), (4, 9, 4), (9, 4, 1), (4, 1, 4)]
+    """
+    return set(zip(*[input_list[i:] for i in range(ngram_value)]))
+
+
+def add_ngram(sequences, token_indice, ngram_range=2):
+    """
+    Augment the input list of list (sequences) by appending n-grams values.
+
+    Example: adding bi-gram
+    >>> sequences = [[1, 3, 4, 5], [1, 3, 7, 9, 2]]
+    >>> token_indice = {(1, 3): 1337, (9, 2): 42, (4, 5): 2017}
+    >>> add_ngram(sequences, token_indice, ngram_range=2)
+    [[1, 3, 4, 5, 1337, 2017], [1, 3, 7, 9, 2, 1337, 42]]
+
+    Example: adding tri-gram
+    >>> sequences = [[1, 3, 4, 5], [1, 3, 7, 9, 2]]
+    >>> token_indice = {(1, 3): 1337, (9, 2): 42, (4, 5): 2017, (7, 9, 2): 2018}
+    >>> add_ngram(sequences, token_indice, ngram_range=3)
+    [[1, 3, 4, 5, 1337], [1, 3, 7, 9, 2, 1337, 2018]]
+    """
+    new_sequences = []
+    for input_list in sequences:
+        new_list = input_list[:]
+        for i in range(len(new_list) - ngram_range + 1):
+            for ngram_value in range(2, ngram_range + 1):
+                ngram = tuple(new_list[i:i + ngram_value])
+                if ngram in token_indice:
+                    new_list.append(token_indice[ngram])
+        new_sequences.append(new_list)
+
+    return new_sequences
+
+# Set parameters:
+# ngram_range = 2 will add bi-grams features
+ngram_range = 1
+max_features = 20000
+maxlen = 400
+batch_size = 32
+embedding_dims = 50
+nb_epoch = 5
+
+print('Loading data...')
+(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features)
+print(len(X_train), 'train sequences')
+print(len(X_test), 'test sequences')
+print('Average train sequence length: {}'.format(np.mean(list(map(len, X_train)), dtype=int)))
+print('Average test sequence length: {}'.format(np.mean(list(map(len, X_test)), dtype=int)))
+
+if ngram_range > 1:
+    print('Adding {}-gram features'.format(ngram_range))
+    # Create set of unique n-gram from the training set.
+    ngram_set = set()
+    for input_list in X_train:
+        for i in range(2, ngram_range + 1):
+            set_of_ngram = create_ngram_set(input_list, ngram_value=i)
+            ngram_set.update(set_of_ngram)
+
+    # Dictionary mapping n-gram token to a unique integer.
+    # Integer values are greater than max_features in order
+    # to avoid collision with existing features.
+    start_index = max_features + 1
+    token_indice = {v: k + start_index for k, v in enumerate(ngram_set)}
+    indice_token = {token_indice[k]: k for k in token_indice}
+
+    # max_features is the highest integer that could be found in the dataset.
+    max_features = np.max(list(indice_token.keys())) + 1
+
+    # Augmenting X_train and X_test with n-grams features
+    X_train = add_ngram(X_train, token_indice, ngram_range)
+    X_test = add_ngram(X_test, token_indice, ngram_range)
+    print('Average train sequence length: {}'.format(np.mean(list(map(len, X_train)), dtype=int)))
+    print('Average test sequence length: {}'.format(np.mean(list(map(len, X_test)), dtype=int)))
+
+print('Pad sequences (samples x time)')
+X_train = sequence.pad_sequences(X_train, maxlen=maxlen)
+X_test = sequence.pad_sequences(X_test, maxlen=maxlen)
+print('X_train shape:', X_train.shape)
+print('X_test shape:', X_test.shape)
+
+print('Build model...')
+model = Sequential()
+
+# we start off with an efficient embedding layer which maps
+# our vocab indices into embedding_dims dimensions
+model.add(Embedding(max_features,
+                    embedding_dims,
+                    input_length=maxlen))
+
+# we add a GlobalAveragePooling1D, which will average the embeddings
+# of all words in the document
+model.add(GlobalAveragePooling1D())
+
+# We project onto a single unit output layer, and squash it with a sigmoid:
+model.add(Dense(1, activation='sigmoid'))
+
+model.compile(loss='binary_crossentropy',
+              optimizer='adam',
+              metrics=['accuracy'])
+
+model.fit(X_train, y_train,
+          batch_size=batch_size,
+          nb_epoch=nb_epoch,
+          validation_data=(X_test, y_test))

examples/imdb_lstm.py

@@ -1,8 +1,6 @@
 '''Trains a LSTM on the IMDB sentiment classification task.
-
 The dataset is actually too small for LSTM to be of any advantage
-compared to simpler, much faster methods such as TF-IDF+LogReg.
-
+compared to simpler, much faster methods such as TF-IDF + LogReg.
 Notes:
 
 - RNNs are tricky. Choice of batch size is important,
@@ -17,11 +15,9 @@ import numpy as np
 np.random.seed(1337)  # for reproducibility
 
 from keras.preprocessing import sequence
-from keras.utils import np_utils
 from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation
-from keras.layers.embeddings import Embedding
-from keras.layers.recurrent import LSTM, SimpleRNN, GRU
+from keras.layers import Dense, Activation, Embedding
+from keras.layers import LSTM
 from keras.datasets import imdb
 
 max_features = 20000
@@ -29,8 +25,7 @@ maxlen = 80  # cut texts after this number of words (among top max_features most
 batch_size = 32
 
 print('Loading data...')
-(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features,
-                                                      test_split=0.2)
+(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features)
 print(len(X_train), 'train sequences')
 print(len(X_test), 'test sequences')
 
@@ -42,7 +37,7 @@ print('X_test shape:', X_test.shape)
 
 print('Build model...')
 model = Sequential()
-model.add(Embedding(max_features, 128, input_length=maxlen, dropout=0.2))
+model.add(Embedding(max_features, 128, dropout=0.2))
 model.add(LSTM(128, dropout_W=0.2, dropout_U=0.2))  # try using a GRU instead, for fun
 model.add(Dense(1))
 model.add(Activation('sigmoid'))
@@ -53,8 +48,6 @@ model.compile(loss='binary_crossentropy',
               metrics=['accuracy'])
 
 print('Train...')
-print(X_train.shape)
-print(y_train.shape)
 model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=15,
           validation_data=(X_test, y_test))
 score, acc = model.evaluate(X_test, y_test,

examples/lstm_benchmark.py

@@ -0,0 +1,83 @@
+'''Compare LSTM implementations on the IMDB sentiment classification task.
+
+consume_less='cpu' preprocesses input to the LSTM which typically results in
+faster computations at the expense of increased peak memory usage as the
+preprocessed input must be kept in memory.
+
+consume_less='mem' does away with the preprocessing, meaning that it might take
+a little longer, but should require less peak memory.
+
+consume_less='gpu' concatenates the input, output and forget gate's weights
+into one, large matrix, resulting in faster computation time as the GPU can
+utilize more cores, at the expense of reduced regularization because the same
+dropout is shared across the gates.
+
+Note that the relative performance of the different `consume_less` modes
+can vary depending on your device, your model and the size of your data.
+'''
+
+import time
+import numpy as np
+import matplotlib.pyplot as plt
+
+from keras.preprocessing import sequence
+from keras.models import Sequential
+from keras.layers import Embedding, Dense, LSTM
+from keras.datasets import imdb
+
+max_features = 20000
+max_length = 80
+embedding_dim = 256
+batch_size = 128
+epochs = 10
+modes = ['cpu', 'mem', 'gpu']
+
+print('Loading data...')
+(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features)
+X_train = sequence.pad_sequences(X_train, max_length)
+X_test = sequence.pad_sequences(X_test, max_length)
+
+# Compile and train different models while meauring performance.
+results = []
+for mode in modes:
+    print('Testing mode: consume_less="{}"'.format(mode))
+
+    model = Sequential()
+    model.add(Embedding(max_features, embedding_dim, input_length=max_length, dropout=0.2))
+    model.add(LSTM(embedding_dim, dropout_W=0.2, dropout_U=0.2, consume_less=mode))
+    model.add(Dense(1, activation='sigmoid'))
+    model.compile(loss='binary_crossentropy',
+                  optimizer='adam',
+                  metrics=['accuracy'])
+
+    start_time = time.time()
+    history = model.fit(X_train, y_train,
+                        batch_size=batch_size,
+                        nb_epoch=epochs,
+                        validation_data=(X_test, y_test))
+    average_time_per_epoch = (time.time() - start_time) / epochs
+
+    results.append((history, average_time_per_epoch))
+
+# Compare models' accuracy, loss and elapsed time per epoch.
+plt.style.use('ggplot')
+ax1 = plt.subplot2grid((2, 2), (0, 0))
+ax1.set_title('Accuracy')
+ax1.set_ylabel('Validation Accuracy')
+ax1.set_xlabel('Epochs')
+ax2 = plt.subplot2grid((2, 2), (1, 0))
+ax2.set_title('Loss')
+ax2.set_ylabel('Validation Loss')
+ax2.set_xlabel('Epochs')
+ax3 = plt.subplot2grid((2, 2), (0, 1), rowspan=2)
+ax3.set_title('Time')
+ax3.set_ylabel('Seconds')
+for mode, result in zip(modes, results):
+    ax1.plot(result[0].epoch, result[0].history['val_acc'], label=mode)
+    ax2.plot(result[0].epoch, result[0].history['val_loss'], label=mode)
+ax1.legend()
+ax2.legend()
+ax3.bar(np.arange(len(results)), [x[1] for x in results],
+        tick_label=modes, align='center')
+plt.tight_layout()
+plt.show()

examples/lstm_text_generation.py

@@ -12,8 +12,9 @@ has at least ~100k characters. ~1M is better.
 
 from __future__ import print_function
 from keras.models import Sequential
-from keras.layers.core import Dense, Activation, Dropout
-from keras.layers.recurrent import LSTM
+from keras.layers import Dense, Activation
+from keras.layers import LSTM
+from keras.optimizers import RMSprop
 from keras.utils.data_utils import get_file
 import numpy as np
 import random
@@ -23,7 +24,7 @@ path = get_file('nietzsche.txt', origin="https://s3.amazonaws.com/text-datasets/
 text = open(path).read().lower()
 print('corpus length:', len(text))
 
-chars = set(text)
+chars = sorted(list(set(text)))
 print('total chars:', len(chars))
 char_indices = dict((c, i) for i, c in enumerate(chars))
 indices_char = dict((i, c) for i, c in enumerate(chars))
@@ -47,24 +48,25 @@ for i, sentence in enumerate(sentences):
     y[i, char_indices[next_chars[i]]] = 1
 
 
-# build the model: 2 stacked LSTM
+# build the model: a single LSTM
 print('Build model...')
 model = Sequential()
-model.add(LSTM(512, return_sequences=True, input_shape=(maxlen, len(chars))))
-model.add(Dropout(0.2))
-model.add(LSTM(512, return_sequences=False))
-model.add(Dropout(0.2))
+model.add(LSTM(128, input_shape=(maxlen, len(chars))))
 model.add(Dense(len(chars)))
 model.add(Activation('softmax'))
 
-model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
+optimizer = RMSprop(lr=0.01)
+model.compile(loss='categorical_crossentropy', optimizer=optimizer)
 
 
-def sample(a, temperature=1.0):
+def sample(preds, temperature=1.0):
     # helper function to sample an index from a probability array
-    a = np.log(a) / temperature
-    a = np.exp(a) / np.sum(np.exp(a))
-    return np.argmax(np.random.multinomial(1, a, 1))
+    preds = np.asarray(preds).astype('float64')
+    preds = np.log(preds) / temperature
+    exp_preds = np.exp(preds)
+    preds = exp_preds / np.sum(exp_preds)
+    probas = np.random.multinomial(1, preds, 1)
+    return np.argmax(probas)
 
 # train the model, output generated text after each iteration
 for iteration in range(1, 60):

examples/mnist_acgan.py

@@ -0,0 +1,310 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Train an Auxiliary Classifier Generative Adversarial Network (ACGAN) on the
+MNIST dataset. See https://arxiv.org/abs/1610.09585 for more details.
+
+You should start to see reasonable images after ~5 epochs, and good images
+by ~15 epochs. You should use a GPU, as the convolution-heavy operations are
+very slow on the CPU. Prefer the TensorFlow backend if you plan on iterating, as
+the compilation time can be a blocker using Theano.
+
+Timings:
+
+Hardware           | Backend | Time / Epoch
+-------------------------------------------
+ CPU               | TF      | 3 hrs
+ Titan X (maxwell) | TF      | 4 min
+ Titan X (maxwell) | TH      | 7 min
+
+Consult https://github.com/lukedeo/keras-acgan for more information and
+example output
+"""
+from __future__ import print_function
+
+from collections import defaultdict
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+from PIL import Image
+
+from six.moves import range
+
+import keras.backend as K
+from keras.datasets import mnist
+from keras.layers import Input, Dense, Reshape, Flatten, Embedding, merge, Dropout
+from keras.layers.advanced_activations import LeakyReLU
+from keras.layers.convolutional import UpSampling2D, Convolution2D
+from keras.models import Sequential, Model
+from keras.optimizers import Adam
+from keras.utils.generic_utils import Progbar
+import numpy as np
+
+np.random.seed(1337)
+
+K.set_image_dim_ordering('th')
+
+
+def build_generator(latent_size):
+    # we will map a pair of (z, L), where z is a latent vector and L is a
+    # label drawn from P_c, to image space (..., 1, 28, 28)
+    cnn = Sequential()
+
+    cnn.add(Dense(1024, input_dim=latent_size, activation='relu'))
+    cnn.add(Dense(128 * 7 * 7, activation='relu'))
+    cnn.add(Reshape((128, 7, 7)))
+
+    # upsample to (..., 14, 14)
+    cnn.add(UpSampling2D(size=(2, 2)))
+    cnn.add(Convolution2D(256, 5, 5, border_mode='same',
+                          activation='relu', init='glorot_normal'))
+
+    # upsample to (..., 28, 28)
+    cnn.add(UpSampling2D(size=(2, 2)))
+    cnn.add(Convolution2D(128, 5, 5, border_mode='same',
+                          activation='relu', init='glorot_normal'))
+
+    # take a channel axis reduction
+    cnn.add(Convolution2D(1, 2, 2, border_mode='same',
+                          activation='tanh', init='glorot_normal'))
+
+    # this is the z space commonly refered to in GAN papers
+    latent = Input(shape=(latent_size, ))
+
+    # this will be our label
+    image_class = Input(shape=(1,), dtype='int32')
+
+    # 10 classes in MNIST
+    cls = Flatten()(Embedding(10, latent_size,
+                              init='glorot_normal')(image_class))
+
+    # hadamard product between z-space and a class conditional embedding
+    h = merge([latent, cls], mode='mul')
+
+    fake_image = cnn(h)
+
+    return Model(input=[latent, image_class], output=fake_image)
+
+
+def build_discriminator():
+    # build a relatively standard conv net, with LeakyReLUs as suggested in
+    # the reference paper
+    cnn = Sequential()
+
+    cnn.add(Convolution2D(32, 3, 3, border_mode='same', subsample=(2, 2),
+                          input_shape=(1, 28, 28)))
+    cnn.add(LeakyReLU())
+    cnn.add(Dropout(0.3))
+
+    cnn.add(Convolution2D(64, 3, 3, border_mode='same', subsample=(1, 1)))
+    cnn.add(LeakyReLU())
+    cnn.add(Dropout(0.3))
+
+    cnn.add(Convolution2D(128, 3, 3, border_mode='same', subsample=(2, 2)))
+    cnn.add(LeakyReLU())
+    cnn.add(Dropout(0.3))
+
+    cnn.add(Convolution2D(256, 3, 3, border_mode='same', subsample=(1, 1)))
+    cnn.add(LeakyReLU())
+    cnn.add(Dropout(0.3))
+
+    cnn.add(Flatten())
+
+    image = Input(shape=(1, 28, 28))
+
+    features = cnn(image)
+
+    # first output (name=generation) is whether or not the discriminator
+    # thinks the image that is being shown is fake, and the second output
+    # (name=auxiliary) is the class that the discriminator thinks the image
+    # belongs to.
+    fake = Dense(1, activation='sigmoid', name='generation')(features)
+    aux = Dense(10, activation='softmax', name='auxiliary')(features)
+
+    return Model(input=image, output=[fake, aux])
+
+if __name__ == '__main__':
+
+    # batch and latent size taken from the paper
+    nb_epochs = 50
+    batch_size = 100
+    latent_size = 100
+
+    # Adam parameters suggested in https://arxiv.org/abs/1511.06434
+    adam_lr = 0.0002
+    adam_beta_1 = 0.5
+
+    # build the discriminator
+    discriminator = build_discriminator()
+    discriminator.compile(
+        optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1),
+        loss=['binary_crossentropy', 'sparse_categorical_crossentropy']
+    )
+
+    # build the generator
+    generator = build_generator(latent_size)
+    generator.compile(optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1),
+                      loss='binary_crossentropy')
+
+    latent = Input(shape=(latent_size, ))
+    image_class = Input(shape=(1,), dtype='int32')
+
+    # get a fake image
+    fake = generator([latent, image_class])
+
+    # we only want to be able to train generation for the combined model
+    discriminator.trainable = False
+    fake, aux = discriminator(fake)
+    combined = Model(input=[latent, image_class], output=[fake, aux])
+
+    combined.compile(
+        optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1),
+        loss=['binary_crossentropy', 'sparse_categorical_crossentropy']
+    )
+
+    # get our mnist data, and force it to be of shape (..., 1, 28, 28) with
+    # range [-1, 1]
+    (X_train, y_train), (X_test, y_test) = mnist.load_data()
+    X_train = (X_train.astype(np.float32) - 127.5) / 127.5
+    X_train = np.expand_dims(X_train, axis=1)
+
+    X_test = (X_test.astype(np.float32) - 127.5) / 127.5
+    X_test = np.expand_dims(X_test, axis=1)
+
+    nb_train, nb_test = X_train.shape[0], X_test.shape[0]
+
+    train_history = defaultdict(list)
+    test_history = defaultdict(list)
+
+    for epoch in range(nb_epochs):
+        print('Epoch {} of {}'.format(epoch + 1, nb_epochs))
+
+        nb_batches = int(X_train.shape[0] / batch_size)
+        progress_bar = Progbar(target=nb_batches)
+
+        epoch_gen_loss = []
+        epoch_disc_loss = []
+
+        for index in range(nb_batches):
+            progress_bar.update(index)
+            # generate a new batch of noise
+            noise = np.random.uniform(-1, 1, (batch_size, latent_size))
+
+            # get a batch of real images
+            image_batch = X_train[index * batch_size:(index + 1) * batch_size]
+            label_batch = y_train[index * batch_size:(index + 1) * batch_size]
+
+            # sample some labels from p_c
+            sampled_labels = np.random.randint(0, 10, batch_size)
+
+            # generate a batch of fake images, using the generated labels as a
+            # conditioner. We reshape the sampled labels to be
+            # (batch_size, 1) so that we can feed them into the embedding
+            # layer as a length one sequence
+            generated_images = generator.predict(
+                [noise, sampled_labels.reshape((-1, 1))], verbose=0)
+
+            X = np.concatenate((image_batch, generated_images))
+            y = np.array([1] * batch_size + [0] * batch_size)
+            aux_y = np.concatenate((label_batch, sampled_labels), axis=0)
+
+            # see if the discriminator can figure itself out...
+            epoch_disc_loss.append(discriminator.train_on_batch(X, [y, aux_y]))
+
+            # make new noise. we generate 2 * batch size here such that we have
+            # the generator optimize over an identical number of images as the
+            # discriminator
+            noise = np.random.uniform(-1, 1, (2 * batch_size, latent_size))
+            sampled_labels = np.random.randint(0, 10, 2 * batch_size)
+
+            # we want to train the genrator to trick the discriminator
+            # For the generator, we want all the {fake, not-fake} labels to say
+            # not-fake
+            trick = np.ones(2 * batch_size)
+
+            epoch_gen_loss.append(combined.train_on_batch(
+                [noise, sampled_labels.reshape((-1, 1))], [trick, sampled_labels]))
+
+        print('\nTesting for epoch {}:'.format(epoch + 1))
+
+        # evaluate the testing loss here
+
+        # generate a new batch of noise
+        noise = np.random.uniform(-1, 1, (nb_test, latent_size))
+
+        # sample some labels from p_c and generate images from them
+        sampled_labels = np.random.randint(0, 10, nb_test)
+        generated_images = generator.predict(
+            [noise, sampled_labels.reshape((-1, 1))], verbose=False)
+
+        X = np.concatenate((X_test, generated_images))
+        y = np.array([1] * nb_test + [0] * nb_test)
+        aux_y = np.concatenate((y_test, sampled_labels), axis=0)
+
+        # see if the discriminator can figure itself out...
+        discriminator_test_loss = discriminator.evaluate(
+            X, [y, aux_y], verbose=False)
+
+        discriminator_train_loss = np.mean(np.array(epoch_disc_loss), axis=0)
+
+        # make new noise
+        noise = np.random.uniform(-1, 1, (2 * nb_test, latent_size))
+        sampled_labels = np.random.randint(0, 10, 2 * nb_test)
+
+        trick = np.ones(2 * nb_test)
+
+        generator_test_loss = combined.evaluate(
+            [noise, sampled_labels.reshape((-1, 1))],
+            [trick, sampled_labels], verbose=False)
+
+        generator_train_loss = np.mean(np.array(epoch_gen_loss), axis=0)
+
+        # generate an epoch report on performance
+        train_history['generator'].append(generator_train_loss)
+        train_history['discriminator'].append(discriminator_train_loss)
+
+        test_history['generator'].append(generator_test_loss)
+        test_history['discriminator'].append(discriminator_test_loss)
+
+        print('{0:<22s} | {1:4s} | {2:15s} | {3:5s}'.format(
+            'component', *discriminator.metrics_names))
+        print('-' * 65)
+
+        ROW_FMT = '{0:<22s} | {1:<4.2f} | {2:<15.2f} | {3:<5.2f}'
+        print(ROW_FMT.format('generator (train)',
+                             *train_history['generator'][-1]))
+        print(ROW_FMT.format('generator (test)',
+                             *test_history['generator'][-1]))
+        print(ROW_FMT.format('discriminator (train)',
+                             *train_history['discriminator'][-1]))
+        print(ROW_FMT.format('discriminator (test)',
+                             *test_history['discriminator'][-1]))
+
+        # save weights every epoch
+        generator.save_weights(
+            'params_generator_epoch_{0:03d}.hdf5'.format(epoch), True)
+        discriminator.save_weights(
+            'params_discriminator_epoch_{0:03d}.hdf5'.format(epoch), True)
+
+        # generate some digits to display
+        noise = np.random.uniform(-1, 1, (100, latent_size))
+
+        sampled_labels = np.array([
+            [i] * 10 for i in range(10)
+        ]).reshape(-1, 1)
+
+        # get a batch to display
+        generated_images = generator.predict(
+            [noise, sampled_labels], verbose=0)
+
+        # arrange them into a grid
+        img = (np.concatenate([r.reshape(-1, 28)
+                               for r in np.split(generated_images, 10)
+                               ], axis=-1) * 127.5 + 127.5).astype(np.uint8)
+
+        Image.fromarray(img).save(
+            'plot_epoch_{0:03d}_generated.png'.format(epoch))
+
+    pickle.dump({'train': train_history, 'test': test_history},
+                open('acgan-history.pkl', 'wb'))

examples/mnist_cnn.py

@@ -11,9 +11,10 @@ np.random.seed(1337)  # for reproducibility
 
 from keras.datasets import mnist
 from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation, Flatten
-from keras.layers.convolutional import Convolution2D, MaxPooling2D
+from keras.layers import Dense, Dropout, Activation, Flatten
+from keras.layers import Convolution2D, MaxPooling2D
 from keras.utils import np_utils
+from keras import backend as K
 
 batch_size = 128
 nb_classes = 10
@@ -24,15 +25,22 @@ img_rows, img_cols = 28, 28
 # number of convolutional filters to use
 nb_filters = 32
 # size of pooling area for max pooling
-nb_pool = 2
+pool_size = (2, 2)
 # convolution kernel size
-nb_conv = 3
+kernel_size = (3, 3)
 
 # the data, shuffled and split between train and test sets
 (X_train, y_train), (X_test, y_test) = mnist.load_data()
 
-X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)
-X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)
+if K.image_dim_ordering() == 'th':
+    X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)
+    X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)
+    input_shape = (1, img_rows, img_cols)
+else:
+    X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1)
+    X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)
+    input_shape = (img_rows, img_cols, 1)
+
 X_train = X_train.astype('float32')
 X_test = X_test.astype('float32')
 X_train /= 255
@@ -47,13 +55,13 @@ Y_test = np_utils.to_categorical(y_test, nb_classes)
 
 model = Sequential()
 
-model.add(Convolution2D(nb_filters, nb_conv, nb_conv,
+model.add(Convolution2D(nb_filters, kernel_size[0], kernel_size[1],
                         border_mode='valid',
-                        input_shape=(1, img_rows, img_cols)))
+                        input_shape=input_shape))
 model.add(Activation('relu'))
-model.add(Convolution2D(nb_filters, nb_conv, nb_conv))
+model.add(Convolution2D(nb_filters, kernel_size[0], kernel_size[1]))
 model.add(Activation('relu'))
-model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
+model.add(MaxPooling2D(pool_size=pool_size))
 model.add(Dropout(0.25))
 
 model.add(Flatten())

examples/mnist_hierarchical_rnn.py

@@ -0,0 +1,87 @@
+"""This is an example of using Hierarchical RNN (HRNN) to classify MNIST digits.
+
+HRNNs can learn across multiple levels of temporal hiearchy over a complex sequence.
+Usually, the first recurrent layer of an HRNN encodes a sentence (e.g. of word vectors)
+into a  sentence vector. The second recurrent layer then encodes a sequence of
+such vectors (encoded by the first layer) into a document vector. This
+document vector is considered to preserve both the word-level and
+sentence-level structure of the context.
+
+# References
+    - [A Hierarchical Neural Autoencoder for Paragraphs and Documents](https://arxiv.org/abs/1506.01057)
+        Encodes paragraphs and documents with HRNN.
+        Results have shown that HRNN outperforms standard
+        RNNs and may play some role in more sophisticated generation tasks like
+        summarization or question answering.
+    - [Hierarchical recurrent neural network for skeleton based action recognition](http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=7298714)
+        Achieved state-of-the-art results on skeleton based action recognition with 3 levels
+        of bidirectional HRNN combined with fully connected layers.
+
+In the below MNIST example the first LSTM layer first encodes every
+column of pixels of shape (28, 1) to a column vector of shape (128,). The second LSTM
+layer encodes then these 28 column vectors of shape (28, 128) to a image vector
+representing the whole image. A final Dense layer is added for prediction.
+
+After 5 epochs: train acc: 0.9858, val acc: 0.9864
+"""
+from __future__ import print_function
+
+from keras.datasets import mnist
+from keras.models import Model
+from keras.layers import Input, Dense, TimeDistributed
+from keras.layers import LSTM
+from keras.utils import np_utils
+
+# Training parameters.
+batch_size = 32
+nb_classes = 10
+nb_epochs = 5
+
+# Embedding dimensions.
+row_hidden = 128
+col_hidden = 128
+
+# The data, shuffled and split between train and test sets.
+(X_train, y_train), (X_test, y_test) = mnist.load_data()
+
+# Reshapes data to 4D for Hierarchical RNN.
+X_train = X_train.reshape(X_train.shape[0], 28, 28, 1)
+X_test = X_test.reshape(X_test.shape[0], 28, 28, 1)
+X_train = X_train.astype('float32')
+X_test = X_test.astype('float32')
+X_train /= 255
+X_test /= 255
+print('X_train shape:', X_train.shape)
+print(X_train.shape[0], 'train samples')
+print(X_test.shape[0], 'test samples')
+
+# Converts class vectors to binary class matrices.
+Y_train = np_utils.to_categorical(y_train, nb_classes)
+Y_test = np_utils.to_categorical(y_test, nb_classes)
+
+row, col, pixel = X_train.shape[1:]
+
+# 4D input.
+x = Input(shape=(row, col, pixel))
+
+# Encodes a row of pixels using TimeDistributed Wrapper.
+encoded_rows = TimeDistributed(LSTM(output_dim=row_hidden))(x)
+
+# Encodes columns of encoded rows.
+encoded_columns = LSTM(col_hidden)(encoded_rows)
+
+# Final predictions and model.
+prediction = Dense(nb_classes, activation='softmax')(encoded_columns)
+model = Model(input=x, output=prediction)
+model.compile(loss='categorical_crossentropy',
+              optimizer='rmsprop',
+              metrics=['accuracy'])
+
+# Training.
+model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epochs,
+          verbose=1, validation_data=(X_test, Y_test))
+
+# Evaluation.
+scores = model.evaluate(X_test, Y_test, verbose=0)
+print('Test loss:', scores[0])
+print('Test accuracy:', scores[1])

examples/mnist_irnn.py

@@ -3,7 +3,7 @@ with pixel-by-pixel sequential MNIST in
 "A Simple Way to Initialize Recurrent Networks of Rectified Linear Units"
 by Quoc V. Le, Navdeep Jaitly, Geoffrey E. Hinton
 
-arXiv:1504.00941v2 [cs.NE] 7 Apr 201
+arXiv:1504.00941v2 [cs.NE] 7 Apr 2015
 http://arxiv.org/pdf/1504.00941v2.pdf
 
 Optimizer is replaced with RMSprop which yields more stable and steady
@@ -17,9 +17,9 @@ from __future__ import print_function
 
 from keras.datasets import mnist
 from keras.models import Sequential
-from keras.layers.core import Dense, Activation
+from keras.layers import Dense, Activation
+from keras.layers import SimpleRNN
 from keras.initializations import normal, identity
-from keras.layers.recurrent import SimpleRNN
 from keras.optimizers import RMSprop
 from keras.utils import np_utils
 

examples/mnist_mlp.py

@@ -12,7 +12,7 @@ np.random.seed(1337)  # for reproducibility
 from keras.datasets import mnist
 from keras.models import Sequential
 from keras.layers.core import Dense, Dropout, Activation
-from keras.optimizers import SGD, Adam, RMSprop
+from keras.optimizers import RMSprop
 from keras.utils import np_utils
 
 

examples/mnist_net2net.py

@@ -0,0 +1,385 @@
+'''This is an implementation of Net2Net experiment with MNIST in
+'Net2Net: Accelerating Learning via Knowledge Transfer'
+by Tianqi Chen, Ian Goodfellow, and Jonathon Shlens
+
+arXiv:1511.05641v4 [cs.LG] 23 Apr 2016
+http://arxiv.org/abs/1511.05641
+
+Notes
+- What:
+  + Net2Net is a group of methods to transfer knowledge from a teacher neural
+    net to a student net,so that the student net can be trained faster than
+    from scratch.
+  + The paper discussed two specific methods of Net2Net, i.e. Net2WiderNet
+    and Net2DeeperNet.
+  + Net2WiderNet replaces a model with an equivalent wider model that has
+    more units in each hidden layer.
+  + Net2DeeperNet replaces a model with an equivalent deeper model.
+  + Both are based on the idea of 'function-preserving transformations of
+    neural nets'.
+- Why:
+  + Enable fast exploration of multiple neural nets in experimentation and
+    design process,by creating a series of wider and deeper models with
+    transferable knowledge.
+  + Enable 'lifelong learning system' by gradually adjusting model complexity
+    to data availability,and reusing transferable knowledge.
+
+Experiments
+- Teacher model: a basic CNN model trained on MNIST for 3 epochs.
+- Net2WiderNet exepriment:
+  + Student model has a wider Conv2D layer and a wider FC layer.
+  + Comparison of 'random-padding' vs 'net2wider' weight initialization.
+  + With both methods, student model should immediately perform as well as
+    teacher model, but 'net2wider' is slightly better.
+- Net2DeeperNet experiment:
+  + Student model has an extra Conv2D layer and an extra FC layer.
+  + Comparison of 'random-init' vs 'net2deeper' weight initialization.
+  + Starting performance of 'net2deeper' is better than 'random-init'.
+- Hyper-parameters:
+  + SGD with momentum=0.9 is used for training teacher and student models.
+  + Learning rate adjustment: it's suggested to reduce learning rate
+    to 1/10 for student model.
+  + Addition of noise in 'net2wider' is used to break weight symmetry
+    and thus enable full capacity of student models. It is optional
+    when a Dropout layer is used.
+
+Results
+- Tested with 'Theano' backend and 'th' image_dim_ordering.
+- Running on GPU GeForce GTX 980M
+- Performance Comparisons - validation loss values during first 3 epochs:
+(1) teacher_model:             0.075    0.041    0.041
+(2) wider_random_pad:          0.036    0.034    0.032
+(3) wider_net2wider:           0.032    0.030    0.030
+(4) deeper_random_init:        0.061    0.043    0.041
+(5) deeper_net2deeper:         0.032    0.031    0.029
+'''
+
+from __future__ import print_function
+from six.moves import xrange
+import numpy as np
+np.random.seed(1337)
+
+from keras.models import Sequential
+from keras.layers import Conv2D, MaxPooling2D, Dense, Flatten
+from keras.optimizers import SGD
+from keras.utils import np_utils
+from keras.datasets import mnist
+
+input_shape = (1, 28, 28)  # image shape
+nb_class = 10  # number of class
+
+
+# load and pre-process data
+def preprocess_input(x):
+    return x.reshape((-1, ) + input_shape) / 255.
+
+
+def preprocess_output(y):
+    return np_utils.to_categorical(y)
+
+(train_x, train_y), (validation_x, validation_y) = mnist.load_data()
+train_x, validation_x = map(preprocess_input, [train_x, validation_x])
+train_y, validation_y = map(preprocess_output, [train_y, validation_y])
+print('Loading MNIST data...')
+print('train_x shape:', train_x.shape, 'train_y shape:', train_y.shape)
+print('validation_x shape:', validation_x.shape,
+      'validation_y shape', validation_y.shape)
+
+
+# knowledge transfer algorithms
+def wider2net_conv2d(teacher_w1, teacher_b1, teacher_w2, new_width, init):
+    '''Get initial weights for a wider conv2d layer with a bigger nb_filter,
+    by 'random-padding' or 'net2wider'.
+
+    # Arguments
+        teacher_w1: `weight` of conv2d layer to become wider,
+          of shape (nb_filter1, nb_channel1, kh1, kw1)
+        teacher_b1: `bias` of conv2d layer to become wider,
+          of shape (nb_filter1, )
+        teacher_w2: `weight` of next connected conv2d layer,
+          of shape (nb_filter2, nb_channel2, kh2, kw2)
+        new_width: new `nb_filter` for the wider conv2d layer
+        init: initialization algorithm for new weights,
+          either 'random-pad' or 'net2wider'
+    '''
+    assert teacher_w1.shape[0] == teacher_w2.shape[1], (
+        'successive layers from teacher model should have compatible shapes')
+    assert teacher_w1.shape[0] == teacher_b1.shape[0], (
+        'weight and bias from same layer should have compatible shapes')
+    assert new_width > teacher_w1.shape[0], (
+        'new width (nb_filter) should be bigger than the existing one')
+
+    n = new_width - teacher_w1.shape[0]
+    if init == 'random-pad':
+        new_w1 = np.random.normal(0, 0.1, size=(n, ) + teacher_w1.shape[1:])
+        new_b1 = np.ones(n) * 0.1
+        new_w2 = np.random.normal(0, 0.1, size=(
+            teacher_w2.shape[0], n) + teacher_w2.shape[2:])
+    elif init == 'net2wider':
+        index = np.random.randint(teacher_w1.shape[0], size=n)
+        factors = np.bincount(index)[index] + 1.
+        new_w1 = teacher_w1[index, :, :, :]
+        new_b1 = teacher_b1[index]
+        new_w2 = teacher_w2[:, index, :, :] / factors.reshape((1, -1, 1, 1))
+    else:
+        raise ValueError('Unsupported weight initializer: %s' % init)
+
+    student_w1 = np.concatenate((teacher_w1, new_w1), axis=0)
+    if init == 'random-pad':
+        student_w2 = np.concatenate((teacher_w2, new_w2), axis=1)
+    elif init == 'net2wider':
+        # add small noise to break symmetry, so that student model will have
+        # full capacity later
+        noise = np.random.normal(0, 5e-2 * new_w2.std(), size=new_w2.shape)
+        student_w2 = np.concatenate((teacher_w2, new_w2 + noise), axis=1)
+        student_w2[:, index, :, :] = new_w2
+    student_b1 = np.concatenate((teacher_b1, new_b1), axis=0)
+
+    return student_w1, student_b1, student_w2
+
+
+def wider2net_fc(teacher_w1, teacher_b1, teacher_w2, new_width, init):
+    '''Get initial weights for a wider fully connected (dense) layer
+       with a bigger nout, by 'random-padding' or 'net2wider'.
+
+    # Arguments
+        teacher_w1: `weight` of fc layer to become wider,
+          of shape (nin1, nout1)
+        teacher_b1: `bias` of fc layer to become wider,
+          of shape (nout1, )
+        teacher_w2: `weight` of next connected fc layer,
+          of shape (nin2, nout2)
+        new_width: new `nout` for the wider fc layer
+        init: initialization algorithm for new weights,
+          either 'random-pad' or 'net2wider'
+    '''
+    assert teacher_w1.shape[1] == teacher_w2.shape[0], (
+        'successive layers from teacher model should have compatible shapes')
+    assert teacher_w1.shape[1] == teacher_b1.shape[0], (
+        'weight and bias from same layer should have compatible shapes')
+    assert new_width > teacher_w1.shape[1], (
+        'new width (nout) should be bigger than the existing one')
+
+    n = new_width - teacher_w1.shape[1]
+    if init == 'random-pad':
+        new_w1 = np.random.normal(0, 0.1, size=(teacher_w1.shape[0], n))
+        new_b1 = np.ones(n) * 0.1
+        new_w2 = np.random.normal(0, 0.1, size=(n, teacher_w2.shape[1]))
+    elif init == 'net2wider':
+        index = np.random.randint(teacher_w1.shape[1], size=n)
+        factors = np.bincount(index)[index] + 1.
+        new_w1 = teacher_w1[:, index]
+        new_b1 = teacher_b1[index]
+        new_w2 = teacher_w2[index, :] / factors[:, np.newaxis]
+    else:
+        raise ValueError('Unsupported weight initializer: %s' % init)
+
+    student_w1 = np.concatenate((teacher_w1, new_w1), axis=1)
+    if init == 'random-pad':
+        student_w2 = np.concatenate((teacher_w2, new_w2), axis=0)
+    elif init == 'net2wider':
+        # add small noise to break symmetry, so that student model will have
+        # full capacity later
+        noise = np.random.normal(0, 5e-2 * new_w2.std(), size=new_w2.shape)
+        student_w2 = np.concatenate((teacher_w2, new_w2 + noise), axis=0)
+        student_w2[index, :] = new_w2
+    student_b1 = np.concatenate((teacher_b1, new_b1), axis=0)
+
+    return student_w1, student_b1, student_w2
+
+
+def deeper2net_conv2d(teacher_w):
+    '''Get initial weights for a deeper conv2d layer by net2deeper'.
+
+    # Arguments
+        teacher_w: `weight` of previous conv2d layer,
+          of shape (nb_filter, nb_channel, kh, kw)
+    '''
+    nb_filter, nb_channel, kh, kw = teacher_w.shape
+    student_w = np.zeros((nb_filter, nb_filter, kh, kw))
+    for i in xrange(nb_filter):
+        student_w[i, i, (kh - 1) / 2, (kw - 1) / 2] = 1.
+    student_b = np.zeros(nb_filter)
+    return student_w, student_b
+
+
+def copy_weights(teacher_model, student_model, layer_names):
+    '''Copy weights from teacher_model to student_model,
+     for layers with names listed in layer_names
+    '''
+    for name in layer_names:
+        weights = teacher_model.get_layer(name=name).get_weights()
+        student_model.get_layer(name=name).set_weights(weights)
+
+
+# methods to construct teacher_model and student_models
+def make_teacher_model(train_data, validation_data, nb_epoch=3):
+    '''Train a simple CNN as teacher model.
+    '''
+    model = Sequential()
+    model.add(Conv2D(64, 3, 3, input_shape=input_shape,
+                     border_mode='same', name='conv1'))
+    model.add(MaxPooling2D(name='pool1'))
+    model.add(Conv2D(64, 3, 3, border_mode='same', name='conv2'))
+    model.add(MaxPooling2D(name='pool2'))
+    model.add(Flatten(name='flatten'))
+    model.add(Dense(64, activation='relu', name='fc1'))
+    model.add(Dense(nb_class, activation='softmax', name='fc2'))
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=SGD(lr=0.01, momentum=0.9),
+                  metrics=['accuracy'])
+
+    train_x, train_y = train_data
+    history = model.fit(train_x, train_y, nb_epoch=nb_epoch,
+                        validation_data=validation_data)
+    return model, history
+
+
+def make_wider_student_model(teacher_model, train_data,
+                             validation_data, init, nb_epoch=3):
+    '''Train a wider student model based on teacher_model,
+       with either 'random-pad' (baseline) or 'net2wider'
+    '''
+    new_conv1_width = 128
+    new_fc1_width = 128
+
+    model = Sequential()
+    # a wider conv1 compared to teacher_model
+    model.add(Conv2D(new_conv1_width, 3, 3, input_shape=input_shape,
+                     border_mode='same', name='conv1'))
+    model.add(MaxPooling2D(name='pool1'))
+    model.add(Conv2D(64, 3, 3, border_mode='same', name='conv2'))
+    model.add(MaxPooling2D(name='pool2'))
+    model.add(Flatten(name='flatten'))
+    # a wider fc1 compared to teacher model
+    model.add(Dense(new_fc1_width, activation='relu', name='fc1'))
+    model.add(Dense(nb_class, activation='softmax', name='fc2'))
+
+    # The weights for other layers need to be copied from teacher_model
+    # to student_model, except for widened layers
+    # and their immediate downstreams, which will be initialized separately.
+    # For this example there are no other layers that need to be copied.
+
+    w_conv1, b_conv1 = teacher_model.get_layer('conv1').get_weights()
+    w_conv2, b_conv2 = teacher_model.get_layer('conv2').get_weights()
+    new_w_conv1, new_b_conv1, new_w_conv2 = wider2net_conv2d(
+        w_conv1, b_conv1, w_conv2, new_conv1_width, init)
+    model.get_layer('conv1').set_weights([new_w_conv1, new_b_conv1])
+    model.get_layer('conv2').set_weights([new_w_conv2, b_conv2])
+
+    w_fc1, b_fc1 = teacher_model.get_layer('fc1').get_weights()
+    w_fc2, b_fc2 = teacher_model.get_layer('fc2').get_weights()
+    new_w_fc1, new_b_fc1, new_w_fc2 = wider2net_fc(
+        w_fc1, b_fc1, w_fc2, new_fc1_width, init)
+    model.get_layer('fc1').set_weights([new_w_fc1, new_b_fc1])
+    model.get_layer('fc2').set_weights([new_w_fc2, b_fc2])
+
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=SGD(lr=0.001, momentum=0.9),
+                  metrics=['accuracy'])
+
+    train_x, train_y = train_data
+    history = model.fit(train_x, train_y, nb_epoch=nb_epoch,
+                        validation_data=validation_data)
+    return model, history
+
+
+def make_deeper_student_model(teacher_model, train_data,
+                              validation_data, init, nb_epoch=3):
+    '''Train a deeper student model based on teacher_model,
+       with either 'random-init' (baseline) or 'net2deeper'
+    '''
+    model = Sequential()
+    model.add(Conv2D(64, 3, 3, input_shape=input_shape,
+                     border_mode='same', name='conv1'))
+    model.add(MaxPooling2D(name='pool1'))
+    model.add(Conv2D(64, 3, 3, border_mode='same', name='conv2'))
+    # add another conv2d layer to make original conv2 deeper
+    if init == 'net2deeper':
+        prev_w, _ = model.get_layer('conv2').get_weights()
+        new_weights = deeper2net_conv2d(prev_w)
+        model.add(Conv2D(64, 3, 3, border_mode='same',
+                         name='conv2-deeper', weights=new_weights))
+    elif init == 'random-init':
+        model.add(Conv2D(64, 3, 3, border_mode='same', name='conv2-deeper'))
+    else:
+        raise ValueError('Unsupported weight initializer: %s' % init)
+    model.add(MaxPooling2D(name='pool2'))
+    model.add(Flatten(name='flatten'))
+    model.add(Dense(64, activation='relu', name='fc1'))
+    # add another fc layer to make original fc1 deeper
+    if init == 'net2deeper':
+        # net2deeper for fc layer with relu, is just an identity initializer
+        model.add(Dense(64, init='identity',
+                        activation='relu', name='fc1-deeper'))
+    elif init == 'random-init':
+        model.add(Dense(64, activation='relu', name='fc1-deeper'))
+    else:
+        raise ValueError('Unsupported weight initializer: %s' % init)
+    model.add(Dense(nb_class, activation='softmax', name='fc2'))
+
+    # copy weights for other layers
+    copy_weights(teacher_model, model, layer_names=[
+                 'conv1', 'conv2', 'fc1', 'fc2'])
+
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=SGD(lr=0.001, momentum=0.9),
+                  metrics=['accuracy'])
+
+    train_x, train_y = train_data
+    history = model.fit(train_x, train_y, nb_epoch=nb_epoch,
+                        validation_data=validation_data)
+    return model, history
+
+
+# experiments setup
+def net2wider_experiment():
+    '''Benchmark performances of
+    (1) a teacher model,
+    (2) a wider student model with `random_pad` initializer
+    (3) a wider student model with `Net2WiderNet` initializer
+    '''
+    train_data = (train_x, train_y)
+    validation_data = (validation_x, validation_y)
+    print('\nExperiment of Net2WiderNet ...')
+    print('\nbuilding teacher model ...')
+    teacher_model, _ = make_teacher_model(train_data,
+                                          validation_data,
+                                          nb_epoch=3)
+
+    print('\nbuilding wider student model by random padding ...')
+    make_wider_student_model(teacher_model, train_data,
+                             validation_data, 'random-pad',
+                             nb_epoch=3)
+    print('\nbuilding wider student model by net2wider ...')
+    make_wider_student_model(teacher_model, train_data,
+                             validation_data, 'net2wider',
+                             nb_epoch=3)
+
+
+def net2deeper_experiment():
+    '''Benchmark performances of
+    (1) a teacher model,
+    (2) a deeper student model with `random_init` initializer
+    (3) a deeper student model with `Net2DeeperNet` initializer
+    '''
+    train_data = (train_x, train_y)
+    validation_data = (validation_x, validation_y)
+    print('\nExperiment of Net2DeeperNet ...')
+    print('\nbuilding teacher model ...')
+    teacher_model, _ = make_teacher_model(train_data,
+                                          validation_data,
+                                          nb_epoch=3)
+
+    print('\nbuilding deeper student model by random init ...')
+    make_deeper_student_model(teacher_model, train_data,
+                              validation_data, 'random-init',
+                              nb_epoch=3)
+    print('\nbuilding deeper student model by net2deeper ...')
+    make_deeper_student_model(teacher_model, train_data,
+                              validation_data, 'net2deeper',
+                              nb_epoch=3)
+
+# run the experiments
+net2wider_experiment()
+net2deeper_experiment()

examples/mnist_siamese_graph.py

@@ -19,7 +19,7 @@ import random
 from keras.datasets import mnist
 from keras.models import Sequential, Model
 from keras.layers import Dense, Dropout, Input, Lambda
-from keras.optimizers import SGD, RMSprop
+from keras.optimizers import RMSprop
 from keras import backend as K
 
 
@@ -28,6 +28,11 @@ def euclidean_distance(vects):
     return K.sqrt(K.sum(K.square(x - y), axis=1, keepdims=True))
 
 
+def eucl_dist_output_shape(shapes):
+    shape1, shape2 = shapes
+    return (shape1[0], 1)
+
+
 def contrastive_loss(y_true, y_pred):
     '''Contrastive loss from Hadsell-et-al.'06
     http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
@@ -45,7 +50,7 @@ def create_pairs(x, digit_indices):
     n = min([len(digit_indices[d]) for d in range(10)]) - 1
     for d in range(10):
         for i in range(n):
-            z1, z2 = digit_indices[d][i], digit_indices[d][i+1]
+            z1, z2 = digit_indices[d][i], digit_indices[d][i + 1]
             pairs += [[x[z1], x[z2]]]
             inc = random.randrange(1, 10)
             dn = (d + inc) % 10
@@ -103,7 +108,7 @@ input_b = Input(shape=(input_dim,))
 processed_a = base_network(input_a)
 processed_b = base_network(input_b)
 
-distance = Lambda(euclidean_distance)([processed_a, processed_b])
+distance = Lambda(euclidean_distance, output_shape=eucl_dist_output_shape)([processed_a, processed_b])
 
 model = Model(input=[input_a, input_b], output=distance)
 

examples/mnist_sklearn_wrapper.py

@@ -0,0 +1,104 @@
+'''Example of how to use sklearn wrapper
+
+Builds simple CNN models on MNIST and uses sklearn's GridSearchCV to find best model
+'''
+
+from __future__ import print_function
+import numpy as np
+np.random.seed(1337)  # for reproducibility
+
+from keras.datasets import mnist
+from keras.models import Sequential
+from keras.layers import Dense, Dropout, Activation, Flatten
+from keras.layers import Convolution2D, MaxPooling2D
+from keras.utils import np_utils
+from keras.wrappers.scikit_learn import KerasClassifier
+from keras import backend as K
+from sklearn.grid_search import GridSearchCV
+
+
+nb_classes = 10
+
+# input image dimensions
+img_rows, img_cols = 28, 28
+
+# load training data and do basic data normalization
+(X_train, y_train), (X_test, y_test) = mnist.load_data()
+
+if K.image_dim_ordering() == 'th':
+    X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)
+    X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)
+    input_shape = (1, img_rows, img_cols)
+else:
+    X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1)
+    X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)
+    input_shape = (img_rows, img_cols, 1)
+
+X_train = X_train.astype('float32')
+X_test = X_test.astype('float32')
+X_train /= 255
+X_test /= 255
+
+# convert class vectors to binary class matrices
+y_train = np_utils.to_categorical(y_train, nb_classes)
+y_test = np_utils.to_categorical(y_test, nb_classes)
+
+
+def make_model(dense_layer_sizes, nb_filters, nb_conv, nb_pool):
+    '''Creates model comprised of 2 convolutional layers followed by dense layers
+
+    dense_layer_sizes: List of layer sizes. This list has one number for each layer
+    nb_filters: Number of convolutional filters in each convolutional layer
+    nb_conv: Convolutional kernel size
+    nb_pool: Size of pooling area for max pooling
+    '''
+
+    model = Sequential()
+
+    model.add(Convolution2D(nb_filters, nb_conv, nb_conv,
+                            border_mode='valid',
+                            input_shape=input_shape))
+    model.add(Activation('relu'))
+    model.add(Convolution2D(nb_filters, nb_conv, nb_conv))
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
+    model.add(Dropout(0.25))
+
+    model.add(Flatten())
+    for layer_size in dense_layer_sizes:
+        model.add(Dense(layer_size))
+    model.add(Activation('relu'))
+    model.add(Dropout(0.5))
+    model.add(Dense(nb_classes))
+    model.add(Activation('softmax'))
+
+    model.compile(loss='categorical_crossentropy',
+                  optimizer='adadelta',
+                  metrics=['accuracy'])
+
+    return model
+
+dense_size_candidates = [[32], [64], [32, 32], [64, 64]]
+my_classifier = KerasClassifier(make_model, batch_size=32)
+validator = GridSearchCV(my_classifier,
+                         param_grid={'dense_layer_sizes': dense_size_candidates,
+                                     # nb_epoch is avail for tuning even when not
+                                     # an argument to model building function
+                                     'nb_epoch': [3, 6],
+                                     'nb_filters': [8],
+                                     'nb_conv': [3],
+                                     'nb_pool': [2]},
+                         scoring='log_loss',
+                         n_jobs=1)
+validator.fit(X_train, y_train)
+
+print('The parameters of the best model are: ')
+print(validator.best_params_)
+
+# validator.best_estimator_ returns sklearn-wrapped version of best model.
+# validator.best_estimator_.model returns the (unwrapped) keras model
+best_model = validator.best_estimator_.model
+metric_names = best_model.metrics_names
+metric_values = best_model.evaluate(X_test, y_test)
+for metric, value in zip(metric_names, metric_values):
+    print(metric, ': ', value)

examples/mnist_swwae.py

@@ -0,0 +1,176 @@
+'''Trains a stacked what-where autoencoder built on residual blocks on the
+MNIST dataset.  It exemplifies two influential methods that have been developed
+in the past few years.
+
+The first is the idea of properly "unpooling." During any max pool, the
+exact location (the "where") of the maximal value in a pooled receptive field
+is lost, however it can be very useful in the overall reconstruction of an
+input image.  Therefore, if the "where" is handed from the encoder
+to the corresponding decoder layer, features being decoded can be "placed" in
+the right location, allowing for reconstructions of much higher fidelity.
+
+References:
+[1]
+"Visualizing and Understanding Convolutional Networks"
+Matthew D Zeiler, Rob Fergus
+https://arxiv.org/abs/1311.2901v3
+
+[2]
+"Stacked What-Where Auto-encoders"
+Junbo Zhao, Michael Mathieu, Ross Goroshin, Yann LeCun
+https://arxiv.org/abs/1506.02351v8
+
+The second idea exploited here is that of residual learning.  Residual blocks
+ease the training process by allowing skip connections that give the network
+the ability to be as linear (or non-linear) as the data sees fit.  This allows
+for much deep networks to be easily trained.  The residual element seems to
+be advantageous in the context of this example as it allows a nice symmetry
+between the encoder and decoder.  Normally, in the decoder, the final
+projection to the space where the image is reconstructed is linear, however
+this does not have to be the case for a residual block as the degree to which
+its output is linear or non-linear is determined by the data it is fed.
+However, in order to cap the reconstruction in this example, a hard softmax is
+applied as a bias because we know the MNIST digits are mapped to [0,1].
+
+References:
+[3]
+"Deep Residual Learning for Image Recognition"
+Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun
+https://arxiv.org/abs/1512.03385v1
+
+[4]
+"Identity Mappings in Deep Residual Networks"
+Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun
+https://arxiv.org/abs/1603.05027v3
+
+'''
+from __future__ import print_function
+import numpy as np
+np.random.seed(1337)  # for reproducibility
+
+from keras.datasets import mnist
+from keras.models import Model
+from keras.layers import Activation, merge
+from keras.layers import UpSampling2D, Convolution2D, MaxPooling2D
+from keras.layers import Input, BatchNormalization
+import matplotlib.pyplot as plt
+import keras.backend as K
+
+
+def convresblock(x, nfeats=8, ksize=3, nskipped=2):
+    ''' The proposed residual block from [4]'''
+    y0 = Convolution2D(nfeats, ksize, ksize, border_mode='same')(x)
+    y = y0
+    for i in range(nskipped):
+        y = BatchNormalization(mode=0, axis=1)(y)
+        y = Activation('relu')(y)
+        y = Convolution2D(nfeats, ksize, ksize, border_mode='same')(y)
+    return merge([y0, y], mode='sum')
+
+
+def getwhere(x):
+    ''' Calculate the "where" mask that contains switches indicating which
+    index contained the max value when MaxPool2D was applied.  Using the
+    gradient of the sum is a nice trick to keep everything high level.'''
+    y_prepool, y_postpool = x
+    return K.gradients(K.sum(y_postpool), y_prepool)
+
+if K.backend() == 'tensorflow':
+    raise RuntimeError('This example can only run with the '
+                       'Theano backend for the time being, '
+                       'because it requires taking the gradient '
+                       'of a gradient, which isn\'t '
+                       'supported for all TF ops.')
+
+# This example assume 'th' dim ordering.
+K.set_image_dim_ordering('th')
+
+# input image dimensions
+img_rows, img_cols = 28, 28
+
+# the data, shuffled and split between train and test sets
+(X_train, _), (X_test, _) = mnist.load_data()
+
+X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)
+X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)
+X_train = X_train.astype('float32')
+X_test = X_test.astype('float32')
+X_train /= 255
+X_test /= 255
+print('X_train shape:', X_train.shape)
+print(X_train.shape[0], 'train samples')
+print(X_test.shape[0], 'test samples')
+
+# The size of the kernel used for the MaxPooling2D
+pool_size = 2
+# The total number of feature maps at each layer
+nfeats = [8, 16, 32, 64, 128]
+# The sizes of the pooling kernel at each layer
+pool_sizes = np.array([1, 1, 1, 1, 1]) * pool_size
+# The convolution kernel size
+ksize = 3
+# Number of epochs to train for
+nb_epoch = 5
+# Batch size during training
+batch_size = 128
+
+if pool_size == 2:
+    # if using a 5 layer net of pool_size = 2
+    X_train = np.pad(X_train, [[0, 0], [0, 0], [2, 2], [2, 2]],
+                     mode='constant')
+    X_test = np.pad(X_test, [[0, 0], [0, 0], [2, 2], [2, 2]], mode='constant')
+    nlayers = 5
+elif pool_size == 3:
+    # if using a 3 layer net of pool_size = 3
+    X_train = X_train[:, :, :-1, :-1]
+    X_test = X_test[:, :, :-1, :-1]
+    nlayers = 3
+else:
+    import sys
+    sys.exit("Script supports pool_size of 2 and 3.")
+
+# Shape of input to train on (note that model is fully convolutional however)
+input_shape = X_train.shape[1:]
+# The final list of the size of axis=1 for all layers, including input
+nfeats_all = [input_shape[0]] + nfeats
+
+# First build the encoder, all the while keeping track of the "where" masks
+img_input = Input(shape=input_shape)
+
+# We push the "where" masks to the following list
+wheres = [None] * nlayers
+y = img_input
+for i in range(nlayers):
+    y_prepool = convresblock(y, nfeats=nfeats_all[i + 1], ksize=ksize)
+    y = MaxPooling2D(pool_size=(pool_sizes[i], pool_sizes[i]))(y_prepool)
+    wheres[i] = merge([y_prepool, y], mode=getwhere,
+                      output_shape=lambda x: x[0])
+
+# Now build the decoder, and use the stored "where" masks to place the features
+for i in range(nlayers):
+    ind = nlayers - 1 - i
+    y = UpSampling2D(size=(pool_sizes[ind], pool_sizes[ind]))(y)
+    y = merge([y, wheres[ind]], mode='mul')
+    y = convresblock(y, nfeats=nfeats_all[ind], ksize=ksize)
+
+# Use hard_simgoid to clip range of reconstruction
+y = Activation('hard_sigmoid')(y)
+
+# Define the model and it's mean square error loss, and compile it with Adam
+model = Model(img_input, y)
+model.compile('adam', 'mse')
+
+# Fit the model
+model.fit(X_train, X_train, validation_data=(X_test, X_test),
+          batch_size=batch_size, nb_epoch=nb_epoch)
+
+# Plot
+X_recon = model.predict(X_test[:25])
+X_plot = np.concatenate((X_test[:25], X_recon), axis=1)
+X_plot = X_plot.reshape((5, 10, input_shape[-2], input_shape[-1]))
+X_plot = np.vstack([np.hstack(x) for x in X_plot])
+plt.figure()
+plt.axis('off')
+plt.title('Test Samples: Originals/Reconstructions')
+plt.imshow(X_plot, interpolation='none', cmap='gray')
+plt.savefig('reconstructions.png')

examples/mnist_transfer_cnn.py

@@ -19,10 +19,10 @@ np.random.seed(1337)  # for reproducibility
 
 from keras.datasets import mnist
 from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation, Flatten
-from keras.layers.convolutional import Convolution2D, MaxPooling2D
+from keras.layers import Dense, Dropout, Activation, Flatten
+from keras.layers import Convolution2D, MaxPooling2D
 from keras.utils import np_utils
-
+from keras import backend as K
 
 now = datetime.datetime.now
 
@@ -35,14 +35,19 @@ img_rows, img_cols = 28, 28
 # number of convolutional filters to use
 nb_filters = 32
 # size of pooling area for max pooling
-nb_pool = 2
+pool_size = 2
 # convolution kernel size
-nb_conv = 3
+kernel_size = 3
+
+if K.image_dim_ordering() == 'th':
+    input_shape = (1, img_rows, img_cols)
+else:
+    input_shape = (img_rows, img_cols, 1)
 
 
 def train_model(model, train, test, nb_classes):
-    X_train = train[0].reshape(train[0].shape[0], 1, img_rows, img_cols)
-    X_test = test[0].reshape(test[0].shape[0], 1, img_rows, img_cols)
+    X_train = train[0].reshape((train[0].shape[0],) + input_shape)
+    X_test = test[0].reshape((test[0].shape[0],) + input_shape)
     X_train = X_train.astype('float32')
     X_test = X_test.astype('float32')
     X_train /= 255
@@ -86,13 +91,13 @@ y_test_gte5 = y_test[y_test >= 5] - 5
 
 # define two groups of layers: feature (convolutions) and classification (dense)
 feature_layers = [
-    Convolution2D(nb_filters, nb_conv, nb_conv,
+    Convolution2D(nb_filters, kernel_size, kernel_size,
                   border_mode='valid',
-                  input_shape=(1, img_rows, img_cols)),
+                  input_shape=input_shape),
     Activation('relu'),
-    Convolution2D(nb_filters, nb_conv, nb_conv),
+    Convolution2D(nb_filters, kernel_size, kernel_size),
     Activation('relu'),
-    MaxPooling2D(pool_size=(nb_pool, nb_pool)),
+    MaxPooling2D(pool_size=(pool_size, pool_size)),
     Dropout(0.25),
     Flatten(),
 ]
@@ -105,9 +110,7 @@ classification_layers = [
 ]
 
 # create complete model
-model = Sequential()
-for l in feature_layers + classification_layers:
-    model.add(l)
+model = Sequential(feature_layers + classification_layers)
 
 # train model for 5-digit classification [0..4]
 train_model(model,

examples/neural_doodle.py

@@ -0,0 +1,366 @@
+'''Neural doodle with Keras
+
+Script Usage:
+    # Arguments:
+    ```
+    --nlabels:              # of regions (colors) in mask images
+    --style-image:          image to learn style from
+    --style-mask:           semantic labels for style image
+    --target-mask:          semantic labels for target image (your doodle)
+    --content-image:        optional image to learn content from
+    --target-image-prefix:  path prefix for generated target images
+    ```
+
+    # Example 1: doodle using a style image, style mask
+    and target mask.
+    ```
+    python neural_doodle.py --nlabels 4 --style-image Monet/style.png \
+    --style-mask Monet/style_mask.png --target-mask Monet/target_mask.png \
+    --target-image-prefix generated/monet
+    ```
+
+    # Example 2: doodle using a style image, style mask,
+    target mask and an optional content image.
+    ```
+    python neural_doodle.py --nlabels 4 --style-image Renoir/style.png \
+    --style-mask Renoir/style_mask.png --target-mask Renoir/target_mask.png \
+    --content-image Renoir/creek.jpg \
+    --target-image-prefix generated/renoir
+    ```
+
+References:
+[Dmitry Ulyanov's blog on fast-neural-doodle](http://dmitryulyanov.github.io/feed-forward-neural-doodle/)
+[Torch code for fast-neural-doodle](https://github.com/DmitryUlyanov/fast-neural-doodle)
+[Torch code for online-neural-doodle](https://github.com/DmitryUlyanov/online-neural-doodle)
+[Paper Texture Networks: Feed-forward Synthesis of Textures and Stylized Images](http://arxiv.org/abs/1603.03417)
+[Discussion on parameter tuning](https://github.com/fchollet/keras/issues/3705)
+
+Resources:
+Example images can be downloaded from
+https://github.com/DmitryUlyanov/fast-neural-doodle/tree/master/data
+'''
+from __future__ import print_function
+import time
+import argparse
+import numpy as np
+from scipy.optimize import fmin_l_bfgs_b
+from scipy.misc import imread, imsave
+
+from keras import backend as K
+from keras.layers import Input, AveragePooling2D
+from keras.models import Model
+from keras.preprocessing.image import load_img, img_to_array
+from keras.applications import vgg19
+
+# Command line arguments
+parser = argparse.ArgumentParser(description='Keras neural doodle example')
+parser.add_argument('--nlabels', type=int,
+                    help='number of semantic labels'
+                    ' (regions in differnet colors)'
+                    ' in style_mask/target_mask')
+parser.add_argument('--style-image', type=str,
+                    help='path to image to learn style from')
+parser.add_argument('--style-mask', type=str,
+                    help='path to semantic mask of style image')
+parser.add_argument('--target-mask', type=str,
+                    help='path to semantic mask of target image')
+parser.add_argument('--content-image', type=str, default=None,
+                    help='path to optional content image')
+parser.add_argument('--target-image-prefix', type=str,
+                    help='path prefix for generated results')
+args = parser.parse_args()
+
+style_img_path = args.style_image
+style_mask_path = args.style_mask
+target_mask_path = args.target_mask
+content_img_path = args.content_image
+target_img_prefix = args.target_image_prefix
+use_content_img = content_img_path is not None
+
+nb_labels = args.nlabels
+nb_colors = 3  # RGB
+# determine image sizes based on target_mask
+ref_img = imread(target_mask_path)
+img_nrows, img_ncols = ref_img.shape[:2]
+
+total_variation_weight = 50.
+style_weight = 1.
+content_weight = 0.1 if use_content_img else 0
+
+content_feature_layers = ['block5_conv2']
+# To get better generation qualities, use more conv layers for style features
+style_feature_layers = ['block1_conv1', 'block2_conv1', 'block3_conv1',
+                        'block4_conv1', 'block5_conv1']
+
+
+# helper functions for reading/processing images
+def preprocess_image(image_path):
+    img = load_img(image_path, target_size=(img_nrows, img_ncols))
+    img = img_to_array(img)
+    img = np.expand_dims(img, axis=0)
+    img = vgg19.preprocess_input(img)
+    return img
+
+
+def deprocess_image(x):
+    if K.image_dim_ordering() == 'th':
+        x = x.reshape((3, img_nrows, img_ncols))
+        x = x.transpose((1, 2, 0))
+    else:
+        x = x.reshape((img_nrows, img_ncols, 3))
+    # Remove zero-center by mean pixel
+    x[:, :, 0] += 103.939
+    x[:, :, 1] += 116.779
+    x[:, :, 2] += 123.68
+    # 'BGR'->'RGB'
+    x = x[:, :, ::-1]
+    x = np.clip(x, 0, 255).astype('uint8')
+    return x
+
+
+def kmeans(xs, k):
+    assert xs.ndim == 2
+    try:
+        from sklearn.cluster import k_means
+        _, labels, _ = k_means(xs.astype("float64"), k)
+    except ImportError:
+        from scipy.cluster.vq import kmeans2
+        _, labels = kmeans2(xs, k, missing='raise')
+    return labels
+
+
+def load_mask_labels():
+    '''Load both target and style masks.
+    A mask image (nr x nc) with m labels/colors will be loaded
+    as a 4D boolean tensor: (1, m, nr, nc) for 'th' or (1, nr, nc, m) for 'tf'
+    '''
+    target_mask_img = load_img(target_mask_path,
+                               target_size=(img_nrows, img_ncols))
+    target_mask_img = img_to_array(target_mask_img)
+    style_mask_img = load_img(style_mask_path,
+                              target_size=(img_nrows, img_ncols))
+    style_mask_img = img_to_array(style_mask_img)
+    if K.image_dim_ordering() == 'th':
+        mask_vecs = np.vstack([style_mask_img.reshape((3, -1)).T,
+                               target_mask_img.reshape((3, -1)).T])
+    else:
+        mask_vecs = np.vstack([style_mask_img.reshape((-1, 3)),
+                               target_mask_img.reshape((-1, 3))])
+
+    labels = kmeans(mask_vecs, nb_labels)
+    style_mask_label = labels[:img_nrows *
+                              img_ncols].reshape((img_nrows, img_ncols))
+    target_mask_label = labels[img_nrows *
+                               img_ncols:].reshape((img_nrows, img_ncols))
+
+    stack_axis = 0 if K.image_dim_ordering() == 'th' else -1
+    style_mask = np.stack([style_mask_label == r for r in xrange(nb_labels)],
+                          axis=stack_axis)
+    target_mask = np.stack([target_mask_label == r for r in xrange(nb_labels)],
+                           axis=stack_axis)
+
+    return (np.expand_dims(style_mask, axis=0),
+            np.expand_dims(target_mask, axis=0))
+
+# Create tensor variables for images
+if K.image_dim_ordering() == 'th':
+    shape = (1, nb_colors, img_nrows, img_ncols)
+else:
+    shape = (1, img_nrows, img_ncols, nb_colors)
+
+style_image = K.variable(preprocess_image(style_img_path))
+target_image = K.placeholder(shape=shape)
+if use_content_img:
+    content_image = K.variable(preprocess_image(content_img_path))
+else:
+    content_image = K.zeros(shape=shape)
+
+images = K.concatenate([style_image, target_image, content_image], axis=0)
+
+# Create tensor variables for masks
+raw_style_mask, raw_target_mask = load_mask_labels()
+style_mask = K.variable(raw_style_mask.astype("float32"))
+target_mask = K.variable(raw_target_mask.astype("float32"))
+masks = K.concatenate([style_mask, target_mask], axis=0)
+
+# index constants for images and tasks variables
+STYLE, TARGET, CONTENT = 0, 1, 2
+
+# Build image model, mask model and use layer outputs as features
+# image model as VGG19
+image_model = vgg19.VGG19(include_top=False, input_tensor=images)
+
+# mask model as a series of pooling
+mask_input = Input(tensor=masks, shape=(None, None, None), name="mask_input")
+x = mask_input
+for layer in image_model.layers[1:]:
+    name = 'mask_%s' % layer.name
+    if 'conv' in layer.name:
+        x = AveragePooling2D((3, 3), strides=(
+            1, 1), name=name, border_mode="same")(x)
+    elif 'pool' in layer.name:
+        x = AveragePooling2D((2, 2), name=name)(x)
+mask_model = Model(mask_input, x)
+
+# Collect features from image_model and task_model
+image_features = {}
+mask_features = {}
+for img_layer, mask_layer in zip(image_model.layers, mask_model.layers):
+    if 'conv' in img_layer.name:
+        assert 'mask_' + img_layer.name == mask_layer.name
+        layer_name = img_layer.name
+        img_feat, mask_feat = img_layer.output, mask_layer.output
+        image_features[layer_name] = img_feat
+        mask_features[layer_name] = mask_feat
+
+
+# Define loss functions
+def gram_matrix(x):
+    assert K.ndim(x) == 3
+    features = K.batch_flatten(x)
+    gram = K.dot(features, K.transpose(features))
+    return gram
+
+
+def region_style_loss(style_image, target_image, style_mask, target_mask):
+    '''Calculate style loss between style_image and target_image,
+    for one common region specified by their (boolean) masks
+    '''
+    assert 3 == K.ndim(style_image) == K.ndim(target_image)
+    assert 2 == K.ndim(style_mask) == K.ndim(target_mask)
+    if K.image_dim_ordering() == 'th':
+        masked_style = style_image * style_mask
+        masked_target = target_image * target_mask
+        nb_channels = K.shape(style_image)[0]
+    else:
+        masked_style = K.permute_dimensions(
+            style_image, (2, 0, 1)) * style_mask
+        masked_target = K.permute_dimensions(
+            target_image, (2, 0, 1)) * target_mask
+        nb_channels = K.shape(style_image)[-1]
+    s = gram_matrix(masked_style) / K.mean(style_mask) / nb_channels
+    c = gram_matrix(masked_target) / K.mean(target_mask) / nb_channels
+    return K.mean(K.square(s - c))
+
+
+def style_loss(style_image, target_image, style_masks, target_masks):
+    '''Calculate style loss between style_image and target_image,
+    in all regions.
+    '''
+    assert 3 == K.ndim(style_image) == K.ndim(target_image)
+    assert 3 == K.ndim(style_masks) == K.ndim(target_masks)
+    loss = K.variable(0)
+    for i in xrange(nb_labels):
+        if K.image_dim_ordering() == 'th':
+            style_mask = style_masks[i, :, :]
+            target_mask = target_masks[i, :, :]
+        else:
+            style_mask = style_masks[:, :, i]
+            target_mask = target_masks[:, :, i]
+        loss += region_style_loss(style_image,
+                                  target_image, style_mask, target_mask)
+    return loss
+
+
+def content_loss(content_image, target_image):
+    return K.sum(K.square(target_image - content_image))
+
+
+def total_variation_loss(x):
+    assert 4 == K.ndim(x)
+    if K.image_dim_ordering() == 'th':
+        a = K.square(x[:, :, :img_nrows - 1, :img_ncols - 1] -
+                     x[:, :, 1:, :img_ncols - 1])
+        b = K.square(x[:, :, :img_nrows - 1, :img_ncols - 1] -
+                     x[:, :, :img_nrows - 1, 1:])
+    else:
+        a = K.square(x[:, :img_nrows - 1, :img_ncols - 1, :] -
+                     x[:, 1:, :img_ncols - 1, :])
+        b = K.square(x[:, :img_nrows - 1, :img_ncols - 1, :] -
+                     x[:, :img_nrows - 1, 1:, :])
+    return K.sum(K.pow(a + b, 1.25))
+
+# Overall loss is the weighted sum of content_loss, style_loss and tv_loss
+# Each individual loss uses features from image/mask models.
+loss = K.variable(0)
+for layer in content_feature_layers:
+    content_feat = image_features[layer][CONTENT, :, :, :]
+    target_feat = image_features[layer][TARGET, :, :, :]
+    loss += content_weight * content_loss(content_feat, target_feat)
+
+for layer in style_feature_layers:
+    style_feat = image_features[layer][STYLE, :, :, :]
+    target_feat = image_features[layer][TARGET, :, :, :]
+    style_masks = mask_features[layer][STYLE, :, :, :]
+    target_masks = mask_features[layer][TARGET, :, :, :]
+    sl = style_loss(style_feat, target_feat, style_masks, target_masks)
+    loss += (style_weight / len(style_feature_layers)) * sl
+
+loss += total_variation_weight * total_variation_loss(target_image)
+loss_grads = K.gradients(loss, target_image)
+
+# Evaluator class for computing efficiency
+outputs = [loss]
+if isinstance(loss_grads, (list, tuple)):
+    outputs += loss_grads
+else:
+    outputs.append(loss_grads)
+
+f_outputs = K.function([target_image], outputs)
+
+
+def eval_loss_and_grads(x):
+    if K.image_dim_ordering() == 'th':
+        x = x.reshape((1, 3, img_nrows, img_ncols))
+    else:
+        x = x.reshape((1, img_nrows, img_ncols, 3))
+    outs = f_outputs([x])
+    loss_value = outs[0]
+    if len(outs[1:]) == 1:
+        grad_values = outs[1].flatten().astype('float64')
+    else:
+        grad_values = np.array(outs[1:]).flatten().astype('float64')
+    return loss_value, grad_values
+
+
+class Evaluator(object):
+
+    def __init__(self):
+        self.loss_value = None
+        self.grads_values = None
+
+    def loss(self, x):
+        assert self.loss_value is None
+        loss_value, grad_values = eval_loss_and_grads(x)
+        self.loss_value = loss_value
+        self.grad_values = grad_values
+        return self.loss_value
+
+    def grads(self, x):
+        assert self.loss_value is not None
+        grad_values = np.copy(self.grad_values)
+        self.loss_value = None
+        self.grad_values = None
+        return grad_values
+
+evaluator = Evaluator()
+
+# Generate images by iterative optimization
+if K.image_dim_ordering() == 'th':
+    x = np.random.uniform(0, 255, (1, 3, img_nrows, img_ncols)) - 128.
+else:
+    x = np.random.uniform(0, 255, (1, img_nrows, img_ncols, 3)) - 128.
+
+for i in range(50):
+    print('Start of iteration', i)
+    start_time = time.time()
+    x, min_val, info = fmin_l_bfgs_b(evaluator.loss, x.flatten(),
+                                     fprime=evaluator.grads, maxfun=20)
+    print('Current loss value:', min_val)
+    # save current generated image
+    img = deprocess_image(x.copy())
+    fname = target_img_prefix + '_at_iteration_%d.png' % i
+    imsave(fname, img)
+    end_time = time.time()
+    print('Image saved as', fname)
+    print('Iteration %d completed in %ds' % (i, end_time - start_time))

examples/neural_style_transfer.py

@@ -1,10 +1,5 @@
 '''Neural style transfer with Keras.
 
-Before running this script, download the weights for the VGG16 model at:
-https://drive.google.com/file/d/0Bz7KyqmuGsilT0J5dmRCM0ROVHc/view?usp=sharing
-(source: https://gist.github.com/baraldilorenzo/07d7802847aaad0a35d3)
-and make sure the variable `weights_path` in this script matches the location of the file.
-
 Run the script with:
 ```
 python neural_style_transfer.py path_to_your_base_image.jpg path_to_your_reference.jpg prefix_for_results
@@ -13,9 +8,15 @@ e.g.:
 ```
 python neural_style_transfer.py img/tuebingen.jpg img/starry_night.jpg results/my_result
 ```
+Optional parameters:
+```
+--iter, To specify the number of iterations the style transfer takes place (Default is 10)
+--content_weight, The weight given to the content loss (Default is 0.025)
+--style_weight, The weight given to the style loss (Default is 1.0)
+--tv_weight, The weight given to the total variation loss (Default is 1.0)
+```
 
-It is preferrable to run this script on GPU, for speed.
-If running on CPU, prefer the TensorFlow backend (much faster).
+It is preferable to run this script on GPU, for speed.
 
 Example result: https://twitter.com/fchollet/status/686631033085677568
 
@@ -34,7 +35,7 @@ the pixels of the combination image, giving it visual coherence.
 
 - The style loss is where the deep learning keeps in --that one is defined
 using a deep convolutional neural network. Precisely, it consists in a sum of
-L2 distances betwen the Gram matrices of the representations of
+L2 distances between the Gram matrices of the representations of
 the base image and the style reference image, extracted from
 different layers of a convnet (trained on ImageNet). The general idea
 is to capture color/texture information at different spatial
@@ -49,16 +50,14 @@ keeping the generated image close enough to the original one.
 '''
 
 from __future__ import print_function
-from scipy.misc import imread, imresize, imsave
+from keras.preprocessing.image import load_img, img_to_array
+from scipy.misc import imsave
 import numpy as np
 from scipy.optimize import fmin_l_bfgs_b
 import time
-import os
 import argparse
-import h5py
 
-from keras.models import Sequential
-from keras.layers.convolutional import Convolution2D, ZeroPadding2D, MaxPooling2D
+from keras.applications import vgg16
 from keras import backend as K
 
 parser = argparse.ArgumentParser(description='Neural style transfer with Keras.')
@@ -68,33 +67,56 @@ parser.add_argument('style_reference_image_path', metavar='ref', type=str,
                     help='Path to the style reference image.')
 parser.add_argument('result_prefix', metavar='res_prefix', type=str,
                     help='Prefix for the saved results.')
+parser.add_argument('--iter', type=int, default=10, required=False,
+                    help='Number of iterations to run.')
+parser.add_argument('--content_weight', type=float, default=0.025, required=False,
+                    help='Content weight.')
+parser.add_argument('--style_weight', type=float, default=1.0, required=False,
+                    help='Style weight.')
+parser.add_argument('--tv_weight', type=float, default=1.0, required=False,
+                    help='Total Variation weight.')
 
 args = parser.parse_args()
 base_image_path = args.base_image_path
 style_reference_image_path = args.style_reference_image_path
 result_prefix = args.result_prefix
-weights_path = 'vgg16_weights.h5'
+iterations = args.iter
 
 # these are the weights of the different loss components
-total_variation_weight = 1.
-style_weight = 1.
-content_weight = 0.025
+total_variation_weight = args.tv_weight
+style_weight = args.style_weight
+content_weight = args.content_weight
 
 # dimensions of the generated picture.
-img_width = 400
-img_height = 400
-assert img_height == img_width, 'Due to the use of the Gram matrix, width and height must match.'
+width, height = load_img(base_image_path).size
+img_nrows = 400
+img_ncols = int(width * img_nrows / height)
 
 # util function to open, resize and format pictures into appropriate tensors
+
+
 def preprocess_image(image_path):
-    img = imresize(imread(image_path), (img_width, img_height))
-    img = img.transpose((2, 0, 1)).astype('float64')
+    img = load_img(image_path, target_size=(img_nrows, img_ncols))
+    img = img_to_array(img)
     img = np.expand_dims(img, axis=0)
+    img = vgg16.preprocess_input(img)
     return img
 
 # util function to convert a tensor into a valid image
+
+
 def deprocess_image(x):
-    x = x.transpose((1, 2, 0))
+    if K.image_dim_ordering() == 'th':
+        x = x.reshape((3, img_nrows, img_ncols))
+        x = x.transpose((1, 2, 0))
+    else:
+        x = x.reshape((img_nrows, img_ncols, 3))
+    # Remove zero-center by mean pixel
+    x[:, :, 0] += 103.939
+    x[:, :, 1] += 116.779
+    x[:, :, 2] += 123.68
+    # 'BGR'->'RGB'
+    x = x[:, :, ::-1]
     x = np.clip(x, 0, 255).astype('uint8')
     return x
 
@@ -103,7 +125,10 @@ base_image = K.variable(preprocess_image(base_image_path))
 style_reference_image = K.variable(preprocess_image(style_reference_image_path))
 
 # this will contain our generated image
-combination_image = K.placeholder((1, 3, img_width, img_height))
+if K.image_dim_ordering() == 'th':
+    combination_image = K.placeholder((1, 3, img_nrows, img_ncols))
+else:
+    combination_image = K.placeholder((1, img_nrows, img_ncols, 3))
 
 # combine the 3 images into a single Keras tensor
 input_tensor = K.concatenate([base_image,
@@ -111,60 +136,9 @@ input_tensor = K.concatenate([base_image,
                               combination_image], axis=0)
 
 # build the VGG16 network with our 3 images as input
-first_layer = ZeroPadding2D((1, 1))
-first_layer.set_input(input_tensor, shape=(3, 3, img_width, img_height))
-
-model = Sequential()
-model.add(first_layer)
-model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(64, 3, 3, activation='relu'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(128, 3, 3, activation='relu'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(256, 3, 3, activation='relu'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu'))
-model.add(ZeroPadding2D((1, 1)))
-model.add(Convolution2D(512, 3, 3, activation='relu'))
-model.add(MaxPooling2D((2, 2), strides=(2, 2)))
-
-# load the weights of the VGG16 networks
-# (trained on ImageNet, won the ILSVRC competition in 2014)
-# note: when there is a complete match between your model definition
-# and your weight savefile, you can simply call model.load_weights(filename)
-assert os.path.exists(weights_path), 'Model weights not found (see "weights_path" variable in script).'
-f = h5py.File(weights_path)
-for k in range(f.attrs['nb_layers']):
-    if k >= len(model.layers):
-        # we don't look at the last (fully-connected) layers in the savefile
-        break
-    g = f['layer_{}'.format(k)]
-    weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]
-    model.layers[k].set_weights(weights)
-f.close()
+# the model will be loaded with pre-trained ImageNet weights
+model = vgg16.VGG16(input_tensor=input_tensor,
+                    weights='imagenet', include_top=False)
 print('Model loaded.')
 
 # get the symbolic outputs of each "key" layer (we gave them unique names).
@@ -174,9 +148,14 @@ outputs_dict = dict([(layer.name, layer.output) for layer in model.layers])
 # first we need to define 4 util functions
 
 # the gram matrix of an image tensor (feature-wise outer product)
+
+
 def gram_matrix(x):
     assert K.ndim(x) == 3
-    features = K.batch_flatten(x)
+    if K.image_dim_ordering() == 'th':
+        features = K.batch_flatten(x)
+    else:
+        features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1)))
     gram = K.dot(features, K.transpose(features))
     return gram
 
@@ -185,38 +164,50 @@ def gram_matrix(x):
 # It is based on the gram matrices (which capture style) of
 # feature maps from the style reference image
 # and from the generated image
+
+
 def style_loss(style, combination):
     assert K.ndim(style) == 3
     assert K.ndim(combination) == 3
     S = gram_matrix(style)
     C = gram_matrix(combination)
     channels = 3
-    size = img_width * img_height
+    size = img_nrows * img_ncols
     return K.sum(K.square(S - C)) / (4. * (channels ** 2) * (size ** 2))
 
 # an auxiliary loss function
 # designed to maintain the "content" of the
 # base image in the generated image
+
+
 def content_loss(base, combination):
     return K.sum(K.square(combination - base))
 
 # the 3rd loss function, total variation loss,
 # designed to keep the generated image locally coherent
+
+
 def total_variation_loss(x):
     assert K.ndim(x) == 4
-    a = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, 1:, :img_height-1])
-    b = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, :img_width-1, 1:])
+    if K.image_dim_ordering() == 'th':
+        a = K.square(x[:, :, :img_nrows - 1, :img_ncols - 1] - x[:, :, 1:, :img_ncols - 1])
+        b = K.square(x[:, :, :img_nrows - 1, :img_ncols - 1] - x[:, :, :img_nrows - 1, 1:])
+    else:
+        a = K.square(x[:, :img_nrows - 1, :img_ncols - 1, :] - x[:, 1:, :img_ncols - 1, :])
+        b = K.square(x[:, :img_nrows - 1, :img_ncols - 1, :] - x[:, :img_nrows - 1, 1:, :])
     return K.sum(K.pow(a + b, 1.25))
 
 # combine these loss functions into a single scalar
 loss = K.variable(0.)
-layer_features = outputs_dict['conv4_2']
+layer_features = outputs_dict['block4_conv2']
 base_image_features = layer_features[0, :, :, :]
 combination_features = layer_features[2, :, :, :]
 loss += content_weight * content_loss(base_image_features,
                                       combination_features)
 
-feature_layers = ['conv1_1', 'conv2_1', 'conv3_1', 'conv4_1', 'conv5_1']
+feature_layers = ['block1_conv1', 'block2_conv1',
+                  'block3_conv1', 'block4_conv1',
+                  'block5_conv1']
 for layer_name in feature_layers:
     layer_features = outputs_dict[layer_name]
     style_reference_features = layer_features[1, :, :, :]
@@ -229,14 +220,19 @@ loss += total_variation_weight * total_variation_loss(combination_image)
 grads = K.gradients(loss, combination_image)
 
 outputs = [loss]
-if type(grads) in {list, tuple}:
+if isinstance(grads, (list, tuple)):
     outputs += grads
 else:
     outputs.append(grads)
 
 f_outputs = K.function([combination_image], outputs)
+
+
 def eval_loss_and_grads(x):
-    x = x.reshape((1, 3, img_width, img_height))
+    if K.image_dim_ordering() == 'th':
+        x = x.reshape((1, 3, img_nrows, img_ncols))
+    else:
+        x = x.reshape((1, img_nrows, img_ncols, 3))
     outs = f_outputs([x])
     loss_value = outs[0]
     if len(outs[1:]) == 1:
@@ -251,7 +247,10 @@ def eval_loss_and_grads(x):
 # "loss" and "grads". This is done because scipy.optimize
 # requires separate functions for loss and gradients,
 # but computing them separately would be inefficient.
+
+
 class Evaluator(object):
+
     def __init__(self):
         self.loss_value = None
         self.grads_values = None
@@ -274,15 +273,19 @@ evaluator = Evaluator()
 
 # run scipy-based optimization (L-BFGS) over the pixels of the generated image
 # so as to minimize the neural style loss
-x = np.random.uniform(0, 255, (1, 3, img_width, img_height))
-for i in range(10):
+if K.image_dim_ordering() == 'th':
+    x = np.random.uniform(0, 255, (1, 3, img_nrows, img_ncols)) - 128.
+else:
+    x = np.random.uniform(0, 255, (1, img_nrows, img_ncols, 3)) - 128.
+
+for i in range(iterations):
     print('Start of iteration', i)
     start_time = time.time()
     x, min_val, info = fmin_l_bfgs_b(evaluator.loss, x.flatten(),
                                      fprime=evaluator.grads, maxfun=20)
     print('Current loss value:', min_val)
     # save current generated image
-    img = deprocess_image(x.reshape((3, img_width, img_height)))
+    img = deprocess_image(x.copy())
     fname = result_prefix + '_at_iteration_%d.png' % i
     imsave(fname, img)
     end_time = time.time()

examples/pretrained_word_embeddings.py

@@ -0,0 +1,144 @@
+'''This script loads pre-trained word embeddings (GloVe embeddings)
+into a frozen Keras Embedding layer, and uses it to
+train a text classification model on the 20 Newsgroup dataset
+(classication of newsgroup messages into 20 different categories).
+
+GloVe embedding data can be found at:
+http://nlp.stanford.edu/data/glove.6B.zip
+(source page: http://nlp.stanford.edu/projects/glove/)
+
+20 Newsgroup data can be found at:
+http://www.cs.cmu.edu/afs/cs.cmu.edu/project/theo-20/www/data/news20.html
+'''
+
+from __future__ import print_function
+import os
+import numpy as np
+np.random.seed(1337)
+
+from keras.preprocessing.text import Tokenizer
+from keras.preprocessing.sequence import pad_sequences
+from keras.utils.np_utils import to_categorical
+from keras.layers import Dense, Input, Flatten
+from keras.layers import Conv1D, MaxPooling1D, Embedding
+from keras.models import Model
+import sys
+
+BASE_DIR = ''
+GLOVE_DIR = BASE_DIR + '/glove.6B/'
+TEXT_DATA_DIR = BASE_DIR + '/20_newsgroup/'
+MAX_SEQUENCE_LENGTH = 1000
+MAX_NB_WORDS = 20000
+EMBEDDING_DIM = 100
+VALIDATION_SPLIT = 0.2
+
+# first, build index mapping words in the embeddings set
+# to their embedding vector
+
+print('Indexing word vectors.')
+
+embeddings_index = {}
+f = open(os.path.join(GLOVE_DIR, 'glove.6B.100d.txt'))
+for line in f:
+    values = line.split()
+    word = values[0]
+    coefs = np.asarray(values[1:], dtype='float32')
+    embeddings_index[word] = coefs
+f.close()
+
+print('Found %s word vectors.' % len(embeddings_index))
+
+# second, prepare text samples and their labels
+print('Processing text dataset')
+
+texts = []  # list of text samples
+labels_index = {}  # dictionary mapping label name to numeric id
+labels = []  # list of label ids
+for name in sorted(os.listdir(TEXT_DATA_DIR)):
+    path = os.path.join(TEXT_DATA_DIR, name)
+    if os.path.isdir(path):
+        label_id = len(labels_index)
+        labels_index[name] = label_id
+        for fname in sorted(os.listdir(path)):
+            if fname.isdigit():
+                fpath = os.path.join(path, fname)
+                if sys.version_info < (3,):
+                    f = open(fpath)
+                else:
+                    f = open(fpath, encoding='latin-1')
+                texts.append(f.read())
+                f.close()
+                labels.append(label_id)
+
+print('Found %s texts.' % len(texts))
+
+# finally, vectorize the text samples into a 2D integer tensor
+tokenizer = Tokenizer(nb_words=MAX_NB_WORDS)
+tokenizer.fit_on_texts(texts)
+sequences = tokenizer.texts_to_sequences(texts)
+
+word_index = tokenizer.word_index
+print('Found %s unique tokens.' % len(word_index))
+
+data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)
+
+labels = to_categorical(np.asarray(labels))
+print('Shape of data tensor:', data.shape)
+print('Shape of label tensor:', labels.shape)
+
+# split the data into a training set and a validation set
+indices = np.arange(data.shape[0])
+np.random.shuffle(indices)
+data = data[indices]
+labels = labels[indices]
+nb_validation_samples = int(VALIDATION_SPLIT * data.shape[0])
+
+x_train = data[:-nb_validation_samples]
+y_train = labels[:-nb_validation_samples]
+x_val = data[-nb_validation_samples:]
+y_val = labels[-nb_validation_samples:]
+
+print('Preparing embedding matrix.')
+
+# prepare embedding matrix
+nb_words = min(MAX_NB_WORDS, len(word_index))
+embedding_matrix = np.zeros((nb_words, EMBEDDING_DIM))
+for word, i in word_index.items():
+    if i >= MAX_NB_WORDS:
+        continue
+    embedding_vector = embeddings_index.get(word)
+    if embedding_vector is not None:
+        # words not found in embedding index will be all-zeros.
+        embedding_matrix[i] = embedding_vector
+
+# load pre-trained word embeddings into an Embedding layer
+# note that we set trainable = False so as to keep the embeddings fixed
+embedding_layer = Embedding(nb_words,
+                            EMBEDDING_DIM,
+                            weights=[embedding_matrix],
+                            input_length=MAX_SEQUENCE_LENGTH,
+                            trainable=False)
+
+print('Training model.')
+
+# train a 1D convnet with global maxpooling
+sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
+embedded_sequences = embedding_layer(sequence_input)
+x = Conv1D(128, 5, activation='relu')(embedded_sequences)
+x = MaxPooling1D(5)(x)
+x = Conv1D(128, 5, activation='relu')(x)
+x = MaxPooling1D(5)(x)
+x = Conv1D(128, 5, activation='relu')(x)
+x = MaxPooling1D(35)(x)
+x = Flatten()(x)
+x = Dense(128, activation='relu')(x)
+preds = Dense(len(labels_index), activation='softmax')(x)
+
+model = Model(sequence_input, preds)
+model.compile(loss='categorical_crossentropy',
+              optimizer='rmsprop',
+              metrics=['acc'])
+
+# happy learning!
+model.fit(x_train, y_train, validation_data=(x_val, y_val),
+          nb_epoch=2, batch_size=128)

examples/reuters_mlp.py

@@ -8,8 +8,7 @@ np.random.seed(1337)  # for reproducibility
 
 from keras.datasets import reuters
 from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation
-from keras.layers.normalization import BatchNormalization
+from keras.layers import Dense, Dropout, Activation
 from keras.utils import np_utils
 from keras.preprocessing.text import Tokenizer
 
@@ -22,7 +21,7 @@ print('Loading data...')
 print(len(X_train), 'train sequences')
 print(len(X_test), 'test sequences')
 
-nb_classes = np.max(y_train)+1
+nb_classes = np.max(y_train) + 1
 print(nb_classes, 'classes')
 
 print('Vectorizing sequence data...')

examples/stateful_lstm.py

@@ -5,8 +5,7 @@ from __future__ import print_function
 import numpy as np
 import matplotlib.pyplot as plt
 from keras.models import Sequential
-from keras.layers.core import Dense
-from keras.layers.recurrent import LSTM
+from keras.layers import Dense, LSTM
 
 
 # since we are using stateful rnn tsteps can be set to 1
@@ -17,7 +16,7 @@ epochs = 25
 lahead = 1
 
 
-def gen_cosine_amp(amp=100, period=25, x0=0, xn=50000, step=1, k=0.0001):
+def gen_cosine_amp(amp=100, period=1000, x0=0, xn=50000, step=1, k=0.0001):
     """Generates an absolute cosine time series with the amplitude
     exponentially decreasing
 
@@ -32,7 +31,7 @@ def gen_cosine_amp(amp=100, period=25, x0=0, xn=50000, step=1, k=0.0001):
     cos = np.zeros(((xn - x0) * step, 1, 1))
     for i in range(len(cos)):
         idx = x0 + i * step
-        cos[i, 0, 0] = amp * np.cos(idx / (2 * np.pi * period))
+        cos[i, 0, 0] = amp * np.cos(2 * np.pi * idx / period)
         cos[i, 0, 0] = cos[i, 0, 0] * np.exp(-k * idx)
     return cos
 
@@ -55,7 +54,6 @@ model.add(LSTM(50,
                return_sequences=True,
                stateful=True))
 model.add(LSTM(50,
-               batch_input_shape=(batch_size, tsteps, 1),
                return_sequences=False,
                stateful=True))
 model.add(Dense(1))
@@ -75,7 +73,7 @@ for i in range(epochs):
 print('Predicting')
 predicted_output = model.predict(cos, batch_size=batch_size)
 
-print('Ploting Results')
+print('Plotting Results')
 plt.subplot(2, 1, 1)
 plt.plot(expected_output)
 plt.title('Expected')

examples/variational_autoencoder.py

@@ -0,0 +1,101 @@
+'''This script demonstrates how to build a variational autoencoder with Keras.
+
+Reference: "Auto-Encoding Variational Bayes" https://arxiv.org/abs/1312.6114
+'''
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.stats import norm
+
+from keras.layers import Input, Dense, Lambda
+from keras.models import Model
+from keras import backend as K
+from keras import objectives
+from keras.datasets import mnist
+
+batch_size = 100
+original_dim = 784
+latent_dim = 2
+intermediate_dim = 256
+nb_epoch = 50
+epsilon_std = 1.0
+
+x = Input(batch_shape=(batch_size, original_dim))
+h = Dense(intermediate_dim, activation='relu')(x)
+z_mean = Dense(latent_dim)(h)
+z_log_var = Dense(latent_dim)(h)
+
+
+def sampling(args):
+    z_mean, z_log_var = args
+    epsilon = K.random_normal(shape=(batch_size, latent_dim), mean=0.,
+                              std=epsilon_std)
+    return z_mean + K.exp(z_log_var / 2) * epsilon
+
+# note that "output_shape" isn't necessary with the TensorFlow backend
+z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_var])
+
+# we instantiate these layers separately so as to reuse them later
+decoder_h = Dense(intermediate_dim, activation='relu')
+decoder_mean = Dense(original_dim, activation='sigmoid')
+h_decoded = decoder_h(z)
+x_decoded_mean = decoder_mean(h_decoded)
+
+
+def vae_loss(x, x_decoded_mean):
+    xent_loss = original_dim * objectives.binary_crossentropy(x, x_decoded_mean)
+    kl_loss = - 0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
+    return xent_loss + kl_loss
+
+vae = Model(x, x_decoded_mean)
+vae.compile(optimizer='rmsprop', loss=vae_loss)
+
+# train the VAE on MNIST digits
+(x_train, y_train), (x_test, y_test) = mnist.load_data()
+
+x_train = x_train.astype('float32') / 255.
+x_test = x_test.astype('float32') / 255.
+x_train = x_train.reshape((len(x_train), np.prod(x_train.shape[1:])))
+x_test = x_test.reshape((len(x_test), np.prod(x_test.shape[1:])))
+
+vae.fit(x_train, x_train,
+        shuffle=True,
+        nb_epoch=nb_epoch,
+        batch_size=batch_size,
+        validation_data=(x_test, x_test))
+
+# build a model to project inputs on the latent space
+encoder = Model(x, z_mean)
+
+# display a 2D plot of the digit classes in the latent space
+x_test_encoded = encoder.predict(x_test, batch_size=batch_size)
+plt.figure(figsize=(6, 6))
+plt.scatter(x_test_encoded[:, 0], x_test_encoded[:, 1], c=y_test)
+plt.colorbar()
+plt.show()
+
+# build a digit generator that can sample from the learned distribution
+decoder_input = Input(shape=(latent_dim,))
+_h_decoded = decoder_h(decoder_input)
+_x_decoded_mean = decoder_mean(_h_decoded)
+generator = Model(decoder_input, _x_decoded_mean)
+
+# display a 2D manifold of the digits
+n = 15  # figure with 15x15 digits
+digit_size = 28
+figure = np.zeros((digit_size * n, digit_size * n))
+# linearly spaced coordinates on the unit square were transformed through the inverse CDF (ppf) of the Gaussian
+# to produce values of the latent variables z, since the prior of the latent space is Gaussian
+grid_x = norm.ppf(np.linspace(0.05, 0.95, n))
+grid_y = norm.ppf(np.linspace(0.05, 0.95, n))
+
+for i, yi in enumerate(grid_x):
+    for j, xi in enumerate(grid_y):
+        z_sample = np.array([[xi, yi]])
+        x_decoded = generator.predict(z_sample)
+        digit = x_decoded[0].reshape(digit_size, digit_size)
+        figure[i * digit_size: (i + 1) * digit_size,
+               j * digit_size: (j + 1) * digit_size] = digit
+
+plt.figure(figsize=(10, 10))
+plt.imshow(figure, cmap='Greys_r')
+plt.show()

examples/variational_autoencoder_deconv.py

@@ -0,0 +1,174 @@
+'''This script demonstrates how to build a variational autoencoder
+with Keras and deconvolution layers.
+
+Reference: "Auto-Encoding Variational Bayes" https://arxiv.org/abs/1312.6114
+'''
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.stats import norm
+
+from keras.layers import Input, Dense, Lambda, Flatten, Reshape
+from keras.layers import Convolution2D, Deconvolution2D
+from keras.models import Model
+from keras import backend as K
+from keras import objectives
+from keras.datasets import mnist
+
+# input image dimensions
+img_rows, img_cols, img_chns = 28, 28, 1
+# number of convolutional filters to use
+nb_filters = 64
+# convolution kernel size
+nb_conv = 3
+
+batch_size = 100
+if K.image_dim_ordering() == 'th':
+    original_img_size = (img_chns, img_rows, img_cols)
+else:
+    original_img_size = (img_rows, img_cols, img_chns)
+latent_dim = 2
+intermediate_dim = 128
+epsilon_std = 1.0
+nb_epoch = 5
+
+x = Input(batch_shape=(batch_size,) + original_img_size)
+conv_1 = Convolution2D(img_chns, 2, 2, border_mode='same', activation='relu')(x)
+conv_2 = Convolution2D(nb_filters, 2, 2,
+                       border_mode='same', activation='relu',
+                       subsample=(2, 2))(conv_1)
+conv_3 = Convolution2D(nb_filters, nb_conv, nb_conv,
+                       border_mode='same', activation='relu',
+                       subsample=(1, 1))(conv_2)
+conv_4 = Convolution2D(nb_filters, nb_conv, nb_conv,
+                       border_mode='same', activation='relu',
+                       subsample=(1, 1))(conv_3)
+flat = Flatten()(conv_4)
+hidden = Dense(intermediate_dim, activation='relu')(flat)
+
+z_mean = Dense(latent_dim)(hidden)
+z_log_var = Dense(latent_dim)(hidden)
+
+
+def sampling(args):
+    z_mean, z_log_var = args
+    epsilon = K.random_normal(shape=(batch_size, latent_dim),
+                              mean=0., std=epsilon_std)
+    return z_mean + K.exp(z_log_var) * epsilon
+
+# note that "output_shape" isn't necessary with the TensorFlow backend
+# so you could write `Lambda(sampling)([z_mean, z_log_var])`
+z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_var])
+
+# we instantiate these layers separately so as to reuse them later
+decoder_hid = Dense(intermediate_dim, activation='relu')
+decoder_upsample = Dense(nb_filters * 14 * 14, activation='relu')
+
+if K.image_dim_ordering() == 'th':
+    output_shape = (batch_size, nb_filters, 14, 14)
+else:
+    output_shape = (batch_size, 14, 14, nb_filters)
+
+decoder_reshape = Reshape(output_shape[1:])
+decoder_deconv_1 = Deconvolution2D(nb_filters, nb_conv, nb_conv,
+                                   output_shape,
+                                   border_mode='same',
+                                   subsample=(1, 1),
+                                   activation='relu')
+decoder_deconv_2 = Deconvolution2D(nb_filters, nb_conv, nb_conv,
+                                   output_shape,
+                                   border_mode='same',
+                                   subsample=(1, 1),
+                                   activation='relu')
+if K.image_dim_ordering() == 'th':
+    output_shape = (batch_size, nb_filters, 29, 29)
+else:
+    output_shape = (batch_size, 29, 29, nb_filters)
+decoder_deconv_3_upsamp = Deconvolution2D(nb_filters, 2, 2,
+                                          output_shape,
+                                          border_mode='valid',
+                                          subsample=(2, 2),
+                                          activation='relu')
+decoder_mean_squash = Convolution2D(img_chns, 2, 2,
+                                    border_mode='valid',
+                                    activation='sigmoid')
+
+hid_decoded = decoder_hid(z)
+up_decoded = decoder_upsample(hid_decoded)
+reshape_decoded = decoder_reshape(up_decoded)
+deconv_1_decoded = decoder_deconv_1(reshape_decoded)
+deconv_2_decoded = decoder_deconv_2(deconv_1_decoded)
+x_decoded_relu = decoder_deconv_3_upsamp(deconv_2_decoded)
+x_decoded_mean_squash = decoder_mean_squash(x_decoded_relu)
+
+
+def vae_loss(x, x_decoded_mean):
+    # NOTE: binary_crossentropy expects a batch_size by dim
+    # for x and x_decoded_mean, so we MUST flatten these!
+    x = K.flatten(x)
+    x_decoded_mean = K.flatten(x_decoded_mean)
+    xent_loss = img_rows * img_cols * objectives.binary_crossentropy(x, x_decoded_mean)
+    kl_loss = - 0.5 * K.mean(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
+    return xent_loss + kl_loss
+
+vae = Model(x, x_decoded_mean_squash)
+vae.compile(optimizer='rmsprop', loss=vae_loss)
+vae.summary()
+
+# train the VAE on MNIST digits
+(x_train, _), (x_test, y_test) = mnist.load_data()
+
+x_train = x_train.astype('float32') / 255.
+x_train = x_train.reshape((x_train.shape[0],) + original_img_size)
+x_test = x_test.astype('float32') / 255.
+x_test = x_test.reshape((x_test.shape[0],) + original_img_size)
+
+print('x_train.shape:', x_train.shape)
+
+vae.fit(x_train, x_train,
+        shuffle=True,
+        nb_epoch=nb_epoch,
+        batch_size=batch_size,
+        validation_data=(x_test, x_test))
+
+# build a model to project inputs on the latent space
+encoder = Model(x, z_mean)
+
+# display a 2D plot of the digit classes in the latent space
+x_test_encoded = encoder.predict(x_test, batch_size=batch_size)
+plt.figure(figsize=(6, 6))
+plt.scatter(x_test_encoded[:, 0], x_test_encoded[:, 1], c=y_test)
+plt.colorbar()
+plt.show()
+
+# build a digit generator that can sample from the learned distribution
+decoder_input = Input(shape=(latent_dim,))
+_hid_decoded = decoder_hid(decoder_input)
+_up_decoded = decoder_upsample(_hid_decoded)
+_reshape_decoded = decoder_reshape(_up_decoded)
+_deconv_1_decoded = decoder_deconv_1(_reshape_decoded)
+_deconv_2_decoded = decoder_deconv_2(_deconv_1_decoded)
+_x_decoded_relu = decoder_deconv_3_upsamp(_deconv_2_decoded)
+_x_decoded_mean_squash = decoder_mean_squash(_x_decoded_relu)
+generator = Model(decoder_input, _x_decoded_mean_squash)
+
+# display a 2D manifold of the digits
+n = 15  # figure with 15x15 digits
+digit_size = 28
+figure = np.zeros((digit_size * n, digit_size * n))
+# linearly spaced coordinates on the unit square were transformed through the inverse CDF (ppf) of the Gaussian
+# to produce values of the latent variables z, since the prior of the latent space is Gaussian
+grid_x = norm.ppf(np.linspace(0.05, 0.95, n))
+grid_y = norm.ppf(np.linspace(0.05, 0.95, n))
+
+for i, yi in enumerate(grid_x):
+    for j, xi in enumerate(grid_y):
+        z_sample = np.array([[xi, yi]])
+        z_sample = np.tile(z_sample, batch_size).reshape(batch_size, 2)
+        x_decoded = generator.predict(z_sample, batch_size=batch_size)
+        digit = x_decoded[0].reshape(digit_size, digit_size)
+        figure[i * digit_size: (i + 1) * digit_size,
+               j * digit_size: (j + 1) * digit_size] = digit
+
+plt.figure(figsize=(10, 10))
+plt.imshow(figure, cmap='Greys_r')
+plt.show()

keras/__init__.py

@@ -1 +1,18 @@
-__version__ = '1.0.0'
+from __future__ import absolute_import
+from . import backend
+from . import datasets
+from . import engine
+from . import layers
+from . import preprocessing
+from . import utils
+from . import wrappers
+from . import callbacks
+from . import constraints
+from . import initializations
+from . import metrics
+from . import models
+from . import objectives
+from . import optimizers
+from . import regularizers
+
+__version__ = '1.2.2'

keras/activations.py

@@ -1,5 +1,6 @@
 from __future__ import absolute_import
 from . import backend as K
+from .utils.generic_utils import get_from_module
 
 
 def softmax(x):
@@ -11,14 +12,23 @@ def softmax(x):
         s = K.sum(e, axis=-1, keepdims=True)
         return e / s
     else:
-        raise Exception('Cannot apply softmax to a tensor that is not 2D or 3D. ' +
-                        'Here, ndim=' + str(ndim))
+        raise ValueError('Cannot apply softmax to a tensor '
+                         'that is not 2D or 3D. '
+                         'Here, ndim=' + str(ndim))
+
+
+def elu(x, alpha=1.0):
+    return K.elu(x, alpha)
 
 
 def softplus(x):
     return K.softplus(x)
 
 
+def softsign(x):
+    return K.softsign(x)
+
+
 def relu(x, alpha=0., max_value=None):
     return K.relu(x, alpha=alpha, max_value=max_value)
 
@@ -36,12 +46,10 @@ def hard_sigmoid(x):
 
 
 def linear(x):
-    '''
-    The function returns the variable that is passed in, so all types work.
-    '''
     return x
 
 
-from .utils.generic_utils import get_from_module
 def get(identifier):
+    if identifier is None:
+        return linear
     return get_from_module(identifier, globals(), 'activation function')

keras/applications/__init__.py

@@ -0,0 +1,5 @@
+from .vgg16 import VGG16
+from .vgg19 import VGG19
+from .resnet50 import ResNet50
+from .inception_v3 import InceptionV3
+from .xception import Xception

keras/applications/audio_conv_utils.py

@@ -0,0 +1,92 @@
+import numpy as np
+from .. import backend as K
+
+try:
+    import librosa
+except ImportError:
+    librosa = None
+
+
+TAGS = ['rock', 'pop', 'alternative', 'indie', 'electronic',
+        'female vocalists', 'dance', '00s', 'alternative rock', 'jazz',
+        'beautiful', 'metal', 'chillout', 'male vocalists',
+        'classic rock', 'soul', 'indie rock', 'Mellow', 'electronica',
+        '80s', 'folk', '90s', 'chill', 'instrumental', 'punk',
+        'oldies', 'blues', 'hard rock', 'ambient', 'acoustic',
+        'experimental', 'female vocalist', 'guitar', 'Hip-Hop',
+        '70s', 'party', 'country', 'easy listening',
+        'sexy', 'catchy', 'funk', 'electro', 'heavy metal',
+        'Progressive rock', '60s', 'rnb', 'indie pop',
+        'sad', 'House', 'happy']
+
+
+def preprocess_input(audio_path, dim_ordering='default'):
+    """Reads an audio file and outputs a Mel-spectrogram.
+
+    # Arguments
+        audio_path: path to the target audio file.
+        dim_ordering: data format for the output spectrogram image.
+
+    # Returns
+        3D Numpy tensor encoding the Mel-spectrogram.
+
+    # Raises
+        ImportError: if librosa is not available.
+    """
+    if dim_ordering == 'default':
+        dim_ordering = K.image_dim_ordering()
+    assert dim_ordering in {'tf', 'th'}
+
+    if librosa is None:
+        raise ImportError('Librosa is required to process audio files. '
+                          'Install it via `pip install librosa` or visit '
+                          'http://librosa.github.io/librosa/ for details.')
+
+    # mel-spectrogram parameters
+    sr = 12000
+    n_fft = 512
+    n_mels = 96
+    hop_length = 256
+    duration = 29.12
+
+    src, sr = librosa.load(audio_path, sr=sr)
+    n_sample = src.shape[0]
+    n_sample_wanted = int(duration * sr)
+
+    # trim the signal at the center
+    if n_sample < n_sample_wanted:  # if too short
+        src = np.hstack((src, np.zeros((int(duration * sr) - n_sample,))))
+    elif n_sample > n_sample_wanted:  # if too long
+        src = src[(n_sample - n_sample_wanted) // 2:
+                  (n_sample + n_sample_wanted) // 2]
+
+    logam = librosa.logamplitude
+    melgram = librosa.feature.melspectrogram
+    x = logam(melgram(y=src, sr=sr, hop_length=hop_length,
+                      n_fft=n_fft, n_mels=n_mels) ** 2,
+              ref_power=1.0)
+
+    if dim_ordering == 'th':
+        x = np.expand_dims(x, axis=0)
+    elif dim_ordering == 'tf':
+        x = np.expand_dims(x, axis=3)
+    return x
+
+
+def decode_predictions(preds, top_n=5):
+    """Decode the output of a music tagger model.
+
+    # Arguments
+        preds: 2-dimensional numpy array
+        top_n: integer in [0, 50], number of items to show.
+
+    # Returns
+        Decoded output.
+    """
+    assert len(preds.shape) == 2 and preds.shape[1] == 50
+    results = []
+    for pred in preds:
+        result = zip(TAGS, pred)
+        result = sorted(result, key=lambda x: x[1], reverse=True)
+        results.append(result[:top_n])
+    return results

keras/applications/imagenet_utils.py

@@ -0,0 +1,139 @@
+import numpy as np
+import json
+
+from ..utils.data_utils import get_file
+from .. import backend as K
+
+CLASS_INDEX = None
+CLASS_INDEX_PATH = 'https://s3.amazonaws.com/deep-learning-models/image-models/imagenet_class_index.json'
+
+
+def preprocess_input(x, dim_ordering='default'):
+    """Preprocesses a tensor encoding a batch of images.
+
+    # Arguments
+        x: input Numpy tensor, 4D.
+        dim_ordering: data format of the image tensor.
+
+    # Returns
+        Preprocessed tensor.
+    """
+    if dim_ordering == 'default':
+        dim_ordering = K.image_dim_ordering()
+    assert dim_ordering in {'tf', 'th'}
+
+    if dim_ordering == 'th':
+        # 'RGB'->'BGR'
+        x = x[:, ::-1, :, :]
+        # Zero-center by mean pixel
+        x[:, 0, :, :] -= 103.939
+        x[:, 1, :, :] -= 116.779
+        x[:, 2, :, :] -= 123.68
+    else:
+        # 'RGB'->'BGR'
+        x = x[:, :, :, ::-1]
+        # Zero-center by mean pixel
+        x[:, :, :, 0] -= 103.939
+        x[:, :, :, 1] -= 116.779
+        x[:, :, :, 2] -= 123.68
+    return x
+
+
+def decode_predictions(preds, top=5):
+    """Decodes the prediction of an ImageNet model.
+
+    # Arguments
+        preds: Numpy tensor encoding a batch of predictions.
+        top: integer, how many top-guesses to return.
+
+    # Returns
+        A list of lists of top class prediction tuples
+        `(class_name, class_description, score)`.
+        One list of tuples per sample in batch input.
+
+    # Raises
+        ValueError: in case of invalid shape of the `pred` array
+            (must be 2D).
+    """
+    global CLASS_INDEX
+    if len(preds.shape) != 2 or preds.shape[1] != 1000:
+        raise ValueError('`decode_predictions` expects '
+                         'a batch of predictions '
+                         '(i.e. a 2D array of shape (samples, 1000)). '
+                         'Found array with shape: ' + str(preds.shape))
+    if CLASS_INDEX is None:
+        fpath = get_file('imagenet_class_index.json',
+                         CLASS_INDEX_PATH,
+                         cache_subdir='models')
+        CLASS_INDEX = json.load(open(fpath))
+    results = []
+    for pred in preds:
+        top_indices = pred.argsort()[-top:][::-1]
+        result = [tuple(CLASS_INDEX[str(i)]) + (pred[i],) for i in top_indices]
+        result.sort(key=lambda x: x[2], reverse=True)
+        results.append(result)
+    return results
+
+
+def _obtain_input_shape(input_shape,
+                        default_size,
+                        min_size,
+                        dim_ordering,
+                        include_top):
+    """Internal utility to compute/validate an ImageNet model's input shape.
+
+    # Arguments
+        input_shape: either None (will return the default network input shape),
+            or a user-provided shape to be validated.
+        default_size: default input width/height for the model.
+        min_size: minimum input width/height accepted by the model.
+        dim_ordering: image data format to use.
+        include_top: whether the model is expected to
+            be linked to a classifier via a Flatten layer.
+
+    # Returns
+        An integer shape tuple (may include None entries).
+
+    # Raises
+        ValueError: in case of invalid argument values.
+    """
+    if dim_ordering == 'th':
+        default_shape = (3, default_size, default_size)
+    else:
+        default_shape = (default_size, default_size, 3)
+    if include_top:
+        if input_shape is not None:
+            if input_shape != default_shape:
+                raise ValueError('When setting`include_top=True`, '
+                                 '`input_shape` should be ' + str(default_shape) + '.')
+        input_shape = default_shape
+    else:
+        if dim_ordering == 'th':
+            if input_shape is not None:
+                if len(input_shape) != 3:
+                    raise ValueError('`input_shape` must be a tuple of three integers.')
+                if input_shape[0] != 3:
+                    raise ValueError('The input must have 3 channels; got '
+                                     '`input_shape=' + str(input_shape) + '`')
+                if ((input_shape[1] is not None and input_shape[1] < min_size) or
+                   (input_shape[2] is not None and input_shape[2] < min_size)):
+                    raise ValueError('Input size must be at least ' +
+                                     str(min_size) + 'x' + str(min_size) + ', got '
+                                     '`input_shape=' + str(input_shape) + '`')
+            else:
+                input_shape = (3, None, None)
+        else:
+            if input_shape is not None:
+                if len(input_shape) != 3:
+                    raise ValueError('`input_shape` must be a tuple of three integers.')
+                if input_shape[-1] != 3:
+                    raise ValueError('The input must have 3 channels; got '
+                                     '`input_shape=' + str(input_shape) + '`')
+                if ((input_shape[0] is not None and input_shape[0] < min_size) or
+                   (input_shape[1] is not None and input_shape[1] < min_size)):
+                    raise ValueError('Input size must be at least ' +
+                                     str(min_size) + 'x' + str(min_size) + ', got '
+                                     '`input_shape=' + str(input_shape) + '`')
+            else:
+                input_shape = (None, None, 3)
+    return input_shape

keras/applications/inception_v3.py

@@ -0,0 +1,330 @@
+# -*- coding: utf-8 -*-
+"""Inception V3 model for Keras.
+
+Note that the ImageNet weights provided are from a model that had not fully converged.
+Inception v3 should be able to reach 6.9% top-5 error, but our model
+only gets to 7.8% (same as a fully-converged ResNet 50).
+For comparison, VGG16 only gets to 9.9%, quite a bit worse.
+
+Also, do note that the input image format for this model is different than for
+the VGG16 and ResNet models (299x299 instead of 224x224), and that the input preprocessing function
+is also different (same as Xception).
+
+# Reference
+
+- [Rethinking the Inception Architecture for Computer Vision](http://arxiv.org/abs/1512.00567)
+
+"""
+from __future__ import print_function
+from __future__ import absolute_import
+
+import warnings
+
+from ..models import Model
+from ..layers import Flatten, Dense, Input, BatchNormalization, merge
+from ..layers import Convolution2D, MaxPooling2D, AveragePooling2D
+from ..engine.topology import get_source_inputs
+from ..utils.layer_utils import convert_all_kernels_in_model
+from ..utils.data_utils import get_file
+from .. import backend as K
+from .imagenet_utils import decode_predictions, _obtain_input_shape
+
+
+TH_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.2/inception_v3_weights_th_dim_ordering_th_kernels.h5'
+TF_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.2/inception_v3_weights_tf_dim_ordering_tf_kernels.h5'
+TH_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.2/inception_v3_weights_th_dim_ordering_th_kernels_notop.h5'
+TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.2/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5'
+
+
+def conv2d_bn(x, nb_filter, nb_row, nb_col,
+              border_mode='same', subsample=(1, 1),
+              name=None):
+    """Utility function to apply conv + BN.
+    """
+    if name is not None:
+        bn_name = name + '_bn'
+        conv_name = name + '_conv'
+    else:
+        bn_name = None
+        conv_name = None
+    if K.image_dim_ordering() == 'th':
+        bn_axis = 1
+    else:
+        bn_axis = 3
+    x = Convolution2D(nb_filter, nb_row, nb_col,
+                      subsample=subsample,
+                      activation='relu',
+                      border_mode=border_mode,
+                      name=conv_name)(x)
+    x = BatchNormalization(axis=bn_axis, name=bn_name)(x)
+    return x
+
+
+def InceptionV3(include_top=True, weights='imagenet',
+                input_tensor=None, input_shape=None,
+                classes=1000):
+    """Instantiate the Inception v3 architecture,
+    optionally loading weights pre-trained
+    on ImageNet. Note that when using TensorFlow,
+    for best performance you should set
+    `image_dim_ordering="tf"` in your Keras config
+    at ~/.keras/keras.json.
+
+    The model and the weights are compatible with both
+    TensorFlow and Theano. The dimension ordering
+    convention used by the model is the one
+    specified in your Keras config file.
+
+    Note that the default input image size for this model is 299x299.
+
+    # Arguments
+        include_top: whether to include the fully-connected
+            layer at the top of the network.
+        weights: one of `None` (random initialization)
+            or "imagenet" (pre-training on ImageNet).
+        input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
+            to use as image input for the model.
+        input_shape: optional shape tuple, only to be specified
+            if `include_top` is False (otherwise the input shape
+            has to be `(299, 299, 3)` (with `tf` dim ordering)
+            or `(3, 299, 299)` (with `th` dim ordering).
+            It should have exactly 3 inputs channels,
+            and width and height should be no smaller than 139.
+            E.g. `(150, 150, 3)` would be one valid value.
+        classes: optional number of classes to classify images
+            into, only to be specified if `include_top` is True, and
+            if no `weights` argument is specified.
+
+    # Returns
+        A Keras model instance.
+    """
+    if weights not in {'imagenet', None}:
+        raise ValueError('The `weights` argument should be either '
+                         '`None` (random initialization) or `imagenet` '
+                         '(pre-training on ImageNet).')
+
+    if weights == 'imagenet' and include_top and classes != 1000:
+        raise ValueError('If using `weights` as imagenet with `include_top`'
+                         ' as true, `classes` should be 1000')
+
+    # Determine proper input shape
+    input_shape = _obtain_input_shape(input_shape,
+                                      default_size=299,
+                                      min_size=139,
+                                      dim_ordering=K.image_dim_ordering(),
+                                      include_top=include_top)
+
+    if input_tensor is None:
+        img_input = Input(shape=input_shape)
+    else:
+        if not K.is_keras_tensor(input_tensor):
+            img_input = Input(tensor=input_tensor, shape=input_shape)
+        else:
+            img_input = input_tensor
+
+    if K.image_dim_ordering() == 'th':
+        channel_axis = 1
+    else:
+        channel_axis = 3
+
+    x = conv2d_bn(img_input, 32, 3, 3, subsample=(2, 2), border_mode='valid')
+    x = conv2d_bn(x, 32, 3, 3, border_mode='valid')
+    x = conv2d_bn(x, 64, 3, 3)
+    x = MaxPooling2D((3, 3), strides=(2, 2))(x)
+
+    x = conv2d_bn(x, 80, 1, 1, border_mode='valid')
+    x = conv2d_bn(x, 192, 3, 3, border_mode='valid')
+    x = MaxPooling2D((3, 3), strides=(2, 2))(x)
+
+    # mixed 0, 1, 2: 35 x 35 x 256
+    for i in range(3):
+        branch1x1 = conv2d_bn(x, 64, 1, 1)
+
+        branch5x5 = conv2d_bn(x, 48, 1, 1)
+        branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)
+
+        branch3x3dbl = conv2d_bn(x, 64, 1, 1)
+        branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
+        branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
+
+        branch_pool = AveragePooling2D(
+            (3, 3), strides=(1, 1), border_mode='same')(x)
+        branch_pool = conv2d_bn(branch_pool, 32, 1, 1)
+        x = merge([branch1x1, branch5x5, branch3x3dbl, branch_pool],
+                  mode='concat', concat_axis=channel_axis,
+                  name='mixed' + str(i))
+
+    # mixed 3: 17 x 17 x 768
+    branch3x3 = conv2d_bn(x, 384, 3, 3, subsample=(2, 2), border_mode='valid')
+
+    branch3x3dbl = conv2d_bn(x, 64, 1, 1)
+    branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)
+    branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3,
+                             subsample=(2, 2), border_mode='valid')
+
+    branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x)
+    x = merge([branch3x3, branch3x3dbl, branch_pool],
+              mode='concat', concat_axis=channel_axis,
+              name='mixed3')
+
+    # mixed 4: 17 x 17 x 768
+    branch1x1 = conv2d_bn(x, 192, 1, 1)
+
+    branch7x7 = conv2d_bn(x, 128, 1, 1)
+    branch7x7 = conv2d_bn(branch7x7, 128, 1, 7)
+    branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
+
+    branch7x7dbl = conv2d_bn(x, 128, 1, 1)
+    branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
+    branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 1, 7)
+    branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)
+    branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
+
+    branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same')(x)
+    branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
+    x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool],
+              mode='concat', concat_axis=channel_axis,
+              name='mixed4')
+
+    # mixed 5, 6: 17 x 17 x 768
+    for i in range(2):
+        branch1x1 = conv2d_bn(x, 192, 1, 1)
+
+        branch7x7 = conv2d_bn(x, 160, 1, 1)
+        branch7x7 = conv2d_bn(branch7x7, 160, 1, 7)
+        branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
+
+        branch7x7dbl = conv2d_bn(x, 160, 1, 1)
+        branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
+        branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 1, 7)
+        branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)
+        branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
+
+        branch_pool = AveragePooling2D(
+            (3, 3), strides=(1, 1), border_mode='same')(x)
+        branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
+        x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool],
+                  mode='concat', concat_axis=channel_axis,
+                  name='mixed' + str(5 + i))
+
+    # mixed 7: 17 x 17 x 768
+    branch1x1 = conv2d_bn(x, 192, 1, 1)
+
+    branch7x7 = conv2d_bn(x, 192, 1, 1)
+    branch7x7 = conv2d_bn(branch7x7, 192, 1, 7)
+    branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)
+
+    branch7x7dbl = conv2d_bn(x, 160, 1, 1)
+    branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
+    branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
+    branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)
+    branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)
+
+    branch_pool = AveragePooling2D((3, 3), strides=(1, 1), border_mode='same')(x)
+    branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
+    x = merge([branch1x1, branch7x7, branch7x7dbl, branch_pool],
+              mode='concat', concat_axis=channel_axis,
+              name='mixed7')
+
+    # mixed 8: 8 x 8 x 1280
+    branch3x3 = conv2d_bn(x, 192, 1, 1)
+    branch3x3 = conv2d_bn(branch3x3, 320, 3, 3,
+                          subsample=(2, 2), border_mode='valid')
+
+    branch7x7x3 = conv2d_bn(x, 192, 1, 1)
+    branch7x7x3 = conv2d_bn(branch7x7x3, 192, 1, 7)
+    branch7x7x3 = conv2d_bn(branch7x7x3, 192, 7, 1)
+    branch7x7x3 = conv2d_bn(branch7x7x3, 192, 3, 3,
+                            subsample=(2, 2), border_mode='valid')
+
+    branch_pool = AveragePooling2D((3, 3), strides=(2, 2))(x)
+    x = merge([branch3x3, branch7x7x3, branch_pool],
+              mode='concat', concat_axis=channel_axis,
+              name='mixed8')
+
+    # mixed 9: 8 x 8 x 2048
+    for i in range(2):
+        branch1x1 = conv2d_bn(x, 320, 1, 1)
+
+        branch3x3 = conv2d_bn(x, 384, 1, 1)
+        branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3)
+        branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1)
+        branch3x3 = merge([branch3x3_1, branch3x3_2],
+                          mode='concat', concat_axis=channel_axis,
+                          name='mixed9_' + str(i))
+
+        branch3x3dbl = conv2d_bn(x, 448, 1, 1)
+        branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3)
+        branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3)
+        branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1)
+        branch3x3dbl = merge([branch3x3dbl_1, branch3x3dbl_2],
+                             mode='concat', concat_axis=channel_axis)
+
+        branch_pool = AveragePooling2D(
+            (3, 3), strides=(1, 1), border_mode='same')(x)
+        branch_pool = conv2d_bn(branch_pool, 192, 1, 1)
+        x = merge([branch1x1, branch3x3, branch3x3dbl, branch_pool],
+                  mode='concat', concat_axis=channel_axis,
+                  name='mixed' + str(9 + i))
+
+    if include_top:
+        # Classification block
+        x = AveragePooling2D((8, 8), strides=(8, 8), name='avg_pool')(x)
+        x = Flatten(name='flatten')(x)
+        x = Dense(classes, activation='softmax', name='predictions')(x)
+
+    # Ensure that the model takes into account
+    # any potential predecessors of `input_tensor`.
+    if input_tensor is not None:
+        inputs = get_source_inputs(input_tensor)
+    else:
+        inputs = img_input
+    # Create model.
+    model = Model(inputs, x, name='inception_v3')
+
+    # load weights
+    if weights == 'imagenet':
+        if K.image_dim_ordering() == 'th':
+            if include_top:
+                weights_path = get_file('inception_v3_weights_th_dim_ordering_th_kernels.h5',
+                                        TH_WEIGHTS_PATH,
+                                        cache_subdir='models',
+                                        md5_hash='b3baf3070cc4bf476d43a2ea61b0ca5f')
+            else:
+                weights_path = get_file('inception_v3_weights_th_dim_ordering_th_kernels_notop.h5',
+                                        TH_WEIGHTS_PATH_NO_TOP,
+                                        cache_subdir='models',
+                                        md5_hash='79aaa90ab4372b4593ba3df64e142f05')
+            model.load_weights(weights_path)
+            if K.backend() == 'tensorflow':
+                warnings.warn('You are using the TensorFlow backend, yet you '
+                              'are using the Theano '
+                              'image dimension ordering convention '
+                              '(`image_dim_ordering="th"`). '
+                              'For best performance, set '
+                              '`image_dim_ordering="tf"` in '
+                              'your Keras config '
+                              'at ~/.keras/keras.json.')
+                convert_all_kernels_in_model(model)
+        else:
+            if include_top:
+                weights_path = get_file('inception_v3_weights_tf_dim_ordering_tf_kernels.h5',
+                                        TF_WEIGHTS_PATH,
+                                        cache_subdir='models',
+                                        md5_hash='fe114b3ff2ea4bf891e9353d1bbfb32f')
+            else:
+                weights_path = get_file('inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5',
+                                        TF_WEIGHTS_PATH_NO_TOP,
+                                        cache_subdir='models',
+                                        md5_hash='2f3609166de1d967d1a481094754f691')
+            model.load_weights(weights_path)
+            if K.backend() == 'theano':
+                convert_all_kernels_in_model(model)
+    return model
+
+
+def preprocess_input(x):
+    x /= 255.
+    x -= 0.5
+    x *= 2.
+    return x

keras/applications/music_tagger_crnn.py

@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+"""MusicTaggerCRNN model for Keras.
+
+# Reference:
+
+- [Music-auto_tagging-keras](https://github.com/keunwoochoi/music-auto_tagging-keras)
+
+"""
+from __future__ import print_function
+from __future__ import absolute_import
+
+from .. import backend as K
+from ..layers import Input, Dense
+from ..models import Model
+from ..layers import Dense, Dropout, Reshape, Permute
+from ..layers.convolutional import Convolution2D
+from ..layers.convolutional import MaxPooling2D, ZeroPadding2D
+from ..layers.normalization import BatchNormalization
+from ..layers.advanced_activations import ELU
+from ..layers.recurrent import GRU
+from ..engine.topology import get_source_inputs
+from ..utils.data_utils import get_file
+from ..utils.layer_utils import convert_all_kernels_in_model
+from .audio_conv_utils import decode_predictions, preprocess_input
+
+TH_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.3/music_tagger_crnn_weights_tf_kernels_th_dim_ordering.h5'
+TF_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.3/music_tagger_crnn_weights_tf_kernels_tf_dim_ordering.h5'
+
+
+def MusicTaggerCRNN(weights='msd', input_tensor=None,
+                    include_top=True, classes=50):
+    """Instantiate the MusicTaggerCRNN architecture,
+    optionally loading weights pre-trained
+    on Million Song Dataset. Note that when using TensorFlow,
+    for best performance you should set
+    `image_dim_ordering="tf"` in your Keras config
+    at ~/.keras/keras.json.
+
+    The model and the weights are compatible with both
+    TensorFlow and Theano. The dimension ordering
+    convention used by the model is the one
+    specified in your Keras config file.
+
+    For preparing mel-spectrogram input, see
+    `audio_conv_utils.py` in [applications](https://github.com/fchollet/keras/tree/master/keras/applications).
+    You will need to install [Librosa](http://librosa.github.io/librosa/)
+    to use it.
+
+    # Arguments
+        weights: one of `None` (random initialization)
+            or "msd" (pre-training on ImageNet).
+        input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
+            to use as image input for the model.
+        include_top: whether to include the 1 fully-connected
+            layer (output layer) at the top of the network.
+            If False, the network outputs 32-dim features.
+        classes: optional number of classes to classify images
+            into, only to be specified if `include_top` is True, and
+            if no `weights` argument is specified.
+
+    # Returns
+        A Keras model instance.
+    """
+    if weights not in {'msd', None}:
+        raise ValueError('The `weights` argument should be either '
+                         '`None` (random initialization) or `msd` '
+                         '(pre-training on Million Song Dataset).')
+
+    if weights == 'msd' and include_top and classes != 50:
+        raise ValueError('If using `weights` as msd with `include_top`'
+                         ' as true, `classes` should be 50')
+    # Determine proper input shape
+    if K.image_dim_ordering() == 'th':
+        input_shape = (1, 96, 1366)
+    else:
+        input_shape = (96, 1366, 1)
+
+    if input_tensor is None:
+        melgram_input = Input(shape=input_shape)
+    else:
+        if not K.is_keras_tensor(input_tensor):
+            melgram_input = Input(tensor=input_tensor, shape=input_shape)
+        else:
+            melgram_input = input_tensor
+
+    # Determine input axis
+    if K.image_dim_ordering() == 'th':
+        channel_axis = 1
+        freq_axis = 2
+        time_axis = 3
+    else:
+        channel_axis = 3
+        freq_axis = 1
+        time_axis = 2
+
+    # Input block
+    x = ZeroPadding2D(padding=(0, 37))(melgram_input)
+    x = BatchNormalization(axis=time_axis, name='bn_0_freq')(x)
+
+    # Conv block 1
+    x = Convolution2D(64, 3, 3, border_mode='same', name='conv1')(x)
+    x = BatchNormalization(axis=channel_axis, mode=0, name='bn1')(x)
+    x = ELU()(x)
+    x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool1')(x)
+
+    # Conv block 2
+    x = Convolution2D(128, 3, 3, border_mode='same', name='conv2')(x)
+    x = BatchNormalization(axis=channel_axis, mode=0, name='bn2')(x)
+    x = ELU()(x)
+    x = MaxPooling2D(pool_size=(3, 3), strides=(3, 3), name='pool2')(x)
+
+    # Conv block 3
+    x = Convolution2D(128, 3, 3, border_mode='same', name='conv3')(x)
+    x = BatchNormalization(axis=channel_axis, mode=0, name='bn3')(x)
+    x = ELU()(x)
+    x = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool3')(x)
+
+    # Conv block 4
+    x = Convolution2D(128, 3, 3, border_mode='same', name='conv4')(x)
+    x = BatchNormalization(axis=channel_axis, mode=0, name='bn4')(x)
+    x = ELU()(x)
+    x = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool4')(x)
+
+    # reshaping
+    if K.image_dim_ordering() == 'th':
+        x = Permute((3, 1, 2))(x)
+    x = Reshape((15, 128))(x)
+
+    # GRU block 1, 2, output
+    x = GRU(32, return_sequences=True, name='gru1')(x)
+    x = GRU(32, return_sequences=False, name='gru2')(x)
+
+    if include_top:
+        x = Dense(classes, activation='sigmoid', name='output')(x)
+
+    # Ensure that the model takes into account
+    # any potential predecessors of `input_tensor`.
+    if input_tensor is not None:
+        inputs = get_source_inputs(input_tensor)
+    else:
+        inputs = melgram_input
+    # Create model.
+    model = Model(inputs, x, name='music_tagger_crnn')
+
+    if weights is None:
+        return model
+    else:
+        # Load weights
+        if K.image_dim_ordering() == 'tf':
+            weights_path = get_file('music_tagger_crnn_weights_tf_kernels_tf_dim_ordering.h5',
+                                    TF_WEIGHTS_PATH,
+                                    cache_subdir='models')
+        else:
+            weights_path = get_file('music_tagger_crnn_weights_tf_kernels_th_dim_ordering.h5',
+                                    TH_WEIGHTS_PATH,
+                                    cache_subdir='models')
+        model.load_weights(weights_path, by_name=True)
+        if K.backend() == 'theano':
+            convert_all_kernels_in_model(model)
+        return model

keras/applications/resnet50.py

@@ -0,0 +1,254 @@
+# -*- coding: utf-8 -*-
+"""ResNet50 model for Keras.
+
+# Reference:
+
+- [Deep Residual Learning for Image Recognition](https://arxiv.org/abs/1512.03385)
+
+Adapted from code contributed by BigMoyan.
+"""
+from __future__ import print_function
+from __future__ import absolute_import
+
+import warnings
+
+from ..layers import merge, Input
+from ..layers import Dense, Activation, Flatten
+from ..layers import Convolution2D, MaxPooling2D, ZeroPadding2D, AveragePooling2D
+from ..layers import BatchNormalization
+from ..models import Model
+from .. import backend as K
+from ..engine.topology import get_source_inputs
+from ..utils.layer_utils import convert_all_kernels_in_model
+from ..utils.data_utils import get_file
+from .imagenet_utils import decode_predictions, preprocess_input, _obtain_input_shape
+
+
+TH_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.2/resnet50_weights_th_dim_ordering_th_kernels.h5'
+TF_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.2/resnet50_weights_tf_dim_ordering_tf_kernels.h5'
+TH_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.2/resnet50_weights_th_dim_ordering_th_kernels_notop.h5'
+TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.2/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5'
+
+
+def identity_block(input_tensor, kernel_size, filters, stage, block):
+    """The identity_block is the block that has no conv layer at shortcut
+
+    # Arguments
+        input_tensor: input tensor
+        kernel_size: defualt 3, the kernel size of middle conv layer at main path
+        filters: list of integers, the nb_filters of 3 conv layer at main path
+        stage: integer, current stage label, used for generating layer names
+        block: 'a','b'..., current block label, used for generating layer names
+    """
+    nb_filter1, nb_filter2, nb_filter3 = filters
+    if K.image_dim_ordering() == 'tf':
+        bn_axis = 3
+    else:
+        bn_axis = 1
+    conv_name_base = 'res' + str(stage) + block + '_branch'
+    bn_name_base = 'bn' + str(stage) + block + '_branch'
+
+    x = Convolution2D(nb_filter1, 1, 1, name=conv_name_base + '2a')(input_tensor)
+    x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
+    x = Activation('relu')(x)
+
+    x = Convolution2D(nb_filter2, kernel_size, kernel_size,
+                      border_mode='same', name=conv_name_base + '2b')(x)
+    x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)
+    x = Activation('relu')(x)
+
+    x = Convolution2D(nb_filter3, 1, 1, name=conv_name_base + '2c')(x)
+    x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)
+
+    x = merge([x, input_tensor], mode='sum')
+    x = Activation('relu')(x)
+    return x
+
+
+def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)):
+    """conv_block is the block that has a conv layer at shortcut
+
+    # Arguments
+        input_tensor: input tensor
+        kernel_size: defualt 3, the kernel size of middle conv layer at main path
+        filters: list of integers, the nb_filters of 3 conv layer at main path
+        stage: integer, current stage label, used for generating layer names
+        block: 'a','b'..., current block label, used for generating layer names
+
+    Note that from stage 3, the first conv layer at main path is with subsample=(2,2)
+    And the shortcut should have subsample=(2,2) as well
+    """
+    nb_filter1, nb_filter2, nb_filter3 = filters
+    if K.image_dim_ordering() == 'tf':
+        bn_axis = 3
+    else:
+        bn_axis = 1
+    conv_name_base = 'res' + str(stage) + block + '_branch'
+    bn_name_base = 'bn' + str(stage) + block + '_branch'
+
+    x = Convolution2D(nb_filter1, 1, 1, subsample=strides,
+                      name=conv_name_base + '2a')(input_tensor)
+    x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x)
+    x = Activation('relu')(x)
+
+    x = Convolution2D(nb_filter2, kernel_size, kernel_size, border_mode='same',
+                      name=conv_name_base + '2b')(x)
+    x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x)
+    x = Activation('relu')(x)
+
+    x = Convolution2D(nb_filter3, 1, 1, name=conv_name_base + '2c')(x)
+    x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x)
+
+    shortcut = Convolution2D(nb_filter3, 1, 1, subsample=strides,
+                             name=conv_name_base + '1')(input_tensor)
+    shortcut = BatchNormalization(axis=bn_axis, name=bn_name_base + '1')(shortcut)
+
+    x = merge([x, shortcut], mode='sum')
+    x = Activation('relu')(x)
+    return x
+
+
+def ResNet50(include_top=True, weights='imagenet',
+             input_tensor=None, input_shape=None,
+             classes=1000):
+    """Instantiate the ResNet50 architecture,
+    optionally loading weights pre-trained
+    on ImageNet. Note that when using TensorFlow,
+    for best performance you should set
+    `image_dim_ordering="tf"` in your Keras config
+    at ~/.keras/keras.json.
+
+    The model and the weights are compatible with both
+    TensorFlow and Theano. The dimension ordering
+    convention used by the model is the one
+    specified in your Keras config file.
+
+    # Arguments
+        include_top: whether to include the 3 fully-connected
+            layers at the top of the network.
+        weights: one of `None` (random initialization)
+            or "imagenet" (pre-training on ImageNet).
+        input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
+            to use as image input for the model.
+        input_shape: optional shape tuple, only to be specified
+            if `include_top` is False (otherwise the input shape
+            has to be `(224, 224, 3)` (with `tf` dim ordering)
+            or `(3, 224, 244)` (with `th` dim ordering).
+            It should have exactly 3 inputs channels,
+            and width and height should be no smaller than 197.
+            E.g. `(200, 200, 3)` would be one valid value.
+        classes: optional number of classes to classify images
+            into, only to be specified if `include_top` is True, and
+            if no `weights` argument is specified.
+
+    # Returns
+        A Keras model instance.
+    """
+    if weights not in {'imagenet', None}:
+        raise ValueError('The `weights` argument should be either '
+                         '`None` (random initialization) or `imagenet` '
+                         '(pre-training on ImageNet).')
+
+    if weights == 'imagenet' and include_top and classes != 1000:
+        raise ValueError('If using `weights` as imagenet with `include_top`'
+                         ' as true, `classes` should be 1000')
+
+    # Determine proper input shape
+    input_shape = _obtain_input_shape(input_shape,
+                                      default_size=224,
+                                      min_size=197,
+                                      dim_ordering=K.image_dim_ordering(),
+                                      include_top=include_top)
+
+    if input_tensor is None:
+        img_input = Input(shape=input_shape)
+    else:
+        if not K.is_keras_tensor(input_tensor):
+            img_input = Input(tensor=input_tensor, shape=input_shape)
+        else:
+            img_input = input_tensor
+    if K.image_dim_ordering() == 'tf':
+        bn_axis = 3
+    else:
+        bn_axis = 1
+
+    x = ZeroPadding2D((3, 3))(img_input)
+    x = Convolution2D(64, 7, 7, subsample=(2, 2), name='conv1')(x)
+    x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
+    x = Activation('relu')(x)
+    x = MaxPooling2D((3, 3), strides=(2, 2))(x)
+
+    x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
+    x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
+    x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
+
+    x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
+    x = identity_block(x, 3, [128, 128, 512], stage=3, block='b')
+    x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')
+    x = identity_block(x, 3, [128, 128, 512], stage=3, block='d')
+
+    x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
+    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b')
+    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c')
+    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d')
+    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e')
+    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f')
+
+    x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
+    x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')
+    x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')
+
+    x = AveragePooling2D((7, 7), name='avg_pool')(x)
+
+    if include_top:
+        x = Flatten()(x)
+        x = Dense(classes, activation='softmax', name='fc1000')(x)
+
+    # Ensure that the model takes into account
+    # any potential predecessors of `input_tensor`.
+    if input_tensor is not None:
+        inputs = get_source_inputs(input_tensor)
+    else:
+        inputs = img_input
+    # Create model.
+    model = Model(inputs, x, name='resnet50')
+
+    # load weights
+    if weights == 'imagenet':
+        if K.image_dim_ordering() == 'th':
+            if include_top:
+                weights_path = get_file('resnet50_weights_th_dim_ordering_th_kernels.h5',
+                                        TH_WEIGHTS_PATH,
+                                        cache_subdir='models',
+                                        md5_hash='1c1f8f5b0c8ee28fe9d950625a230e1c')
+            else:
+                weights_path = get_file('resnet50_weights_th_dim_ordering_th_kernels_notop.h5',
+                                        TH_WEIGHTS_PATH_NO_TOP,
+                                        cache_subdir='models',
+                                        md5_hash='f64f049c92468c9affcd44b0976cdafe')
+            model.load_weights(weights_path)
+            if K.backend() == 'tensorflow':
+                warnings.warn('You are using the TensorFlow backend, yet you '
+                              'are using the Theano '
+                              'image dimension ordering convention '
+                              '(`image_dim_ordering="th"`). '
+                              'For best performance, set '
+                              '`image_dim_ordering="tf"` in '
+                              'your Keras config '
+                              'at ~/.keras/keras.json.')
+                convert_all_kernels_in_model(model)
+        else:
+            if include_top:
+                weights_path = get_file('resnet50_weights_tf_dim_ordering_tf_kernels.h5',
+                                        TF_WEIGHTS_PATH,
+                                        cache_subdir='models',
+                                        md5_hash='a7b3fe01876f51b976af0dea6bc144eb')
+            else:
+                weights_path = get_file('resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5',
+                                        TF_WEIGHTS_PATH_NO_TOP,
+                                        cache_subdir='models',
+                                        md5_hash='a268eb855778b3df3c7506639542a6af')
+            model.load_weights(weights_path)
+            if K.backend() == 'theano':
+                convert_all_kernels_in_model(model)
+    return model

keras/applications/vgg16.py

@@ -0,0 +1,166 @@
+# -*- coding: utf-8 -*-
+"""VGG16 model for Keras.
+
+# Reference
+
+- [Very Deep Convolutional Networks for Large-Scale Image Recognition](https://arxiv.org/abs/1409.1556)
+
+"""
+from __future__ import print_function
+from __future__ import absolute_import
+
+import warnings
+
+from ..models import Model
+from ..layers import Flatten, Dense, Input
+from ..layers import Convolution2D, MaxPooling2D
+from ..engine.topology import get_source_inputs
+from ..utils.layer_utils import convert_all_kernels_in_model
+from ..utils.data_utils import get_file
+from .. import backend as K
+from .imagenet_utils import decode_predictions, preprocess_input, _obtain_input_shape
+
+
+TH_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_th_dim_ordering_th_kernels.h5'
+TF_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_tf_dim_ordering_tf_kernels.h5'
+TH_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_th_dim_ordering_th_kernels_notop.h5'
+TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5'
+
+
+def VGG16(include_top=True, weights='imagenet',
+          input_tensor=None, input_shape=None,
+          classes=1000):
+    """Instantiate the VGG16 architecture,
+    optionally loading weights pre-trained
+    on ImageNet. Note that when using TensorFlow,
+    for best performance you should set
+    `image_dim_ordering="tf"` in your Keras config
+    at ~/.keras/keras.json.
+
+    The model and the weights are compatible with both
+    TensorFlow and Theano. The dimension ordering
+    convention used by the model is the one
+    specified in your Keras config file.
+
+    # Arguments
+        include_top: whether to include the 3 fully-connected
+            layers at the top of the network.
+        weights: one of `None` (random initialization)
+            or "imagenet" (pre-training on ImageNet).
+        input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
+            to use as image input for the model.
+        input_shape: optional shape tuple, only to be specified
+            if `include_top` is False (otherwise the input shape
+            has to be `(224, 224, 3)` (with `tf` dim ordering)
+            or `(3, 224, 244)` (with `th` dim ordering).
+            It should have exactly 3 inputs channels,
+            and width and height should be no smaller than 48.
+            E.g. `(200, 200, 3)` would be one valid value.
+        classes: optional number of classes to classify images
+            into, only to be specified if `include_top` is True, and
+            if no `weights` argument is specified.
+
+    # Returns
+        A Keras model instance.
+    """
+    if weights not in {'imagenet', None}:
+        raise ValueError('The `weights` argument should be either '
+                         '`None` (random initialization) or `imagenet` '
+                         '(pre-training on ImageNet).')
+
+    if weights == 'imagenet' and include_top and classes != 1000:
+        raise ValueError('If using `weights` as imagenet with `include_top`'
+                         ' as true, `classes` should be 1000')
+    # Determine proper input shape
+    input_shape = _obtain_input_shape(input_shape,
+                                      default_size=224,
+                                      min_size=48,
+                                      dim_ordering=K.image_dim_ordering(),
+                                      include_top=include_top)
+
+    if input_tensor is None:
+        img_input = Input(shape=input_shape)
+    else:
+        if not K.is_keras_tensor(input_tensor):
+            img_input = Input(tensor=input_tensor, shape=input_shape)
+        else:
+            img_input = input_tensor
+    # Block 1
+    x = Convolution2D(64, 3, 3, activation='relu', border_mode='same', name='block1_conv1')(img_input)
+    x = Convolution2D(64, 3, 3, activation='relu', border_mode='same', name='block1_conv2')(x)
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
+
+    # Block 2
+    x = Convolution2D(128, 3, 3, activation='relu', border_mode='same', name='block2_conv1')(x)
+    x = Convolution2D(128, 3, 3, activation='relu', border_mode='same', name='block2_conv2')(x)
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
+
+    # Block 3
+    x = Convolution2D(256, 3, 3, activation='relu', border_mode='same', name='block3_conv1')(x)
+    x = Convolution2D(256, 3, 3, activation='relu', border_mode='same', name='block3_conv2')(x)
+    x = Convolution2D(256, 3, 3, activation='relu', border_mode='same', name='block3_conv3')(x)
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
+
+    # Block 4
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block4_conv1')(x)
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block4_conv2')(x)
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block4_conv3')(x)
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
+
+    # Block 5
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block5_conv1')(x)
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block5_conv2')(x)
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block5_conv3')(x)
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)
+
+    if include_top:
+        # Classification block
+        x = Flatten(name='flatten')(x)
+        x = Dense(4096, activation='relu', name='fc1')(x)
+        x = Dense(4096, activation='relu', name='fc2')(x)
+        x = Dense(classes, activation='softmax', name='predictions')(x)
+
+    # Ensure that the model takes into account
+    # any potential predecessors of `input_tensor`.
+    if input_tensor is not None:
+        inputs = get_source_inputs(input_tensor)
+    else:
+        inputs = img_input
+    # Create model.
+    model = Model(inputs, x, name='vgg16')
+
+    # load weights
+    if weights == 'imagenet':
+        if K.image_dim_ordering() == 'th':
+            if include_top:
+                weights_path = get_file('vgg16_weights_th_dim_ordering_th_kernels.h5',
+                                        TH_WEIGHTS_PATH,
+                                        cache_subdir='models')
+            else:
+                weights_path = get_file('vgg16_weights_th_dim_ordering_th_kernels_notop.h5',
+                                        TH_WEIGHTS_PATH_NO_TOP,
+                                        cache_subdir='models')
+            model.load_weights(weights_path)
+            if K.backend() == 'tensorflow':
+                warnings.warn('You are using the TensorFlow backend, yet you '
+                              'are using the Theano '
+                              'image dimension ordering convention '
+                              '(`image_dim_ordering="th"`). '
+                              'For best performance, set '
+                              '`image_dim_ordering="tf"` in '
+                              'your Keras config '
+                              'at ~/.keras/keras.json.')
+                convert_all_kernels_in_model(model)
+        else:
+            if include_top:
+                weights_path = get_file('vgg16_weights_tf_dim_ordering_tf_kernels.h5',
+                                        TF_WEIGHTS_PATH,
+                                        cache_subdir='models')
+            else:
+                weights_path = get_file('vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5',
+                                        TF_WEIGHTS_PATH_NO_TOP,
+                                        cache_subdir='models')
+            model.load_weights(weights_path)
+            if K.backend() == 'theano':
+                convert_all_kernels_in_model(model)
+    return model

keras/applications/vgg19.py

@@ -0,0 +1,169 @@
+# -*- coding: utf-8 -*-
+"""VGG19 model for Keras.
+
+# Reference
+
+- [Very Deep Convolutional Networks for Large-Scale Image Recognition](https://arxiv.org/abs/1409.1556)
+
+"""
+from __future__ import print_function
+from __future__ import absolute_import
+
+import warnings
+
+from ..models import Model
+from ..layers import Flatten, Dense, Input
+from ..layers import Convolution2D, MaxPooling2D
+from ..engine.topology import get_source_inputs
+from ..utils.layer_utils import convert_all_kernels_in_model
+from ..utils.data_utils import get_file
+from .. import backend as K
+from .imagenet_utils import decode_predictions, preprocess_input, _obtain_input_shape
+
+
+TH_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg19_weights_th_dim_ordering_th_kernels.h5'
+TF_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg19_weights_tf_dim_ordering_tf_kernels.h5'
+TH_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg19_weights_th_dim_ordering_th_kernels_notop.h5'
+TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5'
+
+
+def VGG19(include_top=True, weights='imagenet',
+          input_tensor=None, input_shape=None,
+          classes=1000):
+    """Instantiate the VGG19 architecture,
+    optionally loading weights pre-trained
+    on ImageNet. Note that when using TensorFlow,
+    for best performance you should set
+    `image_dim_ordering="tf"` in your Keras config
+    at ~/.keras/keras.json.
+
+    The model and the weights are compatible with both
+    TensorFlow and Theano. The dimension ordering
+    convention used by the model is the one
+    specified in your Keras config file.
+
+    # Arguments
+        include_top: whether to include the 3 fully-connected
+            layers at the top of the network.
+        weights: one of `None` (random initialization)
+            or "imagenet" (pre-training on ImageNet).
+        input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
+            to use as image input for the model.
+        input_shape: optional shape tuple, only to be specified
+            if `include_top` is False (otherwise the input shape
+            has to be `(224, 224, 3)` (with `tf` dim ordering)
+            or `(3, 224, 244)` (with `th` dim ordering).
+            It should have exactly 3 inputs channels,
+            and width and height should be no smaller than 48.
+            E.g. `(200, 200, 3)` would be one valid value.
+        classes: optional number of classes to classify images
+            into, only to be specified if `include_top` is True, and
+            if no `weights` argument is specified.
+
+    # Returns
+        A Keras model instance.
+    """
+    if weights not in {'imagenet', None}:
+        raise ValueError('The `weights` argument should be either '
+                         '`None` (random initialization) or `imagenet` '
+                         '(pre-training on ImageNet).')
+
+    if weights == 'imagenet' and include_top and classes != 1000:
+        raise ValueError('If using `weights` as imagenet with `include_top`'
+                         ' as true, `classes` should be 1000')
+    # Determine proper input shape
+    input_shape = _obtain_input_shape(input_shape,
+                                      default_size=224,
+                                      min_size=48,
+                                      dim_ordering=K.image_dim_ordering(),
+                                      include_top=include_top)
+
+    if input_tensor is None:
+        img_input = Input(shape=input_shape)
+    else:
+        if not K.is_keras_tensor(input_tensor):
+            img_input = Input(tensor=input_tensor, shape=input_shape)
+        else:
+            img_input = input_tensor
+    # Block 1
+    x = Convolution2D(64, 3, 3, activation='relu', border_mode='same', name='block1_conv1')(img_input)
+    x = Convolution2D(64, 3, 3, activation='relu', border_mode='same', name='block1_conv2')(x)
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
+
+    # Block 2
+    x = Convolution2D(128, 3, 3, activation='relu', border_mode='same', name='block2_conv1')(x)
+    x = Convolution2D(128, 3, 3, activation='relu', border_mode='same', name='block2_conv2')(x)
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
+
+    # Block 3
+    x = Convolution2D(256, 3, 3, activation='relu', border_mode='same', name='block3_conv1')(x)
+    x = Convolution2D(256, 3, 3, activation='relu', border_mode='same', name='block3_conv2')(x)
+    x = Convolution2D(256, 3, 3, activation='relu', border_mode='same', name='block3_conv3')(x)
+    x = Convolution2D(256, 3, 3, activation='relu', border_mode='same', name='block3_conv4')(x)
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
+
+    # Block 4
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block4_conv1')(x)
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block4_conv2')(x)
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block4_conv3')(x)
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block4_conv4')(x)
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
+
+    # Block 5
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block5_conv1')(x)
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block5_conv2')(x)
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block5_conv3')(x)
+    x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block5_conv4')(x)
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)
+
+    if include_top:
+        # Classification block
+        x = Flatten(name='flatten')(x)
+        x = Dense(4096, activation='relu', name='fc1')(x)
+        x = Dense(4096, activation='relu', name='fc2')(x)
+        x = Dense(classes, activation='softmax', name='predictions')(x)
+
+    # Ensure that the model takes into account
+    # any potential predecessors of `input_tensor`.
+    if input_tensor is not None:
+        inputs = get_source_inputs(input_tensor)
+    else:
+        inputs = img_input
+    # Create model.
+    model = Model(inputs, x, name='vgg19')
+
+    # load weights
+    if weights == 'imagenet':
+        if K.image_dim_ordering() == 'th':
+            if include_top:
+                weights_path = get_file('vgg19_weights_th_dim_ordering_th_kernels.h5',
+                                        TH_WEIGHTS_PATH,
+                                        cache_subdir='models')
+            else:
+                weights_path = get_file('vgg19_weights_th_dim_ordering_th_kernels_notop.h5',
+                                        TH_WEIGHTS_PATH_NO_TOP,
+                                        cache_subdir='models')
+            model.load_weights(weights_path)
+            if K.backend() == 'tensorflow':
+                warnings.warn('You are using the TensorFlow backend, yet you '
+                              'are using the Theano '
+                              'image dimension ordering convention '
+                              '(`image_dim_ordering="th"`). '
+                              'For best performance, set '
+                              '`image_dim_ordering="tf"` in '
+                              'your Keras config '
+                              'at ~/.keras/keras.json.')
+                convert_all_kernels_in_model(model)
+        else:
+            if include_top:
+                weights_path = get_file('vgg19_weights_tf_dim_ordering_tf_kernels.h5',
+                                        TF_WEIGHTS_PATH,
+                                        cache_subdir='models')
+            else:
+                weights_path = get_file('vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5',
+                                        TF_WEIGHTS_PATH_NO_TOP,
+                                        cache_subdir='models')
+            model.load_weights(weights_path)
+            if K.backend() == 'theano':
+                convert_all_kernels_in_model(model)
+    return model

keras/applications/xception.py

@@ -0,0 +1,233 @@
+# -*- coding: utf-8 -*-
+"""Xception V1 model for Keras.
+
+On ImageNet, this model gets to a top-1 validation accuracy of 0.790
+and a top-5 validation accuracy of 0.945.
+
+Do note that the input image format for this model is different than for
+the VGG16 and ResNet models (299x299 instead of 224x224),
+and that the input preprocessing function
+is also different (same as Inception V3).
+
+Also do note that this model is only available for the TensorFlow backend,
+due to its reliance on `SeparableConvolution` layers.
+
+# Reference
+
+- [Xception: Deep Learning with Depthwise Separable Convolutions](https://arxiv.org/abs/1610.02357)
+
+"""
+from __future__ import print_function
+from __future__ import absolute_import
+
+import warnings
+
+from ..models import Model
+from ..layers import Dense, Input, BatchNormalization, Activation, merge
+from ..layers import Conv2D, SeparableConv2D, MaxPooling2D, GlobalAveragePooling2D
+from ..engine.topology import get_source_inputs
+from ..utils.data_utils import get_file
+from .. import backend as K
+from .imagenet_utils import decode_predictions, _obtain_input_shape
+
+
+TF_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.4/xception_weights_tf_dim_ordering_tf_kernels.h5'
+TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.4/xception_weights_tf_dim_ordering_tf_kernels_notop.h5'
+
+
+def Xception(include_top=True, weights='imagenet',
+             input_tensor=None, input_shape=None,
+             classes=1000):
+    """Instantiate the Xception architecture,
+    optionally loading weights pre-trained
+    on ImageNet. This model is available for TensorFlow only,
+    and can only be used with inputs following the TensorFlow
+    dimension ordering `(width, height, channels)`.
+    You should set `image_dim_ordering="tf"` in your Keras config
+    located at ~/.keras/keras.json.
+
+    Note that the default input image size for this model is 299x299.
+
+    # Arguments
+        include_top: whether to include the fully-connected
+            layer at the top of the network.
+        weights: one of `None` (random initialization)
+            or "imagenet" (pre-training on ImageNet).
+        input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
+            to use as image input for the model.
+        input_shape: optional shape tuple, only to be specified
+            if `include_top` is False (otherwise the input shape
+            has to be `(299, 299, 3)`.
+            It should have exactly 3 inputs channels,
+            and width and height should be no smaller than 71.
+            E.g. `(150, 150, 3)` would be one valid value.
+        classes: optional number of classes to classify images
+            into, only to be specified if `include_top` is True, and
+            if no `weights` argument is specified.
+
+    # Returns
+        A Keras model instance.
+    """
+    if weights not in {'imagenet', None}:
+        raise ValueError('The `weights` argument should be either '
+                         '`None` (random initialization) or `imagenet` '
+                         '(pre-training on ImageNet).')
+
+    if weights == 'imagenet' and include_top and classes != 1000:
+        raise ValueError('If using `weights` as imagenet with `include_top`'
+                         ' as true, `classes` should be 1000')
+
+    if K.backend() != 'tensorflow':
+        raise RuntimeError('The Xception model is only available with '
+                           'the TensorFlow backend.')
+    if K.image_dim_ordering() != 'tf':
+        warnings.warn('The Xception model is only available for the '
+                      'input dimension ordering "tf" '
+                      '(width, height, channels). '
+                      'However your settings specify the default '
+                      'dimension ordering "th" (channels, width, height). '
+                      'You should set `image_dim_ordering="tf"` in your Keras '
+                      'config located at ~/.keras/keras.json. '
+                      'The model being returned right now will expect inputs '
+                      'to follow the "tf" dimension ordering.')
+        K.set_image_dim_ordering('tf')
+        old_dim_ordering = 'th'
+    else:
+        old_dim_ordering = None
+
+    # Determine proper input shape
+    input_shape = _obtain_input_shape(input_shape,
+                                      default_size=299,
+                                      min_size=71,
+                                      dim_ordering=K.image_dim_ordering(),
+                                      include_top=include_top)
+
+    if input_tensor is None:
+        img_input = Input(shape=input_shape)
+    else:
+        if not K.is_keras_tensor(input_tensor):
+            img_input = Input(tensor=input_tensor, shape=input_shape)
+        else:
+            img_input = input_tensor
+
+    x = Conv2D(32, 3, 3, subsample=(2, 2), bias=False, name='block1_conv1')(img_input)
+    x = BatchNormalization(name='block1_conv1_bn')(x)
+    x = Activation('relu', name='block1_conv1_act')(x)
+    x = Conv2D(64, 3, 3, bias=False, name='block1_conv2')(x)
+    x = BatchNormalization(name='block1_conv2_bn')(x)
+    x = Activation('relu', name='block1_conv2_act')(x)
+
+    residual = Conv2D(128, 1, 1, subsample=(2, 2),
+                      border_mode='same', bias=False)(x)
+    residual = BatchNormalization()(residual)
+
+    x = SeparableConv2D(128, 3, 3, border_mode='same', bias=False, name='block2_sepconv1')(x)
+    x = BatchNormalization(name='block2_sepconv1_bn')(x)
+    x = Activation('relu', name='block2_sepconv2_act')(x)
+    x = SeparableConv2D(128, 3, 3, border_mode='same', bias=False, name='block2_sepconv2')(x)
+    x = BatchNormalization(name='block2_sepconv2_bn')(x)
+
+    x = MaxPooling2D((3, 3), strides=(2, 2), border_mode='same', name='block2_pool')(x)
+    x = merge([x, residual], mode='sum')
+
+    residual = Conv2D(256, 1, 1, subsample=(2, 2),
+                      border_mode='same', bias=False)(x)
+    residual = BatchNormalization()(residual)
+
+    x = Activation('relu', name='block3_sepconv1_act')(x)
+    x = SeparableConv2D(256, 3, 3, border_mode='same', bias=False, name='block3_sepconv1')(x)
+    x = BatchNormalization(name='block3_sepconv1_bn')(x)
+    x = Activation('relu', name='block3_sepconv2_act')(x)
+    x = SeparableConv2D(256, 3, 3, border_mode='same', bias=False, name='block3_sepconv2')(x)
+    x = BatchNormalization(name='block3_sepconv2_bn')(x)
+
+    x = MaxPooling2D((3, 3), strides=(2, 2), border_mode='same', name='block3_pool')(x)
+    x = merge([x, residual], mode='sum')
+
+    residual = Conv2D(728, 1, 1, subsample=(2, 2),
+                      border_mode='same', bias=False)(x)
+    residual = BatchNormalization()(residual)
+
+    x = Activation('relu', name='block4_sepconv1_act')(x)
+    x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name='block4_sepconv1')(x)
+    x = BatchNormalization(name='block4_sepconv1_bn')(x)
+    x = Activation('relu', name='block4_sepconv2_act')(x)
+    x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name='block4_sepconv2')(x)
+    x = BatchNormalization(name='block4_sepconv2_bn')(x)
+
+    x = MaxPooling2D((3, 3), strides=(2, 2), border_mode='same', name='block4_pool')(x)
+    x = merge([x, residual], mode='sum')
+
+    for i in range(8):
+        residual = x
+        prefix = 'block' + str(i + 5)
+
+        x = Activation('relu', name=prefix + '_sepconv1_act')(x)
+        x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name=prefix + '_sepconv1')(x)
+        x = BatchNormalization(name=prefix + '_sepconv1_bn')(x)
+        x = Activation('relu', name=prefix + '_sepconv2_act')(x)
+        x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name=prefix + '_sepconv2')(x)
+        x = BatchNormalization(name=prefix + '_sepconv2_bn')(x)
+        x = Activation('relu', name=prefix + '_sepconv3_act')(x)
+        x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name=prefix + '_sepconv3')(x)
+        x = BatchNormalization(name=prefix + '_sepconv3_bn')(x)
+
+        x = merge([x, residual], mode='sum')
+
+    residual = Conv2D(1024, 1, 1, subsample=(2, 2),
+                      border_mode='same', bias=False)(x)
+    residual = BatchNormalization()(residual)
+
+    x = Activation('relu', name='block13_sepconv1_act')(x)
+    x = SeparableConv2D(728, 3, 3, border_mode='same', bias=False, name='block13_sepconv1')(x)
+    x = BatchNormalization(name='block13_sepconv1_bn')(x)
+    x = Activation('relu', name='block13_sepconv2_act')(x)
+    x = SeparableConv2D(1024, 3, 3, border_mode='same', bias=False, name='block13_sepconv2')(x)
+    x = BatchNormalization(name='block13_sepconv2_bn')(x)
+
+    x = MaxPooling2D((3, 3), strides=(2, 2), border_mode='same', name='block13_pool')(x)
+    x = merge([x, residual], mode='sum')
+
+    x = SeparableConv2D(1536, 3, 3, border_mode='same', bias=False, name='block14_sepconv1')(x)
+    x = BatchNormalization(name='block14_sepconv1_bn')(x)
+    x = Activation('relu', name='block14_sepconv1_act')(x)
+
+    x = SeparableConv2D(2048, 3, 3, border_mode='same', bias=False, name='block14_sepconv2')(x)
+    x = BatchNormalization(name='block14_sepconv2_bn')(x)
+    x = Activation('relu', name='block14_sepconv2_act')(x)
+
+    if include_top:
+        x = GlobalAveragePooling2D(name='avg_pool')(x)
+        x = Dense(classes, activation='softmax', name='predictions')(x)
+
+    # Ensure that the model takes into account
+    # any potential predecessors of `input_tensor`.
+    if input_tensor is not None:
+        inputs = get_source_inputs(input_tensor)
+    else:
+        inputs = img_input
+    # Create model.
+    model = Model(inputs, x, name='xception')
+
+    # load weights
+    if weights == 'imagenet':
+        if include_top:
+            weights_path = get_file('xception_weights_tf_dim_ordering_tf_kernels.h5',
+                                    TF_WEIGHTS_PATH,
+                                    cache_subdir='models')
+        else:
+            weights_path = get_file('xception_weights_tf_dim_ordering_tf_kernels_notop.h5',
+                                    TF_WEIGHTS_PATH_NO_TOP,
+                                    cache_subdir='models')
+        model.load_weights(weights_path)
+
+    if old_dim_ordering:
+        K.set_image_dim_ordering(old_dim_ordering)
+    return model
+
+
+def preprocess_input(x):
+    x /= 255.
+    x -= 0.5
+    x *= 2.
+    return x

keras/backend/__init__.py

@@ -9,6 +9,11 @@ from .common import set_epsilon
 from .common import set_floatx
 from .common import get_uid
 from .common import cast_to_floatx
+from .common import image_dim_ordering
+from .common import set_image_dim_ordering
+from .common import is_keras_tensor
+from .common import legacy_weight_ordering
+from .common import set_legacy_weight_ordering
 
 _keras_base_dir = os.path.expanduser('~')
 if not os.access(_keras_base_dir, os.W_OK):
@@ -18,34 +23,42 @@ _keras_dir = os.path.join(_keras_base_dir, '.keras')
 if not os.path.exists(_keras_dir):
     os.makedirs(_keras_dir)
 
-_BACKEND = 'theano'
+# Default backend: TensorFlow.
+_BACKEND = 'tensorflow'
+
 _config_path = os.path.expanduser(os.path.join(_keras_dir, 'keras.json'))
 if os.path.exists(_config_path):
     _config = json.load(open(_config_path))
     _floatx = _config.get('floatx', floatx())
     assert _floatx in {'float16', 'float32', 'float64'}
     _epsilon = _config.get('epsilon', epsilon())
-    assert type(_epsilon) == float
+    assert isinstance(_epsilon, float)
     _backend = _config.get('backend', _BACKEND)
     assert _backend in {'theano', 'tensorflow'}
+    _image_dim_ordering = _config.get('image_dim_ordering',
+                                      image_dim_ordering())
+    assert _image_dim_ordering in {'tf', 'th'}
 
     set_floatx(_floatx)
     set_epsilon(_epsilon)
+    set_image_dim_ordering(_image_dim_ordering)
     _BACKEND = _backend
-else:
-    # save config file, for easy edition
+
+# save config file
+if not os.path.exists(_config_path):
     _config = {'floatx': floatx(),
                'epsilon': epsilon(),
-               'backend': _BACKEND}
+               'backend': _BACKEND,
+               'image_dim_ordering': image_dim_ordering()}
     with open(_config_path, 'w') as f:
-        # add new line in order for bash 'cat' display the content correctly
-        f.write(json.dumps(_config) + '\n')
+        f.write(json.dumps(_config, indent=4))
 
 if 'KERAS_BACKEND' in os.environ:
     _backend = os.environ['KERAS_BACKEND']
     assert _backend in {'theano', 'tensorflow'}
     _BACKEND = _backend
 
+# import backend
 if _BACKEND == 'theano':
     sys.stderr.write('Using Theano backend.\n')
     from .theano_backend import *
@@ -53,4 +66,11 @@ elif _BACKEND == 'tensorflow':
     sys.stderr.write('Using TensorFlow backend.\n')
     from .tensorflow_backend import *
 else:
-    raise Exception('Unknown backend: ' + str(_BACKEND))
+    raise ValueError('Unknown backend: ' + str(_BACKEND))
+
+
+def backend():
+    """Publicly accessible method
+    for determining the current backend.
+    """
+    return _BACKEND

keras/backend/common.py

@@ -1,45 +1,217 @@
 import numpy as np
 
+from collections import defaultdict
+
 # the type of float to use throughout the session.
 _FLOATX = 'float32'
 _EPSILON = 10e-8
-_UID_PREFIXES = {}
+_UID_PREFIXES = defaultdict(int)
+_IMAGE_DIM_ORDERING = 'tf'
+_LEGACY_WEIGHT_ORDERING = False
 
 
 def epsilon():
+    """Returns the value of the fuzz
+    factor used in numeric expressions.
+
+    # Returns
+        A float.
+
+    # Example
+    ```python
+        >>> keras.backend.epsilon()
+        1e-08
+    ```
+    """
     return _EPSILON
 
 
 def set_epsilon(e):
+    """Sets the value of the fuzz
+    factor used in numeric expressions.
+
+    # Arguments
+        e: float. New value of epsilon.
+
+    # Example
+    ```python
+        >>> from keras import backend as K
+        >>> K.epsilon()
+        1e-08
+        >>> K.set_epsilon(1e-05)
+        >>> K.epsilon()
+        1e-05
+    ```
+    """
     global _EPSILON
     _EPSILON = e
 
 
 def floatx():
-    '''Returns the default float type, as a string
+    """Returns the default float type, as a string
     (e.g. 'float16', 'float32', 'float64').
-    '''
+
+    # Returns
+        String, the current default float type.
+
+    # Example
+    ```python
+        >>> keras.backend.floatx()
+        'float32'
+    ```
+    """
     return _FLOATX
 
 
 def set_floatx(floatx):
+    """Sets the default float type.
+
+    # Arguments
+        String: 'float16', 'float32', or 'float64'.
+
+    # Example
+    ```python
+        >>> from keras import backend as K
+        >>> K.floatx()
+        'float32'
+        >>> K.set_floatx('float16')
+        >>> K.floatx()
+        'float16'
+    ```
+    """
     global _FLOATX
     if floatx not in {'float16', 'float32', 'float64'}:
-        raise Exception('Unknown floatx type: ' + str(floatx))
-    floatx = str(floatx)
-    _FLOATX = floatx
+        raise ValueError('Unknown floatx type: ' + str(floatx))
+    _FLOATX = str(floatx)
 
 
 def cast_to_floatx(x):
-    '''Cast a Numpy array to floatx.
-    '''
+    """Cast a Numpy array to the default Keras float type.
+
+    # Arguments
+        x: Numpy array.
+
+    # Returns
+        The same Numpy array, cast to its new type.
+
+    # Example
+    ```python
+        >>> from keras import backend as K
+        >>> K.floatx()
+        'float32'
+        >>> arr = numpy.array([1.0, 2.0], dtype='float64')
+        >>> arr.dtype
+        dtype('float64')
+        >>> new_arr = K.cast_to_floatx(arr)
+        >>> new_arr
+        array([ 1.,  2.], dtype=float32)
+        >>> new_arr.dtype
+        dtype('float32')
+    ```
+    """
     return np.asarray(x, dtype=_FLOATX)
 
 
+def image_dim_ordering():
+    """Returns the default image dimension ordering
+    convention ('th' or 'tf').
+
+    # Returns
+        A string, either `'th'` or `'tf'`
+
+    # Example
+    ```python
+        >>> keras.backend.image_dim_ordering()
+        'th'
+    ```
+    """
+    return _IMAGE_DIM_ORDERING
+
+
+def set_image_dim_ordering(dim_ordering):
+    """Sets the value of the image dimension
+    ordering convention ('th' or 'tf').
+
+    # Arguments
+        dim_ordering: string. `'th'` or `'tf'`.
+
+    # Example
+    ```python
+        >>> from keras import backend as K
+        >>> K.image_dim_ordering()
+        'th'
+        >>> K.set_image_dim_ordering('tf')
+        >>> K.image_dim_ordering()
+        'tf'
+    ```
+    """
+    global _IMAGE_DIM_ORDERING
+    if dim_ordering not in {'tf', 'th'}:
+        raise ValueError('Unknown dim_ordering:', dim_ordering)
+    _IMAGE_DIM_ORDERING = str(dim_ordering)
+
+
 def get_uid(prefix=''):
-    if prefix not in _UID_PREFIXES:
-        _UID_PREFIXES[prefix] = 1
-        return 1
+    """Provides a unique UID given a string prefix.
+
+    # Arguments
+        prefix: string.
+
+    # Returns
+        An integer.
+
+    # Example
+    ```
+        >>> keras.backend.get_uid('dense')
+        >>> 1
+        >>> keras.backend.get_uid('dense')
+        >>> 2
+    ```
+
+    """
+    _UID_PREFIXES[prefix] += 1
+    return _UID_PREFIXES[prefix]
+
+
+def reset_uids():
+    global _UID_PREFIXES
+    _UID_PREFIXES = defaultdict(int)
+
+
+def is_keras_tensor(x):
+    """Returns whether `x` is a Keras tensor.
+
+    # Arguments
+        x: a potential tensor.
+
+    # Returns
+        A boolean: whether the argument is a Keras tensor.
+
+    # Examples
+    ```python
+        >>> from keras import backend as K
+        >>> np_var = numpy.array([1, 2])
+        >>> K.is_keras_tensor(np_var)
+        False
+        >>> keras_var = K.variable(np_var)
+        >>> K.is_keras_tensor(keras_var)  # A variable is not a Tensor.
+        False
+        >>> keras_placeholder = K.placeholder(shape=(2, 4, 5))
+        >>> K.is_keras_tensor(keras_placeholder)  # A placeholder is a Tensor.
+        True
+    ```
+    """
+    if hasattr(x, '_keras_shape'):
+        return True
     else:
-        _UID_PREFIXES[prefix] += 1
-        return _UID_PREFIXES[prefix]
+        return False
+
+
+def set_legacy_weight_ordering(value):
+    global _LEGACY_WEIGHT_ORDERING
+    assert value in {True, False}
+    _LEGACY_WEIGHT_ORDERING = value
+
+
+def legacy_weight_ordering():
+    return _LEGACY_WEIGHT_ORDERING

keras/constraints.py

@@ -1,8 +1,10 @@
 from __future__ import absolute_import
 from . import backend as K
+from .utils.generic_utils import get_from_module
 
 
 class Constraint(object):
+
     def __call__(self, p):
         return p
 
@@ -11,26 +13,29 @@ class Constraint(object):
 
 
 class MaxNorm(Constraint):
-    '''Constrain the weights incident to each hidden unit to have a norm less than or equal to a desired value.
+    """MaxNorm weight constraint.
+
+    Constrains the weights incident to each hidden unit
+    to have a norm less than or equal to a desired value.
 
     # Arguments
         m: the maximum norm for the incoming weights.
-        axis: integer, axis along which to calculate weight norms. For instance,
-            in a `Dense` layer the weight matrix has shape (input_dim, output_dim),
-            set `axis` to `0` to constrain each weight vector of length (input_dim).
-            In a `MaxoutDense` layer the weight tensor has shape (nb_feature, input_dim, output_dim),
-            set `axis` to `1` to constrain each weight vector of length (input_dim),
-            i.e. constrain the filters incident to the `max` operation.
-            In a `Convolution2D` layer with the Theano backend, the weight tensor
-            has shape (nb_filter, stack_size, nb_row, nb_col), set `axis` to `[1,2,3]`
-            to constrain the weights of each filter tensor of size (stack_size, nb_row, nb_col).
-            In a `Convolution2D` layer with the TensorFlow backend, the weight tensor
-            has shape (nb_row, nb_col, stack_size, nb_filter), set `axis` to `[0,1,2]`
-            to constrain the weights of each filter tensor of size (nb_row, nb_col, stack_size).
+        axis: integer, axis along which to calculate weight norms.
+            For instance, in a `Dense` layer the weight matrix
+            has shape `(input_dim, output_dim)`,
+            set `axis` to `0` to constrain each weight vector
+            of length `(input_dim,)`.
+            In a `Convolution2D` layer with `dim_ordering="tf"`,
+            the weight tensor has shape
+            `(rows, cols, input_depth, output_depth)`,
+            set `axis` to `[0, 1, 2]`
+            to constrain the weights of each filter tensor of size
+            `(rows, cols, input_depth)`.
 
     # References
         - [Dropout: A Simple Way to Prevent Neural Networks from Overfitting Srivastava, Hinton, et al. 2014](http://www.cs.toronto.edu/~rsalakhu/papers/srivastava14a.pdf)
-    '''
+    """
+
     def __init__(self, m=2, axis=0):
         self.m = m
         self.axis = axis
@@ -38,7 +43,7 @@ class MaxNorm(Constraint):
     def __call__(self, p):
         norms = K.sqrt(K.sum(K.square(p), axis=self.axis, keepdims=True))
         desired = K.clip(norms, 0, self.m)
-        p = p * (desired / (K.epsilon() + norms))
+        p *= (desired / (K.epsilon() + norms))
         return p
 
     def get_config(self):
@@ -48,46 +53,98 @@ class MaxNorm(Constraint):
 
 
 class NonNeg(Constraint):
-    '''Constrain the weights to be non-negative.
-    '''
+    """Constrains the weights to be non-negative.
+    """
+
     def __call__(self, p):
         p *= K.cast(p >= 0., K.floatx())
         return p
 
 
 class UnitNorm(Constraint):
-    '''Constrain the weights incident to each hidden unit to have unit norm.
+    """Constrains the weights incident to each hidden unit to have unit norm.
 
     # Arguments
-        axis: integer, axis along which to calculate weight norms. For instance,
-            in a `Dense` layer the weight matrix has shape (input_dim, output_dim),
-            set `axis` to `0` to constrain each weight vector of length (input_dim).
-            In a `MaxoutDense` layer the weight tensor has shape (nb_feature, input_dim, output_dim),
-            set `axis` to `1` to constrain each weight vector of length (input_dim),
-            i.e. constrain the filters incident to the `max` operation.
-            In a `Convolution2D` layer with the Theano backend, the weight tensor
-            has shape (nb_filter, stack_size, nb_row, nb_col), set `axis` to `[1,2,3]`
-            to constrain the weights of each filter tensor of size (stack_size, nb_row, nb_col).
-            In a `Convolution2D` layer with the TensorFlow backend, the weight tensor
-            has shape (nb_row, nb_col, stack_size, nb_filter), set `axis` to `[0,1,2]`
-            to constrain the weights of each filter tensor of size (nb_row, nb_col, stack_size).
-    '''
+        axis: integer, axis along which to calculate weight norms.
+            For instance, in a `Dense` layer the weight matrix
+            has shape `(input_dim, output_dim)`,
+            set `axis` to `0` to constrain each weight vector
+            of length `(input_dim,)`.
+            In a `Convolution2D` layer with `dim_ordering="tf"`,
+            the weight tensor has shape
+            `(rows, cols, input_depth, output_depth)`,
+            set `axis` to `[0, 1, 2]`
+            to constrain the weights of each filter tensor of size
+            `(rows, cols, input_depth)`.
+    """
+
     def __init__(self, axis=0):
         self.axis = axis
 
     def __call__(self, p):
-        return p / (K.epsilon() + K.sqrt(K.sum(K.square(p), axis=self.axis, keepdims=True)))
+        return p / (K.epsilon() + K.sqrt(K.sum(K.square(p),
+                                               axis=self.axis,
+                                               keepdims=True)))
 
     def get_config(self):
         return {'name': self.__class__.__name__,
                 'axis': self.axis}
 
 
+class MinMaxNorm(Constraint):
+    """MinMaxNorm weight constraint.
+
+    Constrains the weights incident to each hidden unit
+    to have the norm between a lower bound and an upper bound.
+
+    # Arguments
+        low: the minimum norm for the incoming weights.
+        high: the maximum norm for the incoming weights.
+        rate: rate for enforcing the constraint: weights will be
+            rescaled to yield (1 - rate) * norm + rate * norm.clip(low, high).
+            Effectively, this means that rate=1.0 stands for strict
+            enforcement of the constraint, while rate<1.0 means that
+            weights will be rescaled at each step to slowly move
+            towards a value inside the desired interval.
+        axis: integer, axis along which to calculate weight norms.
+            For instance, in a `Dense` layer the weight matrix
+            has shape `(input_dim, output_dim)`,
+            set `axis` to `0` to constrain each weight vector
+            of length `(input_dim,)`.
+            In a `Convolution2D` layer with `dim_ordering="tf"`,
+            the weight tensor has shape
+            `(rows, cols, input_depth, output_depth)`,
+            set `axis` to `[0, 1, 2]`
+            to constrain the weights of each filter tensor of size
+            `(rows, cols, input_depth)`.
+    """
+    def __init__(self, low=0.0, high=1.0, rate=1.0, axis=0):
+        self.low = low
+        self.high = high
+        self.rate = rate
+        self.axis = axis
+
+    def __call__(self, p):
+        norms = K.sqrt(K.sum(K.square(p), axis=self.axis, keepdims=True))
+        desired = self.rate * K.clip(norms, self.low, self.high) + (1 - self.rate) * norms
+        p *= (desired / (K.epsilon() + norms))
+        return p
+
+    def get_config(self):
+        return {'name': self.__class__.__name__,
+                'low': self.low,
+                'high': self.high,
+                'rate': self.rate,
+                'axis': self.axis}
+
+
+# Aliases.
+
 maxnorm = MaxNorm
 nonneg = NonNeg
 unitnorm = UnitNorm
 
-from .utils.generic_utils import get_from_module
+
 def get(identifier, kwargs=None):
     return get_from_module(identifier, globals(), 'constraint',
                            instantiate=True, kwargs=kwargs)

keras/datasets/cifar.py

@@ -2,21 +2,31 @@
 from __future__ import absolute_import
 import sys
 from six.moves import cPickle
-from six.moves import range
 
 
 def load_batch(fpath, label_key='labels'):
+    """Internal utility for parsing CIFAR data.
+
+    # Arguments
+        fpath: path the file to parse.
+        label_key: key for label data in the retrieve
+            dictionary.
+
+    # Returns
+        A tuple `(data, labels)`.
+    """
     f = open(fpath, 'rb')
     if sys.version_info < (3,):
         d = cPickle.load(f)
     else:
-        d = cPickle.load(f, encoding="bytes")
+        d = cPickle.load(f, encoding='bytes')
         # decode utf8
+        d_decoded = {}
         for k, v in d.items():
-            del(d[k])
-            d[k.decode("utf8")] = v
+            d_decoded[k.decode('utf8')] = v
+        d = d_decoded
     f.close()
-    data = d["data"]
+    data = d['data']
     labels = d[label_key]
 
     data = data.reshape(data.shape[0], 3, 32, 32)

keras/datasets/cifar10.py

@@ -1,30 +1,40 @@
 from __future__ import absolute_import
 from .cifar import load_batch
 from ..utils.data_utils import get_file
+from .. import backend as K
 import numpy as np
 import os
 
 
 def load_data():
-    dirname = "cifar-10-batches-py"
-    origin = "http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz"
+    """Loads CIFAR10 dataset.
+
+    # Returns
+        Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
+    """
+    dirname = 'cifar-10-batches-py'
+    origin = 'http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
     path = get_file(dirname, origin=origin, untar=True)
 
     nb_train_samples = 50000
 
-    X_train = np.zeros((nb_train_samples, 3, 32, 32), dtype="uint8")
-    y_train = np.zeros((nb_train_samples,), dtype="uint8")
+    x_train = np.zeros((nb_train_samples, 3, 32, 32), dtype='uint8')
+    y_train = np.zeros((nb_train_samples,), dtype='uint8')
 
     for i in range(1, 6):
         fpath = os.path.join(path, 'data_batch_' + str(i))
         data, labels = load_batch(fpath)
-        X_train[(i-1)*10000:i*10000, :, :, :] = data
-        y_train[(i-1)*10000:i*10000] = labels
+        x_train[(i - 1) * 10000: i * 10000, :, :, :] = data
+        y_train[(i - 1) * 10000: i * 10000] = labels
 
     fpath = os.path.join(path, 'test_batch')
-    X_test, y_test = load_batch(fpath)
+    x_test, y_test = load_batch(fpath)
 
     y_train = np.reshape(y_train, (len(y_train), 1))
     y_test = np.reshape(y_test, (len(y_test), 1))
 
-    return (X_train, y_train), (X_test, y_test)
+    if K.image_dim_ordering() == 'tf':
+        x_train = x_train.transpose(0, 2, 3, 1)
+        x_test = x_test.transpose(0, 2, 3, 1)
+
+    return (x_train, y_train), (x_test, y_test)

keras/datasets/cifar100.py

@@ -1,28 +1,41 @@
 from __future__ import absolute_import
 from .cifar import load_batch
 from ..utils.data_utils import get_file
+from .. import backend as K
 import numpy as np
 import os
 
 
 def load_data(label_mode='fine'):
-    if label_mode not in ['fine', 'coarse']:
-        raise Exception('label_mode must be one of "fine" "coarse".')
+    """Loads CIFAR100 dataset.
 
-    dirname = "cifar-100-python"
-    origin = "http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz"
+    # Arguments
+        label_mode: one of "fine", "coarse".
+
+    # Returns
+        Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
+
+    # Raises
+        ValueError: in case of invalid `label_mode`.
+    """
+    if label_mode not in ['fine', 'coarse']:
+        raise ValueError('label_mode must be one of "fine" "coarse".')
+
+    dirname = 'cifar-100-python'
+    origin = 'http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz'
     path = get_file(dirname, origin=origin, untar=True)
 
-    nb_test_samples = 10000
-    nb_train_samples = 50000
-
     fpath = os.path.join(path, 'train')
-    X_train, y_train = load_batch(fpath, label_key=label_mode+'_labels')
+    x_train, y_train = load_batch(fpath, label_key=label_mode + '_labels')
 
     fpath = os.path.join(path, 'test')
-    X_test, y_test = load_batch(fpath, label_key=label_mode+'_labels')
+    x_test, y_test = load_batch(fpath, label_key=label_mode + '_labels')
 
     y_train = np.reshape(y_train, (len(y_train), 1))
     y_test = np.reshape(y_test, (len(y_test), 1))
 
-    return (X_train, y_train), (X_test, y_test)
+    if K.image_dim_ordering() == 'tf':
+        x_train = x_train.transpose(0, 2, 3, 1)
+        x_test = x_test.transpose(0, 2, 3, 1)
+
+    return (x_train, y_train), (x_test, y_test)

keras/datasets/data_utils.py

@@ -1,4 +0,0 @@
-from ..utils.data_utils import *
-import warnings
-
-warnings.warn('data_utils has been moved to keras.utils.data_utils.')

keras/datasets/imdb.py

@@ -4,66 +4,130 @@ import gzip
 from ..utils.data_utils import get_file
 from six.moves import zip
 import numpy as np
+import sys
 
 
-def load_data(path="imdb.pkl", nb_words=None, skip_top=0,
-              maxlen=None, test_split=0.2, seed=113,
+def load_data(path='imdb_full.pkl', nb_words=None, skip_top=0,
+              maxlen=None, seed=113,
               start_char=1, oov_char=2, index_from=3):
+    """Loads the IMDB dataset.
 
-    path = get_file(path, origin="https://s3.amazonaws.com/text-datasets/imdb.pkl")
+    # Arguments
+        path: where to cache the data (relative to `~/.keras/dataset`).
+        nb_words: max number of words to include. Words are ranked
+            by how often they occur (in the training set) and only
+            the most frequent words are kept
+        skip_top: skip the top N most frequently occuring words
+            (which may not be informative).
+        maxlen: truncate sequences after this length.
+        seed: random seed for sample shuffling.
+        start_char: The start of a sequence will be marked with this character.
+            Set to 1 because 0 is usually the padding character.
+        oov_char: words that were cut out because of the `nb_words`
+            or `skip_top` limit will be replaced with this character.
+        index_from: index actual words with this index and higher.
 
-    if path.endswith(".gz"):
+    # Returns
+        Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
+
+    # Raises
+        ValueError: in case `maxlen` is so low
+            that no input sequence could be kept.
+
+    Note that the 'out of vocabulary' character is only used for
+    words that were present in the training set but are not included
+    because they're not making the `nb_words` cut here.
+    Words that were not seen in the trining set but are in the test set
+    have simply been skipped.
+    """
+    path = get_file(path,
+                    origin='https://s3.amazonaws.com/text-datasets/imdb_full.pkl',
+                    md5_hash='d091312047c43cf9e4e38fef92437263')
+
+    if path.endswith('.gz'):
         f = gzip.open(path, 'rb')
     else:
         f = open(path, 'rb')
 
-    X, labels = cPickle.load(f)
+    (x_train, labels_train), (x_test, labels_test) = cPickle.load(f)
     f.close()
 
     np.random.seed(seed)
-    np.random.shuffle(X)
+    np.random.shuffle(x_train)
     np.random.seed(seed)
-    np.random.shuffle(labels)
+    np.random.shuffle(labels_train)
+
+    np.random.seed(seed * 2)
+    np.random.shuffle(x_test)
+    np.random.seed(seed * 2)
+    np.random.shuffle(labels_test)
+
+    xs = x_train + x_test
+    labels = labels_train + labels_test
 
     if start_char is not None:
-        X = [[start_char] + [w + index_from for w in x] for x in X]
+        xs = [[start_char] + [w + index_from for w in x] for x in xs]
     elif index_from:
-        X = [[w + index_from for w in x] for x in X]
+        xs = [[w + index_from for w in x] for x in xs]
 
     if maxlen:
-        new_X = []
+        new_xs = []
         new_labels = []
-        for x, y in zip(X, labels):
+        for x, y in zip(xs, labels):
             if len(x) < maxlen:
-                new_X.append(x)
+                new_xs.append(x)
                 new_labels.append(y)
-        X = new_X
+        xs = new_xs
         labels = new_labels
-    if not X:
-        raise Exception('After filtering for sequences shorter than maxlen=' +
-                        str(maxlen) + ', no sequence was kept. '
-                        'Increase maxlen.')
+    if not xs:
+        raise ValueError('After filtering for sequences shorter than maxlen=' +
+                         str(maxlen) + ', no sequence was kept. '
+                         'Increase maxlen.')
     if not nb_words:
-        nb_words = max([max(x) for x in X])
+        nb_words = max([max(x) for x in xs])
 
     # by convention, use 2 as OOV word
-    # reserve 'index_from' (=3 by default) characters: 0 (padding), 1 (start), 2 (OOV)
+    # reserve 'index_from' (=3 by default) characters:
+    # 0 (padding), 1 (start), 2 (OOV)
     if oov_char is not None:
-        X = [[oov_char if (w >= nb_words or w < skip_top) else w for w in x] for x in X]
+        xs = [[oov_char if (w >= nb_words or w < skip_top) else w for w in x] for x in xs]
     else:
-        nX = []
-        for x in X:
+        new_xs = []
+        for x in xs:
             nx = []
             for w in x:
-                if (w >= nb_words or w < skip_top):
+                if w >= nb_words or w < skip_top:
                     nx.append(w)
-            nX.append(nx)
-        X = nX
+            new_xs.append(nx)
+        xs = new_xs
 
-    X_train = np.array(X[:int(len(X) * (1 - test_split))])
-    y_train = np.array(labels[:int(len(X) * (1 - test_split))])
+    x_train = np.array(xs[:len(x_train)])
+    y_train = np.array(labels[:len(x_train)])
 
-    X_test = np.array(X[int(len(X) * (1 - test_split)):])
-    y_test = np.array(labels[int(len(X) * (1 - test_split)):])
+    x_test = np.array(xs[len(x_train):])
+    y_test = np.array(labels[len(x_train):])
 
-    return (X_train, y_train), (X_test, y_test)
+    return (x_train, y_train), (x_test, y_test)
+
+
+def get_word_index(path='imdb_word_index.pkl'):
+    """Retrieves the dictionary mapping word indices back to words.
+
+    # Arguments
+        path: where to cache the data (relative to `~/.keras/dataset`).
+
+    # Returns
+        The word index dictionary.
+    """
+    path = get_file(path,
+                    origin='https://s3.amazonaws.com/text-datasets/imdb_word_index.pkl',
+                    md5_hash='72d94b01291be4ff843198d3b0e1e4d7')
+    f = open(path, 'rb')
+
+    if sys.version_info < (3,):
+        data = cPickle.load(f)
+    else:
+        data = cPickle.load(f, encoding='latin1')
+
+    f.close()
+    return data

keras/datasets/mnist.py

@@ -1,14 +1,22 @@
-# -*- coding: utf-8 -*-
 import gzip
 from ..utils.data_utils import get_file
 from six.moves import cPickle
 import sys
 
 
-def load_data(path="mnist.pkl.gz"):
-    path = get_file(path, origin="https://s3.amazonaws.com/img-datasets/mnist.pkl.gz")
+def load_data(path='mnist.pkl.gz'):
+    """Loads the MNIST dataset.
 
-    if path.endswith(".gz"):
+    # Arguments
+        path: path where to cache the dataset locally
+            (relative to ~/.keras/datasets).
+
+    # Returns
+        Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
+    """
+    path = get_file(path, origin='https://s3.amazonaws.com/img-datasets/mnist.pkl.gz')
+
+    if path.endswith('.gz'):
         f = gzip.open(path, 'rb')
     else:
         f = open(path, 'rb')
@@ -16,7 +24,7 @@ def load_data(path="mnist.pkl.gz"):
     if sys.version_info < (3,):
         data = cPickle.load(f)
     else:
-        data = cPickle.load(f, encoding="bytes")
+        data = cPickle.load(f, encoding='bytes')
 
     f.close()
-    return data  # (X_train, y_train), (X_test, y_test)
+    return data  # (x_train, y_train), (x_test, y_test)

keras/datasets/reuters.py

@@ -4,64 +4,108 @@ from ..utils.data_utils import get_file
 from six.moves import cPickle
 from six.moves import zip
 import numpy as np
+import sys
 
 
-def load_data(path="reuters.pkl", nb_words=None, skip_top=0,
+def load_data(path='reuters.pkl', nb_words=None, skip_top=0,
               maxlen=None, test_split=0.2, seed=113,
               start_char=1, oov_char=2, index_from=3):
+    """Loads the Reuters newswire classification dataset.
 
-    path = get_file(path, origin="https://s3.amazonaws.com/text-datasets/reuters.pkl")
+    # Arguments
+        path: where to cache the data (relative to `~/.keras/dataset`).
+        nb_words: max number of words to include. Words are ranked
+            by how often they occur (in the training set) and only
+            the most frequent words are kept
+        skip_top: skip the top N most frequently occuring words
+            (which may not be informative).
+        maxlen: truncate sequences after this length.
+        test_split: Fraction of the dataset to be used as test data.
+        seed: random seed for sample shuffling.
+        start_char: The start of a sequence will be marked with this character.
+            Set to 1 because 0 is usually the padding character.
+        oov_char: words that were cut out because of the `nb_words`
+            or `skip_top` limit will be replaced with this character.
+        index_from: index actual words with this index and higher.
+
+    # Returns
+        Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
+
+    Note that the 'out of vocabulary' character is only used for
+    words that were present in the training set but are not included
+    because they're not making the `nb_words` cut here.
+    Words that were not seen in the trining set but are in the test set
+    have simply been skipped.
+    """
+
+    path = get_file(path, origin='https://s3.amazonaws.com/text-datasets/reuters.pkl')
     f = open(path, 'rb')
-    X, labels = cPickle.load(f)
+    xs, labels = cPickle.load(f)
     f.close()
 
     np.random.seed(seed)
-    np.random.shuffle(X)
+    np.random.shuffle(xs)
     np.random.seed(seed)
     np.random.shuffle(labels)
 
     if start_char is not None:
-        X = [[start_char] + [w + index_from for w in x] for x in X]
+        xs = [[start_char] + [w + index_from for w in x] for x in xs]
     elif index_from:
-        X = [[w + index_from for w in x] for x in X]
+        xs = [[w + index_from for w in x] for x in xs]
 
     if maxlen:
-        new_X = []
+        new_xs = []
         new_labels = []
-        for x, y in zip(X, labels):
+        for x, y in zip(xs, labels):
             if len(x) < maxlen:
-                new_X.append(x)
+                new_xs.append(x)
                 new_labels.append(y)
-        X = new_X
+        xs = new_xs
         labels = new_labels
 
     if not nb_words:
-        nb_words = max([max(x) for x in X])
+        nb_words = max([max(x) for x in xs])
 
     # by convention, use 2 as OOV word
-    # reserve 'index_from' (=3 by default) characters: 0 (padding), 1 (start), 2 (OOV)
+    # reserve 'index_from' (=3 by default) characters:
+    # 0 (padding), 1 (start), 2 (OOV)
     if oov_char is not None:
-        X = [[oov_char if (w >= nb_words or w < skip_top) else w for w in x] for x in X]
+        xs = [[oov_char if (w >= nb_words or w < skip_top) else w for w in x] for x in xs]
     else:
-        nX = []
-        for x in X:
+        new_xs = []
+        for x in xs:
             nx = []
             for w in x:
-                if (w >= nb_words or w < skip_top):
+                if w >= nb_words or w < skip_top:
                     nx.append(w)
-            nX.append(nx)
-        X = nX
+            new_xs.append(nx)
+        xs = new_xs
 
-    X_train = X[:int(len(X) * (1 - test_split))]
-    y_train = labels[:int(len(X) * (1 - test_split))]
+    x_train = xs[:int(len(xs) * (1 - test_split))]
+    y_train = labels[:int(len(xs) * (1 - test_split))]
 
-    X_test = X[int(len(X) * (1 - test_split)):]
-    y_test = labels[int(len(X) * (1 - test_split)):]
+    x_test = xs[int(len(xs) * (1 - test_split)):]
+    y_test = labels[int(len(xs) * (1 - test_split)):]
 
-    return (X_train, y_train), (X_test, y_test)
+    return (x_train, y_train), (x_test, y_test)
 
 
-def get_word_index(path="reuters_word_index.pkl"):
-    path = get_file(path, origin="https://s3.amazonaws.com/text-datasets/reuters_word_index.pkl")
+def get_word_index(path='reuters_word_index.pkl'):
+    """Retrieves the dictionary mapping word indices back to words.
+
+    # Arguments
+        path: where to cache the data (relative to `~/.keras/dataset`).
+
+    # Returns
+        The word index dictionary.
+    """
+    path = get_file(path, origin='https://s3.amazonaws.com/text-datasets/reuters_word_index.pkl')
     f = open(path, 'rb')
-    return cPickle.load(f)
+
+    if sys.version_info < (3,):
+        data = cPickle.load(f)
+    else:
+        data = cPickle.load(f, encoding='latin1')
+
+    f.close()
+    return data

keras/initializations.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 import numpy as np
 from . import backend as K
+from .utils.generic_utils import get_from_module
 
 
 def get_fans(shape, dim_ordering='th'):
@@ -8,46 +9,52 @@ def get_fans(shape, dim_ordering='th'):
         fan_in = shape[0]
         fan_out = shape[1]
     elif len(shape) == 4 or len(shape) == 5:
-        # assuming convolution kernels (2D or 3D).
+        # Assuming convolution kernels (2D or 3D).
         # TH kernel shape: (depth, input_depth, ...)
         # TF kernel shape: (..., input_depth, depth)
         if dim_ordering == 'th':
-            fan_in = np.prod(shape[1:])
-            fan_out = shape[0]
+            receptive_field_size = np.prod(shape[2:])
+            fan_in = shape[1] * receptive_field_size
+            fan_out = shape[0] * receptive_field_size
         elif dim_ordering == 'tf':
-            fan_in = np.prod(shape[:-1])
-            fan_out = shape[-1]
+            receptive_field_size = np.prod(shape[:2])
+            fan_in = shape[-2] * receptive_field_size
+            fan_out = shape[-1] * receptive_field_size
         else:
-            raise Exception('Invalid dim_ordering: ' + dim_ordering)
+            raise ValueError('Invalid dim_ordering: ' + dim_ordering)
     else:
-        # no specific assumptions
+        # No specific assumptions.
         fan_in = np.sqrt(np.prod(shape))
         fan_out = np.sqrt(np.prod(shape))
     return fan_in, fan_out
 
 
-def uniform(shape, scale=0.05, name=None):
-    return K.variable(np.random.uniform(low=-scale, high=scale, size=shape),
-                      name=name)
+def uniform(shape, scale=0.05, name=None, dim_ordering='th'):
+    return K.random_uniform_variable(shape, -scale, scale, name=name)
 
 
-def normal(shape, scale=0.05, name=None):
-    return K.variable(np.random.normal(loc=0.0, scale=scale, size=shape),
-                      name=name)
+def normal(shape, scale=0.05, name=None, dim_ordering='th'):
+    return K.random_normal_variable(shape, 0.0, scale, name=name)
 
 
 def lecun_uniform(shape, name=None, dim_ordering='th'):
-    ''' Reference: LeCun 98, Efficient Backprop
+    """LeCun uniform variance scaling initializer.
+
+    # References
+        LeCun 98, Efficient Backprop,
         http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf
-    '''
+    """
     fan_in, fan_out = get_fans(shape, dim_ordering=dim_ordering)
     scale = np.sqrt(3. / fan_in)
     return uniform(shape, scale, name=name)
 
 
 def glorot_normal(shape, name=None, dim_ordering='th'):
-    ''' Reference: Glorot & Bengio, AISTATS 2010
-    '''
+    """Glorot normal variance scaling initializer.
+
+    # References
+        Glorot & Bengio, AISTATS 2010
+    """
     fan_in, fan_out = get_fans(shape, dim_ordering=dim_ordering)
     s = np.sqrt(2. / (fan_in + fan_out))
     return normal(shape, s, name=name)
@@ -60,48 +67,55 @@ def glorot_uniform(shape, name=None, dim_ordering='th'):
 
 
 def he_normal(shape, name=None, dim_ordering='th'):
-    ''' Reference:  He et al., http://arxiv.org/abs/1502.01852
-    '''
+    """He normal variance scaling initializer.
+
+    # References
+        He et al., http://arxiv.org/abs/1502.01852
+    """
     fan_in, fan_out = get_fans(shape, dim_ordering=dim_ordering)
     s = np.sqrt(2. / fan_in)
     return normal(shape, s, name=name)
 
 
 def he_uniform(shape, name=None, dim_ordering='th'):
+    """He uniform variance scaling initializer.
+    """
     fan_in, fan_out = get_fans(shape, dim_ordering=dim_ordering)
     s = np.sqrt(6. / fan_in)
     return uniform(shape, s, name=name)
 
 
-def orthogonal(shape, scale=1.1, name=None):
-    ''' From Lasagne. Reference: Saxe et al., http://arxiv.org/abs/1312.6120
-    '''
+def orthogonal(shape, scale=1.1, name=None, dim_ordering='th'):
+    """Orthogonal initializer.
+
+    # References
+        Saxe et al., http://arxiv.org/abs/1312.6120
+    """
     flat_shape = (shape[0], np.prod(shape[1:]))
     a = np.random.normal(0.0, 1.0, flat_shape)
     u, _, v = np.linalg.svd(a, full_matrices=False)
-    # pick the one with the correct shape
+    # Pick the one with the correct shape.
     q = u if u.shape == flat_shape else v
     q = q.reshape(shape)
     return K.variable(scale * q[:shape[0], :shape[1]], name=name)
 
 
-def identity(shape, scale=1, name=None):
+def identity(shape, scale=1, name=None, dim_ordering='th'):
     if len(shape) != 2 or shape[0] != shape[1]:
-        raise Exception('Identity matrix initialization can only be used '
-                        'for 2D square matrices.')
+        raise ValueError('Identity matrix initialization can only be used '
+                         'for 2D square matrices.')
     else:
         return K.variable(scale * np.identity(shape[0]), name=name)
 
 
-def zero(shape, name=None):
+def zero(shape, name=None, dim_ordering='th'):
     return K.zeros(shape, name=name)
 
 
-def one(shape, name=None):
+def one(shape, name=None, dim_ordering='th'):
     return K.ones(shape, name=name)
 
 
-from .utils.generic_utils import get_from_module
 def get(identifier, **kwargs):
     return get_from_module(identifier, globals(),
                            'initialization', kwargs=kwargs)

keras/layers/__init__.py

@@ -2,9 +2,12 @@ from __future__ import absolute_import
 from ..engine import Layer, Input, InputLayer, Merge, merge, InputSpec
 from .core import *
 from .convolutional import *
+from .pooling import *
+from .local import *
 from .recurrent import *
 from .normalization import *
 from .embeddings import *
 from .noise import *
 from .advanced_activations import *
 from .wrappers import *
+from .convolutional_recurrent import *

keras/layers/advanced_activations.py

@@ -5,8 +5,9 @@ import numpy as np
 
 
 class LeakyReLU(Layer):
-    '''Special version of a Rectified Linear Unit
-    that allows a small gradient when the unit is not active:
+    """Leaky version of a Rectified Linear Unit.
+
+    It allows a small gradient when the unit is not active:
     `f(x) = alpha * x for x < 0`,
     `f(x) = x for x >= 0`.
 
@@ -20,7 +21,11 @@ class LeakyReLU(Layer):
 
     # Arguments
         alpha: float >= 0. Negative slope coefficient.
-    '''
+
+    # References
+        - [Rectifier Nonlinearities Improve Neural Network Acoustic Models](https://web.stanford.edu/~awni/papers/relu_hybrid_icml2013_final.pdf)
+    """
+
     def __init__(self, alpha=0.3, **kwargs):
         self.supports_masking = True
         self.alpha = alpha
@@ -36,7 +41,9 @@ class LeakyReLU(Layer):
 
 
 class PReLU(Layer):
-    '''Parametric Rectified Linear Unit:
+    """Parametric Rectified Linear Unit.
+
+    It follows:
     `f(x) = alphas * x for x < 0`,
     `f(x) = x for x >= 0`,
     where `alphas` is a learned array with the same shape as x.
@@ -51,19 +58,39 @@ class PReLU(Layer):
 
     # Arguments
         init: initialization function for the weights.
-        weights: initial weights, as a list of a single numpy array.
+        weights: initial weights, as a list of a single Numpy array.
+        shared_axes: the axes along which to share learnable
+            parameters for the activation function.
+            For example, if the incoming feature maps
+            are from a 2D convolution
+            with output shape `(batch, height, width, channels)`,
+            and you wish to share parameters across space
+            so that each filter only has one set of parameters,
+            set `shared_axes=[1, 2]`.
 
     # References
-        - [Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification](http://arxiv.org/pdf/1502.01852v1.pdf)
-    '''
-    def __init__(self, init='zero', weights=None, **kwargs):
+        - [Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification](https://arxiv.org/abs/1502.01852)
+    """
+
+    def __init__(self, init='zero', weights=None, shared_axes=None, **kwargs):
         self.supports_masking = True
         self.init = initializations.get(init)
         self.initial_weights = weights
+        if not isinstance(shared_axes, (list, tuple)):
+            self.shared_axes = [shared_axes]
+        else:
+            self.shared_axes = list(shared_axes)
         super(PReLU, self).__init__(**kwargs)
 
     def build(self, input_shape):
-        self.alphas = self.init(input_shape[1:],
+        param_shape = list(input_shape[1:])
+        self.param_broadcast = [False] * len(param_shape)
+        if self.shared_axes[0] is not None:
+            for i in self.shared_axes:
+                param_shape[i - 1] = 1
+                self.param_broadcast[i - 1] = True
+
+        self.alphas = self.init(param_shape,
                                 name='{}_alphas'.format(self.name))
         self.trainable_weights = [self.alphas]
 
@@ -73,7 +100,11 @@ class PReLU(Layer):
 
     def call(self, x, mask=None):
         pos = K.relu(x)
-        neg = self.alphas * (x - abs(x)) * 0.5
+        if K.backend() == 'theano':
+            neg = (K.pattern_broadcast(self.alphas, self.param_broadcast) *
+                   (x - K.abs(x)) * 0.5)
+        else:
+            neg = self.alphas * (x - K.abs(x)) * 0.5
         return pos + neg
 
     def get_config(self):
@@ -83,7 +114,9 @@ class PReLU(Layer):
 
 
 class ELU(Layer):
-    '''Exponential Linear Unit:
+    """Exponential Linear Unit.
+
+    It follows:
     `f(x) =  alpha * (exp(x) - 1.) for x < 0`,
     `f(x) = x for x >= 0`.
 
@@ -99,27 +132,28 @@ class ELU(Layer):
         alpha: scale for the negative factor.
 
     # References
-        - [Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs)](http://arxiv.org/pdf/1511.07289v1.pdf)
-    '''
+        - [Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs)](https://arxiv.org/abs/1511.07289v1)
+    """
+
     def __init__(self, alpha=1.0, **kwargs):
         self.supports_masking = True
         self.alpha = K.cast_to_floatx(alpha)
         super(ELU, self).__init__(**kwargs)
 
     def call(self, x, mask=None):
-        pos = K.relu(x)
-        neg = (x - abs(x)) * 0.5
-        return pos + self.alpha * (K.exp(neg) - 1.)
+        return K.elu(x, self.alpha)
 
     def get_config(self):
-        config = {'alpha': self.alpha}
+        config = {'alpha': float(self.alpha)}
         base_config = super(ELU, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
 
 class ParametricSoftplus(Layer):
-    '''Parametric Softplus:
-    `alpha * log(1 + exp(beta * x))`
+    """Parametric Softplus.
+
+    It follows:
+    `f(x) = alpha * log(1 + exp(beta * x))`
 
     # Input shape
         Arbitrary. Use the keyword argument `input_shape`
@@ -133,23 +167,42 @@ class ParametricSoftplus(Layer):
         alpha_init: float. Initial value of the alpha weights.
         beta_init: float. Initial values of the beta weights.
         weights: initial weights, as a list of 2 numpy arrays.
+        shared_axes: the axes along which to share learnable
+            parameters for the activation function.
+            For example, if the incoming feature maps
+            are from a 2D convolution
+            with output shape `(batch, height, width, channels)`,
+            and you wish to share parameters across space
+            so that each filter only has one set of parameters,
+            set `shared_axes=[1, 2]`.
 
     # References
         - [Inferring Nonlinear Neuronal Computation Based on Physiologically Plausible Inputs](http://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1003143)
-    '''
+    """
+
     def __init__(self, alpha_init=0.2, beta_init=5.0,
-                 weights=None, **kwargs):
+                 weights=None, shared_axes=None, **kwargs):
         self.supports_masking = True
         self.alpha_init = K.cast_to_floatx(alpha_init)
         self.beta_init = K.cast_to_floatx(beta_init)
         self.initial_weights = weights
+        if not isinstance(shared_axes, (list, tuple)):
+            self.shared_axes = [shared_axes]
+        else:
+            self.shared_axes = list(shared_axes)
         super(ParametricSoftplus, self).__init__(**kwargs)
 
     def build(self, input_shape):
-        input_shape = input_shape[1:]
-        self.alphas = K.variable(self.alpha_init * np.ones(input_shape),
+        param_shape = list(input_shape[1:])
+        self.param_broadcast = [False] * len(param_shape)
+        if self.shared_axes[0] is not None:
+            for i in self.shared_axes:
+                param_shape[i - 1] = 1
+                self.param_broadcast[i - 1] = True
+
+        self.alphas = K.variable(self.alpha_init * np.ones(param_shape),
                                  name='{}_alphas'.format(self.name))
-        self.betas = K.variable(self.beta_init * np.ones(input_shape),
+        self.betas = K.variable(self.beta_init * np.ones(param_shape),
                                 name='{}_betas'.format(self.name))
         self.trainable_weights = [self.alphas, self.betas]
 
@@ -158,18 +211,25 @@ class ParametricSoftplus(Layer):
             del self.initial_weights
 
     def call(self, x, mask=None):
-        return K.softplus(self.betas * x) * self.alphas
+        if K.backend() == 'theano':
+            return (K.softplus(K.pattern_broadcast(self.betas,
+                                                   self.param_broadcast) * x) *
+                    K.pattern_broadcast(self.alphas, self.param_broadcast))
+        else:
+            return K.softplus(self.betas * x) * self.alphas
 
     def get_config(self):
-        config = {'alpha_init': self.alpha_init,
-                  'beta_init': self.beta_init}
+        config = {'alpha_init': float(self.alpha_init),
+                  'beta_init': float(self.beta_init)}
         base_config = super(ParametricSoftplus, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
 
 class ThresholdedReLU(Layer):
-    '''Thresholded Rectified Linear Unit:
-    `f(x) = x for x > theta`
+    """Thresholded Rectified Linear Unit.
+
+    It follows:
+    `f(x) = x for x > theta`,
     `f(x) = 0 otherwise`.
 
     # Input shape
@@ -184,8 +244,9 @@ class ThresholdedReLU(Layer):
         theta: float >= 0. Threshold location of activation.
 
     # References
-        - [Zero-Bias Autoencoders and the Benefits of Co-Adapting Features](http://arxiv.org/pdf/1402.3337.pdf)
-    '''
+        - [Zero-Bias Autoencoders and the Benefits of Co-Adapting Features](http://arxiv.org/abs/1402.3337)
+    """
+
     def __init__(self, theta=1.0, **kwargs):
         self.supports_masking = True
         self.theta = K.cast_to_floatx(theta)
@@ -195,13 +256,18 @@ class ThresholdedReLU(Layer):
         return x * K.cast(x > self.theta, K.floatx())
 
     def get_config(self):
-        config = {'theta': self.theta}
+        config = {'theta': float(self.theta)}
         base_config = super(ThresholdedReLU, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
 
 class SReLU(Layer):
-    '''S-shaped Rectified Linear Unit.
+    """S-shaped Rectified Linear Unit.
+
+    It follows:
+    `f(x) = t^r + a^r(x - t^r) for x >= t^r`,
+    `f(x) = x for t^r > x > t^l`,
+    `f(x) = t^l + a^l(x - t^l) for x <= t^l`.
 
     # Input shape
         Arbitrary. Use the keyword argument `input_shape`
@@ -216,46 +282,77 @@ class SReLU(Layer):
         a_left_init: initialization function for the left part slope
         t_right_init: initialization function for the right part intercept
         a_right_init: initialization function for the right part slope
+        shared_axes: the axes along which to share learnable
+            parameters for the activation function.
+            For example, if the incoming feature maps
+            are from a 2D convolution
+            with output shape `(batch, height, width, channels)`,
+            and you wish to share parameters across space
+            so that each filter only has one set of parameters,
+            set `shared_axes=[1, 2]`.
 
     # References
         - [Deep Learning with S-shaped Rectified Linear Activation Units](http://arxiv.org/abs/1512.07030)
-    '''
+    """
+
     def __init__(self, t_left_init='zero', a_left_init='glorot_uniform',
-                 t_right_init='glorot_uniform', a_right_init='one', **kwargs):
+                 t_right_init='glorot_uniform', a_right_init='one',
+                 shared_axes=None, **kwargs):
         self.supports_masking = True
         self.t_left_init = t_left_init
         self.a_left_init = a_left_init
         self.t_right_init = t_right_init
         self.a_right_init = a_right_init
+        if not isinstance(shared_axes, (list, tuple)):
+            self.shared_axes = [shared_axes]
+        else:
+            self.shared_axes = list(shared_axes)
         super(SReLU, self).__init__(**kwargs)
 
     def build(self, input_shape):
-        input_shape = input_shape[1:]
+        param_shape = list(input_shape[1:])
+        self.param_broadcast = [False] * len(param_shape)
+        if self.shared_axes[0] is not None:
+            for i in self.shared_axes:
+                param_shape[i - 1] = 1
+                self.param_broadcast[i - 1] = True
 
         t_left_init = initializations.get(self.t_left_init)
         a_left_init = initializations.get(self.a_left_init)
         t_right_init = initializations.get(self.t_right_init)
         a_right_init = initializations.get(self.a_right_init)
 
-        self.t_left = t_left_init(input_shape,
+        self.t_left = t_left_init(param_shape,
                                   name='{}_t_left'.format(self.name))
-        self.a_left = a_left_init(input_shape,
+        self.a_left = a_left_init(param_shape,
                                   name='{}_a_left'.format(self.name))
-        self.t_right = t_right_init(input_shape,
+        self.t_right = t_right_init(param_shape,
                                     name='{}_t_right'.format(self.name))
-        self.a_right = a_right_init(input_shape,
+        self.a_right = a_right_init(param_shape,
                                     name='{}_a_right'.format(self.name))
         # ensure the the right part is always to the right of the left
-        self.t_right_actual = self.t_left + abs(self.t_right)
+        self.t_right_actual = self.t_left + K.abs(self.t_right)
         self.trainable_weights = [self.t_left, self.a_left,
                                   self.t_right, self.a_right]
 
     def call(self, x, mask=None):
-        Y_left_and_center = self.t_left + K.relu(x - self.t_left,
-                                                 self.a_left,
-                                                 self.t_right_actual - self.t_left)
-        Y_right = K.relu(x - self.t_right_actual) * self.a_right
-        return Y_left_and_center + Y_right
+        if K.backend() == 'theano':
+            t_left = K.pattern_broadcast(self.t_left, self.param_broadcast)
+            a_left = K.pattern_broadcast(self.a_left, self.param_broadcast)
+            a_right = K.pattern_broadcast(self.a_right, self.param_broadcast)
+            t_right_actual = K.pattern_broadcast(self.t_right_actual,
+                                                 self.param_broadcast)
+        else:
+            t_left = self.t_left
+            a_left = self.a_left
+            a_right = self.a_right
+            t_right_actual = self.t_right_actual
+
+        y_left_and_center = t_left + K.relu(x - t_left,
+                                            a_left,
+                                            t_right_actual - t_left)
+        y_right = K.relu(x - t_right_actual) * a_right
+        return y_left_and_center + y_right
 
     def get_config(self):
         config = {'t_left_init': self.t_left_init,

keras/layers/convolutional_recurrent.py

@@ -0,0 +1,515 @@
+from .. import backend as K
+from .. import activations
+from .. import initializations
+from .. import regularizers
+
+import numpy as np
+from ..engine import Layer
+from ..engine import InputSpec
+from ..utils.np_utils import conv_output_length
+import warnings
+
+
+class ConvRecurrent2D(Layer):
+    """Abstract base class for convolutional recurrent layers.
+
+    Do not use in a model -- it's not a functional layer!
+
+    ConvLSTM2D
+    follow the specifications of this class and accept
+    the keyword arguments listed below.
+
+    # Input shape
+        5D tensor with shape `(nb_samples, timesteps, channels, rows, cols)`.
+
+    # Output shape
+        - if `return_sequences`: 5D tensor with shape
+            `(nb_samples, timesteps, channels, rows, cols)`.
+        - else, 4D tensor with shape `(nb_samples, channels, rows, cols)`.
+
+    # Arguments
+        weights: list of numpy arrays to set as initial weights.
+            The list should have 3 elements, of shapes:
+            `[(input_dim, nb_filter), (nb_filter, nb_filter), (nb_filter,)]`.
+        return_sequences: Boolean. Whether to return the last output
+            in the output sequence, or the full sequence.
+        go_backwards: Boolean (default False).
+            If True, rocess the input sequence backwards.
+        stateful: Boolean (default False). If True, the last state
+            for each sample at index i in a batch will be used as initial
+            state for the sample of index i in the following batch.
+        nb_filter: Number of convolution filters to use.
+        nb_row: Number of rows in the convolution kernel.
+        nb_col: Number of columns in the convolution kernel.
+            is required when using this layer as the first layer in a model.
+        input_shape: input_shape
+
+    # Masking
+        This layer supports masking for input data with a variable number
+        of timesteps. To introduce masks to your data,
+        use an [Embedding](embeddings.md) layer with the `mask_zero` parameter
+        set to `True`.
+        **Note:** for the time being, masking is only supported with Theano.
+
+    # TensorFlow warning
+        For the time being, when using the TensorFlow backend,
+        the number of timesteps used must be specified in your model.
+        Make sure to pass an `input_length` int argument to your
+        recurrent layer (if it comes first in your model),
+        or to pass a complete `input_shape` argument to the first layer
+        in your model otherwise.
+
+
+    # Note on using statefulness in RNNs
+        You can set RNN layers to be 'stateful', which means that the states
+        computed for the samples in one batch will be reused as initial states
+        for the samples in the next batch.
+        This assumes a one-to-one mapping between
+        samples in different successive batches.
+
+        To enable statefulness:
+            - specify `stateful=True` in the layer constructor.
+            - specify a fixed batch size for your model, by passing
+                a `batch_input_size=(...)` to the first layer in your model.
+                This is the expected shape of your inputs *including the batch
+                size*.
+                It should be a tuple of integers, e.g. `(32, 10, 100)`.
+
+        To reset the states of your model, call `.reset_states()` on either
+        a specific layer, or on your entire model.
+    """
+
+    def __init__(self, weights=None, nb_row=None, nb_col=None, nb_filter=None,
+                 return_sequences=False, go_backwards=False, stateful=False,
+                 dim_ordering=None, **kwargs):
+        self.return_sequences = return_sequences
+        self.go_backwards = go_backwards
+        self.stateful = stateful
+        self.initial_weights = weights
+        self.nb_row = nb_row
+        self.nb_col = nb_col
+        self.nb_filter = nb_filter
+        self.dim_ordering = dim_ordering
+        self.input_spec = [InputSpec(ndim=5)]
+
+        super(ConvRecurrent2D, self).__init__(**kwargs)
+
+    def compute_mask(self, input, mask):
+        if self.return_sequences:
+            return mask
+        else:
+            return None
+
+    def get_output_shape_for(self, input_shape):
+
+        if self.dim_ordering == 'th':
+            rows = input_shape[3]
+            cols = input_shape[4]
+        elif self.dim_ordering == 'tf':
+            rows = input_shape[2]
+            cols = input_shape[3]
+        else:
+            raise ValueError('Invalid dim_ordering:', self.dim_ordering)
+
+        rows = conv_output_length(rows, self.nb_row,
+                                  self.border_mode, self.subsample[0])
+        cols = conv_output_length(cols, self.nb_col,
+                                  self.border_mode, self.subsample[1])
+
+        if self.return_sequences:
+            if self.dim_ordering == 'th':
+                return (input_shape[0], input_shape[1],
+                        self.nb_filter, rows, cols)
+            elif self.dim_ordering == 'tf':
+                return (input_shape[0], input_shape[1],
+                        rows, cols, self.nb_filter)
+        else:
+            if self.dim_ordering == 'th':
+                return (input_shape[0], self.nb_filter, rows, cols)
+            elif self.dim_ordering == 'tf':
+                return (input_shape[0], rows, cols, self.nb_filter)
+
+    def step(self, x, states):
+        raise NotImplementedError
+
+    def get_constants(self, X, train=False):
+        return None
+
+    def get_initial_states(self, X):
+        # (samples, timesteps, row, col, filter)
+        initial_state = K.zeros_like(X)
+        # (samples,row, col, filter)
+        initial_state = K.sum(initial_state, axis=1)
+        initial_state = self.conv_step(initial_state, K.zeros(self.W_shape),
+                                       border_mode=self.border_mode)
+
+        initial_states = [initial_state for _ in range(2)]
+        return initial_states
+
+    def preprocess_input(self, x):
+        return x
+
+    def call(self, x, mask=None):
+        assert K.ndim(x) == 5
+        input_shape = self.input_spec[0].shape
+        unroll = False
+
+        if self.stateful:
+            initial_states = self.states
+        else:
+            initial_states = self.get_initial_states(x)
+
+        constants = self.get_constants(x)
+        preprocessed_input = self.preprocess_input(x)
+
+        last_output, outputs, states = K.rnn(self.step, preprocessed_input,
+                                             initial_states,
+                                             go_backwards=self.go_backwards,
+                                             mask=mask,
+                                             constants=constants,
+                                             unroll=unroll,
+                                             input_length=input_shape[1])
+        if self.stateful:
+            self.updates = []
+            for i in range(len(states)):
+                self.updates.append((self.states[i], states[i]))
+
+        if self.return_sequences:
+            return outputs
+        else:
+            return last_output
+
+    def get_config(self):
+        config = {'return_sequences': self.return_sequences,
+                  'go_backwards': self.go_backwards,
+                  'stateful': self.stateful}
+        if self.stateful:
+            config['batch_input_shape'] = self.input_spec[0].shape
+
+        base_config = super(ConvRecurrent2D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
+class ConvLSTM2D(ConvRecurrent2D):
+    """Convolutional LSTM.
+
+    # Input shape
+        - if dim_ordering='th'
+            5D tensor with shape:
+            `(samples,time, channels, rows, cols)`
+        - if dim_ordering='tf'
+            5D tensor with shape:
+            `(samples,time, rows, cols, channels)`
+
+     # Output shape
+        - if `return_sequences`
+             - if dim_ordering='th'
+                5D tensor with shape:
+                `(samples, time, nb_filter, output_row, output_col)`
+             - if dim_ordering='tf'
+                5D tensor with shape:
+                `(samples, time, output_row, output_col, nb_filter)`
+        - else
+            - if dim_ordering ='th'
+                4D tensor with shape:
+                `(samples, nb_filter, output_row, output_col)`
+            - if dim_ordering='tf'
+                4D tensor with shape:
+                `(samples, output_row, output_col, nb_filter)`
+
+        where o_row and o_col depend on the shape of the filter and
+        the border_mode
+
+        # Arguments
+            nb_filter: Number of convolution filters to use.
+            nb_row: Number of rows in the convolution kernel.
+            nb_col: Number of columns in the convolution kernel.
+            border_mode: 'valid' or 'same'.
+            subsample: tuple of length 2. Factor by which to subsample output.
+                Also called strides elsewhere.
+            dim_ordering: 'tf' if the feature are at the last dimension or 'th'
+            stateful : Boolean (default False). If True, the last state
+                for each sample at index i in a batch will be used as initial
+                state for the sample of index i in the following batch.
+            init: weight initialization function.
+                Can be the name of an existing function (str),
+                or a Theano function
+                (see: [initializations](../initializations.md)).
+            inner_init: initialization function of the inner cells.
+            forget_bias_init: initialization function for the bias of the
+            forget gate.
+                [Jozefowicz et al.](http://www.jmlr.org/proceedings/papers/v37/jozefowicz15.pdf)
+                recommend initializing with ones.
+            activation: activation function.
+                Can be the name of an existing function (str),
+                or a Theano function (see: [activations](../activations.md)).
+            inner_activation: activation function for the inner cells.
+
+    # References
+        - [Convolutional LSTM Network: A Machine Learning Approach for
+        Precipitation Nowcasting](http://arxiv.org/abs/1506.04214v1)
+        The current implementation does not include the feedback loop on the
+        cells output
+    """
+
+    def __init__(self, nb_filter, nb_row, nb_col,
+                 init='glorot_uniform', inner_init='orthogonal',
+                 forget_bias_init='one', activation='tanh',
+                 inner_activation='hard_sigmoid',
+                 dim_ordering='default',
+                 border_mode='valid', subsample=(1, 1),
+                 W_regularizer=None, U_regularizer=None, b_regularizer=None,
+                 dropout_W=0., dropout_U=0., **kwargs):
+
+        if dim_ordering == 'default':
+            dim_ordering = K.image_dim_ordering()
+        if dim_ordering not in {'tf', 'th'}:
+            raise ValueError('dim_ordering must be in {tf,th}', dim_ordering)
+        self.nb_filter = nb_filter
+        self.nb_row = nb_row
+        self.nb_col = nb_col
+        self.init = initializations.get(init)
+        self.inner_init = initializations.get(inner_init)
+        self.forget_bias_init = initializations.get(forget_bias_init)
+        self.activation = activations.get(activation)
+        self.inner_activation = activations.get(inner_activation)
+        self.border_mode = border_mode
+        self.subsample = subsample
+
+        if dim_ordering == 'th':
+            warnings.warn('Be carefull if used with convolution3D layers:\n'
+                          'th in convolution 3D corresponds to '
+                          '(samples, channels, conv_dim1, conv_dim2,'
+                          'conv_dim3)\n'
+                          'while for this network it corresponds to: '
+                          '(samples, time, channels, rows, cols)')
+        self.dim_ordering = dim_ordering
+
+        kwargs['nb_filter'] = nb_filter
+        kwargs['nb_row'] = nb_row
+        kwargs['nb_col'] = nb_col
+        kwargs['dim_ordering'] = dim_ordering
+
+        self.W_regularizer = regularizers.get(W_regularizer)
+        self.U_regularizer = regularizers.get(U_regularizer)
+        self.b_regularizer = regularizers.get(b_regularizer)
+        self.dropout_W, self.dropout_U = dropout_W, dropout_U
+        if self.dropout_W or self.dropout_U:
+            self.uses_learning_phase = True
+
+        super(ConvLSTM2D, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+        self.input_spec = [InputSpec(shape=input_shape)]
+
+        if self.dim_ordering == 'th':
+            stack_size = input_shape[2]
+            self.W_shape = (self.nb_filter, stack_size,
+                            self.nb_row, self.nb_col)
+        elif self.dim_ordering == 'tf':
+            stack_size = input_shape[4]
+            self.W_shape = (self.nb_row, self.nb_col,
+                            stack_size, self.nb_filter)
+        else:
+            raise ValueError('Invalid dim_ordering:', self.dim_ordering)
+
+        if self.dim_ordering == 'th':
+            self.W_shape1 = (self.nb_filter, self.nb_filter,
+                             self.nb_row, self.nb_col)
+        elif self.dim_ordering == 'tf':
+            self.W_shape1 = (self.nb_row, self.nb_col,
+                             self.nb_filter, self.nb_filter)
+
+        if self.stateful:
+            self.reset_states()
+        else:
+            # initial states: 2 all-zero tensor of shape (nb_filter)
+            self.states = [None, None, None, None]
+
+        self.W_i = self.init(self.W_shape, name='{}_W_i'.format(self.name))
+        self.U_i = self.inner_init(self.W_shape1,
+                                   name='{}_U_i'.format(self.name))
+        self.b_i = K.zeros((self.nb_filter,), name='{}_b_i'.format(self.name))
+
+        self.W_f = self.init(self.W_shape, name='{}_W_f'.format(self.name))
+        self.U_f = self.inner_init(self.W_shape1,
+                                   name='{}_U_f'.format(self.name))
+        self.b_f = self.forget_bias_init((self.nb_filter,),
+                                         name='{}_b_f'.format(self.name))
+
+        self.W_c = self.init(self.W_shape, name='{}_W_c'.format(self.name))
+        self.U_c = self.inner_init(self.W_shape1,
+                                   name='{}_U_c'.format(self.name))
+        self.b_c = K.zeros((self.nb_filter,), name='{}_b_c'.format(self.name))
+
+        self.W_o = self.init(self.W_shape, name='{}_W_o'.format(self.name))
+        self.U_o = self.inner_init(self.W_shape1,
+                                   name='{}_U_o'.format(self.name))
+        self.b_o = K.zeros((self.nb_filter,), name='{}_b_o'.format(self.name))
+
+        self.trainable_weights = [self.W_i, self.U_i, self.b_i,
+                                  self.W_c, self.U_c, self.b_c,
+                                  self.W_f, self.U_f, self.b_f,
+                                  self.W_o, self.U_o, self.b_o]
+
+        self.W = K.concatenate([self.W_i, self.W_f, self.W_c, self.W_o])
+        self.U = K.concatenate([self.U_i, self.U_f, self.U_c, self.U_o])
+        self.b = K.concatenate([self.b_i, self.b_f, self.b_c, self.b_o])
+
+        self.regularizers = []
+        if self.W_regularizer:
+            self.W_regularizer.set_param(self.W)
+            self.regularizers.append(self.W_regularizer)
+        if self.U_regularizer:
+            self.U_regularizer.set_param(self.U)
+            self.regularizers.append(self.U_regularizer)
+        if self.b_regularizer:
+            self.b_regularizer.set_param(self.b)
+            self.regularizers.append(self.b_regularizer)
+
+        if self.initial_weights is not None:
+            self.set_weights(self.initial_weights)
+            del self.initial_weights
+        self.built = True
+
+    def reset_states(self):
+        assert self.stateful, 'Layer must be stateful.'
+        input_shape = self.input_spec[0].shape
+        output_shape = self.get_output_shape_for(input_shape)
+        if not input_shape[0]:
+            raise ValueError('If a RNN is stateful, a complete ' +
+                             'input_shape must be provided ' +
+                             '(including batch size).')
+
+        if self.return_sequences:
+            out_row, out_col, out_filter = output_shape[2:]
+        else:
+            out_row, out_col, out_filter = output_shape[1:]
+
+        if hasattr(self, 'states'):
+            K.set_value(self.states[0],
+                        np.zeros((input_shape[0],
+                                  out_row, out_col, out_filter)))
+            K.set_value(self.states[1],
+                        np.zeros((input_shape[0],
+                                  out_row, out_col, out_filter)))
+        else:
+            self.states = [K.zeros((input_shape[0],
+                                    out_row, out_col, out_filter)),
+                           K.zeros((input_shape[0],
+                                    out_row, out_col, out_filter))]
+
+    def conv_step(self, x, W, b=None, border_mode='valid'):
+        input_shape = self.input_spec[0].shape
+
+        conv_out = K.conv2d(x, W, strides=self.subsample,
+                            border_mode=border_mode,
+                            dim_ordering=self.dim_ordering,
+                            image_shape=(input_shape[0],
+                                         input_shape[2],
+                                         input_shape[3],
+                                         input_shape[4]),
+                            filter_shape=self.W_shape)
+        if b:
+            if self.dim_ordering == 'th':
+                conv_out = conv_out + K.reshape(b, (1, self.nb_filter, 1, 1))
+            elif self.dim_ordering == 'tf':
+                conv_out = conv_out + K.reshape(b, (1, 1, 1, self.nb_filter))
+            else:
+                raise ValueError('Invalid dim_ordering:', self.dim_ordering)
+
+        return conv_out
+
+    def conv_step_hidden(self, x, W, border_mode='valid'):
+        # This new function was defined because the
+        # image shape must be hardcoded
+        input_shape = self.input_spec[0].shape
+        output_shape = self.get_output_shape_for(input_shape)
+        if self.return_sequences:
+            out_row, out_col, out_filter = output_shape[2:]
+        else:
+            out_row, out_col, out_filter = output_shape[1:]
+
+        conv_out = K.conv2d(x, W, strides=(1, 1),
+                            border_mode=border_mode,
+                            dim_ordering=self.dim_ordering,
+                            image_shape=(input_shape[0],
+                                         out_row, out_col,
+                                         out_filter),
+                            filter_shape=self.W_shape1)
+
+        return conv_out
+
+    def step(self, x, states):
+        assert len(states) == 4
+        h_tm1 = states[0]
+        c_tm1 = states[1]
+        B_U = states[2]
+        B_W = states[3]
+
+        x_i = self.conv_step(x * B_W[0], self.W_i, self.b_i,
+                             border_mode=self.border_mode)
+        x_f = self.conv_step(x * B_W[1], self.W_f, self.b_f,
+                             border_mode=self.border_mode)
+        x_c = self.conv_step(x * B_W[2], self.W_c, self.b_c,
+                             border_mode=self.border_mode)
+        x_o = self.conv_step(x * B_W[3], self.W_o, self.b_o,
+                             border_mode=self.border_mode)
+
+        # U : from nb_filter to nb_filter
+        # Same because must be stable in the output space
+        h_i = self.conv_step_hidden(h_tm1 * B_U[0], self.U_i,
+                                    border_mode='same')
+        h_f = self.conv_step_hidden(h_tm1 * B_U[1], self.U_f,
+                                    border_mode='same')
+        h_c = self.conv_step_hidden(h_tm1 * B_U[2], self.U_c,
+                                    border_mode='same')
+        h_o = self.conv_step_hidden(h_tm1 * B_U[3], self.U_o,
+                                    border_mode='same')
+
+        i = self.inner_activation(x_i + h_i)
+        f = self.inner_activation(x_f + h_f)
+        c = f * c_tm1 + i * self.activation(x_c + h_c)
+        o = self.inner_activation(x_o + h_o)
+        h = o * self.activation(c)
+
+        return h, [h, c]
+
+    def get_constants(self, x):
+        constants = []
+        if 0 < self.dropout_U < 1:
+            ones = K.zeros_like(x)
+            ones = K.sum(ones, axis=1)
+            ones = self.conv_step(ones, K.zeros(self.W_shape),
+                                  border_mode=self.border_mode)
+            ones += 1
+            B_U = [K.in_train_phase(K.dropout(ones, self.dropout_U), ones)
+                   for _ in range(4)]
+            constants.append(B_U)
+        else:
+            constants.append([K.cast_to_floatx(1.) for _ in range(4)])
+
+        if 0 < self.dropout_W < 1:
+            ones = K.zeros_like(x)
+            ones = K.sum(ones, axis=1)
+            ones += 1
+            B_W = [K.in_train_phase(K.dropout(ones, self.dropout_W), ones)
+                   for _ in range(4)]
+            constants.append(B_W)
+        else:
+            constants.append([K.cast_to_floatx(1.) for _ in range(4)])
+        return constants
+
+    def get_config(self):
+        config = {'nb_filter': self.nb_filter,
+                  'nb_row': self.nb_row,
+                  'nb_col': self.nb_col,
+                  'init': self.init.__name__,
+                  'inner_init': self.inner_init.__name__,
+                  'forget_bias_init': self.forget_bias_init.__name__,
+                  'activation': self.activation.__name__,
+                  'dim_ordering': self.dim_ordering,
+                  'border_mode': self.border_mode,
+                  'inner_activation': self.inner_activation.__name__}
+        base_config = super(ConvLSTM2D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))

keras/layers/embeddings.py

@@ -1,12 +1,14 @@
 from __future__ import absolute_import
 
 from .. import backend as K
-from .. import initializations, regularizers, constraints
+from .. import initializations
+from .. import regularizers
+from .. import constraints
 from ..engine import Layer
 
 
 class Embedding(Layer):
-    '''Turn positive integers (indexes) into dense vectors of fixed size.
+    """Turn positive integers (indexes) into dense vectors of fixed size.
     eg. [[4], [20]] -> [[0.25, 0.1], [0.6, -0.2]]
 
     This layer can only be used as the first layer in a model.
@@ -17,7 +19,7 @@ class Embedding(Layer):
       model = Sequential()
       model.add(Embedding(1000, 64, input_length=10))
       # the model will take as input an integer matrix of size (batch, input_length).
-      # the largest integer (i.e. word index) in the input should be no larger than 1000 (vocabulary size).
+      # the largest integer (i.e. word index) in the input should be no larger than 999 (vocabulary size).
       # now model.output_shape == (None, 10, 64), where None is the batch dimension.
 
       input_array = np.random.randint(1000, size=(32, 10))
@@ -28,14 +30,14 @@ class Embedding(Layer):
     ```
 
     # Arguments
-      input_dim: int >= 0. Size of the vocabulary, ie.
+      input_dim: int > 0. Size of the vocabulary, ie.
           1 + maximum integer index occurring in the input data.
       output_dim: int >= 0. Dimension of the dense embedding.
       init: name of initialization function for the weights
           of the layer (see: [initializations](../initializations.md)),
           or alternatively, Theano function to use for weights initialization.
           This parameter is only relevant if you don't pass a `weights` argument.
-      weights: list of numpy arrays to set as initial weights.
+      weights: list of Numpy arrays to set as initial weights.
           The list should have 1 element, of shape `(input_dim, output_dim)`.
       W_regularizer: instance of the [regularizers](../regularizers.md) module
         (eg. L1 or L2 regularization), applied to the embedding matrix.
@@ -46,6 +48,8 @@ class Embedding(Layer):
           This is useful for [recurrent layers](recurrent.md) which may take
           variable length input. If this is `True` then all subsequent layers
           in the model need to support masking or an exception will be raised.
+          If mask_zero is set to True, as a consequence, index 0 cannot be
+          used in the vocabulary (input_dim should equal |vocabulary| + 2).
       input_length: Length of input sequences, when it is constant.
           This argument is required if you are going to connect
           `Flatten` then `Dense` layers upstream
@@ -60,8 +64,7 @@ class Embedding(Layer):
 
     # References
         - [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](http://arxiv.org/abs/1512.05287)
-    '''
-    input_ndim = 2
+    """
 
     def __init__(self, input_dim, output_dim,
                  init='uniform', input_length=None,
@@ -77,7 +80,6 @@ class Embedding(Layer):
         self.dropout = dropout
 
         self.W_constraint = constraints.get(W_constraint)
-        self.constraints = [self.W_constraint]
 
         self.W_regularizer = regularizers.get(W_regularizer)
         self.activity_regularizer = regularizers.get(activity_regularizer)
@@ -90,20 +92,15 @@ class Embedding(Layer):
         super(Embedding, self).__init__(**kwargs)
 
     def build(self, input_shape):
-        self.W = self.init((self.input_dim, self.output_dim),
-                           name='{}_W'.format(self.name))
-        self.trainable_weights = [self.W]
-        self.regularizers = []
-        if self.W_regularizer:
-            self.W_regularizer.set_param(self.W)
-            self.regularizers.append(self.W_regularizer)
-
-        if self.activity_regularizer:
-            self.activity_regularizer.set_layer(self)
-            self.regularizers.append(self.activity_regularizer)
+        self.W = self.add_weight((self.input_dim, self.output_dim),
+                                 initializer=self.init,
+                                 name='{}_W'.format(self.name),
+                                 regularizer=self.W_regularizer,
+                                 constraint=self.W_constraint)
 
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
+        self.built = True
 
     def compute_mask(self, x, mask=None):
         if not self.mask_zero:
@@ -112,9 +109,15 @@ class Embedding(Layer):
             return K.not_equal(x, 0)
 
     def get_output_shape_for(self, input_shape):
-        return (input_shape[0], self.input_length, self.output_dim)
+        if not self.input_length:
+            input_length = input_shape[1]
+        else:
+            input_length = self.input_length
+        return (input_shape[0], input_length, self.output_dim)
 
     def call(self, x, mask=None):
+        if K.dtype(x) != 'int32':
+            x = K.cast(x, 'int32')
         if 0. < self.dropout < 1.:
             retain_p = 1. - self.dropout
             B = K.random_binomial((self.input_dim,), p=retain_p) * (1. / retain_p)

keras/layers/local.py

@@ -0,0 +1,422 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+from .. import backend as K
+from .. import activations
+from .. import initializations
+from .. import regularizers
+from .. import constraints
+from ..engine import Layer
+from ..engine import InputSpec
+from ..utils.np_utils import conv_output_length
+
+
+class LocallyConnected1D(Layer):
+    """Locally-connected layer for 1D inputs.
+
+    The `LocallyConnected1D` layer works similarly to
+    the `Convolution1D` layer, except that weights are unshared,
+    that is, a different set of filters is applied at each different patch
+    of the input.
+    When using this layer as the first layer in a model,
+    either provide the keyword argument `input_dim`
+    (int, e.g. 128 for sequences of 128-dimensional vectors), or `input_shape`
+    (tuple of integers, e.g. `input_shape=(10, 128)`
+    for sequences of 10 vectors of 128-dimensional vectors).
+    Also, note that this layer can only be used with
+    a fully-specified input shape (`None` dimensions not allowed).
+
+    # Example
+    ```python
+        # apply a unshared weight convolution 1d of length 3 to a sequence with
+        # 10 timesteps, with 64 output filters
+        model = Sequential()
+        model.add(LocallyConnected1D(64, 3, input_shape=(10, 32)))
+        # now model.output_shape == (None, 8, 64)
+        # add a new conv1d on top
+        model.add(LocallyConnected1D(32, 3))
+        # now model.output_shape == (None, 6, 32)
+    ```
+
+    # Arguments
+        nb_filter: Dimensionality of the output.
+        filter_length: The extension (spatial or temporal) of each filter.
+        init: name of initialization function for the weights of the layer
+            (see [initializations](../initializations.md)),
+            or alternatively, Theano function to use for weights initialization.
+            This parameter is only relevant if you don't pass a `weights` argument.
+        activation: name of activation function to use
+            (see [activations](../activations.md)),
+            or alternatively, elementwise Theano function.
+            If you don't specify anything, no activation is applied
+            (ie. "linear" activation: a(x) = x).
+        weights: list of numpy arrays to set as initial weights.
+        border_mode: Only support 'valid'. Please make good use of
+            ZeroPadding1D to achieve same output length.
+        subsample_length: factor by which to subsample output.
+        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
+            (eg. L1 or L2 regularization), applied to the main weights matrix.
+        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
+            applied to the bias.
+        activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
+            applied to the network output.
+        W_constraint: instance of the [constraints](../constraints.md) module
+            (eg. maxnorm, nonneg), applied to the main weights matrix.
+        b_constraint: instance of the [constraints](../constraints.md) module,
+            applied to the bias.
+        bias: whether to include a bias (i.e. make the layer affine rather than linear).
+        input_dim: Number of channels/dimensions in the input.
+            Either this argument or the keyword argument `input_shape`must be
+            provided when using this layer as the first layer in a model.
+        input_length: Length of input sequences, when it is constant.
+            This argument is required if you are going to connect
+            `Flatten` then `Dense` layers upstream
+            (without it, the shape of the dense outputs cannot be computed).
+
+    # Input shape
+        3D tensor with shape: `(samples, steps, input_dim)`.
+
+    # Output shape
+        3D tensor with shape: `(samples, new_steps, nb_filter)`.
+        `steps` value might have changed due to padding.
+    """
+
+    def __init__(self, nb_filter, filter_length,
+                 init='glorot_uniform', activation=None, weights=None,
+                 border_mode='valid', subsample_length=1,
+                 W_regularizer=None, b_regularizer=None, activity_regularizer=None,
+                 W_constraint=None, b_constraint=None,
+                 bias=True, input_dim=None, input_length=None, **kwargs):
+        if border_mode != 'valid':
+            raise ValueError('Invalid border mode for LocallyConnected1D '
+                             '(only "valid" is supported):', border_mode)
+        self.nb_filter = nb_filter
+        self.filter_length = filter_length
+        self.init = initializations.get(init, dim_ordering='th')
+        self.activation = activations.get(activation)
+
+        self.border_mode = border_mode
+        self.subsample_length = subsample_length
+
+        self.W_regularizer = regularizers.get(W_regularizer)
+        self.b_regularizer = regularizers.get(b_regularizer)
+        self.activity_regularizer = regularizers.get(activity_regularizer)
+
+        self.W_constraint = constraints.get(W_constraint)
+        self.b_constraint = constraints.get(b_constraint)
+
+        self.bias = bias
+        self.input_spec = [InputSpec(ndim=3)]
+        self.initial_weights = weights
+        self.input_dim = input_dim
+        self.input_length = input_length
+        if self.input_dim:
+            kwargs['input_shape'] = (self.input_length, self.input_dim)
+        super(LocallyConnected1D, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+        input_dim = input_shape[2]
+        _, output_length, nb_filter = self.get_output_shape_for(input_shape)
+        self.W_shape = (output_length,
+                        self.filter_length * input_dim,
+                        nb_filter)
+        self.W = self.add_weight(self.W_shape,
+                                 initializer=self.init,
+                                 name='{}_W'.format(self.name),
+                                 regularizer=self.W_regularizer,
+                                 constraint=self.W_constraint)
+        if self.bias:
+            self.b = self.add_weight((output_length, self.nb_filter),
+                                     initializer='zero',
+                                     name='{}_b'.format(self.name),
+                                     regularizer=self.b_regularizer,
+                                     constraint=self.b_constraint)
+        else:
+            self.b = None
+
+        if self.initial_weights is not None:
+            self.set_weights(self.initial_weights)
+            del self.initial_weights
+        self.built = True
+
+    def get_output_shape_for(self, input_shape):
+        length = conv_output_length(input_shape[1],
+                                    self.filter_length,
+                                    self.border_mode,
+                                    self.subsample_length)
+        return (input_shape[0], length, self.nb_filter)
+
+    def call(self, x, mask=None):
+        stride = self.subsample_length
+        output_length, feature_dim, nb_filter = self.W_shape
+
+        xs = []
+        for i in range(output_length):
+            slice_length = slice(i * stride, i * stride + self.filter_length)
+            xs.append(K.reshape(x[:, slice_length, :], (1, -1, feature_dim)))
+        x_aggregate = K.concatenate(xs, axis=0)
+        # (output_length, batch_size, nb_filter)
+        output = K.batch_dot(x_aggregate, self.W)
+        output = K.permute_dimensions(output, (1, 0, 2))
+
+        if self.bias:
+            output += K.reshape(self.b, (1, output_length, nb_filter))
+
+        output = self.activation(output)
+        return output
+
+    def get_config(self):
+        config = {'nb_filter': self.nb_filter,
+                  'filter_length': self.filter_length,
+                  'init': self.init.__name__,
+                  'activation': self.activation.__name__,
+                  'border_mode': self.border_mode,
+                  'subsample_length': self.subsample_length,
+                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
+                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
+                  'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
+                  'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
+                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
+                  'bias': self.bias,
+                  'input_dim': self.input_dim,
+                  'input_length': self.input_length}
+        base_config = super(LocallyConnected1D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
+class LocallyConnected2D(Layer):
+    """Locally-connected layer for 2D inputs.
+
+    The `LocallyConnected2D` layer works similarly
+    to the `Convolution2D` layer, except that weights are unshared,
+    that is, a different set of filters is applied at each
+    different patch of the input.
+    When using this layer as the
+    first layer in a model, provide the keyword argument `input_shape` (tuple
+    of integers, does not include the sample axis), e.g.
+    `input_shape=(3, 128, 128)` for 128x128 RGB pictures.
+    Also, note that this layer can only be used with
+    a fully-specified input shape (`None` dimensions not allowed).
+
+    # Examples
+    ```python
+        # apply a 3x3 unshared weights convolution with 64 output filters on a 32x32 image:
+        model = Sequential()
+        model.add(LocallyConnected2D(64, 3, 3, input_shape=(3, 32, 32)))
+        # now model.output_shape == (None, 64, 30, 30)
+        # notice that this layer will consume (30*30)*(3*3*3*64) + (30*30)*64 parameters
+
+        # add a 3x3 unshared weights convolution on top, with 32 output filters:
+        model.add(LocallyConnected2D(32, 3, 3))
+        # now model.output_shape == (None, 32, 28, 28)
+    ```
+
+    # Arguments
+        nb_filter: Number of convolution filters to use.
+        nb_row: Number of rows in the convolution kernel.
+        nb_col: Number of columns in the convolution kernel.
+        init: name of initialization function for the weights of the layer
+            (see [initializations](../initializations.md)), or alternatively,
+            Theano function to use for weights initialization.
+            This parameter is only relevant if you don't pass
+            a `weights` argument.
+        activation: name of activation function to use
+            (see [activations](../activations.md)),
+            or alternatively, elementwise Theano function.
+            If you don't specify anything, no activation is applied
+            (ie. "linear" activation: a(x) = x).
+        weights: list of numpy arrays to set as initial weights.
+        border_mode: Only support 'valid'. Please make good use of
+            ZeroPadding2D to achieve same output shape.
+        subsample: tuple of length 2. Factor by which to subsample output.
+            Also called strides elsewhere.
+        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
+            (eg. L1 or L2 regularization), applied to the main weights matrix.
+        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
+            applied to the bias.
+        activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
+            applied to the network output.
+        W_constraint: instance of the [constraints](../constraints.md) module
+            (eg. maxnorm, nonneg), applied to the main weights matrix.
+        b_constraint: instance of the [constraints](../constraints.md) module,
+            applied to the bias.
+        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
+            (the depth) is at index 1, in 'tf' mode is it at index 3.
+        bias: whether to include a bias (i.e. make the layer affine rather than linear).
+
+    # Input shape
+        4D tensor with shape:
+        `(samples, channels, rows, cols)` if dim_ordering='th'
+        or 4D tensor with shape:
+        `(samples, rows, cols, channels)` if dim_ordering='tf'.
+
+    # Output shape
+        4D tensor with shape:
+        `(samples, nb_filter, new_rows, new_cols)` if dim_ordering='th'
+        or 4D tensor with shape:
+        `(samples, new_rows, new_cols, nb_filter)` if dim_ordering='tf'.
+        `rows` and `cols` values might have changed due to padding.
+    """
+
+    def __init__(self, nb_filter, nb_row, nb_col,
+                 init='glorot_uniform', activation=None, weights=None,
+                 border_mode='valid', subsample=(1, 1),
+                 dim_ordering='default',
+                 W_regularizer=None, b_regularizer=None, activity_regularizer=None,
+                 W_constraint=None, b_constraint=None,
+                 bias=True, **kwargs):
+        if dim_ordering == 'default':
+            dim_ordering = K.image_dim_ordering()
+        if border_mode != 'valid':
+            raise ValueError('Invalid border mode for LocallyConnected2D '
+                             '(only "valid" is supported):', border_mode)
+        self.nb_filter = nb_filter
+        self.nb_row = nb_row
+        self.nb_col = nb_col
+        self.init = initializations.get(init, dim_ordering=dim_ordering)
+        self.activation = activations.get(activation)
+
+        self.border_mode = border_mode
+        self.subsample = tuple(subsample)
+        if dim_ordering not in {'tf', 'th'}:
+            raise ValueError('`dim_ordering` must be in {tf, th}.')
+        self.dim_ordering = dim_ordering
+
+        self.W_regularizer = regularizers.get(W_regularizer)
+        self.b_regularizer = regularizers.get(b_regularizer)
+        self.activity_regularizer = regularizers.get(activity_regularizer)
+
+        self.W_constraint = constraints.get(W_constraint)
+        self.b_constraint = constraints.get(b_constraint)
+
+        self.bias = bias
+        self.input_spec = [InputSpec(ndim=4)]
+        self.initial_weights = weights
+        super(LocallyConnected2D, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+        output_shape = self.get_output_shape_for(input_shape)
+        if self.dim_ordering == 'th':
+            _, nb_filter, output_row, output_col = output_shape
+            input_filter = input_shape[1]
+        elif self.dim_ordering == 'tf':
+            _, output_row, output_col, nb_filter = output_shape
+            input_filter = input_shape[3]
+        else:
+            raise ValueError('Invalid dim_ordering:', self.dim_ordering)
+
+        self.output_row = output_row
+        self.output_col = output_col
+        self.W_shape = (output_row * output_col,
+                        self.nb_row * self.nb_col * input_filter,
+                        nb_filter)
+        self.W = self.add_weight(self.W_shape,
+                                 initializer=self.init,
+                                 name='{}_W'.format(self.name),
+                                 regularizer=self.W_regularizer,
+                                 constraint=self.W_constraint)
+        if self.bias:
+            self.b = self.add_weight((output_row, output_col, nb_filter),
+                                     initializer='zero',
+                                     name='{}_b'.format(self.name),
+                                     regularizer=self.b_regularizer,
+                                     constraint=self.b_constraint)
+        else:
+            self.b = None
+
+        if self.initial_weights is not None:
+            self.set_weights(self.initial_weights)
+            del self.initial_weights
+        self.built = True
+
+    def get_output_shape_for(self, input_shape):
+        if self.dim_ordering == 'th':
+            rows = input_shape[2]
+            cols = input_shape[3]
+        elif self.dim_ordering == 'tf':
+            rows = input_shape[1]
+            cols = input_shape[2]
+        else:
+            raise ValueError('Invalid dim_ordering:', self.dim_ordering)
+
+        rows = conv_output_length(rows, self.nb_row,
+                                  self.border_mode, self.subsample[0])
+        cols = conv_output_length(cols, self.nb_col,
+                                  self.border_mode, self.subsample[1])
+
+        if self.dim_ordering == 'th':
+            return (input_shape[0], self.nb_filter, rows, cols)
+        elif self.dim_ordering == 'tf':
+            return (input_shape[0], rows, cols, self.nb_filter)
+
+    def call(self, x, mask=None):
+        stride_row, stride_col = self.subsample
+        _, feature_dim, nb_filter = self.W_shape
+
+        if self.dim_ordering == 'th':
+            if K.backend() == 'theano':
+                output = []
+                for i in range(self.output_row):
+                    for j in range(self.output_col):
+                        slice_row = slice(i * stride_row,
+                                          i * stride_row + self.nb_row)
+                        slice_col = slice(j * stride_col,
+                                          j * stride_col + self.nb_col)
+                        x_flatten = K.reshape(x[:, :, slice_row, slice_col], (1, -1, feature_dim))
+                        output.append(K.dot(x_flatten, self.W[i * self.output_col + j, :, :]))
+                output = K.concatenate(output, axis=0)
+            else:
+                xs = []
+                for i in range(self.output_row):
+                    for j in range(self.output_col):
+                        slice_row = slice(i * stride_row,
+                                          i * stride_row + self.nb_row)
+                        slice_col = slice(j * stride_col,
+                                          j * stride_col + self.nb_col)
+                        xs.append(K.reshape(x[:, :, slice_row, slice_col], (1, -1, feature_dim)))
+                x_aggregate = K.concatenate(xs, axis=0)
+                output = K.batch_dot(x_aggregate, self.W)
+            output = K.reshape(output, (self.output_row, self.output_col, -1, nb_filter))
+            output = K.permute_dimensions(output, (2, 3, 0, 1))
+        elif self.dim_ordering == 'tf':
+            xs = []
+            for i in range(self.output_row):
+                for j in range(self.output_col):
+                    slice_row = slice(i * stride_row,
+                                      i * stride_row + self.nb_row)
+                    slice_col = slice(j * stride_col,
+                                      j * stride_col + self.nb_col)
+                    xs.append(K.reshape(x[:, slice_row, slice_col, :], (1, -1, feature_dim)))
+            x_aggregate = K.concatenate(xs, axis=0)
+            output = K.batch_dot(x_aggregate, self.W)
+            output = K.reshape(output, (self.output_row, self.output_col, -1, nb_filter))
+            output = K.permute_dimensions(output, (2, 0, 1, 3))
+        else:
+            raise ValueError('Invalid dim_ordering:', self.dim_ordering)
+
+        if self.bias:
+            if self.dim_ordering == 'th':
+                output += K.reshape(self.b, (1, nb_filter, self.output_row, self.output_col))
+            elif self.dim_ordering == 'tf':
+                output += K.reshape(self.b, (1, self.output_row, self.output_col, nb_filter))
+
+        output = self.activation(output)
+        return output
+
+    def get_config(self):
+        config = {'nb_filter': self.nb_filter,
+                  'nb_row': self.nb_row,
+                  'nb_col': self.nb_col,
+                  'init': self.init.__name__,
+                  'activation': self.activation.__name__,
+                  'border_mode': self.border_mode,
+                  'subsample': self.subsample,
+                  'dim_ordering': self.dim_ordering,
+                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
+                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
+                  'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
+                  'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
+                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
+                  'bias': self.bias}
+        base_config = super(LocallyConnected2D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))