Update backend strings (#7214)

* Improve docstring on order of transformations

* Add code markers and be more precise

.github/stale.yml

@@ -0,0 +1,19 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 90
+# Number of days of inactivity before a stale Issue or Pull Request is closed
+daysUntilClose: 30
+# Issues or Pull Requests with these labels will never be considered stale
+exemptLabels:
+  - bug
+  - Announcement
+  - help wanted
+  - To investigate
+# Label to use when marking as stale
+staleLabel: stale
+# Comment to post when marking as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed after 30 days if no further activity 
+  occurs, but feel free to re-open a closed issue if needed.

.gitignore

@@ -1,6 +1,22 @@
 *.DS_Store
 *.pyc
+*.swp
 temp/*
+dist/*
 build/*
 keras/datasets/data/*
-keras/datasets/temp/*
+keras/datasets/temp/*
+docs/site/*
+docs/theme/*
+docs/sources/*
+tags
+Keras.egg-info
+examples/img/*
+
+# test-related
+.coverage
+.cache
+
+# developer environments
+.idea
+.vscode

.travis.yml

@@ -0,0 +1,90 @@
+sudo: required
+dist: trusty
+language: python
+matrix:
+    include:
+        - python: 2.7
+          env: KERAS_BACKEND=tensorflow TEST_MODE=PEP8
+        - python: 2.7
+          env: KERAS_BACKEND=tensorflow TEST_MODE=INTEGRATION_TESTS
+        - python: 3.5
+          env: KERAS_BACKEND=tensorflow TEST_MODE=DOC
+        - python: 2.7
+          env: KERAS_BACKEND=tensorflow
+        - python: 3.5
+          env: KERAS_BACKEND=tensorflow
+        - python: 2.7
+          env: KERAS_BACKEND=theano THEANO_FLAGS=optimizer=fast_compile
+        - python: 3.5
+          env: KERAS_BACKEND=theano THEANO_FLAGS=optimizer=fast_compile
+        - python: 2.7
+          env: KERAS_BACKEND=cntk
+        - python: 3.5
+          env: KERAS_BACKEND=cntk
+install:
+  # code below is taken from http://conda.pydata.org/docs/travis.html
+  # We do this conditionally because it saves us some downloading if the
+  # version is the same.
+  - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
+      wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh;
+    else
+      wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
+    fi
+  - bash miniconda.sh -b -p $HOME/miniconda
+  - export PATH="$HOME/miniconda/bin:$PATH"
+  - hash -r
+  - conda config --set always_yes yes --set changeps1 no
+  - conda update -q conda
+  # Useful for debugging any issues with conda
+  - conda info -a
+
+  - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION numpy scipy matplotlib pandas pytest h5py
+  - source activate test-environment
+  - pip install theano
+
+  # install PIL for preprocessing tests
+  - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
+      conda install pil;
+    elif [[ "$TRAVIS_PYTHON_VERSION" == "3.5" ]]; then
+      conda install Pillow;
+    fi
+
+  - pip install -e .[tests]
+
+  # install TensorFlow (CPU version).
+  - pip install tensorflow
+  
+  # install cntk
+  - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
+      pip install https://cntk.ai/PythonWheel/CPU-Only/cntk-2.0-cp27-cp27mu-linux_x86_64.whl;
+    elif [[ "$TRAVIS_PYTHON_VERSION" == "3.5" ]]; then
+      pip install https://cntk.ai/PythonWheel/CPU-Only/cntk-2.0-cp35-cp35m-linux_x86_64.whl;
+    fi
+
+  #install open mpi
+  - rm -rf ~/mpi
+  - mkdir ~/mpi
+  - pushd ~/mpi
+  - wget http://cntk.ai/PythonWheel/ForKeras/depends/openmpi_1.10-3.zip
+  - unzip ./openmpi_1.10-3.zip
+  - sudo dpkg -i openmpi_1.10-3.deb
+  - popd
+
+# command to run tests
+script:
+  # run keras backend init to initialize backend config
+  - python -c "import keras.backend"
+  # create dataset directory to avoid concurrent directory creation at runtime
+  - mkdir ~/.keras/datasets
+  # set up keras backend
+  - sed -i -e 's/"backend":[[:space:]]*"[^"]*/"backend":\ "'$KERAS_BACKEND'/g' ~/.keras/keras.json;
+  - echo -e "Running tests with the following config:\n$(cat ~/.keras/keras.json)"
+  - if [[ "$TEST_MODE" == "INTEGRATION_TESTS" ]]; then
+       PYTHONPATH=$PWD:$PYTHONPATH py.test tests/integration_tests;
+    elif [[ "$TEST_MODE" == "PEP8" ]]; then
+       PYTHONPATH=$PWD:$PYTHONPATH py.test --pep8 -m pep8 -n0;
+    elif [[ "$TEST_MODE" == "DOC" ]]; then
+        PYTHONPATH=$PWD:$PYTHONPATH py.test tests/test_documentation.py;
+    else
+       PYTHONPATH=$PWD:$PYTHONPATH py.test tests/ --ignore=tests/integration_tests --ignore=tests/test_documentation.py --cov=keras tests/ --cov-fail-under 80 --cov-report term-missing;
+    fi

CONTRIBUTING.md

@@ -0,0 +1,90 @@
+# On Github Issues and Pull Requests
+
+Found a bug? Have a new feature to suggest? Want to contribute changes to the codebase? Make sure to read this first.
+
+## Bug reporting
+
+Your code doesn't work, and you have determined that the issue lies with Keras? Follow these steps to report a bug.
+
+1. Your bug may already be fixed. Make sure to update to the current Keras master branch, as well as the latest Theano/TensorFlow master branch.
+To easily update Theano: `pip install git+git://github.com/Theano/Theano.git --upgrade`
+
+2. Search for similar issues. Make sure to delete `is:open` on the issue search to find solved tickets as well. It's possible somebody has encountered this bug already. Also remember to check out Keras' [FAQ](http://keras.io/faq/). Still having a problem? Open an issue on Github to let us know.
+
+3. Make sure you provide us with useful information about your configuration: what OS are you using? What Keras backend are you using? Are you running on GPU? If so, what is your version of Cuda, of cuDNN? What is your GPU?
+
+4. Provide us with a script to reproduce the issue. This script should be runnable as-is and should not require external data download (use randomly generated data if you need to run a model on some test data). We recommend that you use Github Gists to post your code. Any issue that cannot be reproduced is likely to be closed.
+
+5. If possible, take a stab at fixing the bug yourself --if you can!
+
+The more information you provide, the easier it is for us to validate that there is a bug and the faster we'll be able to take action. If you want your issue to be resolved quickly, following the steps above is crucial.
+
+---
+
+## Requesting a Feature
+
+You can also use Github issues to request features you would like to see in Keras, or changes in the Keras API.
+
+1. Provide a clear and detailed explanation of the feature you want and why it's important to add. Keep in mind that we want features that will be useful to the majority of our users and not just a small subset. If you're just targeting a minority of users, consider writing an add-on library for Keras. It is crucial for Keras to avoid bloating the API and codebase.
+
+2. Provide code snippets demonstrating the API you have in mind and illustrating the use cases of your feature. Of course, you don't need to write any real code at this point!
+
+3. After discussing the feature you may choose to attempt a Pull Request. If you're at all able, start writing some code. We always have more work to do than time to do it. If you can write some code then that will speed the process along.
+
+
+---
+
+## Requests for Contributions
+
+[This is the board](https://github.com/fchollet/keras/projects/1) where we list current outstanding issues and features to be added. If you want to start contributing to Keras, this is the place to start.
+
+
+---
+
+## Pull Requests
+
+**Where should I submit my pull request?**
+
+1. **Keras improvements and bugfixes** go to the [Keras `master` branch](https://github.com/fchollet/keras/tree/master).
+2. **Experimental new features** such as layers and datasets go to [keras-contrib](https://github.com/farizrahman4u/keras-contrib). Unless it is a new feature listed in [Requests for Contributions](https://github.com/fchollet/keras/projects/1), in which case it belongs in core Keras. If you think your feature belongs in core Keras, you can submit a design doc to explain your feature and argue for it (see explainations below).
+
+Here's a quick guide to submitting your improvements:
+
+1. If your PR introduces a change in functionality, make sure you start by writing a design doc and sending it to the Keras mailing list to discuss whether the change should be made, and how to handle it. This will save you from having your PR closed down the road! Of course, if your PR is a simple bug fix, you don't need to do that. The process for writing and submitting design docs is as follow:
+    - Start from [this Google Doc template](https://docs.google.com/document/d/1ZXNfce77LDW9tFAj6U5ctaJmI5mT7CQXOFMEAZo-mAA/edit#), and copy it to new Google doc.
+    - Fill in the content. Note that you will need to insert code examples. To insert code, use a Google Doc extension such as [CodePretty](https://chrome.google.com/webstore/detail/code-pretty/igjbncgfgnfpbnifnnlcmjfbnidkndnh?hl=en) (there are several such extensions available).
+    - Set sharing settings to "everyone with the link is allowed to comment"
+    - Send the document to `keras-users@googlegroups.com` with a subject that starts with `[API DESIGN REVIEW]` (all caps) so that we notice it.
+    - Wait for comments, and answer them as they come. Edit the proposal as necessary.
+    - The proposal will finally be approved or rejected. Once approved, you can send out Pull Requests or ask others to write Pull Requests.
+
+
+2. Write the code (or get others to write it). This is the hard part!
+
+3. Make sure any new function or class you introduce has proper docstrings. Make sure any code you touch still has up-to-date docstrings and documentation. **Docstring style should be respected.** In particular, they should be formatted in MarkDown, and there should be sections for `Arguments`, `Returns`, `Raises` (if applicable). Look at other docstrings in the codebase for examples.
+
+4. Write tests. Your code should have full unit test coverage. If you want to see your PR merged promptly, this is crucial.
+
+5. Run our test suite locally. It's easy: from the Keras folder, simply run: `py.test tests/`.
+    - You will need to install the test requirements as well: `pip install -e .[tests]`.
+
+6. Make sure all tests are passing:
+    - with the Theano backend, on Python 2.7 and Python 3.5. Make sure you have the development version of Theano.
+    - with the TensorFlow backend, on Python 2.7 and Python 3.5. Make sure you have the development version of TensorFlow.
+
+7. We use PEP8 syntax conventions, but we aren't dogmatic when it comes to line length. Make sure your lines stay reasonably sized, though. To make your life easier, we recommend running a PEP8 linter:
+    - Install PEP8 packages: `pip install pep8 pytest-pep8 autopep8`
+    - Run a standalone PEP8 check: `py.test --pep8 -m pep8`
+    - You can automatically fix some PEP8 error by running: `autopep8 -i --select <errors> <FILENAME>` for example: `autopep8 -i --select E128 tests/keras/backend/test_backends.py`
+
+8. When committing, use appropriate, descriptive commit messages.
+
+9. Update the documentation. If introducing new functionality, make sure you include code snippets demonstrating the usage of your new feature.
+
+10. Submit your PR. If your changes have been approved in a previous discussion, and if you have complete (and passing) unit tests as well as proper docstrings/documentation, your PR is likely to be merged promptly. Otherwise, well...
+
+---
+
+## Adding new examples
+
+Even if you don't contribute to the Keras source code, if you have an application of Keras that is concise and powerful, please consider adding it to our collection of examples. [Existing examples](https://github.com/fchollet/keras/tree/master/examples) show idiomatic Keras code: make sure to keep your own script in the same spirit.

ISSUE_TEMPLATE.md

@@ -0,0 +1,13 @@
+Please make sure that the boxes below are checked before you submit your issue. If your issue is an implementation question, please ask your question on [StackOverflow](http://stackoverflow.com/questions/tagged/keras) or [join the Keras Slack channel](https://keras-slack-autojoin.herokuapp.com/) and ask there instead of filing a GitHub issue.
+
+Thank you!
+
+- [ ] Check that you are up-to-date with the master branch of Keras. You can update with:
+pip install git+git://github.com/fchollet/keras.git --upgrade --no-deps
+
+- [ ] If running on TensorFlow, check that you are up-to-date with the latest version. The installation instructions can be found [here](https://www.tensorflow.org/get_started/os_setup).
+
+- [ ] If running on Theano, check that you are up-to-date with the master branch of Theano. You can update with:
+pip install git+git://github.com/Theano/Theano.git --upgrade --no-deps
+
+- [ ] Provide a link to a GitHub Gist of a Python script that can reproduce your issue (or just copy the script here if it is short).

LICENSE

@@ -1,6 +1,27 @@
-The MIT License (MIT)
+COPYRIGHT
 
-Copyright (c) 2015 
+All contributions by François Chollet:
+Copyright (c) 2015, François Chollet.
+All rights reserved.
+
+All contributions by Google:
+Copyright (c) 2015, Google, Inc.
+All rights reserved.
+
+All contributions by Microsoft:
+Copyright (c) 2017, Microsoft, Inc.
+All rights reserved.
+
+All other contributions:
+Copyright (c) 2015 - 2017, the respective contributors.
+All rights reserved.
+
+Each contributor holds copyright over their respective contributions.
+The project versioning (Git) records all such contribution source information.
+
+LICENSE
+
+The MIT License (MIT)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,209 +1,179 @@
-# Keras: Theano-based Deep Learning library
+# Keras: Deep Learning for Python
+
+[![Build Status](https://travis-ci.org/fchollet/keras.svg?branch=master)](https://travis-ci.org/fchollet/keras)
+[![license](https://img.shields.io/github/license/mashape/apistatus.svg?maxAge=2592000)](https://github.com/fchollet/keras/blob/master/LICENSE)
 
 ## You have just found Keras.
 
-Keras is a minimalist, highly modular neural network library in the spirit of Torch, written in Python / Theano so as not to have to deal with the dearth of ecosystem in Lua. It was developed with a focus on enabling fast experimentation. Being able to go from idea to result with the least possible delay is key to doing good research.
+Keras is a high-level neural networks API, written in Python and capable of running on top of [TensorFlow](https://github.com/tensorflow/tensorflow), [CNTK](https://github.com/Microsoft/cntk), or [Theano](https://github.com/Theano/Theano). It was developed with a focus on enabling fast experimentation. *Being able to go from idea to result with the least possible delay is key to doing good research.*
 
 Use Keras if you need a deep learning library that:
-- allows for easy and fast prototyping (through total modularity, minimalism, and extensibility).
-- supports both convolutional networks (for vision) and recurrent networks (for sequence data). As well as combinations of the two. 
-- runs seamlessly on the CPU and the GPU.
+
+- Allows for easy and fast prototyping (through user friendliness, modularity, and extensibility).
+- Supports both convolutional networks and recurrent networks, as well as combinations of the two.
+- Runs seamlessly on CPU and GPU.
 
 Read the documentation at [Keras.io](http://keras.io).
 
-Keras is compatible with __Python 2.7-3.4__.
+Keras is compatible with: __Python 2.7-3.5__.
+
+
+------------------
+
 
 ## Guiding principles
 
-- __Modularity.__ A model is understood as a sequence of standalone, fully-configurable modules that can be plugged together with as little restrictions as possible. In particular, neural layers, cost functions, optimizers, initialization schemes, activation functions and dropout are all standalone modules that you can combine to create new models. 
+- __User friendliness.__ Keras is an API designed for human beings, not machines. It puts user experience front and center. Keras follows best practices for reducing cognitive load: it offers consistent & simple APIs, it minimizes the number of user actions required for common use cases, and it provides clear and actionable feedback upon user error.
 
-- __Minimalism.__ Each module should be kept short and simple (<100 lines of code). Every piece of code should be transparent upon first reading. No black magic: it hurts iteration speed and ability to innovate. 
+- __Modularity.__ A model is understood as a sequence or a graph of standalone, fully-configurable modules that can be plugged together with as little restrictions as possible. In particular, neural layers, cost functions, optimizers, initialization schemes, activation functions, regularization schemes are all standalone modules that you can combine to create new models.
 
-- __Easy extensibility.__ New features (a new module, per the above definition, or a new way to combine modules together) are dead simple to add (as new classes/functions), and existing modules provide ample examples.
+- __Easy extensibility.__ New modules are simple to add (as new classes and functions), and existing modules provide ample examples. To be able to easily create new modules allows for total expressiveness, making Keras suitable for advanced research.
 
-- __Work with Python__. No separate models configuration files in a declarative format (like in Caffe or PyLearn2). Models are described in Python code, which is compact, easier to debug, benefits from syntax highlighting, and most of all, allows for ease of extensibility. See for yourself with the examples below.
+- __Work with Python__. No separate models configuration files in a declarative format. Models are described in Python code, which is compact, easier to debug, and allows for ease of extensibility.
 
-## Examples
 
-### Multilayer Perceptron (MLP):
+------------------
+
+
+## Getting started: 30 seconds to Keras
+
+The core data structure of Keras is a __model__, a way to organize layers. The simplest type of model is the [`Sequential`](http://keras.io/getting-started/sequential-model-guide) model, a linear stack of layers. For more complex architectures, you should use the [Keras functional API](http://keras.io/getting-started/functional-api-guide), which allows to build arbitrary graphs of layers.
+
+Here is the `Sequential` model:
 
 ```python
 from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation
-from keras.optimizers import SGD
 
 model = Sequential()
-model.add(Dense(20, 64, init='uniform'))
-model.add(Activation('tanh'))
-model.add(Dropout(0.5))
-model.add(Dense(64, 64, init='uniform'))
-model.add(Activation('tanh'))
-model.add(Dropout(0.5))
-model.add(Dense(64, 2, init='uniform'))
+```
+
+Stacking layers is as easy as `.add()`:
+
+```python
+from keras.layers import Dense, Activation
+
+model.add(Dense(units=64, input_dim=100))
+model.add(Activation('relu'))
+model.add(Dense(units=10))
 model.add(Activation('softmax'))
-
-sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
-model.compile(loss='mean_squared_error', optimizer=sgd)
-
-model.fit(X_train, y_train, nb_epoch=20, batch_size=16)
-score = model.evaluate(X_test, y_test, batch_size=16)
 ```
 
-### Alternative implementation of MLP:
+Once your model looks good, configure its learning process with `.compile()`:
 
 ```python
-model = Sequential()
-model.add(Dense(20, 64, init='uniform', activation='tanh'))
-model.add(Dropout(0.5))
-model.add(Dense(64, 64, init='uniform', activation='tanh'))
-model.add(Dropout(0.5))
-model.add(Dense(64, 2, init='uniform', activation='softmax'))
-
-sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
-model.compile(loss='mean_squared_error', optimizer=sgd)
+model.compile(loss='categorical_crossentropy',
+              optimizer='sgd',
+              metrics=['accuracy'])
 ```
 
-### VGG-like convnet:
+If you need to, you can further configure your optimizer. A core principle of Keras is to make things reasonably simple, while allowing the user to be fully in control when they need to (the ultimate control being the easy extensibility of the source code).
+```python
+model.compile(loss=keras.losses.categorical_crossentropy,
+              optimizer=keras.optimizers.SGD(lr=0.01, momentum=0.9, nesterov=True))
+```
+
+You can now iterate on your training data in batches:
 
 ```python
-from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation, Flatten
-from keras.layers.convolutional import Convolution2D, MaxPooling2D
-from keras.optimizers import SGD
-
-model = Sequential()
-model.add(Convolution2D(32, 3, 3, 3, border_mode='full')) 
-model.add(Activation('relu'))
-model.add(Convolution2D(32, 32, 3, 3))
-model.add(Activation('relu'))
-model.add(MaxPooling2D(poolsize=(2, 2)))
-model.add(Dropout(0.25))
-
-model.add(Convolution2D(64, 32, 3, 3, border_mode='full')) 
-model.add(Activation('relu'))
-model.add(Convolution2D(64, 64, 3, 3)) 
-model.add(Activation('relu'))
-model.add(MaxPooling2D(poolsize=(2, 2)))
-model.add(Dropout(0.25))
-
-model.add(Flatten())
-model.add(Dense(64*8*8, 256))
-model.add(Activation('relu'))
-model.add(Dropout(0.5))
-
-model.add(Dense(256, 10))
-model.add(Activation('softmax'))
-
-sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
-model.compile(loss='categorical_crossentropy', optimizer=sgd)
-
-model.fit(X_train, Y_train, batch_size=32, nb_epoch=1)
-
+# x_train and y_train are Numpy arrays --just like in the Scikit-Learn API.
+model.fit(x_train, y_train, epochs=5, batch_size=32)
 ```
 
-### Sequence classification with LSTM:
+Alternatively, you can feed batches to your model manually:
 
 ```python
-from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation
-from keras.layers.embeddings import Embedding
-from keras.layers.recurrent import LSTM
-
-model = Sequential()
-model.add(Embedding(max_features, 256))
-model.add(LSTM(256, 128, activation='sigmoid', inner_activation='hard_sigmoid'))
-model.add(Dropout(0.5))
-model.add(Dense(128, 1))
-model.add(Activation('sigmoid'))
-
-model.compile(loss='binary_crossentropy', optimizer='rmsprop')
-
-model.fit(X_train, Y_train, batch_size=16, nb_epoch=10)
-score = model.evaluate(X_test, Y_test, batch_size=16)
+model.train_on_batch(x_batch, y_batch)
 ```
 
-### Architecture for learning image captions with a convnet and a Gated Recurrent Unit:
-(word-level embedding, caption of maximum length 16 words).
-
-Note that getting this to actually "work" will require using a bigger convnet, initialized with pre-trained weights.
-Displaying readable results will also require an embedding decoder.
+Evaluate your performance in one line:
 
 ```python
-max_caption_len = 16
-
-model = Sequential()
-model.add(Convolution2D(32, 3, 3, 3, border_mode='full')) 
-model.add(Activation('relu'))
-model.add(Convolution2D(32, 32, 3, 3))
-model.add(Activation('relu'))
-model.add(MaxPooling2D(poolsize=(2, 2)))
-
-model.add(Convolution2D(64, 32, 3, 3, border_mode='full')) 
-model.add(Activation('relu'))
-model.add(Convolution2D(64, 64, 3, 3)) 
-model.add(Activation('relu'))
-model.add(MaxPooling2D(poolsize=(2, 2)))
-
-model.add(Convolution2D(128, 64, 3, 3, border_mode='full')) 
-model.add(Activation('relu'))
-model.add(Convolution2D(128, 128, 3, 3)) 
-model.add(Activation('relu'))
-model.add(MaxPooling2D(poolsize=(2, 2)))
-
-model.add(Flatten())
-model.add(Dense(128*4*4, 256))
-model.add(Activation('relu'))
-model.add(Dropout(0.5))
-
-model.add(Repeat(max_caption_len)) 
-# the GRU below returns sequences of max_caption_len vectors of size 256 (our word embedding size)
-model.add(GRU(256, 256, return_sequences=True))
-
-model.compile(loss='mean_squared_error', optimizer='rmsprop')
-
-# "images" is a numpy array of shape (nb_samples, nb_channels=3, width, height) 
-# "captions" is a numpy array of shape (nb_samples, max_caption_len=16, embedding_dim=256)
-# captions are supposed already embedded (dense vectors).
-model.fit(images, captions, batch_size=16, nb_epoch=100)
-    
+loss_and_metrics = model.evaluate(x_test, y_test, batch_size=128)
 ```
 
-In the examples folder, you will find example models for real datasets:
-- CIFAR10 small images classification: Convnet with realtime data augmentation
-- IMDB movie review sentiment classification: LSTM over sequences of words
-- Reuters newswires topic classification: Multilayer Perceptron
-- MNIST handwritten digits classification: Multilayer Perceptron
+Or generate predictions on new data:
+
+```python
+classes = model.predict(x_test, batch_size=128)
+```
+
+Building a question answering system, an image classification model, a Neural Turing Machine, or any other model is just as fast. The ideas behind deep learning are simple, so why should their implementation be painful?
+
+For a more in-depth tutorial about Keras, you can check out:
+
+- [Getting started with the Sequential model](http://keras.io/getting-started/sequential-model-guide)
+- [Getting started with the functional API](http://keras.io/getting-started/functional-api-guide)
+
+In the [examples folder](https://github.com/fchollet/keras/tree/master/examples) of the repository, you will find more advanced models: question-answering with memory networks, text generation with stacked LSTMs, etc.
 
 
-## Current capabilities
+------------------
 
-For complete coverage of the API, check out [the Keras documentation](http://keras.io).
-
-A few highlights: convnets, LSTM, GRU, word2vec-style embeddings, PReLU, batch normalization...
 
 ## Installation
 
 Keras uses the following dependencies:
 
 - numpy, scipy
-
-- Theano
-    - See installation instructions: http://deeplearning.net/software/theano/install.html#install
-
+- yaml
 - HDF5 and h5py (optional, required if you use model saving/loading functions)
-
 - Optional but recommended if you use CNNs: cuDNN.
 
-Once you have the dependencies installed, cd to the Keras folder and run the install command:
-```
+
+*When using the TensorFlow backend:*
+
+- TensorFlow
+    - [See installation instructions](https://www.tensorflow.org/install/).
+
+*When using the CNTK backend:*
+
+- CNTK
+    - [See installation instructions](https://docs.microsoft.com/en-us/cognitive-toolkit/setup-cntk-on-your-machine).
+
+*When using the Theano backend:*
+
+- Theano
+    - [See installation instructions](http://deeplearning.net/software/theano/install.html#install).
+
+To install Keras, `cd` to the Keras folder and run the install command:
+```sh
 sudo python setup.py install
 ```
 
+You can also install Keras from PyPI:
+```sh
+sudo pip install keras
+```
+
+------------------
+
+
+## Switching from TensorFlow to CNTK or Theano
+
+By default, Keras will use TensorFlow as its tensor manipulation library. [Follow these instructions](http://keras.io/backend/) to configure the Keras backend.
+
+------------------
+
+
+## Support
+
+You can ask questions and join the development discussion:
+
+- On the [Keras Google group](https://groups.google.com/forum/#!forum/keras-users).
+- On the [Keras Slack channel](https://kerasteam.slack.com). Use [this link](https://keras-slack-autojoin.herokuapp.com/) to request an invitation to the channel.
+
+You can also post **bug reports and feature requests** (only) in [Github issues](https://github.com/fchollet/keras/issues). Make sure to read [our guidelines](https://github.com/fchollet/keras/blob/master/CONTRIBUTING.md) first.
+
+
+------------------
+
+
 ## Why this name, Keras?
 
 Keras (κέρας) means _horn_ in Greek. It is a reference to a literary image from ancient Greek and Latin literature, first found in the _Odyssey_, where dream spirits (_Oneiroi_, singular _Oneiros_) are divided between those who deceive men with false visions, who arrive to Earth through a gate of ivory, and those who announce a future that will come to pass, who arrive through a gate of horn. It's a play on the words κέρας (horn) / κραίνω (fulfill), and ἐλέφας (ivory) / ἐλεφαίρομαι (deceive).
 
-Keras was developed as part of the research effort of project ONEIROS (Open-ended Neuro-Electronic Intelligent Robot Operating System).
+Keras was initially developed as part of the research effort of project ONEIROS (Open-ended Neuro-Electronic Intelligent Robot Operating System).
 
-_"Oneiroi are beyond our unravelling --who can be sure what tale they tell? Not all that men look for comes to pass. Two gates there are that give passage to fleeting Oneiroi; one is made of horn, one of ivory. The Oneiroi that pass through sawn ivory are deceitful, bearing a message that will not be fulfilled; those that come out through polished horn have truth behind them, to be accomplished for men who see them."_ Homer, Odyssey 19. 562 ff (Shewring translation).
+>_"Oneiroi are beyond our unravelling --who can be sure what tale they tell? Not all that men look for comes to pass. Two gates there are that give passage to fleeting Oneiroi; one is made of horn, one of ivory. The Oneiroi that pass through sawn ivory are deceitful, bearing a message that will not be fulfilled; those that come out through polished horn have truth behind them, to be accomplished for men who see them."_ Homer, Odyssey 19. 562 ff (Shewring translation).
 
+------------------

docker/Dockerfile

@@ -0,0 +1,47 @@
+FROM nvidia/cuda:8.0-cudnn5-devel
+
+ENV CONDA_DIR /opt/conda
+ENV PATH $CONDA_DIR/bin:$PATH
+
+RUN mkdir -p $CONDA_DIR && \
+    echo export PATH=$CONDA_DIR/bin:'$PATH' > /etc/profile.d/conda.sh && \
+    apt-get update && \
+    apt-get install -y wget git libhdf5-dev g++ graphviz && \
+    wget --quiet https://repo.continuum.io/miniconda/Miniconda3-4.2.12-Linux-x86_64.sh && \
+    echo "c59b3dd3cad550ac7596e0d599b91e75d88826db132e4146030ef471bb434e9a *Miniconda3-4.2.12-Linux-x86_64.sh" | sha256sum -c - && \
+    /bin/bash /Miniconda3-4.2.12-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
+    rm Miniconda3-4.2.12-Linux-x86_64.sh
+
+ENV NB_USER keras
+ENV NB_UID 1000
+
+RUN useradd -m -s /bin/bash -N -u $NB_UID $NB_USER && \
+    mkdir -p $CONDA_DIR && \
+    chown keras $CONDA_DIR -R && \
+    mkdir -p /src && \
+    chown keras /src
+
+USER keras
+
+# Python
+ARG python_version=3.5
+
+RUN conda install -y python=${python_version} && \
+    pip install --upgrade pip && \
+    pip install tensorflow-gpu && \
+    conda install Pillow scikit-learn notebook pandas matplotlib mkl nose pyyaml six h5py && \
+    conda install theano pygpu && \
+    git clone git://github.com/fchollet/keras.git /src && pip install -e /src[tests] && \
+    pip install git+git://github.com/fchollet/keras.git && \
+    conda clean -yt
+
+ADD theanorc /home/keras/.theanorc
+
+ENV PYTHONPATH='/src/:$PYTHONPATH'
+
+WORKDIR /src
+
+EXPOSE 8888
+
+CMD jupyter notebook --port=8888 --ip=0.0.0.0
+

docker/Makefile

@@ -0,0 +1,26 @@
+help:
+	@cat Makefile
+
+DATA?="${HOME}/Data"
+GPU?=0
+DOCKER_FILE=Dockerfile
+DOCKER=GPU=$(GPU) nvidia-docker
+BACKEND=tensorflow
+TEST=tests/
+SRC=$(shell dirname `pwd`)
+
+build:
+	docker build -t keras --build-arg python_version=3.5 -f $(DOCKER_FILE) .
+
+bash: build
+	$(DOCKER) run -it -v $(SRC):/src -v $(DATA):/data --env KERAS_BACKEND=$(BACKEND) keras bash
+
+ipython: build
+	$(DOCKER) run -it -v $(SRC):/src -v $(DATA):/data --env KERAS_BACKEND=$(BACKEND) keras ipython
+
+notebook: build
+	$(DOCKER) run -it -v $(SRC):/src -v $(DATA):/data --net=host --env KERAS_BACKEND=$(BACKEND) keras
+
+test: build
+	$(DOCKER) run -it -v $(SRC):/src -v $(DATA):/data --env KERAS_BACKEND=$(BACKEND) keras py.test $(TEST)
+

docker/README.md

@@ -0,0 +1,58 @@
+# Using Keras via Docker
+
+This directory contains `Dockerfile` to make it easy to get up and running with
+Keras via [Docker](http://www.docker.com/).
+
+## Installing Docker
+
+General installation instructions are
+[on the Docker site](https://docs.docker.com/installation/), but we give some
+quick links here:
+
+* [OSX](https://docs.docker.com/installation/mac/): [docker toolbox](https://www.docker.com/toolbox)
+* [ubuntu](https://docs.docker.com/installation/ubuntulinux/)
+
+## Running the container
+
+We are using `Makefile` to simplify docker commands within make commands.
+
+Build the container and start a jupyter notebook
+
+    $ make notebook
+
+Build the container and start an iPython shell
+
+    $ make ipython
+
+Build the container and start a bash
+
+    $ make bash
+
+For GPU support install NVidia drivers (ideally latest) and
+[nvidia-docker](https://github.com/NVIDIA/nvidia-docker). Run using
+
+    $ make notebook GPU=0 # or [ipython, bash]
+
+Switch between Theano and TensorFlow
+
+    $ make notebook BACKEND=theano
+    $ make notebook BACKEND=tensorflow
+
+Mount a volume for external data sets
+
+    $ make DATA=~/mydata
+
+Prints all make tasks
+
+    $ make help
+
+You can change Theano parameters by editing `/docker/theanorc`.
+
+
+Note: If you would have a problem running nvidia-docker you may try the old way
+we have used. But it is not recommended. If you find a bug in the nvidia-docker report
+it there please and try using the nvidia-docker as described above.
+
+    $ export CUDA_SO=$(\ls /usr/lib/x86_64-linux-gnu/libcuda.* | xargs -I{} echo '-v {}:{}')
+    $ export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}')
+    $ docker run -it -p 8888:8888 $CUDA_SO $DEVICES gcr.io/tensorflow/tensorflow:latest-gpu

docker/theanorc

@@ -0,0 +1,5 @@
+[global]
+floatX = float32
+optimizer=None
+device = cuda
+

docs/README.md

@@ -5,5 +5,8 @@ Our documentation uses extended Markdown, as implemented by [MkDocs](http://mkdo
 
 ## Building the documentation
 
-- install MkDocs: `sudo pip install mkdocs`
-- `cd` to the `docs/` folder and run: `mkdocs serve`
+- install MkDocs: `pip install mkdocs`
+- `cd` to the `docs/` folder and run:
+    - `python autogen.py`
+    - `mkdocs serve`    # Starts a local webserver:  [localhost:8000](localhost:8000)
+    - `mkdocs build`    # Builds a static site in "site" directory

docs/autogen.py

@@ -0,0 +1,521 @@
+# -*- coding: utf-8 -*-
+'''
+General documentation architecture:
+
+Home
+Index
+
+- Getting started
+    Getting started with the sequential model
+    Getting started with the functional api
+    FAQ
+
+- Models
+    About Keras models
+        explain when one should use Sequential or functional API
+        explain compilation step
+        explain weight saving, weight loading
+        explain serialization, deserialization
+    Sequential
+    Model (functional API)
+
+- Layers
+    About Keras layers
+        explain common layer functions: get_weights, set_weights, get_config
+        explain input_shape
+        explain usage on non-Keras tensors
+    Core Layers
+    Convolutional Layers
+    Pooling Layers
+    Locally-connected Layers
+    Recurrent Layers
+    Embedding Layers
+    Merge Layers
+    Advanced Activations Layers
+    Normalization Layers
+    Noise Layers
+    Layer Wrappers
+    Writing your own Keras layers
+
+- Preprocessing
+    Sequence Preprocessing
+    Text Preprocessing
+    Image Preprocessing
+
+Losses
+Metrics
+Optimizers
+Activations
+Callbacks
+Datasets
+Applications
+Backend
+Initializers
+Regularizers
+Constraints
+Visualization
+Scikit-learn API
+Utils
+Contributing
+
+'''
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import re
+import inspect
+import os
+import shutil
+import sys
+if sys.version[0] == '2':
+    reload(sys)
+    sys.setdefaultencoding('utf8')
+
+import keras
+from keras import utils
+from keras import layers
+from keras import initializers
+from keras.layers import pooling
+from keras.layers import local
+from keras.layers import recurrent
+from keras.layers import core
+from keras.layers import noise
+from keras.layers import normalization
+from keras.layers import advanced_activations
+from keras.layers import embeddings
+from keras.layers import wrappers
+from keras import optimizers
+from keras import callbacks
+from keras import models
+from keras.engine import topology
+from keras import losses
+from keras import metrics
+from keras import backend
+from keras import constraints
+from keras import activations
+from keras import regularizers
+from keras.utils import data_utils
+from keras.utils import io_utils
+from keras.utils import layer_utils
+from keras.utils import np_utils
+from keras.utils import generic_utils
+
+
+EXCLUDE = {
+    'Optimizer',
+    'Wrapper',
+    'get_session',
+    'set_session',
+    'CallbackList',
+    'serialize',
+    'deserialize',
+    'get',
+}
+
+PAGES = [
+    {
+        'page': 'models/sequential.md',
+        'functions': [
+            models.Sequential.compile,
+            models.Sequential.fit,
+            models.Sequential.evaluate,
+            models.Sequential.predict,
+            models.Sequential.train_on_batch,
+            models.Sequential.test_on_batch,
+            models.Sequential.predict_on_batch,
+            models.Sequential.fit_generator,
+            models.Sequential.evaluate_generator,
+            models.Sequential.predict_generator,
+            models.Sequential.get_layer,
+        ],
+    },
+    {
+        'page': 'models/model.md',
+        'functions': [
+            models.Model.compile,
+            models.Model.fit,
+            models.Model.evaluate,
+            models.Model.predict,
+            models.Model.train_on_batch,
+            models.Model.test_on_batch,
+            models.Model.predict_on_batch,
+            models.Model.fit_generator,
+            models.Model.evaluate_generator,
+            models.Model.predict_generator,
+            models.Model.get_layer,
+        ]
+    },
+    {
+        'page': 'layers/core.md',
+        'classes': [
+            layers.Dense,
+            layers.Activation,
+            layers.Dropout,
+            layers.Flatten,
+            layers.Reshape,
+            layers.Permute,
+            layers.RepeatVector,
+            layers.Lambda,
+            layers.ActivityRegularization,
+            layers.Masking,
+        ],
+    },
+    {
+        'page': 'layers/convolutional.md',
+        'classes': [
+            layers.Conv1D,
+            layers.Conv2D,
+            layers.SeparableConv2D,
+            layers.Conv2DTranspose,
+            layers.Conv3D,
+            layers.Cropping1D,
+            layers.Cropping2D,
+            layers.Cropping3D,
+            layers.UpSampling1D,
+            layers.UpSampling2D,
+            layers.UpSampling3D,
+            layers.ZeroPadding1D,
+            layers.ZeroPadding2D,
+            layers.ZeroPadding3D,
+        ],
+    },
+    {
+        'page': 'layers/pooling.md',
+        'classes': [
+            pooling.MaxPooling1D,
+            pooling.MaxPooling2D,
+            pooling.MaxPooling3D,
+            pooling.AveragePooling1D,
+            pooling.AveragePooling2D,
+            pooling.AveragePooling3D,
+            pooling.GlobalMaxPooling1D,
+            pooling.GlobalAveragePooling1D,
+            pooling.GlobalMaxPooling2D,
+            pooling.GlobalAveragePooling2D,
+        ],
+    },
+    {
+        'page': 'layers/local.md',
+        'classes': [
+            local.LocallyConnected1D,
+            local.LocallyConnected2D,
+        ],
+    },
+    {
+        'page': 'layers/recurrent.md',
+        'classes': [
+            recurrent.Recurrent,
+            recurrent.SimpleRNN,
+            recurrent.GRU,
+            recurrent.LSTM,
+        ],
+    },
+    {
+        'page': 'layers/embeddings.md',
+        'classes': [
+            embeddings.Embedding,
+        ],
+    },
+    {
+        'page': 'layers/normalization.md',
+        'classes': [
+            normalization.BatchNormalization,
+        ],
+    },
+    {
+        'page': 'layers/advanced-activations.md',
+        'all_module_classes': [advanced_activations],
+    },
+    {
+        'page': 'layers/noise.md',
+        'all_module_classes': [noise],
+    },
+    {
+        'page': 'layers/merge.md',
+        'classes': [
+            layers.Add,
+            layers.Multiply,
+            layers.Average,
+            layers.Maximum,
+            layers.Concatenate,
+            layers.Dot,
+        ],
+        'functions': [
+            layers.add,
+            layers.multiply,
+            layers.average,
+            layers.maximum,
+            layers.concatenate,
+            layers.dot,
+        ]
+    },
+    {
+        'page': 'layers/wrappers.md',
+        'all_module_classes': [wrappers],
+    },
+    {
+        'page': 'metrics.md',
+        'all_module_functions': [metrics],
+    },
+    {
+        'page': 'losses.md',
+        'all_module_functions': [losses],
+    },
+    {
+        'page': 'initializers.md',
+        'all_module_functions': [initializers],
+        'all_module_classes': [initializers],
+    },
+    {
+        'page': 'optimizers.md',
+        'all_module_classes': [optimizers],
+    },
+    {
+        'page': 'callbacks.md',
+        'all_module_classes': [callbacks],
+    },
+    {
+        'page': 'activations.md',
+        'all_module_functions': [activations],
+    },
+    {
+        'page': 'backend.md',
+        'all_module_functions': [backend],
+    },
+    {
+        'page': 'utils.md',
+        'all_module_functions': [utils],
+        'classes': [utils.CustomObjectScope,
+                    utils.HDF5Matrix,
+                    utils.Sequence]
+    },
+]
+
+ROOT = 'http://keras.io/'
+
+
+def get_earliest_class_that_defined_member(member, cls):
+    ancestors = get_classes_ancestors([cls])
+    result = None
+    for ancestor in ancestors:
+        if member in dir(ancestor):
+            result = ancestor
+    if not result:
+        return cls
+    return result
+
+
+def get_classes_ancestors(classes):
+    ancestors = []
+    for cls in classes:
+        ancestors += cls.__bases__
+    filtered_ancestors = []
+    for ancestor in ancestors:
+        if ancestor.__name__ in ['object']:
+            continue
+        filtered_ancestors.append(ancestor)
+    if filtered_ancestors:
+        return filtered_ancestors + get_classes_ancestors(filtered_ancestors)
+    else:
+        return filtered_ancestors
+
+
+def get_function_signature(function, method=True):
+    wrapped = getattr(function, '_original_function', None)
+    if wrapped is None:
+        signature = inspect.getargspec(function)
+    else:
+        signature = inspect.getargspec(wrapped)
+    defaults = signature.defaults
+    if method:
+        args = signature.args[1:]
+    else:
+        args = signature.args
+    if defaults:
+        kwargs = zip(args[-len(defaults):], defaults)
+        args = args[:-len(defaults)]
+    else:
+        kwargs = []
+    st = '%s.%s(' % (function.__module__, function.__name__)
+    for a in args:
+        st += str(a) + ', '
+    for a, v in kwargs:
+        if isinstance(v, str):
+            v = '\'' + v + '\''
+        st += str(a) + '=' + str(v) + ', '
+    if kwargs or args:
+        return st[:-2] + ')'
+    else:
+        return st + ')'
+
+
+def get_class_signature(cls):
+    try:
+        class_signature = get_function_signature(cls.__init__)
+        class_signature = class_signature.replace('__init__', cls.__name__)
+    except:
+        # in case the class inherits from object and does not
+        # define __init__
+        class_signature = cls.__module__ + '.' + cls.__name__ + '()'
+    return class_signature
+
+
+def class_to_docs_link(cls):
+    module_name = cls.__module__
+    assert module_name[:6] == 'keras.'
+    module_name = module_name[6:]
+    link = ROOT + module_name.replace('.', '/') + '#' + cls.__name__.lower()
+    return link
+
+
+def class_to_source_link(cls):
+    module_name = cls.__module__
+    assert module_name[:6] == 'keras.'
+    path = module_name.replace('.', '/')
+    path += '.py'
+    line = inspect.getsourcelines(cls)[-1]
+    link = 'https://github.com/fchollet/keras/blob/master/' + path + '#L' + str(line)
+    return '[[source]](' + link + ')'
+
+
+def code_snippet(snippet):
+    result = '```python\n'
+    result += snippet + '\n'
+    result += '```\n'
+    return result
+
+
+def process_class_docstring(docstring):
+    docstring = re.sub(r'\n    # (.*)\n',
+                       r'\n    __\1__\n\n',
+                       docstring)
+
+    docstring = re.sub(r'    ([^\s\\\(]+):(.*)\n',
+                       r'    - __\1__:\2\n',
+                       docstring)
+
+    docstring = docstring.replace('    ' * 5, '\t\t')
+    docstring = docstring.replace('    ' * 3, '\t')
+    docstring = docstring.replace('    ', '')
+    return docstring
+
+
+def process_function_docstring(docstring):
+    docstring = re.sub(r'\n    # (.*)\n',
+                       r'\n    __\1__\n\n',
+                       docstring)
+    docstring = re.sub(r'\n        # (.*)\n',
+                       r'\n        __\1__\n\n',
+                       docstring)
+
+    docstring = re.sub(r'    ([^\s\\\(]+):(.*)\n',
+                       r'    - __\1__:\2\n',
+                       docstring)
+
+    docstring = docstring.replace('    ' * 6, '\t\t')
+    docstring = docstring.replace('    ' * 4, '\t')
+    docstring = docstring.replace('    ', '')
+    return docstring
+
+print('Cleaning up existing sources directory.')
+if os.path.exists('sources'):
+    shutil.rmtree('sources')
+
+print('Populating sources directory with templates.')
+for subdir, dirs, fnames in os.walk('templates'):
+    for fname in fnames:
+        new_subdir = subdir.replace('templates', 'sources')
+        if not os.path.exists(new_subdir):
+            os.makedirs(new_subdir)
+        if fname[-3:] == '.md':
+            fpath = os.path.join(subdir, fname)
+            new_fpath = fpath.replace('templates', 'sources')
+            shutil.copy(fpath, new_fpath)
+
+# Take care of index page.
+readme = open('../README.md').read()
+index = open('templates/index.md').read()
+index = index.replace('{{autogenerated}}', readme[readme.find('##'):])
+f = open('sources/index.md', 'w')
+f.write(index)
+f.close()
+
+print('Starting autogeneration.')
+for page_data in PAGES:
+    blocks = []
+    classes = page_data.get('classes', [])
+    for module in page_data.get('all_module_classes', []):
+        module_classes = []
+        for name in dir(module):
+            if name[0] == '_' or name in EXCLUDE:
+                continue
+            module_member = getattr(module, name)
+            if inspect.isclass(module_member):
+                cls = module_member
+                if cls.__module__ == module.__name__:
+                    if cls not in module_classes:
+                        module_classes.append(cls)
+        module_classes.sort(key=lambda x: id(x))
+        classes += module_classes
+
+    for cls in classes:
+        subblocks = []
+        signature = get_class_signature(cls)
+        subblocks.append('<span style="float:right;">' + class_to_source_link(cls) + '</span>')
+        subblocks.append('### ' + cls.__name__ + '\n')
+        subblocks.append(code_snippet(signature))
+        docstring = cls.__doc__
+        if docstring:
+            subblocks.append(process_class_docstring(docstring))
+        blocks.append('\n'.join(subblocks))
+
+    functions = page_data.get('functions', [])
+    for module in page_data.get('all_module_functions', []):
+        module_functions = []
+        for name in dir(module):
+            if name[0] == '_' or name in EXCLUDE:
+                continue
+            module_member = getattr(module, name)
+            if inspect.isfunction(module_member):
+                function = module_member
+                if module.__name__ in function.__module__:
+                    if function not in module_functions:
+                        module_functions.append(function)
+        module_functions.sort(key=lambda x: id(x))
+        functions += module_functions
+
+    for function in functions:
+        subblocks = []
+        signature = get_function_signature(function, method=False)
+        signature = signature.replace(function.__module__ + '.', '')
+        subblocks.append('### ' + function.__name__ + '\n')
+        subblocks.append(code_snippet(signature))
+        docstring = function.__doc__
+        if docstring:
+            subblocks.append(process_function_docstring(docstring))
+        blocks.append('\n\n'.join(subblocks))
+
+    if not blocks:
+        raise RuntimeError('Found no content for page ' +
+                           page_data['page'])
+
+    mkdown = '\n----\n\n'.join(blocks)
+    # save module page.
+    # Either insert content into existing page,
+    # or create page otherwise
+    page_name = page_data['page']
+    path = os.path.join('sources', page_name)
+    if os.path.exists(path):
+        template = open(path).read()
+        assert '{{autogenerated}}' in template, ('Template found for ' + path +
+                                                 ' but missing {{autogenerated}} tag.')
+        mkdown = template.replace('{{autogenerated}}', mkdown)
+        print('...inserting autogenerated content into template:', path)
+    else:
+        print('...creating new page with autogenerated content:', path)
+    subdir = os.path.dirname(path)
+    if not os.path.exists(subdir):
+        os.makedirs(subdir)
+    open(path, 'w').write(mkdown)
+
+shutil.copyfile('../CONTRIBUTING.md', 'sources/contributing.md')

docs/mkdocs.yml

@@ -1,43 +1,54 @@
 site_name: Keras Documentation
 theme: readthedocs
+#theme_dir: theme
 docs_dir: sources
 repo_url: http://github.com/fchollet/keras
-site_url: /
-#theme_dir: theme
-site_description: Documentation for fast and lightweight Keras Deep Learning library.
-include_404: true
-include_search: true
+site_url: http://keras.io/
+site_description: 'Documentation for Keras, the Python Deep Learning library.'
 
 dev_addr: '0.0.0.0:8000'
 google_analytics: ['UA-61785484-1', 'keras.io']
 
-
 pages:
-- [index.md, Home]
-- [documentation.md, Index]
-
-- [examples.md, Examples]
-- [optimizers.md, Optimizers]
-- [objectives.md, Objectives]
-- [models.md, Models]
-- [activations.md, Activations]
-- [initializations.md, Initializations]
-- [regularizers.md, Regularizers]
-- [constraints.md, Constraints]
-- [callbacks.md, Callbacks]
-
-- [datasets.md, Datasets]
-
-- [layers/core.md, Layers, Core Layers]
-- [layers/convolutional.md, Layers, Convolutional Layers]
-- [layers/recurrent.md, Layers, Recurrent Layers]
-- [layers/advanced_activations.md, Layers, Advanced Activations Layers]
-- [layers/normalization.md, Layers, Normalization Layers]
-- [layers/embeddings.md, Layers, Embedding Layers]
-- [layers/containers.md, Layers, Containers]
-
-- [preprocessing/sequence.md, Preprocessing, Sequence Preprocessing]
-- [preprocessing/text.md, Preprocessing, Text Preprocessing]
-- [preprocessing/image.md, Preprocessing, Image Preprocessing]
-
-- [utils/visualization.md, Utils, Visualization Utilities]
+- Home: index.md
+- Getting started:
+  - Guide to the Sequential model: getting-started/sequential-model-guide.md
+  - Guide to the Functional API: getting-started/functional-api-guide.md
+  - FAQ: getting-started/faq.md
+- Models:
+  - About Keras models: models/about-keras-models.md
+  - Sequential: models/sequential.md
+  - Model (functional API): models/model.md
+- Layers:
+  - About Keras layers: layers/about-keras-layers.md
+  - Core Layers: layers/core.md
+  - Convolutional Layers: layers/convolutional.md
+  - Pooling Layers: layers/pooling.md
+  - Locally-connected Layers: layers/local.md
+  - Recurrent Layers: layers/recurrent.md
+  - Embedding Layers: layers/embeddings.md
+  - Merge Layers: layers/merge.md
+  - Advanced Activations Layers: layers/advanced-activations.md
+  - Normalization Layers: layers/normalization.md
+  - Noise layers: layers/noise.md
+  - Layer wrappers: layers/wrappers.md
+  - Writing your own Keras layers: layers/writing-your-own-keras-layers.md
+- Preprocessing:
+  - Sequence Preprocessing: preprocessing/sequence.md
+  - Text Preprocessing: preprocessing/text.md
+  - Image Preprocessing: preprocessing/image.md
+- Losses: losses.md
+- Metrics: metrics.md
+- Optimizers: optimizers.md
+- Activations: activations.md
+- Callbacks: callbacks.md
+- Datasets: datasets.md
+- Applications: applications.md
+- Backend: backend.md
+- Initializers: initializers.md
+- Regularizers: regularizers.md
+- Constraints: constraints.md
+- Visualization: visualization.md
+- Scikit-learn API: scikit-learn-api.md
+- Utils: utils.md
+- Contributing: contributing.md

docs/sources/activations.md

@@ -1,40 +0,0 @@
-
-## Usage of activations
-
-Activations can either be used through an `Activation` layer, or through the `activation` argument supported by all forward layers:
-
-```python
-from keras.layers.core import Activation, Dense
-
-model.add(Dense(64, 64, init='uniform'))
-model.add(Activation('tanh'))
-```
-is equivalent to:
-```python
-model.add(Dense(20, 64, init='uniform', activation='tanh'))
-```
-
-You can also pass an element-wise Theano function as an activation:
-
-```python
-def tanh(x):
-    return theano.tensor.tanh(x)
-
-model.add(Dense(20, 64, init='uniform', activation=tanh))
-model.add(Activation(tanh))
-```
-
-## Available activations
-
-- __softmax__: Should only be applied to 2D layers (expected shape: `(nb_samples, nb_dims)`).
-- __time_distributed_softmax__: Softmax applied to every sample at every timestep of a layer of shape `(nb_samples, nb_timesteps, nb_dims)`.
-- __softplus__
-- __relu__
-- __tanh__
-- __sigmoid__
-- __hard_sigmoid__
-- __linear__
-
-## On Advanced Activations
-
-Activations that are more complex than a simple Theano function (eg. learnable activations, configurable activations, etc.) are available as [Advanced Activation layers](layers/advanced_activations.md), and can be found in the module `keras.layers.advanced_activations`. These include PReLU and LeakyReLU.

docs/sources/callbacks.md

@@ -1,91 +0,0 @@
-## Usage of callbacks
-
-A callback is a set of functions to be applied at given stages of the training procedure. You can use callbacks to get a view on internal states and statistics of the model during training. You can pass a list of callback (as the keyword argument `callbacks`) to the `.fit()` method of the `Sequential` model. The relevant methods of the callbacks will then be called at each stage of the training. 
-
----
-
-## Base class
-
-```python
-keras.callbacks.Callback()
-```
-- __Properties__:
-    - __params__: dict. Training parameters (eg. verbosity, batch size, number of epochs...).
-    - __model__: `keras.models.Model`. Reference of the model being trained.
-- __Methods__:
-    - __on_train_begin__(logs={}): Method called at the beginning of training.
-    - __on_train_end__(logs={}): Method called at the end of training.
-    - __on_epoch_begin__(epoch, logs={}): Method called at the beginning of epoch `epoch`.
-    - __on_epoch_end__(epoch, logs={}): Method called at the end of epoch `epoch`.
-    - __on_batch_begin__(batch, logs={}): Method called at the beginning of batch `batch`.
-    - __on_batch_end__(batch, logs={}): Method called at the end of batch `batch`.
-
-The `logs` dictionary will contain keys for quantities relevant to the current batch or epoch. Currently, the `.fit()` method of the `Sequential` model class will include the following quantities in the `logs` that it passes to its callbacks:
-- __on_epoch_end__: logs optionally include `val_loss` (if validation is enabled in `fit`), and `val_accuracy` (if validation and accuracy monitoring are enabled).
-- __on_batch_begin__: logs include `size`, the number of samples in the current batch.
-- __on_batch_end__: logs include `loss`, and optionally `accuracy` (if accuracy monitoring is enabled).
-
----
-
-
-## Create a callback
-
-You can create a custom callback by extending the base class `keras.callbacks.Callback`. A callback has access to its associated model through the class property `self.model`.
-
-Here's a simple example saving a list of losses over each batch during training:
-```python
-class LossHistory(keras.callbacks.Callback):
-    def on_train_begin(self):
-        self.losses = []
-
-    def on_batch_end(self, batch, logs={}):
-        self.losses.append(logs.get('loss'))
-```
-
----
-
-### Example to record the loss history
-
-```python
-class LossHistory(keras.callbacks.Callback):
-    def on_train_begin(self):
-        self.losses = []
-
-    def on_batch_end(self, batch, logs={}):
-        self.losses.append(logs.get('loss'))
-
-model = Sequential()
-model.add(Dense(784, 10, init='uniform'))
-model.add(Activation('softmax'))
-model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
-
-history = LossHistory()
-model.fit(X_train, Y_train, batch_size=128, nb_epoch=20, verbose=0, callbacks=[history])
-
-print history.losses
-# outputs
-'''
-[0.66047596406559383, 0.3547245744908703, ..., 0.25953155204159617, 0.25901699725311789]
-'''
-```
-
----
-
-### Example to checkpoint models
-
-```python
-from keras.callbacks import ModelCheckpoint
-
-model = Sequential()
-model.add(Dense(784, 10, init='uniform'))
-model.add(Activation('softmax'))
-model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
-
-'''
-saves the model weights after each epoch if the validation loss decreased
-'''
-checkpointer = ModelCheckpoint(filepath="/tmp/weights.hdf5", verbose=1, save_best_only=True)
-model.fit(X_train, Y_train, batch_size=128, nb_epoch=20, verbose=0, validation_data=(X_test, Y_test), callbacks=[checkpointer])
-
-```
-

docs/sources/constraints.md

@@ -1,19 +0,0 @@
-## Usage of constraints
-
-Functions from the `constraints` module allow setting constraints (eg. non-negativity) on network parameters during optimization.
-
-The keyword arguments used for passing constraints to parameters in a layer will depend on the layer. 
-
-In the `Dense` layer it is simply `W_constraint` for the main weights matrix, and `b_constraint` for the bias.
-
-
-```python
-from keras.constraints import maxnorm
-model.add(Dense(64, 64, W_constraint = maxnorm(2)))
-```
-
-## Available constraints
-
-- __maxnorm__(m=2): maximum-norm constraint
-- __nonneg__(): non-negativity constraint
-- __unitnorm__(): unit-norm constraint, enforces the matrix to have unit norm along the last axis

docs/sources/datasets.md

@@ -1,121 +0,0 @@
-# Datasets
-
-## CIFAR10 small image classification
-
-`keras.datasets.cifar10`
-
-Dataset of 50,000 32x32 color training images, labeled over 10 categories, and 10,000 test images.
-
-### Usage:
-
-```python
-(X_train, y_train), (X_test, y_test) = cifar10.load_data()
-```
-
-- __Return:__
-    - 2 tuples:
-        - __X_train, X_test__: uint8 array of RGB image data with shape (nb_samples, 3, 32, 32).
-        - __y_train, y_test__: uint8 array of category labels (integers in range 0-9) with shape (nb_samples,).
-
----
-
-## CIFAR100 small image classification
-
-`keras.datasets.cifar100`
-
-Dataset of 50,000 32x32 color training images, labeled over 100 categories, and 10,000 test images.
-
-### Usage:
-
-```python
-(X_train, y_train), (X_test, y_test) = cifar100.load_data(label_mode='fine')
-```
-
-- __Return:__
-    - 2 tuples:
-        - __X_train, X_test__: uint8 array of RGB image data with shape (nb_samples, 3, 32, 32).
-        - __y_train, y_test__: uint8 array of category labels with shape (nb_samples,).
-
-- __Arguments:__
-
-    - __label_mode__: "fine" or "coarse".
-
----
-
-## IMDB Movie reviews sentiment classification
-
-`keras.datasets.imdb`
-
-Dataset of 25,000 movies reviews from IMDB, labeled by sentiment (positive/negative). Reviews have been preprocessed, and each review is encoded as a [sequence](preprocessing/sequence.md) of word indexes (integers). For convenience, words are indexed by overall frequency in the dataset, so that for instance the integer "3" encodes the 3rd most frequent word in the data. This allows for quick filtering operations such as: "only consider the top 10,000 most common words, but eliminate the top 20 most common words".
-
-As a convention, "0" does not stand for a specific word, but instead is used to encode any unknown word.
-
-### Usage:
-
-```python
-(X_train, y_train), (X_test, y_test) = imdb.load_data(path="imdb.pkl", \
-nb_words=None, skip_top=0, maxlen=None, test_split=0.1, seed=113)
-```
-- __Return:__
-    - 2 tuples:
-        - __X_train, X_test__: list of sequences, which are lists of indexes (integers). If the nb_words argument was specific, the maximum possible index value is nb_words-1. If the maxlen argument was specified, the largest possible sequence length is maxlen.
-        - __y_train, y_test__: list of integer labels (1 or 0). 
-
-- __Arguments:__
-
-    - __path__: if you do have the data locally (at `'~/.keras/datasets/' + path`), if will be downloaded to this location (in cPickle format).
-    - __nb_words__: integer or None. Top most frequent words to consider. Any less frequent word will appear as 0 in the sequence data.
-    - __skip_top__: integer. Top most frequent words to ignore (they will appear as 0s in the sequence data).
-    - __maxlen__: int. Maximum sequence length. Any longer sequence will be truncated.
-    - __test_split__: float. Fraction of the dataset to be used as test data.
-    - __seed__: int. Seed for reproducible data shuffling.
-
----
-
-## Reuters newswire topics classification
-
-`keras.datasets.reuters`
-
-Dataset of 11,228 newswires from Reuters, labeled over 46 topics. As with the IMDB dataset, each wire is encoded as a sequence of word indexes (same conventions).
-
-### Usage:
-
-```python
-(X_train, y_train), (X_test, y_test) = reuters.load_data(path="reuters.pkl", \
-nb_words=None, skip_top=0, maxlen=None, test_split=0.1, seed=113)
-```
-
-The specifications are the same as that of the IMDB dataset.
-
-This dataset also makes available the word index used for encoding the sequences:
-
-```python
-word_index = reuters.get_word_index(path="reuters_word_index.pkl")
-```
-
-- __Return:__ A dictionary where key are words (str) and values are indexes (integer). eg. `word_index["giraffe"]` might return `1234`. 
-
-- __Arguments:__
-
-    - __path__: if you do have the index file locally (at `'~/.keras/datasets/' + path`), if will be downloaded to this location (in cPickle format).
-    
-## MNIST database of handwritten digits
-
-`keras.datasets.mnist`
-
-Dataset of 60,000 28x28 grayscale images of the 10 digits, along with a test set of 10,000 images.
-
-### Usage:
-
-```python
-(X_train, y_train), (X_test, y_test) = mnist.load_data()
-```
-
-- __Return:__
-    - 2 tuples:
-        - __X_train, X_test__: uint8 array of grayscale image data with shape (nb_samples, 28, 28).
-        - __y_train, y_test__: uint8 array of digit labels (integers in range 0-9) with shape (nb_samples,).
-
-- __Arguments:__
-
-    - __path__: if you do have the index file locally (at `'~/.keras/datasets/' + path`), if will be downloaded to this location (in cPickle format).

docs/sources/documentation.md

@@ -1,35 +0,0 @@
-# Keras Documentation Index
-
-## Introduction
-
-- [Home](index.md)
-- [Index](documentation.md)
-- [Examples](examples.md)
-
----
-
-## Base functionality
-
-- [Optimizers](optimizers.md)
-- [Objectives](objectives.md)
-- [Models](models.md)
-- [Activations](activations.md)
-- [Initializations](initializations.md)
-- [Datasets](datasets.md)
-
----
-
-## Layers
-- [Core](layers/core.md)
-- [Convolutional](layers/convolutional.md)
-- [Recurrent](layers/recurrent.md)
-- [Advanced Activations](layers/advanced_activations.md)
-- [Normalization](layers/normalization.md)
-- [Embeddings](layers/embeddings.md)
-
----
-
-## Preprocessing
-- [Sequence](preprocessing/sequence.md)
-- [Text](preprocessing/text.md)
-- [Image](preprocessing/image.md)

docs/sources/examples.md

@@ -1,161 +0,0 @@
-
-Here are a few examples to get you started!
-
-### Multilayer Perceptron (MLP)
-
-```python
-from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation
-from keras.optimizers import SGD
-
-model = Sequential()
-model.add(Dense(20, 64, init='uniform'))
-model.add(Activation('tanh'))
-model.add(Dropout(0.5))
-model.add(Dense(64, 64, init='uniform'))
-model.add(Activation('tanh'))
-model.add(Dropout(0.5))
-model.add(Dense(64, 2, init='uniform'))
-model.add(Activation('softmax'))
-
-sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
-model.compile(loss='mean_squared_error', optimizer=sgd)
-
-model.fit(X_train, y_train, nb_epoch=20, batch_size=16)
-score = model.evaluate(X_test, y_test, batch_size=16)
-```
-
----
-
-### Alternative implementation of MLP
-
-```python
-model = Sequential()
-model.add(Dense(20, 64, init='uniform', activation='tanh'))
-model.add(Dropout(0.5))
-model.add(Dense(64, 64, init='uniform', activation='tanh'))
-model.add(Dropout(0.5))
-model.add(Dense(64, 2, init='uniform', activation='softmax')
-
-sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
-model.compile(loss='mean_squared_error', optimizer=sgd)
-```
-
----
-
-### VGG-like convnet
-
-```python
-from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation, Flatten
-from keras.layers.convolutional import Convolution2D, MaxPooling2D
-from keras.optimizers import SGD
-
-model = Sequential()
-model.add(Convolution2D(32, 3, 3, 3, border_mode='full')) 
-model.add(Activation('relu'))
-model.add(Convolution2D(32, 32, 3, 3))
-model.add(Activation('relu'))
-model.add(MaxPooling2D(poolsize=(2, 2)))
-model.add(Dropout(0.25))
-
-model.add(Convolution2D(64, 32, 3, 3, border_mode='full')) 
-model.add(Activation('relu'))
-model.add(Convolution2D(64, 64, 3, 3)) 
-model.add(Activation('relu'))
-model.add(MaxPooling2D(poolsize=(2, 2)))
-model.add(Dropout(0.25))
-
-model.add(Flatten())
-model.add(Dense(64*8*8, 256))
-model.add(Activation('relu'))
-model.add(Dropout(0.5))
-
-model.add(Dense(256, 10))
-model.add(Activation('softmax'))
-
-sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
-model.compile(loss='categorical_crossentropy', optimizer=sgd)
-
-model.fit(X_train, Y_train, batch_size=32, nb_epoch=1)
-
-```
-
----
-
-### Sequence classification with LSTM
-
-```python
-from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation
-from keras.layers.embeddings import Embedding
-from keras.layers.recurrent import LSTM
-
-model = Sequential()
-model.add(Embedding(max_features, 256))
-model.add(LSTM(256, 128, activation='sigmoid', inner_activation='hard_sigmoid'))
-model.add(Dropout(0.5))
-model.add(Dense(128, 1))
-model.add(Activation('sigmoid'))
-
-model.compile(loss='binary_crossentropy', optimizer='rmsprop')
-
-model.fit(X_train, Y_train, batch_size=16, nb_epoch=10)
-score = model.evaluate(X_test, Y_test, batch_size=16)
-```
-
----
-
-### Architecture for learning image captions with a convnet and a Gated Recurrent Unit
-(word-level embedding, caption of maximum length 16 words).
-
-Note that getting this to actually "work" will require using a bigger convnet, initialized with pre-trained weights.
-Displaying readable results will also require an embedding decoder.
-
-```python
-max_caption_len = 16
-
-model = Sequential()
-model.add(Convolution2D(32, 3, 3, 3, border_mode='full')) 
-model.add(Activation('relu'))
-model.add(Convolution2D(32, 32, 3, 3))
-model.add(Activation('relu'))
-model.add(MaxPooling2D(poolsize=(2, 2)))
-
-model.add(Convolution2D(64, 32, 3, 3, border_mode='full')) 
-model.add(Activation('relu'))
-model.add(Convolution2D(64, 64, 3, 3)) 
-model.add(Activation('relu'))
-model.add(MaxPooling2D(poolsize=(2, 2)))
-
-model.add(Convolution2D(128, 64, 3, 3, border_mode='full')) 
-model.add(Activation('relu'))
-model.add(Convolution2D(128, 128, 3, 3)) 
-model.add(Activation('relu'))
-model.add(MaxPooling2D(poolsize=(2, 2)))
-
-model.add(Flatten())
-model.add(Dense(128*4*4, 256))
-model.add(Activation('relu'))
-model.add(Dropout(0.5))
-
-model.add(Repeat(max_caption_len)) 
-# the GRU below returns sequences of max_caption_len vectors of size 256 (our word embedding size)
-model.add(GRU(256, 256, return_sequences=True))
-
-model.compile(loss='mean_squared_error', optimizer='rmsprop')
-
-# "images" is a numpy array of shape (nb_samples, nb_channels=3, width, height) 
-# "captions" is a numpy array of shape (nb_samples, max_caption_len=16, embedding_dim=256)
-# captions are supposed already embedded (dense vectors).
-model.fit(images, captions, batch_size=16, nb_epoch=100)
-    
-```
-
----
-
-In the [examples folder](https://github.com/fchollet/keras/tree/master/examples), you will find example models for real datasets:
-
-- CIFAR10 small images classification: Convnet with realtime data augmentation
-- IMDB movie review sentiment classification: LSTM over sequences of words
-- Reuters newswires topic classification: Multilayer Perceptron

docs/sources/index.md

@@ -1,131 +0,0 @@
-# Keras: Theano-based Deep Learning library
-
-## Overview
-
-Keras is a minimalist, highly modular neural network library in the spirit of Torch, written in Python, that uses [Theano](http://deeplearning.net/software/theano/) under the hood for fast tensor manipulation on GPU and CPU. It was developed with a focus on enabling fast experimentation. 
-
-Use Keras if you need a deep learning library that:
-
-- allows for easy and fast prototyping (through total modularity, minimalism, and extensibility).
-- supports both __convolutional networks__ and __recurrent networks__ (LSTM, GRU, etc). As well as combinations of the two. 
-- runs seamlessly on the CPU and the GPU.
-
-## Guiding principles
-
-- __Modularity.__ A model is understood as a sequence of standalone, fully-configurable modules that can be plugged together with as little restrictions as possible. In particular, neural layers, cost functions, optimizers, initialization schemes, activation functions and dropout are all standalone modules that you can combine to create new models. 
-
-- __Minimalism.__ Each module should be kept short and simple (<100 lines of code). Every piece of code should be transparent upon first reading. No black magic: it hurts iteration speed and ability to innovate. 
-
-- __Easy extensibility.__ A new feature (a new module, per the above definition, or a new way to combine modules together) are dead simple to add (as new classes/functions), and existing modules provide ample examples.
-
-- __Work with Python__. No separate models configuration files in a declarative format (like in Caffe or PyLearn2). Models are described in Python code, which is compact, easier to debug, benefits from syntax highlighting, and most of all, allows for ease of extensibility.
-
-## Code
-
-Find the code on Github: [fchollet/keras](https://github.com/fchollet/keras).
-
-## License
-
-Keras is licensed under the [MIT license](http://opensource.org/licenses/MIT). 
-
-## Getting started: 30 seconds to Keras
-
-The core datastructure of Keras is a __model__, a way to organize layers. Here's a sequential model (a linear pile of layers).
-
-```python
-from keras.models import Sequential
-
-model = Sequential()
-```
-
-Stacking layers is as easy as `.add()`:
-
-```python
-from keras.layers.core import Dense, Activation
-
-model.add(Dense(input_dim=100, output_dim=64, init="uniform"))
-model.add(Activation("relu"))
-model.add(Dense(input_dim=64, output_dim=10, init="uniform"))
-model.add(Activation("softmax"))
-```
-
-Once your model looks good, configure its learning process with `.compile()`:
-```python
-model.compile(loss='categorical_crossentropy', optimizer='sgd')
-```
-
-If you need to, you can further configure your optimizer. A core principle of Keras is make things things reasonably simple, while allowing the user to be fully in control when they need to (the ultimate control being the easy extensibility of the source code).
-```python
-from keras.optimizers import SGD
-model.compile(loss='categorical_crossentropy', optimizer=SGD(lr=0.01, momentum=0.9, nesterov=True))
-```
-
-You can now iterate on your training data in batches:
-```python
-model.fit(X_train, Y_train, nb_epoch=5, batch_size=32)
-```
-
-Alternatively, you can feed batches to your model manually:
-```python
-model.train(X_batch, Y_batch)
-```
-
-Evaluate your performance in one line:
-```python
-objective_score = model.evaluate(X_test, Y_test, batch_size=32)
-```
-
-Or generate predictions on new data:
-```python
-classes = model.predict_classes(X_test, batch_size=32)
-proba = model.predict_proba(X_test, batch_size=32)
-```
-
-Building a network of LSTMs, a deep CNN, a word2vec embedder or any other model is just as fast. The ideas behind deep learning are simple, so why should their implementation be painful?
-
-Have a look at the [examples](examples.md).
-
-## Installation
-
-Keras uses the following dependencies:
-
-- numpy, scipy
-- Theano
-    - See [installation instructions](http://deeplearning.net/software/theano/install.html#install).
-- HDF5 and h5py (optional, required if you use model saving/loading functions)
-- Optional but recommended if you use CNNs: cuDNN.
-
-Once you have the dependencies installed, clone the repo:
-```bash
-git clone https://github.com/fchollet/keras.git
-```
-Go to the Keras folder and run the install command:
-```bash
-cd keras
-sudo python setup.py install
-```
-
-## Support
-
-You can ask questions and join the development discussion on the [Keras Google group](https://groups.google.com/forum/#!forum/keras-users).
-
-## Contribution Guidelines
-
-Keras welcomes all contributions from the community. 
-
-- Keep a pragmatic mindset and avoid bloat. Only add to the source if that is the only path forward.
-- New features should be documented. Make sure you update the documentation along with your Pull Request.
-- The documentation for every new feature should include a usage example in the form of a code snippet. 
-- All changes should be tested. A formal test process will be introduced very soon.
-- Even if you don't contribute to the Keras source code, if you have an application of Keras that is concise and powerful, please consider adding it to our collection of [examples](https://github.com/fchollet/keras/tree/master/examples).
-
-
-## Why this name, Keras?
-
-Keras (κέρας) means _horn_ in Greek. It is a reference to a literary image from ancient Greek and Latin literature, first found in the _Odyssey_, where dream spirits (_Oneiroi_, singular _Oneiros_) are divided between those who deceive men with false visions, who arrive to Earth through a gate of ivory, and those who announce a future that will come to pass, who arrive through a gate of horn. It's a play on the words κέρας (horn) / κραίνω (fulfill), and ἐλέφας (ivory) / ἐλεφαίρομαι (deceive).
-
-Keras was developed as part of the research effort of project ONEIROS (Open-ended Neuro-Electronic Intelligent Robot Operating System).
-
-> _"Oneiroi are beyond our unravelling --who can be sure what tale they tell? Not all that men look for comes to pass. Two gates there are that give passage to fleeting Oneiroi; one is made of horn, one of ivory. The Oneiroi that pass through sawn ivory are deceitful, bearing a message that will not be fulfilled; those that come out through polished horn have truth behind them, to be accomplished for men who see them."_ 
-
-> -- Homer, Odyssey 19. 562 ff (Shewring translation).

docs/sources/initializations.md

@@ -1,23 +0,0 @@
-# Initializations
-
-## Usage of initializations
-
-Initializations define the probability distribution used to set the initial random weights of Keras layers.
-
-The keyword arguments used for passing initializations to layers will depend on the layer. Usually it is simply `init`:
-
-```python
-model.add(Dense(64, 64, init='uniform'))
-```
-
-## Available initializations
-
-- __uniform__
-- __lecun_uniform__: Uniform initialization scaled by the square root of the number of inputs (LeCun 98).
-- __normal__
-- __orthogonal__: Use with square 2D layers (`shape[0] == shape[1]`).
-- __zero__
-- __glorot_normal__: Gaussian initialization scaled by fan_in + fan_out (Glorot 2010)
-- __glorot_uniform__
-- __he_normal__: Gaussian initialization scaled by fan_in (He et al., 2014)
-- __he_uniform__

docs/sources/layers/advanced_activations.md

@@ -1,35 +0,0 @@
-
-## LeakyReLU
-
-```python
-keras.layers.advanced_activations.LeakyReLU(alpha=0.3)
-```
-
-Special version of a Rectified Linear Unit that allows a small gradient when the unit is not active (`f(x) = alpha*x for x < 0`).
-
-- __Input shape__: This layer does not assume a specific input shape. As a result, it cannot be used as the first layer in a model.
-
-- __Output shape__: Same as input.
-
-- __Arguments__:
-    - __alpha__: float >= 0. Negative slope coefficient.
-
----
-
-## PReLU
-
-```python
-keras.layers.advanced_activations.PReLU(input_shape)
-```
-
-Parametrized linear unit. Similar to a LeakyReLU, where each input unit has its alpha coefficient, and where these coefficients are learned during training.
-
-- __Input shape__: Same as `input_shape`. This layer cannot be used as first layer in a model.
-
-- __Output shape__: Same as input.
-
-- __Arguments__:
-    - __input_shape__: tuple.
-
-- __References__:
-    - [Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification](http://arxiv.org/pdf/1502.01852v1.pdf)

docs/sources/layers/containers.md

@@ -1,23 +0,0 @@
-# Containers
-
-Containers are ensembles of layers that can be interacted with through the same API as `Layer` objects.
-
-## Sequential
-
-```python
-keras.layers.containers.Sequential(layers=[])
-```
-
-The Sequential container is a linear stack of layers. Apart from the `add` methods and the `layers` constructor argument, the API is identical to that of the `Layer` class.
-
-This class is also the basis for the `keras.models.Sequential` architecture.
-
-The `layers` constructor argument is a list of Layer instances.
-
-__Methods__:
-
-```python
-add(layer)
-```
-
-Add a new layer to the stack.

docs/sources/layers/convolutional.md

@@ -1,20 +0,0 @@
-
-## Convolution2D
-
-```python
-keras.layers.convolutional.Convolution2D(nb_filter, stack_size, nb_row, nb_col, 
-        init='glorot_uniform', activation='linear', weights=None, 
-        image_shape=None, border_mode='valid', subsample=(1,1))
-```
-
-This is a wrapper for Theano's [conv2d](http://deeplearning.net/software/theano/library/tensor/nnet/conv.html#theano.tensor.nnet.conv.conv2d). 
-
----
-
-## MaxPooling2D
-
-```python
-keras.layers.convolutional.MaxPooling2D(poolsize=(2, 2), ignore_border=True)
-```
-
-This is a wrapper for Theano's [max_pool_2d](http://deeplearning.net/software/theano/library/tensor/signal/downsample.html).

docs/sources/layers/core.md

@@ -1,355 +0,0 @@
-## Base class
-
-```python
-keras.layers.core.Layer()
-```
-
-__Methods__:
-
-```python
-connect(previous_layer)
-```
-
-Connect the input of the current layer to the output of the argument layer.
-
-- __Return__: None.
-
-- __Arguments__: 
-    - __previous_layer__: Layer object.
-
-
-
-```python
-output(train)
-```
-
-Get the output of the layer.
-
-- __Return__: Theano tensor.
-
-- __Arguments__: 
-    - __train__: Boolean. Specifies whether output is computed in training mode or in testing mode, which can change the logic, for instance in there are any `Dropout` layers in the network. 
-
-
-
-```python
-get_input(train)
-```
-
-Get the input of the layer.
-
-- __Return__: Theano tensor.
-
-- __Arguments__: 
-    - __train__: Boolean. Specifies whether output is computed in training mode or in testing mode, which can change the logic, for instance in there are any `Dropout` layers in the network. 
-
-
-
-```python
-get_weights()
-```
-
-Get the weights of the parameters of the layer.
-
-- __Return__: List of numpy arrays (one per layer parameter). 
-
-
-
-```python
-set_weights(weights)
-```
-
-Set the weights of the parameters of the layer.
-
-- __Arguments__: 
-    - __weights__: List of numpy arrays (one per layer parameter). Should be in the same order as what `get_weights(self)` returns.
-
-
-
----
-
-## Dense
-```python
-keras.layers.core.Dense(input_dim, output_dim, init='glorot_uniform', activation='linear', weights=None \
-W_regularizer=None, b_regularizer=None, W_constraint=None, b_constraint=None)
-```
-
-Standard 1D fully-connect layer. 
-
-- __Input shape__: 2D tensor with shape: `(nb_samples, input_dim)`.
-
-- __Output shape__: 2D tensor with shape: `(nb_samples, output_dim)`.
-
-- __Arguments__:
-
-    - __input_dim__: int >= 0. 
-    - __output_dim__: int >= 0. 
-    - __init__: name of initialization function for the weights of the layer (see: [initializations](../initializations.md)), or alternatively, Theano function to use for weights initialization. This parameter is only relevant if you don't pass a `weights` argument.
-    - __activation__: name of activation function to use (see: [activations](../activations.md)), or alternatively, elementwise Theano function. If you don't specify anything, no activation is applied (ie. "linear" activation: a(x) = x).
-    - __weights__: list of numpy arrays to set as initial weights. The list should have 1 element, of shape `(input_dim, output_dim)`.
-    - __W_regularizer__: instance of the [regularizers](../regularizers.md) module (eg. L1 or L2 regularization), applied to the main weights matrix.
-    - __b_regularizer__: instance of the [regularizers](../regularizers.md) module, applied to the bias.
-    - __W_constraint__: instance of the [constraints](../constraints.md) module (eg. maxnorm, nonneg), applied to the main weights matrix.
-    - __b_constraint__: instance of the [constraints](../constraints.md) module, applied to the bias.
-
----
-
-## TimeDistributedDense
-```python
-keras.layers.core.TimeDistributedDense(input_dim, output_dim, init='glorot_uniform', activation='linear', weights=None \
-W_regularizer=None, b_regularizer=None, W_constraint=None, b_constraint=None)
-```
-
-Fully-connected layer distributed over the time dimension. Useful after a recurrent network set to `return_sequences=True`.
-
-- __Input shape__: 3D tensor with shape: `(nb_samples, nb_timesteps, input_dim)`.
-
-- __Arguments__:
-    - __input_dim__: int >= 0. 
-    - __output_dim__: int >= 0. 
-    - __init__: name of initialization function for the weights of the layer (see: [initializations](../initializations.md)), or alternatively, Theano function to use for weights initialization. This parameter is only relevant if you don't pass a `weights` argument.
-    - __activation__: name of activation function to use (see: [activations](../activations.md)), or alternatively, elementwise Theano function. If you don't specify anything, no activation is applied (ie. "linear" activation: a(x) = x).
-    - __weights__: list of numpy arrays to set as initial weights. The list should have 1 element, of shape `(input_dim, output_dim)`.
-    - __W_regularizer__: instance of the [regularizers](../regularizers.md) module (eg. L1 or L2 regularization), applied to the main weights matrix.
-    - __b_regularizer__: instance of the [regularizers](../regularizers.md) module, applied to the bias.
-    - __W_constraint__: instance of the [constraints](../constraints.md) module (eg. maxnorm, nonneg), applied to the main weights matrix.
-    - __b_constraint__: instance of the [constraints](../constraints.md) module, applied to the bias.
-
-- __Example__:
-```python
-# input shape: (nb_samples, nb_timesteps, 10)
-model.add(LSTM(10, 5, return_sequences=True)) # output shape: (nb_samples, nb_timesteps, 5)
-model.add(TimeDistributedDense(5, 10)) # output shape: (nb_samples, nb_timesteps, 10)
-```
-
-
----
-
-## AutoEncoder
-```python
-keras.layers.core.AutoEncoder(encoder, decoder, output_reconstruction=True, tie_weights=False, weights=None):
-```
-
-A customizable autoencoder model. If `output_reconstruction = True` then dim(input) = dim(output) else dim(output) = dim(hidden)
-
-
-- __Input shape__: The layer shape is defined by the encoder definitions
-
-- __Output shape__: The layer shape is defined by the decoder definitions
-
-- __Arguments__:
-
-    - __encoder__: A [layer](./) or [layer container](./containers.md).
-
-    - __decoder__: A [layer](./) or [layer container](./containers.md).
-    
-    - __output_reconstruction__: If this is False the when .predict() is called the output is the deepest hidden layer's activation. Otherwise the output of the final decoder layer is presented. Be sure your validation data confirms to this logic if you decide to use any.
-    
-    - __tie_weights__: If True then the encoder bias is tied to the decoder bias. **Note**: This required the encoder layer corresponding to this decoder layer to be of the same time, eg: Dense:Dense
-    
-    - __weights__: list of numpy arrays to set as initial weights. The list should have 1 element, of shape `(input_dim, output_dim)`.
-
-- __Example__:
-```python
-from keras.layers import containers
-
-# input shape: (nb_samples, 32)
-encoder = containers.Sequential([Dense(32, 16), Dense(16, 8)])
-decoder = containers.Sequential([Dense(8, 16), Dense(16, 32)])
-autoencoder.add(AutoEncoder(encoder=encoder, decoder=decoder, output_reconstruction=False, tie_weights=True))
-```
-
----
-
-## DenoisingAutoEncoder
-```python
-keras.layers.core.AutoEncoder(encoder, decoder, output_reconstruction=True, tie_weights=False, weights=None, corruption_level=0.3):
-```
-
-A denoising autoencoder model that inherits the base features from autoencoder.
-Since this layer uses similar logic to Dropout it cannot be the first layer in a pipeline.
-
-- __Input shape__: The layer shape is defined by the encoder definitions
-
-- __Output shape__: The layer shape is defined by the decoder definitions
-
-- __Arguments__:
-
-    - __encoder__: A [layer](./) or [layer container](./containers.md).
-
-    - __decoder__: A [layer](./) or [layer container](./containers.md).
-    
-    - __output_reconstruction__: If this is False the when .predict() is called the output is the deepest hidden layer's activation. Otherwise the output of the final decoder layer is presented. Be sure your validation data confirms to this logic if you decide to use any.
-    
-    - __tie_weights__: If True then the encoder bias is tied to the decoder bias. **Note**: This required the encoder layer corresponding to this decoder layer to be of the same time, eg: Dense:Dense
-    
-    - __weights__: list of numpy arrays to set as initial weights. The list should have 1 element, of shape `(input_dim, output_dim)`.
-    
-    - __corruption_level__: the amount of binomial noise added to the input layer of the model.
-
-- __Example__:
-```python
-# input shape: (nb_samples, 32)
-autoencoder.add(Dense(32, 32))
-autoencoder.add(DenoisingAutoEncoder(encoder=Dense(32, 16),
-                                     decoder=Dense(16, 32),
-                                     output_reconstruction=False, tie_weights=True,
-                                     corruption_level=0.3))
-```
-
-
----
-
-## Activation
-```python
-keras.layers.core.Activation(activation)
-```
-Apply an activation function to the input. 
-
-- __Input shape__: This layer does not assume a specific input shape. As a result, it cannot be used as the first layer in a model.
-
-- __Output shape__: Same as input.
-
-- __Arguments__:
-
-    - __activation__: name of activation function to use (see: [activations](../activations.md)), or alternatively, elementwise Theano function.
-
-
----
-
-## Dropout
-```python
-keras.layers.core.Dropout(p)
-```
-Apply dropout to the input. Dropout consists in randomly setting a fraction `p` of input units to 0 at each update during training time, which helps prevent overfitting. Reference: [Dropout: A Simple Way to Prevent Neural Networks from Overfitting](http://www.cs.toronto.edu/~rsalakhu/papers/srivastava14a.pdf)
-
-- __Input shape__: This layer does not assume a specific input shape. 
-
-- __Output shape__: Same as input.
-
-- __Arguments__:
-
-    - __p__: float (0 <= p < 1). Fraction of the input that gets dropped out at training time. 
-
-
----
-
-## Reshape
-```python
-keras.layers.core.Reshape(*dims)
-```
-
-Reshape the input to a new shape containing the same number of units. 
-
-- __Input shape__: This layer does not assume a specific input shape. 
-
-- __Output shape__: `(nb_samples, *dims)`.
-
-- __Arguments__:
-
-    - *dims: integers. Dimensions of the new shape.
-
-- __Example__:
-```python
-# input shape: (nb_samples, 10)
-model.add(Dense(10, 100)) # output shape: (nb_samples, 100)
-model.add(Reshape(10, 10))  # output shape: (nb_samples, 10, 10)
-```
-
----
-
-## Flatten
-```python
-keras.layers.core.Flatten()
-```
-
-Convert a nD input to 1D. 
-
-- __Input shape__: (nb_samples, *). This layer cannot be used as the first layer in a model.
-
-- __Output shape__: `(nb_samples, nb_input_units)`.
-
----
-
-## RepeatVector
-```python
-keras.layers.core.RepeatVector(n)
-```
-
-Repeat the 1D input n times. Dimensions of input are assumed to be (nb_samples, dim). Output will have the shape (nb_samples, n, dim).
-
-- __Input shape__: This layer does not assume a specific input shape. This layer cannot be used as the first layer in a model.
-
-- __Output shape__: `(nb_samples, n, input_dims)`.
-
-- __Arguments__:
-    - __n__: int. 
-
-- __Example__:
-
----
-
-## MaxoutDense
-```python
-keras.layers.core.MaxoutDense(input_dim, output_dim, nb_feature=4, init='glorot_uniform', weights=None, \
-        W_regularizer=None, b_regularizer=None, W_constraint=None, b_constraint=None)
-```
-
-A dense maxout layer. A `MaxoutDense` layer takes the element-wise maximum of `nb_feature` `Dense(input_dim, output_dim)` linear layers. This allows the layer to learn a convex, piecewise linear activation function over the inputs. See [this paper](http://arxiv.org/pdf/1302.4389.pdf) for more details. Note that this is a *linear* layer -- if you wish to apply activation function (you shouldn't need to -- they are universal function approximators), an `Activation` layer must be added after.
-
-- __Input shape__: 2D tensor with shape: `(nb_samples, input_dim)`.
-
-- __Output shape__: 2D tensor with shape: `(nb_samples, output_dim)`.
-
-- __Arguments__:
-
-    - __input_dim__: int >= 0. 
-    - __output_dim__: int >= 0. 
-    - __nb_feature__: int >= 0. the number of features to create for the maxout. This is equivalent to the number of piecewise elements to be allowed for the activation function. 
-    - __init__: name of initialization function for the weights of the layer (see: [initializations](../initializations.md)), or alternatively, Theano function to use for weights initialization. This parameter is only relevant if you don't pass a `weights` argument.
-    - __weights__: list of numpy arrays to set as initial weights. The list should have 1 element, of shape `(input_dim, output_dim)`.
-    - __W_regularizer__: instance of the [regularizers](../regularizers.md) module (eg. L1 or L2 regularization), applied to the main weights matrix.
-    - __b_regularizer__: instance of the [regularizers](../regularizers.md) module, applied to the bias.
-    - __W_constraint__: instance of the [constraints](../constraints.md) module (eg. maxnorm, nonneg), applied to the main weights matrix.
-    - __b_constraint__: instance of the [constraints](../constraints.md) module, applied to the bias.
-
-```python
-# input shape: (nb_samples, 10)
-model.add(Dense(10, 100)) # output shape: (nb_samples, 100)
-model.add(MaxoutDense(100, 100, nb_feature=10)) # output shape: (nb_samples, 100)
-model.add(RepeatVector(2))  # output shape: (nb_samples, 2, 10)
-```
-
-## Merge
-```python
-keras.layers.core.Merge(models, mode='sum')
-```
-
-Merge the output of a list of models into a single tensor, following one of two modes: `sum` or `concat`. 
-
-- __Arguments__:
-    - __models__: List of `Sequential` models.
-    - __mode__: String, one of `{'sum', 'concat'}`. `sum` will simply sum the outputs of the models (therefore all models should have an output with the same shape). `concat` will concatenate the outputs along the last dimension (therefore all models should have an output that only differ along the last dimension). 
-
-- __Example__:
-
-```python
-left = Sequential()
-left.add(Dense(784, 50))
-left.add(Activation('relu'))
-
-right = Sequential()
-right.add(Dense(784, 50))
-right.add(Activation('relu'))
-
-model = Sequential()
-model.add(Merge([left, right], mode='sum'))
-
-model.add(Dense(50, 10))
-model.add(Activation('softmax'))
-
-model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
-
-model.fit([X_train, X_train], Y_train, batch_size=128, nb_epoch=20, validation_data=([X_test, X_test], Y_test))
-```
-

docs/sources/layers/embeddings.md

@@ -1,48 +0,0 @@
-
-## Embedding
-
-```python
-keras.layers.embeddings.Embedding(input_dim, output_dim, init='uniform', weights=None, W_regularizer=None, W_constraint=None)
-```
-
-Turn positive integers (indexes) into denses vectors of fixed size,
-eg. `[[4], [20]] -> [[0.25, 0.1], [0.6, -0.2]]`
-
-- __Input shape__: 2D tensor with shape: `(nb_samples, maxlen)`.
-
-- __Output shape__: 3D tensor with shape: `(nb_samples, maxlen, output_dim)`.
-
-- __Arguments__:
-
-    - __input_dim__: int >= 0. Size of the vocabulary, ie. 1+maximum integer index occuring in the input data.
-    - __output_dim__: int >= 0. Dimension of the dense embedding.
-    - __init__: name of initialization function for the weights of the layer (see: [initializations](../initializations.md)), or alternatively, Theano function to use for weights initialization. This parameter is only relevant if you don't pass a `weights` argument.
-    - __weights__: list of numpy arrays to set as initial weights. The list should have 1 element, of shape `(input_dim, output_dim)`.
-    - __W_regularizer__: instance of the [regularizers](../regularizers.md) module (eg. L1 or L2 regularization), applied to the embedding matrix.
-    - __W_constraint__: instance of the [constraints](../constraints.md) module (eg. maxnorm, nonneg), applied to the embedding matrix.
-
-
-## WordContextProduct
-
-```python
-keras.layers.embeddings.WordContextProduct(input_dim, proj_dim=128, 
-        init='uniform', activation='sigmoid', weights=None)
-```
-
-This layer turns a pair of words (a pivot word + a context word, ie. a word from the same context as a pivot, or a random, out-of-context word), indentified by their indices in a vocabulary, into two dense reprensentations (word representation and context representation).
-
-Then it returns `activation(dot(pivot_embedding, context_embedding))`, which can be trained to encode the probability of finding the context word in the context of the pivot word (or reciprocally depending on your training procedure).
-
-For more context, see Mikolov et al.: [Efficient Estimation of Word reprensentations in Vector Space](http://arxiv.org/pdf/1301.3781v3.pdf)
-
-- __Input shape__: 2D tensor with shape: `(nb_samples, 2)`.
-
-- __Output shape__: 2D tensor with shape: `(nb_samples, 1)`.
-
-- __Arguments__:
-
-    - __input_dim__: int >= 0. Size of the vocabulary, ie. 1+maximum integer index occuring in the input data.
-    - __proj_dim__: int >= 0. Dimension of the dense embedding used internally.
-    - __init__: name of initialization function for the embeddings (see: [initializations](../initializations.md)), or alternatively, Theano function to use for weights initialization. This parameter is only relevant if you don't pass a `weights` argument.
-    - __activation__: name of activation function to use (see: [activations](../activations.md)), or alternatively, elementwise Theano function.
-    - __weights__: list of numpy arrays to set as initial weights. The list should have 2 element, both of shape `(input_dim, proj_dim)`. The first element is the word embedding weights, the second one is the context embedding weights.

docs/sources/layers/normalization.md

@@ -1,20 +0,0 @@
-
-## BatchNormalization
-
-```python
-keras.layers.normalization.BatchNormalization(input_shape, epsilon=1e-6, weights=None)
-```
-
-Normalize the activations of the previous layer at each batch.
-
-- __Input shape__: Same as `input_shape`. This layer cannot be used as first layer in a model.
-
-- __Output shape__: Same as input.
-
-- __Arguments__:
-    - __input_shape__: tuple.
-    - __epsilon__: small float > 0. Fuzz parameter.
-    - __weights__: Initialization weights. List of 2 numpy arrays, with shapes: `[(input_shape,), (input_shape,)]`
-
-- __References__:
-    - [Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift](http://arxiv.org/pdf/1502.03167v3.pdf)

docs/sources/layers/recurrent.md

@@ -1,133 +0,0 @@
-
-## SimpleRNN
-
-```python
-keras.layers.recurrent.SimpleRNN(input_dim, output_dim, 
-        init='glorot_uniform', inner_init='orthogonal', activation='sigmoid', weights=None,
-        truncate_gradient=-1, return_sequences=False)
-```
-Fully connected RNN where output is to fed back to input. Not a particularly useful model, included for demonstration purposes.
-
-- __Input shape__: 3D tensor with shape: `(nb_samples, timesteps, input_dim)`.
-
-- __Output shape__: 
-    - if `return_sequences`: 3D tensor with shape: `(nb_samples, timesteps, ouput_dim)`.
-    - else: 2D tensor with shape: `(nb_samples, output_dim)`.
-
-- __Arguments__:
-    - __input_dim__: dimension of the input.
-    - __output_dim__: dimension of the internal projections and the final output.
-    - __init__: weight initialization function. Can be the name of an existing function (str), or a Theano function (see: [initializations](../initializations.md)).
-    - __activation__: activation function. Can be the name of an existing function (str), or a Theano function (see: [activations](../activations.md)).
-    - __weights__: list of numpy arrays to set as initial weights. The list should have 3 elements, of shapes: `[(input_dim, output_dim), (output_dim, output_dim), (output_dim,)]`.
-    - __truncate_gradient__: Number of steps to use in truncated BPTT. See: [Theano "scan"](http://deeplearning.net/software/theano/library/scan.html).
-    - __return_sequences__: Boolean. Whether to return the last output in the output sequence, or the full sequence.
-
----
-
-## SimpleDeepRNN
-
-```python
-keras.layers.recurrent.SimpleDeepRNN(input_dim, output_dim, depth=3,
-        init='glorot_uniform', inner_init='orthogonal', 
-        activation='sigmoid', inner_activation='hard_sigmoid',
-        weights=None, truncate_gradient=-1, return_sequences=False)
-```
-Fully connected RNN where the output of multiple timesteps (up to "depth" steps in the past) is fed back to the input: 
-
-```
-output = activation( W.x_t + b + inner_activation(U_1.h_tm1) + inner_activation(U_2.h_tm2) + ... )
-```
-
-Not a particularly useful model, included for demonstration purposes.
-
-- __Input shape__: 3D tensor with shape: `(nb_samples, timesteps, input_dim)`.
-
-- __Output shape__:
-    - if `return_sequences`: 3D tensor with shape: `(nb_samples, timesteps, ouput_dim)`.
-    - else: 2D tensor with shape: `(nb_samples, output_dim)`.
-
-- __Arguments__:
-    - __input_dim__: dimension of the input.
-    - __output_dim__: dimension of the internal projections and the final output.
-    - __depth__: int >= 1. Lookback depth (eg. depth=1 is equivalent to SimpleRNN).
-    - __init__: weight initialization function for the output cell. Can be the name of an existing function (str), or a Theano function (see: [initializations](../initializations.md)).
-    - __inner_init__: weight initialization function for the inner cells.
-    - __activation__: activation function for the output. Can be the name of an existing function (str), or a Theano function (see: [activations](../activations.md)).
-    - __inner_activation__: activation function for the inner cells.
-    - __weights__: list of numpy arrays to set as initial weights. The list should have depth+2 elements.
-    - __truncate_gradient__: Number of steps to use in truncated BPTT. See: [Theano "scan"](http://deeplearning.net/software/theano/library/scan.html).
-    - __return_sequences__: Boolean. Whether to return the last output in the output sequence, or the full sequence.
-
-
----
-
-## GRU
-
-```python
-keras.layers.recurrent.GRU(input_dim, output_dim=128, 
-        init='glorot_uniform', inner_init='orthogonal',
-        activation='sigmoid', inner_activation='hard_sigmoid',
-        weights=None, truncate_gradient=-1, return_sequences=False)
-```
-
-Gated Recurrent Unit - Cho et al. 2014.
-
-- __Input shape__: 3D tensor with shape: `(nb_samples, timesteps, input_dim)`.
-
-- __Output shape__:
-    - if `return_sequences`: 3D tensor with shape: `(nb_samples, timesteps, ouput_dim)`.
-    - else: 2D tensor with shape: `(nb_samples, output_dim)`.
-
-- __Arguments__:
-    - __input_dim__: dimension of the input.
-    - __output_dim__: dimension of the internal projections and the final output.
-    - __init__: weight initialization function for the output cell. Can be the name of an existing function (str), or a Theano function (see: [initializations](../initializations.md)).
-    - __inner_init__: weight initialization function for the inner cells.
-    - __activation__: activation function for the output. Can be the name of an existing function (str), or a Theano function (see: [activations](../activations.md)).
-    - __inner_activation__: activation function for the inner cells.
-    - __weights__: list of numpy arrays to set as initial weights. The list should have 9 elements.
-    - __truncate_gradient__: Number of steps to use in truncated BPTT. See: [Theano "scan"](http://deeplearning.net/software/theano/library/scan.html).
-    - __return_sequences__: Boolean. Whether to return the last output in the output sequence, or the full sequence.
-
-- __References__: 
-    - [On the Properties of Neural Machine Translation: Encoder–Decoder Approaches](http://www.aclweb.org/anthology/W14-4012)
-    - [Empirical Evaluation of Gated Recurrent Neural Networks on Sequence Modeling](http://arxiv.org/pdf/1412.3555v1.pdf)
-
----
-
-## LSTM
-
-```python
-keras.layers.recurrent.LSTM(input_dim, output_dim=128, 
-        init='glorot_uniform', inner_init='orthogonal', 
-        activation='tanh', inner_activation='hard_sigmoid',
-        weights=None, truncate_gradient=-1, return_sequences=False)
-```
-
-Long-Short Term Memory unit - Hochreiter 1997.
-
-- __Input shape__: 3D tensor with shape: `(nb_samples, timesteps, input_dim)`.
-
-- __Output shape__:
-    - if `return_sequences`: 3D tensor with shape: `(nb_samples, timesteps, ouput_dim)`.
-    - else: 2D tensor with shape: `(nb_samples, output_dim)`.
-
-- __Arguments__:
-- __input_dim__: dimension of the input.
-    - __output_dim__: dimension of the internal projections and the final output.
-    - __init__: weight initialization function for the output cell. Can be the name of an existing function (str), or a Theano function (see: [initializations](../initializations.md)).
-    - __inner_init__: weight initialization function for the inner cells.
-    - __activation__: activation function for the output. Can be the name of an existing function (str), or a Theano function (see: [activations](../activations.md)).
-    - __inner_activation__: activation function for the inner cells.
-    - __weights__: list of numpy arrays to set as initial weights. The list should have 12 elements.
-    - __truncate_gradient__: Number of steps to use in truncated BPTT. See: [Theano "scan"](http://deeplearning.net/software/theano/library/scan.html).
-    - __return_sequences__: Boolean. Whether to return the last output in the output sequence, or the full sequence.
-
-- __References__: 
-    - [Long short-term memory](http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf) (original 1997 paper)
-    - [Learning to forget: Continual prediction with LSTM](http://www.mitpressjournals.org/doi/pdf/10.1162/089976600300015015)
-    - [Supervised sequence labelling with recurrent neural networks](http://www.cs.toronto.edu/~graves/preprint.pdf)
-            
-            
-                

docs/sources/models.md

@@ -1,114 +0,0 @@
-## Sequential
-
-Linear stack of layers.
-
-```python
-model = keras.models.Sequential()
-```
-- __Methods__:
-    - __add__(layer): Add a layer to the model.
-    - __compile__(optimizer, loss, class_mode="categorical"): 
-        - __Arguments__: 
-            - __optimizer__: str (name of optimizer) or optimizer object. See [optimizers](optimizers.md).
-            - __loss__: str (name of objective function) or objective function. See [objectives](objectives.md).
-            - __class_mode__: one of "categorical", "binary". This is only used for computing classification accuracy or using the predict_classes method. 
-            - __theano_mode__: A `theano.compile.mode.Mode` ([reference](http://deeplearning.net/software/theano/library/compile/mode.html)) instance controlling specifying compilation options.
-    - __fit__(X, y, batch_size=128, nb_epoch=100, verbose=1, validation_split=0., validation_data=None, shuffle=True, show_accuracy=False, callbacks=[]): Train a model for a fixed number of epochs.
-        - __Return__: a history dictionary with a record of training loss values at successive epochs, as well as validation loss values (if applicable), accuracy (if applicable), etc.
-        - __Arguments__: 
-            - __X__: data.
-            - __y__: labels.
-            - __batch_size__: int. Number of samples per gradient update.
-            - __nb_epoch__: int. 
-            - __verbose__: 0 for no logging to stdout, 1 for progress bar logging, 2 for one log line per epoch.
-            - __validation_split__: float (0. < x < 1). Fraction of the data to use as held-out validation data.
-            - __validation_data__: tuple (X, y) to be used as held-out validation data. Will override validation_split.
-            - __shuffle__: boolean. Whether to shuffle the samples at each epoch.
-            - __show_accuracy__: boolean. Whether to display class accuracy in the logs to stdout at each epoch.
-            - __callbacks__: `keras.callbacks.Callback` list. List of callbacks to apply during training. See [callbacks](callbacks.md).
-    - __evaluate__(X, y, batch_size=128, show_accuracy=False, verbose=1): Show performance of the model over some validation data.
-        - __Return__: The loss score over the data.
-        - __Arguments__: Same meaning as fit method above. verbose is used as a binary flag (progress bar or nothing).
-    - __predict__(X, batch_size=128, verbose=1): 
-        - __Return__: An array of predictions for some test data.
-        - __Arguments__: Same meaning as fit method above.
-    - __predict_classes__(X, batch_size=128, verbose=1): Return an array of class predictions for some test data.
-        - __Return__: An array of labels for some test data.
-        - __Arguments__: Same meaning as fit method above. verbose is used as a binary flag (progress bar or nothing).
-    - __train__(X, y, accuracy=False): Single gradient update on one batch. if accuracy==False, return tuple (loss_on_batch, accuracy_on_batch). Else, return loss_on_batch.
-        - __Return__: loss over the data, or tuple `(loss, accuracy)` if `accuracy=True`.
-    - __test__(X, y, accuracy=False): Single performance evaluation on one batch. if accuracy==False, return tuple (loss_on_batch, accuracy_on_batch). Else, return loss_on_batch.
-        - __Return__: loss over the data, or tuple `(loss, accuracy)` if `accuracy=True`.
-    - __save_weights__(fname, overwrite=False): Store the weights of all layers to a HDF5 file. If overwrite==False and the file already exists, an exception will be thrown.
-    - __load_weights__(fname): Sets the weights of a model, based to weights stored by __save__weights__. You can only __load__weights__ on a savefile from a model with an identical architecture. __load_weights__ can be called either before or after the __compile__ step.
-
-__Examples__:
-
-```python
-from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation
-from keras.optimizers import SGD
-
-model = Sequential()
-model.add(Dense(64, 2, init='uniform'))
-model.add(Activation('softmax'))
-
-model.compile(loss='mse', optimizer='sgd')
-
-'''
-Demonstration of verbose modes 1 and 2
-'''
-model.fit(X_train, y_train, nb_epoch=3, batch_size=16, verbose=1)
-# outputs
-'''
-Train on 37800 samples, validate on 4200 samples
-Epoch 0
-37800/37800 [==============================] - 7s - loss: 0.0385
-Epoch 1
-37800/37800 [==============================] - 8s - loss: 0.0140
-Epoch 2
-10960/37800 [=======>......................] - ETA: 4s - loss: 0.0109
-'''
-
-model.fit(X_train, y_train, nb_epoch=3, batch_size=16, verbose=2)
-# outputs
-'''
-Train on 37800 samples, validate on 4200 samples
-Epoch 0
-loss: 0.0190
-Epoch 1
-loss: 0.0146
-Epoch 2
-loss: 0.0049
-'''
-
-'''
-Demonstration of show_accuracy
-'''
-model.fit(X_train, y_train, nb_epoch=3, batch_size=16, verbose=2, show_accuracy=True)
-# outputs
-'''
-Train on 37800 samples, validate on 4200 samples
-Epoch 0
-loss: 0.0190 - acc.: 0.8750
-Epoch 1
-loss: 0.0146 - acc.: 0.8750
-Epoch 2
-loss: 0.0049 - acc.: 1.0000
-'''
-
-'''
-Demonstration of validation_split
-'''
-model.fit(X_train, y_train, nb_epoch=3, batch_size=16, validation_split=0.1, show_accuracy=True, verbose=1)
-# outputs
-'''
-Train on 37800 samples, validate on 4200 samples
-Epoch 0
-37800/37800 [==============================] - 7s - loss: 0.0385 - acc.: 0.7258 - val. loss: 0.0160 - val. acc.: 0.9136
-Epoch 1
-37800/37800 [==============================] - 8s - loss: 0.0140 - acc.: 0.9265 - val. loss: 0.0109 - val. acc.: 0.9383
-Epoch 2
-10960/37800 [=======>......................] - ETA: 4s - loss: 0.0109 - acc.: 0.9420
-'''
-```

docs/sources/objectives.md

@@ -1,24 +0,0 @@
-
-## Usage of objectives
-
-An objective function (or loss function, or optimization score function) is one of the two parameters required to compile a model:
-
-```python
-model.compile(loss='mean_squared_error', optimizer='sgd')
-```
-
-You can either pass the name of an existing objective, or pass a Theano symbolic function that returns a scalar and takes the following two arguments:
-
-- __y_true__: True labels. Theano tensor.
-- __y_pred__: Predictions. Theano tensor of the same shape as y_true.
-
-For a few examples of such functions, check out the [objectives source](https://github.com/fchollet/keras/blob/master/keras/objectives.py).
-
-## Available objectives
-
-- __mean_squared_error__ / __mse__
-- __mean_absolute_error__ / __mae__
-- __squared_hinge__
-- __hinge__
-- __binary_crossentropy__: Also known as logloss. 
-- __categorical_crossentropy__: Also known as multiclass logloss. __Note__: using this objective requires that your labels are binary arrays of shape `(nb_samples, nb_classes)`.

docs/sources/optimizers.md

@@ -1,118 +0,0 @@
-
-## Usage of optimizers
-
-An optimizer is one of the two arguments required for compiling a Keras model:
-
-```python
-model = Sequential()
-model.add(Dense(20, 64, init='uniform'))
-model.add(Activation('tanh'))
-model.add(Activation('softmax'))
-
-sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
-model.compile(loss='mean_squared_error', optimizer=sgd)
-```
-
-You can either instantiate an optimizer before passing it to `model.compile()` , as in the above example, or you can call it by its name. In the latter case, the default parameters for the optimizer will be used.
-
-```python
-# pass optimizer by name: default parameters will be used
-model.compile(loss='mean_squared_error', optimizer='sgd')
-```
-
----
-
-## Base class
-
-```python
-keras.optimizers.Optimizer(**kwargs)
-```
-
-All optimizers descended from this class support the following keyword argument:
-
-- __clipnorm__: float >= 0.
-
-Note: this is base class for building optimizers, not an actual optimizer that can be used for training models.
-
----
-
-##  SGD
-
-```python
-keras.optimizers.SGD(lr=0.01, momentum=0., decay=0., nesterov=False)
-``` 
-
-__Arguments__:
-
-- __lr__: float >= 0. Learning rate.
-- __momentum__: float >= 0. Parameter updates momentum.
-- __decay__: float >= 0. Learning rate decay over each update.
-- __nesterov__: boolean. Whether to apply Nesterov momentum.
-
----
-
-##  Adagrad
-
-```python
-keras.optimizers.Adagrad(lr=0.01, epsilon=1e-6)
-```
-
-It is recommended to leave the parameters of this optimizer at their default values.
-
-__Arguments__:
-
-- __lr__: float >= 0. Learning rate. 
-- __epsilon__: float >= 0. 
-
----
-
-##  Adadelta
-
-```python
-keras.optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=1e-6)
-```
-
-It is recommended to leave the parameters of this optimizer at their default values.
-
-__Arguments__:
-
-- __lr__: float >= 0. Learning rate. It is recommended to leave it at the default value.
-- __rho__: float >= 0. 
-- __epsilon__: float >= 0. Fuzz factor.
-
-For more info, see *"Adadelta: an adaptive learning rate method"* by Matthew Zeiler.
-
----
-
-##  RMSprop 
-
-```python
-keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-6)
-```
-
-It is recommended to leave the parameters of this optimizer at their default values.
-
-__Arguments__:
-
-- __lr__: float >= 0. Learning rate. 
-- __rho__: float >= 0.
-- __epsilon__: float >= 0. Fuzz factor.
-
----
-
-## Adam
-
-```python
-keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8, kappa=1-1e-8)
-```
-
-Adam optimizer, proposed by Kingma and Lei Ba in [Adam: A Method For Stochastic Optimization](http://arxiv.org/pdf/1412.6980v4.pdf). Default parameters are those suggested in the paper. The parameter "lambda" from the paper has been renamed kappa, for syntactic reasons.
-
-__Arguments__:
-
-- __lr__: float >= 0. Learning rate. 
-- __beta_1__, __beta_2__: floats, 0 < beta < 1. Generally close to 1.
-- __epsilon__: float >= 0. Fuzz factor.
-- __kappa__: float 0 < kappa < 1. Lambda parameter in the original paper.
-
----

docs/sources/preprocessing/image.md

@@ -1,70 +0,0 @@
-
-## ImageDataGenerator
-
-```python
-keras.preprocessing.image.ImageDataGenerator(featurewise_center=True,
-    samplewise_center=False,
-    featurewise_std_normalization=True,
-    samplewise_std_normalization=False,
-    zca_whitening=False,
-    rotation_range=0.,
-    width_shift_range=0.,
-    height_shift_range=0.,
-    horizontal_flip=False,
-    vertical_flip=False)
-```
-
-Generate batches of tensor image data with real-time data augmentation.
-
-- __Arguments__:
-    - __featurewise_center__: Boolean. Set input mean to 0 over the dataset.
-    - __samplewise_center__: Boolean. Set each sample mean to 0.
-    - __featurewise_std_normalization__: Boolean. Divide inputs by std of the dataset.
-    - __samplewise_std_normalization__: Boolean. Divide each input by its std.
-    - __zca_whitening__: Boolean. Apply ZCA whitening.
-    - __rotation_range__: Int. Degree range for random rotations.
-    - __width_shift_range__: Float (fraction of total width). Range for random horizontal shifts.
-    - __height_shift_range__: Float (fraction of total height). Range for random vertical shifts.
-    - __horizontal_flip__: Boolean. Randomly flip inputs horizontally.
-    - __vertical_flip__: Boolean. Randomly flip inputs vertically.
-
-- __Methods__:
-    - __fit(X)__: Required if featurewise_center or featurewise_std_normalization or zca_whitening. Compute necessary quantities on some sample data.
-        - __Arguments__: 
-            - __X__: sample data.
-            - __augment__: Boolean (default: False). Whether to fit on randomly augmented samples.
-            - __rounds__: int (default: 1). If augment, how many augmentation passes over the data to use.
-    - __flow(X, y)__:
-        - __Arguments__: 
-            - __X__: data.
-            - __y__: labels.
-            - __batch_size__: int (default: 32).
-            - __shuffle__: boolean (defaut: False).
-            - __save_to_dir__: None or str. This allows you to optimally specify a directory to which to save the augmented pictures being generated (useful for visualizing what you are doing). 
-            - __save_prefix__: str. Prefix to use for filenames of saved pictures.
-            - __save_format__: one of "png", jpeg". 
-
-- __Example__:
-```python
-(X_train, y_train), (X_test, y_test) = cifar10.load_data(test_split=0.1)
-Y_train = np_utils.to_categorical(y_train, nb_classes)
-Y_test = np_utils.to_categorical(y_test, nb_classes)
-
-datagen = ImageDataGenerator(
-    featurewise_center=True,
-    featurewise_std_normalization=True,
-    rotation_range=20,
-    width_shift_range=0.2,
-    height_shift_range=0.2,
-    horizontal_flip=True)
-
-# compute quantities required for featurewise normalization 
-# (std, mean, and principal components if ZCA whitening is applied)
-datagen.fit(X_train)
-
-for e in range(nb_epoch):
-    print 'Epoch', e
-    # batch train with realtime data augmentation
-    for X_batch, Y_batch in datagen.flow(X_train, Y_train):
-        loss = model.train(X_batch, Y_batch)
-```

docs/sources/preprocessing/text.md

@@ -1,81 +0,0 @@
-
-## text_to_word_sequence
-
-```python
-keras.preprocessing.text.text_to_word_sequence(text, 
-    filters=base_filter(), lower=True, split=" ")
-```
-
-Split a sentence into a list of words.
-
-- __Return__: List of words (str).
-
-- __Arguments__:
-    - __text__: str.
-    - __filters__: list (or concatenation) of characters to filter out, such as punctuation. Default: base_filter(), includes basic punctuation, tabs, and newlines.
-    - __lower__: boolean. Whether to set the text to lowercase.
-    - __split__: str. Separator for word splitting.
-
-## one_hot
-
-```python
-keras.preprocessing.text.one_hot(text, n,
-    filters=base_filter(), lower=True, split=" ")
-```
-
-One-hot encode a text into a list of word indexes in a vocabulary of size n.
-
-- __Return__: List of integers in [1, n]. Each integer encodes a word (unicity non-guaranteed).
-
-- __Arguments__: Same as `text_to_word_sequence` above.
-    - __n__: int. Size of vocabulary.
-
-## Tokenizer
-
-```python
-keras.preprocessing.text.Tokenizer(nb_words=None, filters=base_filter(), 
-    lower=True, split=" ")
-```
-
-Class for vectorizing texts, or/and turning texts into sequences (=list of word indexes, where the word of rank i in the dataset (starting at 1) has index i).
-
-- __Arguments__: Same as `text_to_word_sequence` above.
-    - __nb_words__: None or int. Maximum number of words to work with (if set, tokenization will be restricted to the top nb_words most common words in the dataset).
-
-- __Methods__:
-
-    - __fit_on_texts(texts)__: 
-        - __Arguments__:
-            - __texts__: list of texts to train on.
-
-    - __texts_to_sequences(texts)__
-        - __Arguments__: 
-            - __texts__: list of texts to turn to sequences.
-        - __Return__: list of sequences (one per text input).
-
-    - __texts_to_sequences_generator(texts)__: generator version of the above. 
-        - __Return__: yield one sequence per input text.
-
-    - __texts_to_matrix(texts)__:
-        - __Return__: numpy array of shape `(len(texts), nb_words)`.
-        - __Arguments__:
-            - __texts__: list of texts to vectorize.
-            - __mode__: one of "binary", "count", "tfidf", "freq" (default: "binary").
-
-    - __fit_on_sequences(sequences)__: 
-        - __Arguments__:
-            - __sequences__: list of sequences to train on. 
-
-    - __sequences_to_matrix(sequences)__:
-        - __Return__: numpy array of shape `(len(sequences), nb_words)`.
-        - __Arguments__:
-            - __sequences__: list of sequences to vectorize.
-            - __mode__: one of "binary", "count", "tfidf", "freq" (default: "binary").
-
-- __Attributes__:
-    - __word_counts__: dictionary mapping words (str) to the number of times they appeared on during fit. Only set after fit_on_texts was called. 
-    - __word_docs__: dictionary mapping words (str) to the number of documents/texts they appeared on during fit. Only set after fit_on_texts was called.
-    - __word_index__: dictionary mapping words (str) to their rank/index (int). Only set after fit_on_texts was called.
-    - __document_count__: int. Number of documents (texts/sequences) the tokenizer was trained on. Only set after fit_on_texts or fit_on_sequences was called.
-
-

docs/sources/regularizers.md

@@ -1,18 +0,0 @@
-## Usage of regularizers
-
-Regularizers allow to apply penalties on network parameters during optimization.
-
-The keyword arguments used for passing penalties to parameters in a layer will depend on the layer. 
-
-In the `Dense` layer it is simply `W_regularizer` for the main weights matrix, and `b_regularizer` for the bias.
-
-```python
-from keras.regularizers import l2
-model.add(Dense(64, 64, W_regularizer = l2(.01)))
-```
-
-## Available penalties
-
-- __l1__(l=0.01): L1 regularization penalty, also known as LASSO
-- __l2__(l=0.01): L2 regularization penalty, also known as weight decay, or Ridge
-- __l1l2__(l1=0.01, l2=0.01): L1-L2 regularization penalty, also known as ElasticNet

docs/sources/utils/visualization.md

@@ -1,28 +0,0 @@
-## Grapher
-
-Creates a visualization of the model structure using `pydot`.
-
-```python
-grapher = keras.utils.dot_utils.Grapher()
-```
-- __Methods__:
-    - __plot__(model, to_file): creates a graph visualizing the structure of `model` and writes it to `to_file`.
-      - __Arguments__:
-        - __model__: an instance of a Keras model (e.g. `Sequential`)
-        - __to_file__: the filename to save the visualization png to.
-
-__Examples__:
-
-```python
-from keras.models import Sequential
-from keras.layers.core import Dense, Activation
-from keras.utils.dot_utils import Grapher
-
-grapher = Grapher()
-
-model = Sequential()
-model.add(Dense(64, 2, init='uniform'))
-model.add(Activation('softmax'))
-grapher.plot(model, 'model.png')
-```
-

docs/templates/activations.md

@@ -0,0 +1,34 @@
+
+## Usage of activations
+
+Activations can either be used through an `Activation` layer, or through the `activation` argument supported by all forward layers:
+
+```python
+from keras.layers import Activation, Dense
+
+model.add(Dense(64))
+model.add(Activation('tanh'))
+```
+
+This is equivalent to:
+
+```python
+model.add(Dense(64, activation='tanh'))
+```
+
+You can also pass an element-wise TensorFlow/Theano/CNTK function as an activation:
+
+```python
+from keras import backend as K
+
+model.add(Dense(64, activation=K.tanh))
+model.add(Activation(K.tanh))
+```
+
+## Available activations
+
+{{autogenerated}}
+
+## On "Advanced Activations"
+
+Activations that are more complex than a simple TensorFlow/Theano/CNTK function (eg. learnable activations, which maintain a state) are available as [Advanced Activation layers](layers/advanced-activations.md), and can be found in the module `keras.layers.advanced_activations`. These include `PReLU` and `LeakyReLU`.

docs/templates/applications.md

@@ -0,0 +1,455 @@
+# Applications
+
+Keras Applications are deep learning models that are made available alongside pre-trained weights.
+These models can be used for prediction, feature extraction, and fine-tuning.
+
+Weights are downloaded automatically when instantiating a model. They are stored at `~/.keras/models/`.
+
+## Available models
+
+### Models for image classification with weights trained on ImageNet:
+
+- [Xception](#xception)
+- [VGG16](#vgg16)
+- [VGG19](#vgg19)
+- [ResNet50](#resnet50)
+- [InceptionV3](#inceptionv3)
+
+All of these architectures (except Xception) are compatible with both TensorFlow and Theano, and upon instantiation the models will be built according to the image data format set in your Keras configuration file at `~/.keras/keras.json`. For instance, if you have set `image_data_format=channels_last`, then any model loaded from this repository will get built according to the TensorFlow data format convention, "Width-Height-Depth".
+
+The Xception model is only available for TensorFlow, due to its reliance on `SeparableConvolution` layers.
+
+-----
+
+## Usage examples for image classification models
+
+### Classify ImageNet classes with ResNet50
+
+```python
+from keras.applications.resnet50 import ResNet50
+from keras.preprocessing import image
+from keras.applications.resnet50 import preprocess_input, decode_predictions
+import numpy as np
+
+model = ResNet50(weights='imagenet')
+
+img_path = 'elephant.jpg'
+img = image.load_img(img_path, target_size=(224, 224))
+x = image.img_to_array(img)
+x = np.expand_dims(x, axis=0)
+x = preprocess_input(x)
+
+preds = model.predict(x)
+# decode the results into a list of tuples (class, description, probability)
+# (one such list for each sample in the batch)
+print('Predicted:', decode_predictions(preds, top=3)[0])
+# Predicted: [(u'n02504013', u'Indian_elephant', 0.82658225), (u'n01871265', u'tusker', 0.1122357), (u'n02504458', u'African_elephant', 0.061040461)]
+```
+
+### Extract features with VGG16
+
+```python
+from keras.applications.vgg16 import VGG16
+from keras.preprocessing import image
+from keras.applications.vgg16 import preprocess_input
+import numpy as np
+
+model = VGG16(weights='imagenet', include_top=False)
+
+img_path = 'elephant.jpg'
+img = image.load_img(img_path, target_size=(224, 224))
+x = image.img_to_array(img)
+x = np.expand_dims(x, axis=0)
+x = preprocess_input(x)
+
+features = model.predict(x)
+```
+
+### Extract features from an arbitrary intermediate layer with VGG19
+
+```python
+from keras.applications.vgg19 import VGG19
+from keras.preprocessing import image
+from keras.applications.vgg19 import preprocess_input
+from keras.models import Model
+import numpy as np
+
+base_model = VGG19(weights='imagenet')
+model = Model(inputs=base_model.input, outputs=base_model.get_layer('block4_pool').output)
+
+img_path = 'elephant.jpg'
+img = image.load_img(img_path, target_size=(224, 224))
+x = image.img_to_array(img)
+x = np.expand_dims(x, axis=0)
+x = preprocess_input(x)
+
+block4_pool_features = model.predict(x)
+```
+
+### Fine-tune InceptionV3 on a new set of classes
+
+```python
+from keras.applications.inception_v3 import InceptionV3
+from keras.preprocessing import image
+from keras.models import Model
+from keras.layers import Dense, GlobalAveragePooling2D
+from keras import backend as K
+
+# create the base pre-trained model
+base_model = InceptionV3(weights='imagenet', include_top=False)
+
+# add a global spatial average pooling layer
+x = base_model.output
+x = GlobalAveragePooling2D()(x)
+# let's add a fully-connected layer
+x = Dense(1024, activation='relu')(x)
+# and a logistic layer -- let's say we have 200 classes
+predictions = Dense(200, activation='softmax')(x)
+
+# this is the model we will train
+model = Model(inputs=base_model.input, outputs=predictions)
+
+# first: train only the top layers (which were randomly initialized)
+# i.e. freeze all convolutional InceptionV3 layers
+for layer in base_model.layers:
+    layer.trainable = False
+
+# compile the model (should be done *after* setting layers to non-trainable)
+model.compile(optimizer='rmsprop', loss='categorical_crossentropy')
+
+# train the model on the new data for a few epochs
+model.fit_generator(...)
+
+# at this point, the top layers are well trained and we can start fine-tuning
+# convolutional layers from inception V3. We will freeze the bottom N layers
+# and train the remaining top layers.
+
+# let's visualize layer names and layer indices to see how many layers
+# we should freeze:
+for i, layer in enumerate(base_model.layers):
+   print(i, layer.name)
+
+# we chose to train the top 2 inception blocks, i.e. we will freeze
+# the first 249 layers and unfreeze the rest:
+for layer in model.layers[:249]:
+   layer.trainable = False
+for layer in model.layers[249:]:
+   layer.trainable = True
+
+# we need to recompile the model for these modifications to take effect
+# we use SGD with a low learning rate
+from keras.optimizers import SGD
+model.compile(optimizer=SGD(lr=0.0001, momentum=0.9), loss='categorical_crossentropy')
+
+# we train our model again (this time fine-tuning the top 2 inception blocks
+# alongside the top Dense layers
+model.fit_generator(...)
+```
+
+
+### Build InceptionV3 over a custom input tensor
+
+```python
+from keras.applications.inception_v3 import InceptionV3
+from keras.layers import Input
+
+# this could also be the output a different Keras model or layer
+input_tensor = Input(shape=(224, 224, 3))  # this assumes K.image_data_format() == 'channels_last'
+
+model = InceptionV3(input_tensor=input_tensor, weights='imagenet', include_top=True)
+```
+
+-----
+
+# Documentation for individual models
+
+- [Xception](#xception)
+- [VGG16](#vgg16)
+- [VGG19](#vgg19)
+- [ResNet50](#resnet50)
+- [InceptionV3](#inceptionv3)
+
+-----
+
+
+## Xception
+
+
+```python
+keras.applications.xception.Xception(include_top=True, weights='imagenet', input_tensor=None, input_shape=None)
+```
+
+Xception V1 model, with weights pre-trained on ImageNet.
+
+On ImageNet, this model gets to a top-1 validation accuracy of 0.790
+and a top-5 validation accuracy of 0.945.
+
+Note that this model is only available for the TensorFlow backend,
+due to its reliance on `SeparableConvolution` layers. Additionally it only supports
+the data format "channels_last" (height, width, channels).
+
+The default input size for this model is 299x299.
+
+### Arguments
+
+- include_top: whether to include the fully-connected layer at the top of the network.
+- weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+- input_shape: optional shape tuple, only to be specified
+    if `include_top` is False (otherwise the input shape
+    has to be `(299, 299, 3)`.
+    It should have exactly 3 inputs channels,
+    and width and height should be no smaller than 71.
+    E.g. `(150, 150, 3)` would be one valid value.
+- pooling: Optional pooling mode for feature extraction
+    when `include_top` is `False`.
+    - `None` means that the output of the model will be
+        the 4D tensor output of the
+        last convolutional layer.
+    - `avg` means that global average pooling
+        will be applied to the output of the
+        last convolutional layer, and thus
+        the output of the model will be a 2D tensor.
+    - `max` means that global max pooling will
+        be applied.
+- classes: optional number of classes to classify images 
+    into, only to be specified if `include_top` is True, and 
+    if no `weights` argument is specified.
+
+### Returns
+
+A Keras model instance.
+
+### References
+
+- [Xception: Deep Learning with Depthwise Separable Convolutions](https://arxiv.org/abs/1610.02357)
+
+### License
+
+These weights are trained by ourselves and are released under the MIT license.
+
+
+-----
+
+
+## VGG16
+
+```python
+keras.applications.vgg16.VGG16(include_top=True, weights='imagenet', input_tensor=None, input_shape=None)
+```
+
+VGG16 model, with weights pre-trained on ImageNet.
+
+This model is available for both the Theano and TensorFlow backend, and can be built both
+with "channels_first" data format (channels, height, width) or "channels_last" data format (height, width, channels).
+
+The default input size for this model is 224x224.
+
+### Arguments
+
+- include_top: whether to include the 3 fully-connected layers at the top of the network.
+- weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+- input_shape: optional shape tuple, only to be specified
+    if `include_top` is False (otherwise the input shape
+    has to be `(224, 224, 3)` (with `channels_last` data format)
+    or `(3, 224, 224)` (with `channels_first` data format).
+    It should have exactly 3 inputs channels,
+    and width and height should be no smaller than 48.
+    E.g. `(200, 200, 3)` would be one valid value.
+- pooling: Optional pooling mode for feature extraction
+    when `include_top` is `False`.
+    - `None` means that the output of the model will be
+        the 4D tensor output of the
+        last convolutional layer.
+    - `avg` means that global average pooling
+        will be applied to the output of the
+        last convolutional layer, and thus
+        the output of the model will be a 2D tensor.
+    - `max` means that global max pooling will
+        be applied.
+- classes: optional number of classes to classify images 
+    into, only to be specified if `include_top` is True, and 
+    if no `weights` argument is specified.
+    
+### Returns
+
+A Keras model instance.
+
+### References
+
+- [Very Deep Convolutional Networks for Large-Scale Image Recognition](https://arxiv.org/abs/1409.1556): please cite this paper if you use the VGG models in your work.
+
+### License
+
+These weights are ported from the ones [released by VGG at Oxford](http://www.robots.ox.ac.uk/~vgg/research/very_deep/) under the [Creative Commons Attribution License](https://creativecommons.org/licenses/by/4.0/).
+
+-----
+
+## VGG19
+
+
+```python
+keras.applications.vgg19.VGG19(include_top=True, weights='imagenet', input_tensor=None, input_shape=None)
+```
+
+
+VGG19 model, with weights pre-trained on ImageNet.
+
+This model is available for both the Theano and TensorFlow backend, and can be built both
+with "channels_first" data format (channels, height, width) or "channels_last" data format (height, width, channels).
+
+The default input size for this model is 224x224.
+
+### Arguments
+
+- include_top: whether to include the 3 fully-connected layers at the top of the network.
+- weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+- input_shape: optional shape tuple, only to be specified
+    if `include_top` is False (otherwise the input shape
+    has to be `(224, 224, 3)` (with `channels_last` data format)
+    or `(3, 224, 224)` (with `channels_first` data format).
+    It should have exactly 3 inputs channels,
+    and width and height should be no smaller than 48.
+    E.g. `(200, 200, 3)` would be one valid value.
+- pooling: Optional pooling mode for feature extraction
+    when `include_top` is `False`.
+    - `None` means that the output of the model will be
+        the 4D tensor output of the
+        last convolutional layer.
+    - `avg` means that global average pooling
+        will be applied to the output of the
+        last convolutional layer, and thus
+        the output of the model will be a 2D tensor.
+    - `max` means that global max pooling will
+        be applied.
+- classes: optional number of classes to classify images 
+    into, only to be specified if `include_top` is True, and 
+    if no `weights` argument is specified.
+    
+### Returns
+
+A Keras model instance.
+
+
+### References
+
+- [Very Deep Convolutional Networks for Large-Scale Image Recognition](https://arxiv.org/abs/1409.1556)
+
+### License
+
+These weights are ported from the ones [released by VGG at Oxford](http://www.robots.ox.ac.uk/~vgg/research/very_deep/) under the [Creative Commons Attribution License](https://creativecommons.org/licenses/by/4.0/).
+
+-----
+
+## ResNet50
+
+
+```python
+keras.applications.resnet50.ResNet50(include_top=True, weights='imagenet', input_tensor=None, input_shape=None)
+```
+
+
+ResNet50 model, with weights pre-trained on ImageNet.
+
+This model is available for both the Theano and TensorFlow backend, and can be built both
+with "channels_first" data format (channels, height, width) or "channels_last" data format (height, width, channels).
+
+The default input size for this model is 224x224.
+
+
+### Arguments
+
+- include_top: whether to include the fully-connected layer at the top of the network.
+- weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+- input_shape: optional shape tuple, only to be specified
+    if `include_top` is False (otherwise the input shape
+    has to be `(224, 224, 3)` (with `channels_last` data format)
+    or `(3, 224, 224)` (with `channels_first` data format).
+    It should have exactly 3 inputs channels,
+    and width and height should be no smaller than 197.
+    E.g. `(200, 200, 3)` would be one valid value.
+- pooling: Optional pooling mode for feature extraction
+    when `include_top` is `False`.
+    - `None` means that the output of the model will be
+        the 4D tensor output of the
+        last convolutional layer.
+    - `avg` means that global average pooling
+        will be applied to the output of the
+        last convolutional layer, and thus
+        the output of the model will be a 2D tensor.
+    - `max` means that global max pooling will
+        be applied.
+- classes: optional number of classes to classify images 
+    into, only to be specified if `include_top` is True, and 
+    if no `weights` argument is specified.
+    
+### Returns
+
+A Keras model instance.
+
+### References
+
+- [Deep Residual Learning for Image Recognition](https://arxiv.org/abs/1512.03385)
+
+### License
+
+These weights are ported from the ones [released by Kaiming He](https://github.com/KaimingHe/deep-residual-networks) under the [MIT license](https://github.com/KaimingHe/deep-residual-networks/blob/master/LICENSE).
+
+-----
+
+## InceptionV3
+
+
+```python
+keras.applications.inception_v3.InceptionV3(include_top=True, weights='imagenet', input_tensor=None, input_shape=None)
+```
+
+Inception V3 model, with weights pre-trained on ImageNet.
+
+This model is available for both the Theano and TensorFlow backend, and can be built both
+with "channels_first" data format (channels, height, width) or "channels_last" data format (height, width, channels).
+
+The default input size for this model is 299x299.
+
+
+### Arguments
+
+- include_top: whether to include the fully-connected layer at the top of the network.
+- weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet).
+- input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model.
+- input_shape: optional shape tuple, only to be specified
+    if `include_top` is False (otherwise the input shape
+    has to be `(299, 299, 3)` (with `channels_last` data format)
+    or `(3, 299, 299)` (with `channels_first` data format).
+    It should have exactly 3 inputs channels,
+    and width and height should be no smaller than 139.
+    E.g. `(150, 150, 3)` would be one valid value.
+- pooling: Optional pooling mode for feature extraction
+    when `include_top` is `False`.
+    - `None` means that the output of the model will be
+        the 4D tensor output of the
+        last convolutional layer.
+    - `avg` means that global average pooling
+        will be applied to the output of the
+        last convolutional layer, and thus
+        the output of the model will be a 2D tensor.
+    - `max` means that global max pooling will
+        be applied.
+- classes: optional number of classes to classify images 
+    into, only to be specified if `include_top` is True, and 
+    if no `weights` argument is specified.
+    
+### Returns
+
+A Keras model instance.
+
+### References
+
+- [Rethinking the Inception Architecture for Computer Vision](http://arxiv.org/abs/1512.00567)
+
+### License
+
+These weights are released under [the Apache License](https://github.com/tensorflow/models/blob/master/LICENSE).

docs/templates/backend.md

@@ -0,0 +1,131 @@
+# Keras backends
+
+## What is a "backend"?
+
+Keras is a model-level library, providing high-level building blocks for developing deep learning models. It does not handle itself low-level operations such as tensor products, convolutions and so on. Instead, it relies on a specialized, well-optimized tensor manipulation library to do so, serving as the "backend engine" of Keras. Rather than picking one single tensor library and making the implementation of Keras tied to that library, Keras handles the problem in a modular way, and several different backend engines can be plugged seamlessly into Keras.
+
+At this time, Keras has three backend implementations available: the **TensorFlow** backend, the **Theano** backend, and the **CNTK** backend.
+
+- [TensorFlow](http://www.tensorflow.org/) is an open-source symbolic tensor manipulation framework developed by Google, Inc.
+- [Theano](http://deeplearning.net/software/theano/) is an open-source symbolic tensor manipulation framework developed by LISA/MILA Lab at Université de Montréal.
+- [CNTK](https://www.microsoft.com/en-us/cognitive-toolkit/) is an open-source, commercial-grade toolkit for deep learning developed by Microsoft.
+
+In the future, we are likely to add more backend options.
+
+----
+
+## Switching from one backend to another
+
+If you have run Keras at least once, you will find the Keras configuration file at:
+
+`$HOME/.keras/keras.json`
+
+If it isn't there, you can create it.
+
+**NOTE for Windows Users:** Please change `$HOME` with `%USERPROFILE%`.
+
+The default configuration file looks like this:
+
+```
+{
+    "image_data_format": "channels_last",
+    "epsilon": 1e-07,
+    "floatx": "float32",
+    "backend": "tensorflow"
+}
+```
+
+Simply change the field `backend` to `"theano"`, `"tensorflow"`, or `"cntk"`, and Keras will use the new configuration next time you run any Keras code.
+
+You can also define the environment variable ``KERAS_BACKEND`` and this will
+override what is defined in your config file :
+
+```bash
+KERAS_BACKEND=tensorflow python -c "from keras import backend"
+Using TensorFlow backend.
+```
+
+----
+
+## keras.json details
+
+
+```
+{
+    "image_data_format": "channels_last",
+    "epsilon": 1e-07,
+    "floatx": "float32",
+    "backend": "tensorflow"
+}
+```
+
+You can change these settings by editing `$HOME/.keras/keras.json`. 
+
+* `image_data_format`: string, either `"channels_last"` or `"channels_first"`. It specifies which data format convention Keras will follow. (`keras.backend.image_data_format()` returns it.)
+  - For 2D data (e.g. image), `"channels_last"` assumes `(rows, cols, channels)` while `"channels_first"` assumes `(channels, rows, cols)`. 
+  - For 3D data, `"channels_last"` assumes `(conv_dim1, conv_dim2, conv_dim3, channels)` while `"channels_first"` assumes `(channels, conv_dim1, conv_dim2, conv_dim3)`.
+* `epsilon`: float, a numeric fuzzing constant used to avoid dividing by zero in some operations.
+* `floatx`: string, `"float16"`, `"float32"`, or `"float64"`. Default float precision.
+* `backend`: string, `"tensorflow"`, `"theano"`, or `"cntk"`.
+
+----
+
+## Using the abstract Keras backend to write new code
+
+If you want the Keras modules you write to be compatible with both Theano (`th`) and TensorFlow (`tf`), you have to write them via the abstract Keras backend API. Here's an intro.
+
+You can import the backend module via:
+```python
+from keras import backend as K
+```
+
+The code below instantiates an input placeholder. It's equivalent to `tf.placeholder()` or `th.tensor.matrix()`, `th.tensor.tensor3()`, etc.
+
+```python
+input = K.placeholder(shape=(2, 4, 5))
+# also works:
+input = K.placeholder(shape=(None, 4, 5))
+# also works:
+input = K.placeholder(ndim=3)
+```
+
+The code below instantiates a shared variable. It's equivalent to `tf.Variable()` or `th.shared()`.
+
+```python
+import numpy as np
+val = np.random.random((3, 4, 5))
+var = K.variable(value=val)
+
+# all-zeros variable:
+var = K.zeros(shape=(3, 4, 5))
+# all-ones:
+var = K.ones(shape=(3, 4, 5))
+```
+
+Most tensor operations you will need can be done as you would in TensorFlow or Theano:
+
+```python
+# Initializing Tensors with Random Numbers
+b = K.random_uniform_variable(shape=(3, 4)). # Uniform distribution
+c = K.random_normal_variable(shape=(3, 4)). # Gaussian distribution
+d = K.random_normal_variable(shape=(3, 4)).
+# Tensor Arithmetics
+a = b + c * K.abs(d)
+c = K.dot(a, K.transpose(b))
+a = K.sum(b, axis=1)
+a = K.softmax(b)
+a = K.concatenate([b, c], axis=-1)
+# etc...
+```
+
+----
+
+## Backend functions
+
+
+{{autogenerated}}
+
+
+
+
+

docs/templates/callbacks.md

@@ -0,0 +1,70 @@
+## Usage of callbacks
+
+A callback is a set of functions to be applied at given stages of the training procedure. You can use callbacks to get a view on internal states and statistics of the model during training. You can pass a list of callbacks (as the keyword argument `callbacks`) to the `.fit()` method of the `Sequential` or `Model` classes. The relevant methods of the callbacks will then be called at each stage of the training. 
+
+---
+
+{{autogenerated}}
+
+---
+
+
+# Create a callback
+
+You can create a custom callback by extending the base class `keras.callbacks.Callback`. A callback has access to its associated model through the class property `self.model`.
+
+Here's a simple example saving a list of losses over each batch during training:
+```python
+class LossHistory(keras.callbacks.Callback):
+    def on_train_begin(self, logs={}):
+        self.losses = []
+
+    def on_batch_end(self, batch, logs={}):
+        self.losses.append(logs.get('loss'))
+```
+
+---
+
+### Example: recording loss history
+
+```python
+class LossHistory(keras.callbacks.Callback):
+    def on_train_begin(self, logs={}):
+        self.losses = []
+
+    def on_batch_end(self, batch, logs={}):
+        self.losses.append(logs.get('loss'))
+
+model = Sequential()
+model.add(Dense(10, input_dim=784, kernel_initializer='uniform'))
+model.add(Activation('softmax'))
+model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
+
+history = LossHistory()
+model.fit(x_train, y_train, batch_size=128, epochs=20, verbose=0, callbacks=[history])
+
+print(history.losses)
+# outputs
+'''
+[0.66047596406559383, 0.3547245744908703, ..., 0.25953155204159617, 0.25901699725311789]
+'''
+```
+
+---
+
+### Example: model checkpoints
+
+```python
+from keras.callbacks import ModelCheckpoint
+
+model = Sequential()
+model.add(Dense(10, input_dim=784, kernel_initializer='uniform'))
+model.add(Activation('softmax'))
+model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
+
+'''
+saves the model weights after each epoch if the validation loss decreased
+'''
+checkpointer = ModelCheckpoint(filepath='/tmp/weights.hdf5', verbose=1, save_best_only=True)
+model.fit(x_train, y_train, batch_size=128, epochs=20, verbose=0, validation_data=(X_test, Y_test), callbacks=[checkpointer])
+```

docs/templates/constraints.md

@@ -0,0 +1,23 @@
+## Usage of constraints
+
+Functions from the `constraints` module allow setting constraints (eg. non-negativity) on network parameters during optimization.
+
+The penalties are applied on a per-layer basis. The exact API will depend on the layer, but the layers `Dense`, `Conv1D`, `Conv2D` and `Conv3D` have a unified API.
+
+These layers expose 2 keyword arguments:
+
+- `kernel_constraint` for the main weights matrix
+- `bias_constraint` for the bias.
+
+
+```python
+from keras.constraints import maxnorm
+model.add(Dense(64, kernel_constraint=max_norm(2.)))
+```
+
+## Available constraints
+
+- __max_norm(max_value=2, axis=0)__: maximum-norm constraint
+- __non_neg()__: non-negativity constraint
+- __unit_norm(axis=0)__: unit-norm constraint
+- __min_max_norm(min_value=0.0, max_value=1.0, rate=1.0, axis=0)__:  minimum/maximum-norm constraint

docs/templates/datasets.md

@@ -0,0 +1,176 @@
+# Datasets
+
+## CIFAR10 small image classification
+
+Dataset of 50,000 32x32 color training images, labeled over 10 categories, and 10,000 test images.
+
+### Usage:
+
+```python
+from keras.datasets import cifar10
+
+(x_train, y_train), (x_test, y_test) = cifar10.load_data()
+```
+
+- __Returns:__
+    - 2 tuples:
+        - __x_train, x_test__: uint8 array of RGB image data with shape (num_samples, 3, 32, 32).
+        - __y_train, y_test__: uint8 array of category labels (integers in range 0-9) with shape (num_samples,).
+
+
+---
+
+## CIFAR100 small image classification
+
+Dataset of 50,000 32x32 color training images, labeled over 100 categories, and 10,000 test images.
+
+### Usage:
+
+```python
+from keras.datasets import cifar100
+
+(x_train, y_train), (x_test, y_test) = cifar100.load_data(label_mode='fine')
+```
+
+- __Returns:__
+    - 2 tuples:
+        - __x_train, x_test__: uint8 array of RGB image data with shape (num_samples, 3, 32, 32).
+        - __y_train, y_test__: uint8 array of category labels with shape (num_samples,).
+
+- __Arguments:__
+
+    - __label_mode__: "fine" or "coarse".
+
+
+---
+
+## IMDB Movie reviews sentiment classification
+
+Dataset of 25,000 movies reviews from IMDB, labeled by sentiment (positive/negative). Reviews have been preprocessed, and each review is encoded as a [sequence](preprocessing/sequence.md) of word indexes (integers). For convenience, words are indexed by overall frequency in the dataset, so that for instance the integer "3" encodes the 3rd most frequent word in the data. This allows for quick filtering operations such as: "only consider the top 10,000 most common words, but eliminate the top 20 most common words".
+
+As a convention, "0" does not stand for a specific word, but instead is used to encode any unknown word.
+
+### Usage:
+
+```python
+from keras.datasets import imdb
+
+(x_train, y_train), (x_test, y_test) = imdb.load_data(path="imdb.npz",
+                                                      num_words=None,
+                                                      skip_top=0,
+                                                      maxlen=None,
+                                                      seed=113,
+                                                      start_char=1,
+                                                      oov_char=2,
+                                                      index_from=3)
+```
+- __Returns:__
+    - 2 tuples:
+        - __x_train, x_test__: list of sequences, which are lists of indexes (integers). If the num_words argument was specific, the maximum possible index value is num_words-1. If the maxlen argument was specified, the largest possible sequence length is maxlen.
+        - __y_train, y_test__: list of integer labels (1 or 0). 
+
+- __Arguments:__
+
+    - __path__: if you do not have the data locally (at `'~/.keras/datasets/' + path`), it will be downloaded to this location.
+    - __num_words__: integer or None. Top most frequent words to consider. Any less frequent word will appear as `oov_char` value in the sequence data.
+    - __skip_top__: integer. Top most frequent words to ignore (they will appear as `oov_char` value in the sequence data).
+    - __maxlen__: int. Maximum sequence length. Any longer sequence will be truncated.
+    - __seed__: int. Seed for reproducible data shuffling.
+    - __start_char__: int. The start of a sequence will be marked with this character.
+        Set to 1 because 0 is usually the padding character.
+    - __oov_char__: int. words that were cut out because of the `num_words`
+        or `skip_top` limit will be replaced with this character.
+    - __index_from__: int. Index actual words with this index and higher.
+
+
+---
+
+## Reuters newswire topics classification
+
+Dataset of 11,228 newswires from Reuters, labeled over 46 topics. As with the IMDB dataset, each wire is encoded as a sequence of word indexes (same conventions).
+
+### Usage:
+
+```python
+from keras.datasets import reuters
+
+(x_train, y_train), (x_test, y_test) = reuters.load_data(path="reuters.npz",
+                                                         num_words=None,
+                                                         skip_top=0,
+                                                         maxlen=None,
+                                                         test_split=0.2,
+                                                         seed=113,
+                                                         start_char=1,
+                                                         oov_char=2,
+                                                         index_from=3)
+```
+
+The specifications are the same as that of the IMDB dataset, with the addition of:
+
+- __test_split__: float. Fraction of the dataset to be used as test data.
+
+This dataset also makes available the word index used for encoding the sequences:
+
+```python
+word_index = reuters.get_word_index(path="reuters_word_index.json")
+```
+
+- __Returns:__ A dictionary where key are words (str) and values are indexes (integer). eg. `word_index["giraffe"]` might return `1234`. 
+
+- __Arguments:__
+
+    - __path__: if you do not have the index file locally (at `'~/.keras/datasets/' + path`), it will be downloaded to this location.
+    
+
+---
+
+## MNIST database of handwritten digits
+
+Dataset of 60,000 28x28 grayscale images of the 10 digits, along with a test set of 10,000 images.
+
+### Usage:
+
+```python
+from keras.datasets import mnist
+
+(x_train, y_train), (x_test, y_test) = mnist.load_data()
+```
+
+- __Returns:__
+    - 2 tuples:
+        - __x_train, x_test__: uint8 array of grayscale image data with shape (num_samples, 28, 28).
+        - __y_train, y_test__: uint8 array of digit labels (integers in range 0-9) with shape (num_samples,).
+
+- __Arguments:__
+
+    - __path__: if you do not have the index file locally (at `'~/.keras/datasets/' + path`), it will be downloaded to this location.
+
+
+---
+
+## Boston housing price regression dataset
+
+
+Dataset taken from the StatLib library which is maintained at Carnegie Mellon University. 
+
+Samples contain 13 attributes of houses at different locations around the Boston suburbs in the late 1970s.
+Targets are the median values of the houses at a location (in k$).
+
+
+### Usage:
+
+```python
+from keras.datasets import boston_housing
+
+(x_train, y_train), (x_test, y_test) = boston_housing.load_data()
+```
+
+- __Arguments:__
+    - __path__: path where to cache the dataset locally
+        (relative to ~/.keras/datasets).
+    - __seed__: Random seed for shuffling the data
+        before computing the test split.
+    - __test_split__: fraction of the data to reserve as test set.
+
+- __Returns:__
+    Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.

docs/templates/getting-started/faq.md

@@ -0,0 +1,456 @@
+# Keras FAQ: Frequently Asked Keras Questions
+
+- [How should I cite Keras?](#how-should-i-cite-keras)
+- [How can I run Keras on GPU?](#how-can-i-run-keras-on-gpu)
+- [What does "sample", "batch", "epoch" mean?](#what-does-sample-batch-epoch-mean)
+- [How can I save a Keras model?](#how-can-i-save-a-keras-model)
+- [Why is the training loss much higher than the testing loss?](#why-is-the-training-loss-much-higher-than-the-testing-loss)
+- [How can I obtain the output of an intermediate layer?](#how-can-i-obtain-the-output-of-an-intermediate-layer)
+- [How can I use Keras with datasets that don't fit in memory?](#how-can-i-use-keras-with-datasets-that-dont-fit-in-memory)
+- [How can I interrupt training when the validation loss isn't decreasing anymore?](#how-can-i-interrupt-training-when-the-validation-loss-isnt-decreasing-anymore)
+- [How is the validation split computed?](#how-is-the-validation-split-computed)
+- [Is the data shuffled during training?](#is-the-data-shuffled-during-training)
+- [How can I record the training / validation loss / accuracy at each epoch?](#how-can-i-record-the-training-validation-loss-accuracy-at-each-epoch)
+- [How can I "freeze" layers?](#how-can-i-freeze-keras-layers)
+- [How can I use stateful RNNs?](#how-can-i-use-stateful-rnns)
+- [How can I remove a layer from a Sequential model?](#how-can-i-remove-a-layer-from-a-sequential-model)
+- [How can I use pre-trained models in Keras?](#how-can-i-use-pre-trained-models-in-keras)
+- [How can I use HDF5 inputs with Keras?](#how-can-i-use-hdf5-inputs-with-keras)
+- [Where is the Keras configuration file stored?](#where-is-the-keras-configuration-file-stored)
+
+---
+
+### How should I cite Keras?
+
+Please cite Keras in your publications if it helps your research. Here is an example BibTeX entry:
+
+```
+@misc{chollet2015keras,
+  title={Keras},
+  author={Chollet, Fran\c{c}ois and others},
+  year={2015},
+  publisher={GitHub},
+  howpublished={\url{https://github.com/fchollet/keras}},
+}
+```
+
+---
+
+### How can I run Keras on GPU?
+
+If you are running on the TensorFlow or CNTK backends, your code will automatically run on GPU if any available GPU is detected.
+
+If you are running on the Theano backend, you can use one of the following methods:
+
+Method 1: use Theano flags.
+```bash
+THEANO_FLAGS=device=gpu,floatX=float32 python my_keras_script.py
+```
+
+The name 'gpu' might have to be changed depending on your device's identifier (e.g. `gpu0`, `gpu1`, etc).
+
+Method 2: set up your `.theanorc`: [Instructions](http://deeplearning.net/software/theano/library/config.html)
+
+Method 3: manually set `theano.config.device`, `theano.config.floatX` at the beginning of your code:
+```python
+import theano
+theano.config.device = 'gpu'
+theano.config.floatX = 'float32'
+```
+
+---
+
+### What does "sample", "batch", "epoch" mean?
+
+Below are some common definitions that are necessary to know and understand to correctly utilize Keras:
+
+- **Sample**: one element of a dataset.
+  - *Example:* one image is a **sample** in a convolutional network
+  - *Example:* one audio file is a **sample** for a speech recognition model
+- **Batch**: a set of *N* samples. The samples in a **batch** are processed independently, in parallel. If training, a batch results in only one update to the model.
+  - A **batch** generally approximates the distribution of the input data better than a single input. The larger the batch, the better the approximation; however, it is also true that the batch will take longer to processes and will still result in only one update. For inference (evaluate/predict), it is recommended to pick a batch size that is as large as you can afford without going out of memory (since larger batches will usually result in faster evaluating/prediction).
+- **Epoch**: an arbitrary cutoff, generally defined as "one pass over the entire dataset", used to separate training into distinct phases, which is useful for logging and periodic evaluation.
+  - When using `evaluation_data` or `evaluation_split` with the `fit` method of Keras models, evaluation will be run at the end of every **epoch**.
+  - Within Keras, there is the ability to add [callbacks](https://keras.io/callbacks/) specifically designed to be run at the end of an **epoch**. Examples of these are learning rate changes and model checkpointing (saving).
+
+---
+
+### How can I save a Keras model?
+
+*It is not recommended to use pickle or cPickle to save a Keras model.*
+
+You can use `model.save(filepath)` to save a Keras model into a single HDF5 file which will contain:
+
+- the architecture of the model, allowing to re-create the model
+- the weights of the model
+- the training configuration (loss, optimizer)
+- the state of the optimizer, allowing to resume training exactly where you left off.
+
+You can then use `keras.models.load_model(filepath)` to reinstantiate your model.
+`load_model` will also take care of compiling the model using the saved training configuration
+(unless the model was never compiled in the first place).
+
+Example:
+
+```python
+from keras.models import load_model
+
+model.save('my_model.h5')  # creates a HDF5 file 'my_model.h5'
+del model  # deletes the existing model
+
+# returns a compiled model
+# identical to the previous one
+model = load_model('my_model.h5')
+```
+
+If you only need to save the **architecture of a model**, and not its weights or its training configuration, you can do:
+
+```python
+# save as JSON
+json_string = model.to_json()
+
+# save as YAML
+yaml_string = model.to_yaml()
+```
+
+The generated JSON / YAML files are human-readable and can be manually edited if needed.
+
+You can then build a fresh model from this data:
+
+```python
+# model reconstruction from JSON:
+from keras.models import model_from_json
+model = model_from_json(json_string)
+
+# model reconstruction from YAML
+from keras.models import model_from_yaml
+model = model_from_yaml(yaml_string)
+```
+
+If you need to save the **weights of a model**, you can do so in HDF5 with the code below.
+
+Note that you will first need to install HDF5 and the Python library h5py, which do not come bundled with Keras.
+
+```python
+model.save_weights('my_model_weights.h5')
+```
+
+Assuming you have code for instantiating your model, you can then load the weights you saved into a model with the *same* architecture:
+
+```python
+model.load_weights('my_model_weights.h5')
+```
+
+If you need to load weights into a *different* architecture (with some layers in common), for instance for fine-tuning or transfer-learning, you can load weights by *layer name*:
+
+```python
+model.load_weights('my_model_weights.h5', by_name=True)
+```
+
+For example:
+
+```python
+"""
+Assume original model looks like this:
+    model = Sequential()
+    model.add(Dense(2, input_dim=3, name='dense_1'))
+    model.add(Dense(3, name='dense_2'))
+    ...
+    model.save_weights(fname)
+"""
+
+# new model
+model = Sequential()
+model.add(Dense(2, input_dim=3, name='dense_1'))  # will be loaded
+model.add(Dense(10, name='new_dense'))  # will not be loaded
+
+# load weights from first model; will only affect the first layer, dense_1.
+model.load_weights(fname, by_name=True)
+```
+
+---
+
+### Why is the training loss much higher than the testing loss?
+
+A Keras model has two modes: training and testing. Regularization mechanisms, such as Dropout and L1/L2 weight regularization, are turned off at testing time.
+
+Besides, the training loss is the average of the losses over each batch of training data. Because your model is changing over time, the loss over the first batches of an epoch is generally higher than over the last batches. On the other hand, the testing loss for an epoch is computed using the model as it is at the end of the epoch, resulting in a lower loss.
+
+---
+
+### How can I obtain the output of an intermediate layer?
+
+One simple way is to create a new `Model` that will output the layers that you are interested in:
+
+```python
+from keras.models import Model
+
+model = ...  # create the original model
+
+layer_name = 'my_layer'
+intermediate_layer_model = Model(inputs=model.input,
+                                 outputs=model.get_layer(layer_name).output)
+intermediate_output = intermediate_layer_model.predict(data)
+```
+
+Alternatively, you can build a Keras function that will return the output of a certain layer given a certain input, for example:
+
+```python
+from keras import backend as K
+
+# with a Sequential model
+get_3rd_layer_output = K.function([model.layers[0].input],
+                                  [model.layers[3].output])
+layer_output = get_3rd_layer_output([x])[0]
+```
+
+Similarly, you could build a Theano and TensorFlow function directly.
+
+Note that if your model has a different behavior in training and testing phase (e.g. if it uses `Dropout`, `BatchNormalization`, etc.), you will need
+to pass the learning phase flag to your function:
+
+```python
+get_3rd_layer_output = K.function([model.layers[0].input, K.learning_phase()],
+                                  [model.layers[3].output])
+
+# output in test mode = 0
+layer_output = get_3rd_layer_output([x, 0])[0]
+
+# output in train mode = 1
+layer_output = get_3rd_layer_output([x, 1])[0]
+```
+
+---
+
+### How can I use Keras with datasets that don't fit in memory?
+
+You can do batch training using `model.train_on_batch(x, y)` and `model.test_on_batch(x, y)`. See the [models documentation](/models/sequential).
+
+Alternatively, you can write a generator that yields batches of training data and use the method `model.fit_generator(data_generator, steps_per_epoch, epochs)`.
+
+You can see batch training in action in our [CIFAR10 example](https://github.com/fchollet/keras/blob/master/examples/cifar10_cnn.py).
+
+---
+
+### How can I interrupt training when the validation loss isn't decreasing anymore?
+
+You can use an `EarlyStopping` callback:
+
+```python
+from keras.callbacks import EarlyStopping
+early_stopping = EarlyStopping(monitor='val_loss', patience=2)
+model.fit(x, y, validation_split=0.2, callbacks=[early_stopping])
+```
+
+Find out more in the [callbacks documentation](/callbacks).
+
+---
+
+### How is the validation split computed?
+
+If you set the `validation_split` argument in `model.fit` to e.g. 0.1, then the validation data used will be the *last 10%* of the data. If you set it to 0.25, it will be the last 25% of the data, etc. Note that the data isn't shuffled before extracting the validation split, so the validation is literally just the *last* x% of samples in the input you passed.
+
+The same validation set is used for all epochs (within a same call to `fit`).
+
+---
+
+### Is the data shuffled during training?
+
+Yes, if the `shuffle` argument in `model.fit` is set to `True` (which is the default), the training data will be randomly shuffled at each epoch.
+
+Validation data is never shuffled.
+
+---
+
+
+### How can I record the training / validation loss / accuracy at each epoch?
+
+The `model.fit` method returns an `History` callback, which has a `history` attribute containing the lists of successive losses and other metrics.
+
+```python
+hist = model.fit(x, y, validation_split=0.2)
+print(hist.history)
+```
+
+---
+
+### How can I "freeze" Keras layers?
+
+To "freeze" a layer means to exclude it from training, i.e. its weights will never be updated. This is useful in the context of fine-tuning a model, or using fixed embeddings for a text input.
+
+You can pass a `trainable` argument (boolean) to a layer constructor to set a layer to be non-trainable:
+
+```python
+frozen_layer = Dense(32, trainable=False)
+```
+
+Additionally, you can set the `trainable` property of a layer to `True` or `False` after instantiation. For this to take effect, you will need to call `compile()` on your model after modifying the `trainable` property. Here's an example:
+
+```python
+x = Input(shape=(32,))
+layer = Dense(32)
+layer.trainable = False
+y = layer(x)
+
+frozen_model = Model(x, y)
+# in the model below, the weights of `layer` will not be updated during training
+frozen_model.compile(optimizer='rmsprop', loss='mse')
+
+layer.trainable = True
+trainable_model = Model(x, y)
+# with this model the weights of the layer will be updated during training
+# (which will also affect the above model since it uses the same layer instance)
+trainable_model.compile(optimizer='rmsprop', loss='mse')
+
+frozen_model.fit(data, labels)  # this does NOT update the weights of `layer`
+trainable_model.fit(data, labels)  # this updates the weights of `layer`
+```
+
+---
+
+### How can I use stateful RNNs?
+
+Making a RNN stateful means that the states for the samples of each batch will be reused as initial states for the samples in the next batch.
+
+When using stateful RNNs, it is therefore assumed that:
+
+- all batches have the same number of samples
+- If `x1` and `x2` are successive batches of samples, then `x2[i]` is the follow-up sequence to `x1[i]`, for every `i`.
+
+To use statefulness in RNNs, you need to:
+
+- explicitly specify the batch size you are using, by passing a `batch_size` argument to the first layer in your model. E.g. `batch_size=32` for a 32-samples batch of sequences of 10 timesteps with 16 features per timestep.
+- set `stateful=True` in your RNN layer(s).
+- specify `shuffle=False` when calling fit().
+
+To reset the states accumulated:
+
+- use `model.reset_states()` to reset the states of all layers in the model
+- use `layer.reset_states()` to reset the states of a specific stateful RNN layer
+
+Example:
+
+```python
+
+x  # this is our input data, of shape (32, 21, 16)
+# we will feed it to our model in sequences of length 10
+
+model = Sequential()
+model.add(LSTM(32, input_shape=(10, 16), batch_size=32, stateful=True))
+model.add(Dense(16, activation='softmax'))
+
+model.compile(optimizer='rmsprop', loss='categorical_crossentropy')
+
+# we train the network to predict the 11th timestep given the first 10:
+model.train_on_batch(x[:, :10, :], np.reshape(x[:, 10, :], (32, 16)))
+
+# the state of the network has changed. We can feed the follow-up sequences:
+model.train_on_batch(x[:, 10:20, :], np.reshape(x[:, 20, :], (32, 16)))
+
+# let's reset the states of the LSTM layer:
+model.reset_states()
+
+# another way to do it in this case:
+model.layers[0].reset_states()
+```
+
+Notes that the methods `predict`, `fit`, `train_on_batch`, `predict_classes`, etc. will *all* update the states of the stateful layers in a model. This allows you to do not only stateful training, but also stateful prediction.
+
+---
+
+### How can I remove a layer from a Sequential model?
+
+You can remove the last added layer in a Sequential model by calling `.pop()`:
+
+```python
+model = Sequential()
+model.add(Dense(32, activation='relu', input_dim=784))
+model.add(Dense(32, activation='relu'))
+
+print(len(model.layers))  # "2"
+
+model.pop()
+print(len(model.layers))  # "1"
+```
+
+---
+
+### How can I use pre-trained models in Keras?
+
+Code and pre-trained weights are available for the following image classification models:
+
+- Xception
+- VGG16
+- VGG19
+- ResNet50
+- Inception v3
+
+They can be imported from the module `keras.applications`:
+
+```python
+from keras.applications.xception import Xception
+from keras.applications.vgg16 import VGG16
+from keras.applications.vgg19 import VGG19
+from keras.applications.resnet50 import ResNet50
+from keras.applications.inception_v3 import InceptionV3
+
+model = VGG16(weights='imagenet', include_top=True)
+```
+
+For a few simple usage examples, see [the documentation for the Applications module](/applications).
+
+For a detailed example of how to use such a pre-trained model for feature extraction or for fine-tuning, see [this blog post](http://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html).
+
+The VGG16 model is also the basis for several Keras example scripts:
+
+- [Style transfer](https://github.com/fchollet/keras/blob/master/examples/neural_style_transfer.py)
+- [Feature visualization](https://github.com/fchollet/keras/blob/master/examples/conv_filter_visualization.py)
+- [Deep dream](https://github.com/fchollet/keras/blob/master/examples/deep_dream.py)
+
+---
+
+### How can I use HDF5 inputs with Keras?
+
+You can use the `HDF5Matrix` class from `keras.utils.io_utils`. See [the HDF5Matrix documentation](/utils/#hdf5matrix) for details.
+
+You can also directly use a HDF5 dataset:
+
+```python
+import h5py
+with h5py.File('input/file.hdf5', 'r') as f:
+    x_data = f['x_data']
+    model.predict(x_data)
+```
+
+---
+
+### Where is the Keras configuration file stored?
+
+The default directory where all Keras data is stored is:
+
+```bash
+$HOME/.keras/
+```
+
+Note that Windows users should replace `$HOME` with `%USERPROFILE%`.
+In case Keras cannot create the above directory (e.g. due to permission issues), `/tmp/.keras/` is used as a backup.
+
+The Keras configuration file is a JSON file stored at `$HOME/.keras/keras.json`. The default configuration file looks like this:
+
+```
+{
+    "image_data_format": "channels_last",
+    "epsilon": 1e-07,
+    "floatx": "float32",
+    "backend": "tensorflow"
+}
+```
+
+It contains the following fields:
+
+- The image data format to be used as default by image processing layers and utilities (either `channels_last` or `channels_first`).
+- The `epsilon` numerical fuzz factor to be used to prevent division by zero in some operations.
+- The default float data type.
+- The default backend. See the [backend documentation](/backend).
+
+Likewise, cached dataset files, such as those downloaded with [`get_file()`](/utils/#get_file), are stored by default in `$HOME/.keras/datasets/`.

docs/templates/getting-started/functional-api-guide.md

@@ -0,0 +1,422 @@
+# Getting started with the Keras functional API
+
+The Keras functional API is the way to go for defining complex models, such as multi-output models, directed acyclic graphs, or models with shared layers.
+
+This guide assumes that you are already familiar with the `Sequential` model.
+
+Let's start with something simple.
+
+-----
+
+## First example: a densely-connected network
+
+The `Sequential` model is probably a better choice to implement such a network, but it helps to start with something really simple.
+
+- A layer instance is callable (on a tensor), and it returns a tensor
+- Input tensor(s) and output tensor(s) can then be used to define a `Model`
+- Such a model can be trained just like Keras `Sequential` models.
+
+```python
+from keras.layers import Input, Dense
+from keras.models import Model
+
+# This returns a tensor
+inputs = Input(shape=(784,))
+
+# a layer instance is callable on a tensor, and returns a tensor
+x = Dense(64, activation='relu')(inputs)
+x = Dense(64, activation='relu')(x)
+predictions = Dense(10, activation='softmax')(x)
+
+# This creates a model that includes
+# the Input layer and three Dense layers
+model = Model(inputs=inputs, outputs=predictions)
+model.compile(optimizer='rmsprop',
+              loss='categorical_crossentropy',
+              metrics=['accuracy'])
+model.fit(data, labels)  # starts training
+```
+
+-----
+
+## All models are callable, just like layers
+
+With the functional API, it is easy to re-use trained models: you can treat any model as if it were a layer, by calling it on a tensor. Note that by calling a model you aren't just re-using the *architecture* of the model, you are also re-using its weights.
+
+```python
+x = Input(shape=(784,))
+# This works, and returns the 10-way softmax we defined above.
+y = model(x)
+```
+
+This can allow, for instance, to quickly create models that can process *sequences* of inputs. You could turn an image classification model into a video classification model, in just one line.
+
+```python
+from keras.layers import TimeDistributed
+
+# Input tensor for sequences of 20 timesteps,
+# each containing a 784-dimensional vector
+input_sequences = Input(shape=(20, 784))
+
+# This applies our previous model to every timestep in the input sequences.
+# the output of the previous model was a 10-way softmax,
+# so the output of the layer below will be a sequence of 20 vectors of size 10.
+processed_sequences = TimeDistributed(model)(input_sequences)
+```
+
+-----
+
+## Multi-input and multi-output models
+
+Here's a good use case for the functional API: models with multiple inputs and outputs. The functional API makes it easy to manipulate a large number of intertwined datastreams.
+
+Let's consider the following model. We seek to predict how many retweets and likes a news headline will receive on Twitter. The main input to the model will be the headline itself, as a sequence of words, but to spice things up, our model will also have an auxiliary input, receiving extra data such as the time of day when the headline was posted, etc.
+The model will also be supervised via two loss functions. Using the main loss function earlier in a model is a good regularization mechanism for deep models.
+
+Here's what our model looks like:
+
+<img src="https://s3.amazonaws.com/keras.io/img/multi-input-multi-output-graph.png" alt="multi-input-multi-output-graph" style="width: 400px;"/>
+
+Let's implement it with the functional API.
+
+The main input will receive the headline, as a sequence of integers (each integer encodes a word).
+The integers will be between 1 and 10,000 (a vocabulary of 10,000 words) and the sequences will be 100 words long.
+
+```python
+from keras.layers import Input, Embedding, LSTM, Dense
+from keras.models import Model
+
+# Headline input: meant to receive sequences of 100 integers, between 1 and 10000.
+# Note that we can name any layer by passing it a "name" argument.
+main_input = Input(shape=(100,), dtype='int32', name='main_input')
+
+# This embedding layer will encode the input sequence
+# into a sequence of dense 512-dimensional vectors.
+x = Embedding(output_dim=512, input_dim=10000, input_length=100)(main_input)
+
+# A LSTM will transform the vector sequence into a single vector,
+# containing information about the entire sequence
+lstm_out = LSTM(32)(x)
+```
+
+Here we insert the auxiliary loss, allowing the LSTM and Embedding layer to be trained smoothly even though the main loss will be much higher in the model.
+
+```python
+auxiliary_output = Dense(1, activation='sigmoid', name='aux_output')(lstm_out)
+```
+
+At this point, we feed into the model our auxiliary input data by concatenating it with the LSTM output:
+
+```python
+auxiliary_input = Input(shape=(5,), name='aux_input')
+x = keras.layers.concatenate([lstm_out, auxiliary_input])
+
+# We stack a deep densely-connected network on top
+x = Dense(64, activation='relu')(x)
+x = Dense(64, activation='relu')(x)
+x = Dense(64, activation='relu')(x)
+
+# And finally we add the main logistic regression layer
+main_output = Dense(1, activation='sigmoid', name='main_output')(x)
+```
+
+This defines a model with two inputs and two outputs:
+
+```python
+model = Model(inputs=[main_input, auxiliary_input], outputs=[main_output, auxiliary_output])
+```
+
+We compile the model and assign a weight of 0.2 to the auxiliary loss.
+To specify different `loss_weights` or `loss` for each different output, you can use a list or a dictionary.
+Here we pass a single loss as the `loss` argument, so the same loss will be used on all outputs.
+
+```python
+model.compile(optimizer='rmsprop', loss='binary_crossentropy',
+              loss_weights=[1., 0.2])
+```
+
+We can train the model by passing it lists of input arrays and target arrays:
+
+```python
+model.fit([headline_data, additional_data], [labels, labels],
+          epochs=50, batch_size=32)
+```
+
+Since our inputs and outputs are named (we passed them a "name" argument),
+We could also have compiled the model via:
+
+```python
+model.compile(optimizer='rmsprop',
+              loss={'main_output': 'binary_crossentropy', 'aux_output': 'binary_crossentropy'},
+              loss_weights={'main_output': 1., 'aux_output': 0.2})
+
+# And trained it via:
+model.fit({'main_input': headline_data, 'aux_input': additional_data},
+          {'main_output': labels, 'aux_output': labels},
+          epochs=50, batch_size=32)
+```
+
+-----
+
+## Shared layers
+
+Another good use for the functional API are models that use shared layers. Let's take a look at shared layers.
+
+Let's consider a dataset of tweets. We want to build a model that can tell whether two tweets are from the same person or not (this can allow us to compare users by the similarity of their tweets, for instance).
+
+One way to achieve this is to build a model that encodes two tweets into two vectors, concatenates the vectors and adds a logistic regression of top, outputting a probability that the two tweets share the same author. The model would then be trained on positive tweet pairs and negative tweet pairs.
+
+Because the problem is symmetric, the mechanism that encodes the first tweet should be reused (weights and all) to encode the second tweet. Here we use a shared LSTM layer to encode the tweets.
+
+Let's build this with the functional API. We will take as input for a tweet a binary matrix of shape `(140, 256)`, i.e. a sequence of 140 vectors of size 256, where each dimension in the 256-dimensional vector encodes the presence/absence of a character (out of an alphabet of 256 frequent characters).
+
+```python
+import keras
+from keras.layers import Input, LSTM, Dense
+from keras.models import Model
+
+tweet_a = Input(shape=(140, 256))
+tweet_b = Input(shape=(140, 256))
+```
+
+To share a layer across different inputs, simply instantiate the layer once, then call it on as many inputs as you want:
+
+```python
+# This layer can take as input a matrix
+# and will return a vector of size 64
+shared_lstm = LSTM(64)
+
+# When we reuse the same layer instance
+# multiple times, the weights of the layer
+# are also being reused
+# (it is effectively *the same* layer)
+encoded_a = shared_lstm(tweet_a)
+encoded_b = shared_lstm(tweet_b)
+
+# We can then concatenate the two vectors:
+merged_vector = keras.layers.concatenate([encoded_a, encoded_b], axis=-1)
+
+# And add a logistic regression on top
+predictions = Dense(1, activation='sigmoid')(merged_vector)
+
+# We define a trainable model linking the
+# tweet inputs to the predictions
+model = Model(inputs=[tweet_a, tweet_b], outputs=predictions)
+
+model.compile(optimizer='rmsprop',
+              loss='binary_crossentropy',
+              metrics=['accuracy'])
+model.fit([data_a, data_b], labels, epochs=10)
+```
+
+Let's pause to take a look at how to read the shared layer's output or output shape.
+
+-----
+
+## The concept of layer "node"
+
+Whenever you are calling a layer on some input, you are creating a new tensor (the output of the layer), and you are adding a "node" to the layer, linking the input tensor to the output tensor. When you are calling the same layer multiple times, that layer owns multiple nodes indexed as 0, 1, 2...
+
+In previous versions of Keras, you could obtain the output tensor of a layer instance via `layer.get_output()`, or its output shape via `layer.output_shape`. You still can (except `get_output()` has been replaced by the property `output`). But what if a layer is connected to multiple inputs?
+
+As long as a layer is only connected to one input, there is no confusion, and `.output` will return the one output of the layer:
+
+```python
+a = Input(shape=(140, 256))
+
+lstm = LSTM(32)
+encoded_a = lstm(a)
+
+assert lstm.output == encoded_a
+```
+
+Not so if the layer has multiple inputs:
+```python
+a = Input(shape=(140, 256))
+b = Input(shape=(140, 256))
+
+lstm = LSTM(32)
+encoded_a = lstm(a)
+encoded_b = lstm(b)
+
+lstm.output
+```
+```
+>> AssertionError: Layer lstm_1 has multiple inbound nodes,
+hence the notion of "layer output" is ill-defined.
+Use `get_output_at(node_index)` instead.
+```
+
+Okay then. The following works:
+
+```python
+assert lstm.get_output_at(0) == encoded_a
+assert lstm.get_output_at(1) == encoded_b
+```
+
+Simple enough, right?
+
+The same is true for the properties `input_shape` and `output_shape`: as long as the layer has only one node, or as long as all nodes have the same input/output shape, then the notion of "layer output/input shape" is well defined, and that one shape will be returned by `layer.output_shape`/`layer.input_shape`. But if, for instance, you apply a same `Conv2D` layer to an input of shape `(3, 32, 32)`, and then to an input of shape `(3, 64, 64)`, the layer will have multiple input/output shapes, and you will have to fetch them by specifying the index of the node they belong to:
+
+```python
+a = Input(shape=(3, 32, 32))
+b = Input(shape=(3, 64, 64))
+
+conv = Conv2D(16, (3, 3), padding='same')
+conved_a = conv(a)
+
+# Only one input so far, the following will work:
+assert conv.input_shape == (None, 3, 32, 32)
+
+conved_b = conv(b)
+# now the `.input_shape` property wouldn't work, but this does:
+assert conv.get_input_shape_at(0) == (None, 3, 32, 32)
+assert conv.get_input_shape_at(1) == (None, 3, 64, 64)
+```
+
+-----
+
+## More examples
+
+Code examples are still the best way to get started, so here are a few more.
+
+### Inception module
+
+For more information about the Inception architecture, see [Going Deeper with Convolutions](http://arxiv.org/abs/1409.4842).
+
+```python
+from keras.layers import Conv2D, MaxPooling2D, Input
+
+input_img = Input(shape=(3, 256, 256))
+
+tower_1 = Conv2D(64, (1, 1), padding='same', activation='relu')(input_img)
+tower_1 = Conv2D(64, (3, 3), padding='same', activation='relu')(tower_1)
+
+tower_2 = Conv2D(64, (1, 1), padding='same', activation='relu')(input_img)
+tower_2 = Conv2D(64, (5, 5), padding='same', activation='relu')(tower_2)
+
+tower_3 = MaxPooling2D((3, 3), strides=(1, 1), padding='same')(input_img)
+tower_3 = Conv2D(64, (1, 1), padding='same', activation='relu')(tower_3)
+
+output = keras.layers.concatenate([tower_1, tower_2, tower_3], axis=1)
+```
+
+### Residual connection on a convolution layer
+
+For more information about residual networks, see [Deep Residual Learning for Image Recognition](http://arxiv.org/abs/1512.03385).
+
+```python
+from keras.layers import Conv2D, Input
+
+# input tensor for a 3-channel 256x256 image
+x = Input(shape=(3, 256, 256))
+# 3x3 conv with 3 output channels (same as input channels)
+y = Conv2D(3, (3, 3), padding='same')(x)
+# this returns x + y.
+z = keras.layers.add([x, y])
+```
+
+### Shared vision model
+
+This model re-uses the same image-processing module on two inputs, to classify whether two MNIST digits are the same digit or different digits.
+
+```python
+from keras.layers import Conv2D, MaxPooling2D, Input, Dense, Flatten
+from keras.models import Model
+
+# First, define the vision modules
+digit_input = Input(shape=(1, 27, 27))
+x = Conv2D(64, (3, 3))(digit_input)
+x = Conv2D(64, (3, 3))(x)
+x = MaxPooling2D((2, 2))(x)
+out = Flatten()(x)
+
+vision_model = Model(digit_input, out)
+
+# Then define the tell-digits-apart model
+digit_a = Input(shape=(1, 27, 27))
+digit_b = Input(shape=(1, 27, 27))
+
+# The vision model will be shared, weights and all
+out_a = vision_model(digit_a)
+out_b = vision_model(digit_b)
+
+concatenated = keras.layers.concatenate([out_a, out_b])
+out = Dense(1, activation='sigmoid')(concatenated)
+
+classification_model = Model([digit_a, digit_b], out)
+```
+
+### Visual question answering model
+
+This model can select the correct one-word answer when asked a natural-language question about a picture.
+
+It works by encoding the question into a vector, encoding the image into a vector, concatenating the two, and training on top a logistic regression over some vocabulary of potential answers.
+
+```python
+from keras.layers import Conv2D, MaxPooling2D, Flatten
+from keras.layers import Input, LSTM, Embedding, Dense
+from keras.models import Model, Sequential
+
+# First, let's define a vision model using a Sequential model.
+# This model will encode an image into a vector.
+vision_model = Sequential()
+vision_model.add(Conv2D(64, (3, 3), activation='relu', padding='same', input_shape=(3, 224, 224)))
+vision_model.add(Conv2D(64, (3, 3), activation='relu'))
+vision_model.add(MaxPooling2D((2, 2)))
+vision_model.add(Conv2D(128, (3, 3), activation='relu', padding='same'))
+vision_model.add(Conv2D(128, (3, 3), activation='relu'))
+vision_model.add(MaxPooling2D((2, 2)))
+vision_model.add(Conv2D(256, (3, 3), activation='relu', padding='same'))
+vision_model.add(Conv2D(256, (3, 3), activation='relu'))
+vision_model.add(Conv2D(256, (3, 3), activation='relu'))
+vision_model.add(MaxPooling2D((2, 2)))
+vision_model.add(Flatten())
+
+# Now let's get a tensor with the output of our vision model:
+image_input = Input(shape=(3, 224, 224))
+encoded_image = vision_model(image_input)
+
+# Next, let's define a language model to encode the question into a vector.
+# Each question will be at most 100 word long,
+# and we will index words as integers from 1 to 9999.
+question_input = Input(shape=(100,), dtype='int32')
+embedded_question = Embedding(input_dim=10000, output_dim=256, input_length=100)(question_input)
+encoded_question = LSTM(256)(embedded_question)
+
+# Let's concatenate the question vector and the image vector:
+merged = keras.layers.concatenate([encoded_question, encoded_image])
+
+# And let's train a logistic regression over 1000 words on top:
+output = Dense(1000, activation='softmax')(merged)
+
+# This is our final model:
+vqa_model = Model(inputs=[image_input, question_input], outputs=output)
+
+# The next stage would be training this model on actual data.
+```
+
+### Video question answering model
+
+Now that we have trained our image QA model, we can quickly turn it into a video QA model. With appropriate training, you will be able to show it a short video (e.g. 100-frame human action) and ask a natural language question about the video (e.g. "what sport is the boy playing?" -> "football").
+
+```python
+from keras.layers import TimeDistributed
+
+video_input = Input(shape=(100, 3, 224, 224))
+# This is our video encoded via the previously trained vision_model (weights are reused)
+encoded_frame_sequence = TimeDistributed(vision_model)(video_input)  # the output will be a sequence of vectors
+encoded_video = LSTM(256)(encoded_frame_sequence)  # the output will be a vector
+
+# This is a model-level representation of the question encoder, reusing the same weights as before:
+question_encoder = Model(inputs=question_input, outputs=encoded_question)
+
+# Let's use it to encode the question:
+video_question_input = Input(shape=(100,), dtype='int32')
+encoded_video_question = question_encoder(video_question_input)
+
+# And this is our video question answering model:
+merged = keras.layers.concatenate([encoded_video, encoded_video_question])
+output = Dense(1000, activation='softmax')(merged)
+video_qa_model = Model(inputs=[video_input, video_question_input], outputs=output)
+```

docs/templates/getting-started/sequential-model-guide.md

@@ -0,0 +1,394 @@
+# Getting started with the Keras Sequential model
+
+The `Sequential` model is a linear stack of layers.
+
+You can create a `Sequential` model by passing a list of layer instances to the constructor:
+
+```python
+from keras.models import Sequential
+from keras.layers import Dense, Activation
+
+model = Sequential([
+    Dense(32, input_shape=(784,)),
+    Activation('relu'),
+    Dense(10),
+    Activation('softmax'),
+])
+```
+
+You can also simply add layers via the `.add()` method:
+
+```python
+model = Sequential()
+model.add(Dense(32, input_dim=784))
+model.add(Activation('relu'))
+```
+
+----
+
+## Specifying the input shape
+
+The model needs to know what input shape it should expect. For this reason, the first layer in a `Sequential` model (and only the first, because following layers can do automatic shape inference) needs to receive information about its input shape. There are several possible ways to do this:
+
+- Pass an `input_shape` argument to the first layer. This is a shape tuple (a tuple of integers or `None` entries, where `None` indicates that any positive integer may be expected). In `input_shape`, the batch dimension is not included.
+- Some 2D layers, such as `Dense`, support the specification of their input shape via the argument `input_dim`, and some 3D temporal layers support the arguments `input_dim` and `input_length`.
+- If you ever need to specify a fixed batch size for your inputs (this is useful for stateful recurrent networks), you can pass a `batch_size` argument to a layer. If you pass both `batch_size=32` and `input_shape=(6, 8)` to a layer, it will then expect every batch of inputs to have the batch shape `(32, 6, 8)`.
+
+As such, the following snippets are strictly equivalent:
+```python
+model = Sequential()
+model.add(Dense(32, input_shape=(784,)))
+```
+```python
+model = Sequential()
+model.add(Dense(32, input_dim=784))
+```
+
+----
+
+## Compilation
+
+Before training a model, you need to configure the learning process, which is done via the `compile` method. It receives three arguments:
+
+- An optimizer. This could be the string identifier of an existing optimizer (such as `rmsprop` or `adagrad`), or an instance of the `Optimizer` class. See: [optimizers](/optimizers).
+- A loss function. This is the objective that the model will try to minimize. It can be the string identifier of an existing loss function (such as `categorical_crossentropy` or `mse`), or it can be an objective function. See: [losses](/losses).
+- A list of metrics. For any classification problem you will want to set this to `metrics=['accuracy']`. A metric could be the string identifier of an existing metric or a custom metric function.
+
+```python
+# For a multi-class classification problem
+model.compile(optimizer='rmsprop',
+              loss='categorical_crossentropy',
+              metrics=['accuracy'])
+
+# For a binary classification problem
+model.compile(optimizer='rmsprop',
+              loss='binary_crossentropy',
+              metrics=['accuracy'])
+
+# For a mean squared error regression problem
+model.compile(optimizer='rmsprop',
+              loss='mse')
+
+# For custom metrics
+import keras.backend as K
+
+def mean_pred(y_true, y_pred):
+    return K.mean(y_pred)
+
+model.compile(optimizer='rmsprop',
+              loss='binary_crossentropy',
+              metrics=['accuracy', mean_pred])
+```
+
+----
+
+## Training
+
+Keras models are trained on Numpy arrays of input data and labels. For training a model, you will typically use the `fit` function. [Read its documentation here](/models/sequential).
+
+```python
+# For a single-input model with 2 classes (binary classification):
+
+model = Sequential()
+model.add(Dense(32, activation='relu', input_dim=100))
+model.add(Dense(1, activation='sigmoid'))
+model.compile(optimizer='rmsprop',
+              loss='binary_crossentropy',
+              metrics=['accuracy'])
+
+# Generate dummy data
+import numpy as np
+data = np.random.random((1000, 100))
+labels = np.random.randint(2, size=(1000, 1))
+
+# Train the model, iterating on the data in batches of 32 samples
+model.fit(data, labels, epochs=10, batch_size=32)
+```
+
+```python
+# For a single-input model with 10 classes (categorical classification):
+
+model = Sequential()
+model.add(Dense(32, activation='relu', input_dim=100))
+model.add(Dense(10, activation='softmax'))
+model.compile(optimizer='rmsprop',
+              loss='categorical_crossentropy',
+              metrics=['accuracy'])
+
+# Generate dummy data
+import numpy as np
+data = np.random.random((1000, 100))
+labels = np.random.randint(10, size=(1000, 1))
+
+# Convert labels to categorical one-hot encoding
+one_hot_labels = keras.utils.to_categorical(labels, num_classes=10)
+
+# Train the model, iterating on the data in batches of 32 samples
+model.fit(data, one_hot_labels, epochs=10, batch_size=32)
+```
+
+----
+
+
+## Examples
+
+Here are a few examples to get you started!
+
+In the [examples folder](https://github.com/fchollet/keras/tree/master/examples), you will also find example models for real datasets:
+
+- CIFAR10 small images classification: Convolutional Neural Network (CNN) with realtime data augmentation
+- IMDB movie review sentiment classification: LSTM over sequences of words
+- Reuters newswires topic classification: Multilayer Perceptron (MLP)
+- MNIST handwritten digits classification: MLP & CNN
+- Character-level text generation with LSTM
+
+...and more.
+
+
+### Multilayer Perceptron (MLP) for multi-class softmax classification:
+
+```python
+from keras.models import Sequential
+from keras.layers import Dense, Dropout, Activation
+from keras.optimizers import SGD
+
+# Generate dummy data
+import numpy as np
+x_train = np.random.random((1000, 20))
+y_train = keras.utils.to_categorical(np.random.randint(10, size=(1000, 1)), num_classes=10)
+x_test = np.random.random((100, 20))
+y_test = keras.utils.to_categorical(np.random.randint(10, size=(100, 1)), num_classes=10)
+
+model = Sequential()
+# Dense(64) is a fully-connected layer with 64 hidden units.
+# in the first layer, you must specify the expected input data shape:
+# here, 20-dimensional vectors.
+model.add(Dense(64, activation='relu', input_dim=20))
+model.add(Dropout(0.5))
+model.add(Dense(64, activation='relu'))
+model.add(Dropout(0.5))
+model.add(Dense(10, activation='softmax'))
+
+sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
+model.compile(loss='categorical_crossentropy',
+              optimizer=sgd,
+              metrics=['accuracy'])
+
+model.fit(x_train, y_train,
+          epochs=20,
+          batch_size=128)
+score = model.evaluate(x_test, y_test, batch_size=128)
+```
+
+
+### MLP for binary classification:
+
+```python
+import numpy as np
+from keras.models import Sequential
+from keras.layers import Dense, Dropout
+
+# Generate dummy data
+x_train = np.random.random((1000, 20))
+y_train = np.random.randint(2, size=(1000, 1))
+x_test = np.random.random((100, 20))
+y_test = np.random.randint(2, size=(100, 1))
+
+model = Sequential()
+model.add(Dense(64, input_dim=20, activation='relu'))
+model.add(Dropout(0.5))
+model.add(Dense(64, activation='relu'))
+model.add(Dropout(0.5))
+model.add(Dense(1, activation='sigmoid'))
+
+model.compile(loss='binary_crossentropy',
+              optimizer='rmsprop',
+              metrics=['accuracy'])
+
+model.fit(x_train, y_train,
+          epochs=20,
+          batch_size=128)
+score = model.evaluate(x_test, y_test, batch_size=128)
+```
+
+
+### VGG-like convnet:
+
+```python
+import numpy as np
+import keras
+from keras.models import Sequential
+from keras.layers import Dense, Dropout, Flatten
+from keras.layers import Conv2D, MaxPooling2D
+from keras.optimizers import SGD
+
+# Generate dummy data
+x_train = np.random.random((100, 100, 100, 3))
+y_train = keras.utils.to_categorical(np.random.randint(10, size=(100, 1)), num_classes=10)
+x_test = np.random.random((20, 100, 100, 3))
+y_test = keras.utils.to_categorical(np.random.randint(10, size=(20, 1)), num_classes=10)
+
+model = Sequential()
+# input: 100x100 images with 3 channels -> (100, 100, 3) tensors.
+# this applies 32 convolution filters of size 3x3 each.
+model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(100, 100, 3)))
+model.add(Conv2D(32, (3, 3), activation='relu'))
+model.add(MaxPooling2D(pool_size=(2, 2)))
+model.add(Dropout(0.25))
+
+model.add(Conv2D(64, (3, 3), activation='relu'))
+model.add(Conv2D(64, (3, 3), activation='relu'))
+model.add(MaxPooling2D(pool_size=(2, 2)))
+model.add(Dropout(0.25))
+
+model.add(Flatten())
+model.add(Dense(256, activation='relu'))
+model.add(Dropout(0.5))
+model.add(Dense(10, activation='softmax'))
+
+sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
+model.compile(loss='categorical_crossentropy', optimizer=sgd)
+
+model.fit(x_train, y_train, batch_size=32, epochs=10)
+score = model.evaluate(x_test, y_test, batch_size=32)
+```
+
+
+### Sequence classification with LSTM:
+
+```python
+from keras.models import Sequential
+from keras.layers import Dense, Dropout
+from keras.layers import Embedding
+from keras.layers import LSTM
+
+model = Sequential()
+model.add(Embedding(max_features, output_dim=256))
+model.add(LSTM(128))
+model.add(Dropout(0.5))
+model.add(Dense(1, activation='sigmoid'))
+
+model.compile(loss='binary_crossentropy',
+              optimizer='rmsprop',
+              metrics=['accuracy'])
+
+model.fit(x_train, y_train, batch_size=16, epochs=10)
+score = model.evaluate(x_test, y_test, batch_size=16)
+```
+
+### Sequence classification with 1D convolutions:
+
+```python
+from keras.models import Sequential
+from keras.layers import Dense, Dropout
+from keras.layers import Embedding
+from keras.layers import Conv1D, GlobalAveragePooling1D, MaxPooling1D
+
+model = Sequential()
+model.add(Conv1D(64, 3, activation='relu', input_shape=(seq_length, 100)))
+model.add(Conv1D(64, 3, activation='relu'))
+model.add(MaxPooling1D(3))
+model.add(Conv1D(128, 3, activation='relu'))
+model.add(Conv1D(128, 3, activation='relu'))
+model.add(GlobalAveragePooling1D())
+model.add(Dropout(0.5))
+model.add(Dense(1, activation='sigmoid'))
+
+model.compile(loss='binary_crossentropy',
+              optimizer='rmsprop',
+              metrics=['accuracy'])
+
+model.fit(x_train, y_train, batch_size=16, epochs=10)
+score = model.evaluate(x_test, y_test, batch_size=16)
+```
+
+### Stacked LSTM for sequence classification
+
+In this model, we stack 3 LSTM layers on top of each other,
+making the model capable of learning higher-level temporal representations.
+
+The first two LSTMs return their full output sequences, but the last one only returns
+the last step in its output sequence, thus dropping the temporal dimension
+(i.e. converting the input sequence into a single vector).
+
+<img src="https://keras.io/img/regular_stacked_lstm.png" alt="stacked LSTM" style="width: 300px;"/>
+
+```python
+from keras.models import Sequential
+from keras.layers import LSTM, Dense
+import numpy as np
+
+data_dim = 16
+timesteps = 8
+num_classes = 10
+
+# expected input data shape: (batch_size, timesteps, data_dim)
+model = Sequential()
+model.add(LSTM(32, return_sequences=True,
+               input_shape=(timesteps, data_dim)))  # returns a sequence of vectors of dimension 32
+model.add(LSTM(32, return_sequences=True))  # returns a sequence of vectors of dimension 32
+model.add(LSTM(32))  # return a single vector of dimension 32
+model.add(Dense(10, activation='softmax'))
+
+model.compile(loss='categorical_crossentropy',
+              optimizer='rmsprop',
+              metrics=['accuracy'])
+
+# Generate dummy training data
+x_train = np.random.random((1000, timesteps, data_dim))
+y_train = np.random.random((1000, num_classes))
+
+# Generate dummy validation data
+x_val = np.random.random((100, timesteps, data_dim))
+y_val = np.random.random((100, num_classes))
+
+model.fit(x_train, y_train,
+          batch_size=64, epochs=5,
+          validation_data=(x_val, y_val))
+```
+
+
+### Same stacked LSTM model, rendered "stateful"
+
+A stateful recurrent model is one for which the internal states (memories) obtained after processing a batch
+of samples are reused as initial states for the samples of the next batch. This allows to process longer sequences
+while keeping computational complexity manageable.
+
+[You can read more about stateful RNNs in the FAQ.](/getting-started/faq/#how-can-i-use-stateful-rnns)
+
+```python
+from keras.models import Sequential
+from keras.layers import LSTM, Dense
+import numpy as np
+
+data_dim = 16
+timesteps = 8
+num_classes = 10
+batch_size = 32
+
+# Expected input batch shape: (batch_size, timesteps, data_dim)
+# Note that we have to provide the full batch_input_shape since the network is stateful.
+# the sample of index i in batch k is the follow-up for the sample i in batch k-1.
+model = Sequential()
+model.add(LSTM(32, return_sequences=True, stateful=True,
+               batch_input_shape=(batch_size, timesteps, data_dim)))
+model.add(LSTM(32, return_sequences=True, stateful=True))
+model.add(LSTM(32, stateful=True))
+model.add(Dense(10, activation='softmax'))
+
+model.compile(loss='categorical_crossentropy',
+              optimizer='rmsprop',
+              metrics=['accuracy'])
+
+# Generate dummy training data
+x_train = np.random.random((batch_size * 10, timesteps, data_dim))
+y_train = np.random.random((batch_size * 10, num_classes))
+
+# Generate dummy validation data
+x_val = np.random.random((batch_size * 3, timesteps, data_dim))
+y_val = np.random.random((batch_size * 3, num_classes))
+
+model.fit(x_train, y_train,
+          batch_size=batch_size, epochs=5, shuffle=False,
+          validation_data=(x_val, y_val))
+```

docs/templates/index.md

@@ -0,0 +1,3 @@
+# Keras: The Python Deep Learning library
+
+{{autogenerated}}

docs/templates/initializers.md

@@ -0,0 +1,43 @@
+## Usage of initializers
+
+Initializations define the way to set the initial random weights of Keras layers.
+
+The keyword arguments used for passing initializers to layers will depend on the layer. Usually it is simply `kernel_initializer` and `bias_initializer`:
+
+```python
+model.add(Dense(64,
+                kernel_initializer='random_uniform',
+                bias_initializer='zeros'))
+```
+
+## Available initializers
+
+The following built-in initializers are available as part of the `keras.initializers` module:
+
+{{autogenerated}}
+
+
+An initializer may be passed as a string (must match one of the available initializers above), or as a callable:
+
+```python
+from keras import initializers
+
+model.add(Dense(64, kernel_initializer=initializers.random_normal(stddev=0.01)))
+
+# also works; will use the default parameters.
+model.add(Dense(64, kernel_initializer='random_normal'))
+```
+
+
+## Using custom initializers
+
+If passing a custom callable, then it must take the argument `shape` (shape of the variable to initialize) and `dtype` (dtype of generated values):
+
+```python
+from keras import backend as K
+
+def my_init(shape, dtype=None):
+    return K.random_normal(shape, dtype=dtype)
+
+model.add(Dense(64, kernel_initializer=my_init))
+```

docs/templates/layers/about-keras-layers.md

@@ -0,0 +1,37 @@
+# About Keras layers
+
+All Keras layers have a number of methods in common:
+
+- `layer.get_weights()`: returns the weights of the layer as a list of Numpy arrays.
+- `layer.set_weights(weights)`: sets the weights of the layer from a list of Numpy arrays (with the same shapes as the output of `get_weights`).
+- `layer.get_config()`: returns a dictionary containing the configuration of the layer. The layer can be reinstantiated from its config via:
+
+```python
+layer = Dense(32)
+config = layer.get_config()
+reconstructed_layer = Dense.from_config(config)
+```
+
+Or:
+
+```python
+from keras import layers
+
+config = layer.get_config()
+layer = layers.deserialize({'class_name': layer.__class__.__name__,
+                            'config': config})
+```
+
+If a layer has a single node (i.e. if it isn't a shared layer), you can get its input tensor, output tensor, input shape and output shape via:
+
+- `layer.input`
+- `layer.output`
+- `layer.input_shape`
+- `layer.output_shape`
+
+If the layer has multiple nodes (see: [the concept of layer node and shared layers](/getting-started/functional-api-guide/#the-concept-of-layer-node)), you can use the following methods:
+
+- `layer.get_input_at(node_index)`
+- `layer.get_output_at(node_index)`
+- `layer.get_input_shape_at(node_index)`
+- `layer.get_output_shape_at(node_index)`

docs/templates/layers/writing-your-own-keras-layers.md

@@ -0,0 +1,37 @@
+# Writing your own Keras layers
+
+For simple, stateless custom operations, you are probably better off using `layers.core.Lambda` layers. But for any custom operation that has trainable weights, you should implement your own layer.
+
+Here is the skeleton of a Keras layer, **as of Keras 2.0** (if you have an older version, please upgrade). There are only three methods you need to implement:
+
+- `build(input_shape)`: this is where you will define your weights. This method must set `self.built = True`, which can be done by calling `super([Layer], self).build()`.
+- `call(x)`: this is where the layer's logic lives. Unless you want your layer to support masking, you only have to care about the first argument passed to `call`: the input tensor.
+- `compute_output_shape(input_shape)`: in case your layer modifies the shape of its input, you should specify here the shape transformation logic. This allows Keras to do automatic shape inference.
+
+```python
+from keras import backend as K
+from keras.engine.topology import Layer
+import numpy as np
+
+class MyLayer(Layer):
+
+    def __init__(self, output_dim, **kwargs):
+        self.output_dim = output_dim
+        super(MyLayer, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+        # Create a trainable weight variable for this layer.
+        self.kernel = self.add_weight(name='kernel', 
+                                      shape=(input_shape[1], self.output_dim),
+                                      initializer='uniform',
+                                      trainable=True)
+        super(MyLayer, self).build(input_shape)  # Be sure to call this somewhere!
+
+    def call(self, x):
+        return K.dot(x, self.kernel)
+
+    def compute_output_shape(self, input_shape):
+        return (input_shape[0], self.output_dim)
+```
+
+The existing Keras layers provide examples of how to implement almost anything. Never hesitate to read the source code!

docs/templates/losses.md

@@ -0,0 +1,37 @@
+
+## Usage of loss functions
+
+A loss function (or objective function, or optimization score function) is one of the two parameters required to compile a model:
+
+```python
+model.compile(loss='mean_squared_error', optimizer='sgd')
+```
+
+```python
+from keras import losses
+
+model.compile(loss=losses.mean_squared_error, optimizer='sgd')
+```
+
+You can either pass the name of an existing loss function, or pass a TensorFlow/Theano symbolic function that returns a scalar for each data-point and takes the following two arguments:
+
+- __y_true__: True labels. TensorFlow/Theano tensor.
+- __y_pred__: Predictions. TensorFlow/Theano tensor of the same shape as y_true.
+
+The actual optimized objective is the mean of the output array across all datapoints.
+
+For a few examples of such functions, check out the [losses source](https://github.com/fchollet/keras/blob/master/keras/losses.py).
+
+## Available loss functions
+
+{{autogenerated}}
+
+----
+
+**Note**: when using the `categorical_crossentropy` loss, your targets should be in categorical format (e.g. if you have 10 classes, the target for each sample should be a 10-dimensional vector that is all-zeros expect for a 1 at the index corresponding to the class of the sample). In order to convert *integer targets* into *categorical targets*, you can use the Keras utility `to_categorical`:
+
+```python
+from keras.utils.np_utils import to_categorical
+
+categorical_labels = to_categorical(int_labels, num_classes=None)
+```

docs/templates/metrics.md

@@ -0,0 +1,56 @@
+
+## Usage of metrics
+
+A metric is a function that is used to judge the performance of your model. Metric functions are to be supplied in the `metrics` parameter when a model is compiled.
+
+```python
+model.compile(loss='mean_squared_error',
+              optimizer='sgd',
+              metrics=['mae', 'acc'])
+```
+
+```python
+from keras import metrics
+
+model.compile(loss='mean_squared_error',
+              optimizer='sgd',
+              metrics=[metrics.mae, metrics.categorical_accuracy])
+```
+
+A metric function is similar to an [loss function](/losses), except that the results from evaluating a metric are not used when training the model.
+
+You can either pass the name of an existing metric, or pass a Theano/TensorFlow symbolic function (see [Custom metrics](#custom-metrics)).
+
+#### Arguments
+  - __y_true__: True labels. Theano/TensorFlow tensor.
+  - __y_pred__: Predictions. Theano/TensorFlow tensor of the same shape as y_true.
+
+#### Returns
+  Single tensor value representing the mean of the output array across all
+  datapoints.
+
+----
+
+## Available metrics
+
+
+{{autogenerated}}
+
+----
+
+## Custom metrics
+
+Custom metrics can be passed at the compilation step. The
+function would need to take `(y_true, y_pred)` as arguments and return
+a single tensor value.
+
+```python
+import keras.backend as K
+
+def mean_pred(y_true, y_pred):
+    return K.mean(y_pred)
+
+model.compile(optimizer='rmsprop',
+              loss='binary_crossentropy',
+              metrics=['accuracy', mean_pred])
+```

docs/templates/models/about-keras-models.md

@@ -0,0 +1,33 @@
+# About Keras models
+
+There are two types of models available in Keras: [the Sequential model](/models/sequential) and [the Model class used with functional API](/models/model).
+
+These models have a number of methods in common:
+
+- `model.summary()`: prints a summary representation of your model.
+- `model.get_config()`: returns a dictionary containing the configuration of the model. The model can be reinstantiated from its config via:
+```python
+config = model.get_config()
+model = Model.from_config(config)
+# or, for Sequential:
+model = Sequential.from_config(config)
+```
+
+- `model.get_weights()`: returns a list of all weight tensors in the model, as Numpy arrays.
+- `model.set_weights(weights)`: sets the values of the weights of the model, from a list of Numpy arrays. The arrays in the list should have the same shape as those returned by `get_weights()`.
+- `model.to_json()`: returns a representation of the model as a JSON string. Note that the representation does not include the weights, only the architecture. You can reinstantiate the same model (with reinitialized weights) from the JSON string via:
+```python
+from models import model_from_json
+
+json_string = model.to_json()
+model = model_from_json(json_string)
+```
+- `model.to_yaml()`: returns a representation of the model as a YAML string. Note that the representation does not include the weights, only the architecture. You can reinstantiate the same model (with reinitialized weights) from the YAML string via:
+```python
+from models import model_from_yaml
+
+yaml_string = model.to_yaml()
+model = model_from_yaml(yaml_string)
+```
+- `model.save_weights(filepath)`: saves the weights of the model as a HDF5 file.
+- `model.load_weights(filepath, by_name=False)`: loads the weights of the model from a HDF5 file (created by `save_weights`). By default, the architecture is expected to be unchanged. To load weights into a different architecture (with some layers in common), use `by_name=True` to load only those layers with the same name.

docs/templates/models/model.md

@@ -0,0 +1,32 @@
+# Model class API
+
+In the functional API, given some input tensor(s) and output tensor(s), you can instantiate a `Model` via:
+
+```python
+from keras.models import Model
+from keras.layers import Input, Dense
+
+a = Input(shape=(32,))
+b = Dense(32)(a)
+model = Model(inputs=a, outputs=b)
+```
+
+This model will include all layers required in the computation of `b` given `a`.
+
+In the case of multi-input or multi-output models, you can use lists as well:
+
+```python
+model = Model(inputs=[a1, a2], outputs=[b1, b3, b3])
+```
+
+For a detailed introduction of what `Model` can do, read [this guide to the Keras functional API](/getting-started/functional-api-guide).
+
+## Useful attributes of Model
+
+- `model.layers` is a flattened list of the layers comprising the model graph.
+- `model.inputs` is the list of input tensors.
+- `model.outputs` is the list of output tensors.
+
+## Methods
+
+{{autogenerated}}

docs/templates/models/sequential.md

@@ -0,0 +1,14 @@
+# The Sequential model API
+
+To get started, read [this guide to the Keras Sequential model](/getting-started/sequential-model-guide).
+
+## Useful attributes of Model
+
+- `model.layers` is a list of the layers added to the model.
+
+
+----
+
+## Sequential model methods
+
+{{autogenerated}}

docs/templates/optimizers.md

@@ -0,0 +1,50 @@
+
+## Usage of optimizers
+
+An optimizer is one of the two arguments required for compiling a Keras model:
+
+```python
+from keras import optimizers
+
+model = Sequential()
+model.add(Dense(64, kernel_initializer='uniform', input_shape=(10,)))
+model.add(Activation('tanh'))
+model.add(Activation('softmax'))
+
+sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
+model.compile(loss='mean_squared_error', optimizer=sgd)
+```
+
+You can either instantiate an optimizer before passing it to `model.compile()` , as in the above example, or you can call it by its name. In the latter case, the default parameters for the optimizer will be used.
+
+```python
+# pass optimizer by name: default parameters will be used
+model.compile(loss='mean_squared_error', optimizer='sgd')
+```
+
+---
+
+## Parameters common to all Keras optimizers
+
+The parameters `clipnorm` and `clipvalue` can be used with all optimizers to control gradient clipping:
+
+```python
+from keras import optimizers
+
+# All parameter gradients will be clipped to
+# a maximum norm of 1.
+sgd = optimizers.SGD(lr=0.01, clipnorm=1.)
+```
+
+```python
+from keras import optimizers
+
+# All parameter gradients will be clipped to
+# a maximum value of 0.5 and
+# a minimum value of -0.5.
+sgd = optimizers.SGD(lr=0.01, clipvalue=0.5)
+```
+
+---
+
+{{autogenerated}}

docs/templates/preprocessing/image.md

@@ -0,0 +1,208 @@
+
+## ImageDataGenerator
+
+```python
+keras.preprocessing.image.ImageDataGenerator(featurewise_center=False,
+    samplewise_center=False,
+    featurewise_std_normalization=False,
+    samplewise_std_normalization=False,
+    zca_whitening=False,
+    zca_epsilon=1e-6,
+    rotation_range=0.,
+    width_shift_range=0.,
+    height_shift_range=0.,
+    shear_range=0.,
+    zoom_range=0.,
+    channel_shift_range=0.,
+    fill_mode='nearest',
+    cval=0.,
+    horizontal_flip=False,
+    vertical_flip=False,
+    rescale=None,
+    preprocessing_function=None,
+    data_format=K.image_data_format())
+```
+
+Generate batches of tensor image data with real-time data augmentation. The data will be looped over (in batches) indefinitely.
+
+- __Arguments__:
+    - __featurewise_center__: Boolean. Set input mean to 0 over the dataset, feature-wise.
+    - __samplewise_center__: Boolean. Set each sample mean to 0.
+    - __featurewise_std_normalization__: Boolean. Divide inputs by std of the dataset, feature-wise.
+    - __samplewise_std_normalization__: Boolean. Divide each input by its std.
+    - __zca_epsilon__: epsilon for ZCA whitening. Default is 1e-6.
+    - __zca_whitening__: Boolean. Apply ZCA whitening.
+    - __rotation_range__: Int. Degree range for random rotations.
+    - __width_shift_range__: Float (fraction of total width). Range for random horizontal shifts.
+    - __height_shift_range__: Float (fraction of total height). Range for random vertical shifts.
+    - __shear_range__: Float. Shear Intensity (Shear angle in counter-clockwise direction as radians)
+    - __zoom_range__: Float or [lower, upper]. Range for random zoom. If a float, `[lower, upper] = [1-zoom_range, 1+zoom_range]`.
+    - __channel_shift_range__: Float. Range for random channel shifts.
+    - __fill_mode__: One of {"constant", "nearest", "reflect" or "wrap"}.  Points outside the boundaries of the input are filled according to the given mode.
+    - __cval__: Float or Int. Value used for points outside the boundaries when `fill_mode = "constant"`.
+    - __horizontal_flip__: Boolean. Randomly flip inputs horizontally.
+    - __vertical_flip__: Boolean. Randomly flip inputs vertically.
+    - __rescale__: rescaling factor. Defaults to None. If None or 0, no rescaling is applied,
+            otherwise we multiply the data by the value provided (before applying
+            any other transformation).
+    - __preprocessing_function__: function that will be implied on each input.
+            The function will run before any other modification on it.
+            The function should take one argument:
+            one image (Numpy tensor with rank 3),
+            and should output a Numpy tensor with the same shape.
+    - _data_format_: One of {"channels_first", "channels_last"}.
+        "channels_last" mode means that the images should have shape `(samples, height, width, channels)`,
+        "channels_first" mode means that the images should have shape `(samples, channels, height, width)`.
+        It defaults to the `image_data_format` value found in your
+        Keras config file at `~/.keras/keras.json`.
+        If you never set it, then it will be "channels_last".
+
+- __Methods__:
+    - __fit(x)__: Compute the internal data stats related to the data-dependent transformations, based on an array of sample data.
+        Only required if featurewise_center or featurewise_std_normalization or zca_whitening.
+        - __Arguments__:
+            - __x__: sample data. Should have rank 4.
+                In case of grayscale data,
+                the channels axis should have value 1, and in case
+                of RGB data, it should have value 3.
+            - __augment__: Boolean (default: False). Whether to fit on randomly augmented samples.
+            - __rounds__: int (default: 1). If augment, how many augmentation passes over the data to use.
+            - __seed__: int (default: None). Random seed.
+    - __flow(x, y)__: Takes numpy data & label arrays, and generates batches of augmented/normalized data. Yields batches indefinitely, in an infinite loop.
+        - __Arguments__:
+            - __x__: data. Should have rank 4.
+                In case of grayscale data,
+                the channels axis should have value 1, and in case
+                of RGB data, it should have value 3.
+            - __y__: labels.
+            - __batch_size__: int (default: 32).
+            - __shuffle__: boolean (defaut: True).
+            - __seed__: int (default: None).
+            - __save_to_dir__: None or str (default: None). This allows you to optimally specify a directory to which to save the augmented pictures being generated (useful for visualizing what you are doing).
+            - __save_prefix__: str (default: `''`). Prefix to use for filenames of saved pictures (only relevant if `save_to_dir` is set).
+            - __save_format__: one of "png", "jpeg" (only relevant if `save_to_dir` is set). Default: "png".
+        - __yields__: Tuples of `(x, y)` where `x` is a numpy array of image data and `y` is a numpy array of corresponding labels.
+            The generator loops indefinitely.
+    - __flow_from_directory(directory)__: Takes the path to a directory, and generates batches of augmented/normalized data. Yields batches indefinitely, in an infinite loop.
+        - __Arguments__:
+            - __directory__: path to the target directory. It should contain one subdirectory per class.
+                Any PNG, JPG or BMP images inside each of the subdirectories directory tree will be included in the generator.
+                See [this script](https://gist.github.com/fchollet/0830affa1f7f19fd47b06d4cf89ed44d) for more details.
+            - __target_size__: tuple of integers, default: `(256, 256)`. The dimensions to which all images found will be resized.
+            - __color_mode__: one of "grayscale", "rbg". Default: "rgb". Whether the images will be converted to have 1 or 3 color channels.
+            - __classes__: optional list of class subdirectories (e.g. `['dogs', 'cats']`). Default: None. If not provided, the list of classes will be automatically inferred from the subdirectory names/structure under `directory`, where each subdirectory will be treated as a different class (and the order of the classes, which will map to the label indices, will be alphanumeric). The dictionary containing the mapping from class names to class indices can be obtained via the attribute `class_indices`.
+            - __class_mode__: one of "categorical", "binary", "sparse" or None. Default: "categorical". Determines the type of label arrays that are returned: "categorical" will be 2D one-hot encoded labels, "binary" will be 1D binary labels, "sparse" will be 1D integer labels. If None, no labels are returned (the generator will only yield batches of image data, which is useful to use `model.predict_generator()`, `model.evaluate_generator()`, etc.). Please note that in case of class_mode None, the data still needs to reside in a subdirectory of `directory` for it to work correctly.
+            - __batch_size__: size of the batches of data (default: 32).
+            - __shuffle__: whether to shuffle the data (default: True)
+            - __seed__: optional random seed for shuffling and transformations.
+            - __save_to_dir__: None or str (default: None). This allows you to optimally specify a directory to which to save the augmented pictures being generated (useful for visualizing what you are doing).
+            - __save_prefix__: str. Prefix to use for filenames of saved pictures (only relevant if `save_to_dir` is set).
+            - __save_format__: one of "png", "jpeg" (only relevant if `save_to_dir` is set). Default: "png".
+            - __follow_links__: whether to follow symlinks inside class subdirectories (default: False).
+
+
+- __Examples__:
+
+Example of using `.flow(x, y)`:
+
+```python
+(x_train, y_train), (x_test, y_test) = cifar10.load_data()
+y_train = np_utils.to_categorical(y_train, num_classes)
+y_test = np_utils.to_categorical(y_test, num_classes)
+
+datagen = ImageDataGenerator(
+    featurewise_center=True,
+    featurewise_std_normalization=True,
+    rotation_range=20,
+    width_shift_range=0.2,
+    height_shift_range=0.2,
+    horizontal_flip=True)
+
+# compute quantities required for featurewise normalization
+# (std, mean, and principal components if ZCA whitening is applied)
+datagen.fit(x_train)
+
+# fits the model on batches with real-time data augmentation:
+model.fit_generator(datagen.flow(x_train, y_train, batch_size=32),
+                    steps_per_epoch=len(x_train) / 32, epochs=epochs)
+
+# here's a more "manual" example
+for e in range(epochs):
+    print('Epoch', e)
+    batches = 0
+    for x_batch, y_batch in datagen.flow(x_train, y_train, batch_size=32):
+        model.fit(x_batch, y_batch)
+        batches += 1
+        if batches >= len(x_train) / 32:
+            # we need to break the loop by hand because
+            # the generator loops indefinitely
+            break
+```
+
+Example of using `.flow_from_directory(directory)`:
+
+```python
+train_datagen = ImageDataGenerator(
+        rescale=1./255,
+        shear_range=0.2,
+        zoom_range=0.2,
+        horizontal_flip=True)
+
+test_datagen = ImageDataGenerator(rescale=1./255)
+
+train_generator = train_datagen.flow_from_directory(
+        'data/train',
+        target_size=(150, 150),
+        batch_size=32,
+        class_mode='binary')
+
+validation_generator = test_datagen.flow_from_directory(
+        'data/validation',
+        target_size=(150, 150),
+        batch_size=32,
+        class_mode='binary')
+
+model.fit_generator(
+        train_generator,
+        steps_per_epoch=2000,
+        epochs=50,
+        validation_data=validation_generator,
+        validation_steps=800)
+```
+
+Example of transforming images and masks together.
+
+```python
+# we create two instances with the same arguments
+data_gen_args = dict(featurewise_center=True,
+                     featurewise_std_normalization=True,
+                     rotation_range=90.,
+                     width_shift_range=0.1,
+                     height_shift_range=0.1,
+                     zoom_range=0.2)
+image_datagen = ImageDataGenerator(**data_gen_args)
+mask_datagen = ImageDataGenerator(**data_gen_args)
+
+# Provide the same seed and keyword arguments to the fit and flow methods
+seed = 1
+image_datagen.fit(images, augment=True, seed=seed)
+mask_datagen.fit(masks, augment=True, seed=seed)
+
+image_generator = image_datagen.flow_from_directory(
+    'data/images',
+    class_mode=None,
+    seed=seed)
+
+mask_generator = mask_datagen.flow_from_directory(
+    'data/masks',
+    class_mode=None,
+    seed=seed)
+
+# combine generators into one which yields image and masks
+train_generator = zip(image_generator, mask_generator)
+
+model.fit_generator(
+    train_generator,
+    steps_per_epoch=2000,
+    epochs=50)
+```

docs/templates/preprocessing/sequence.md

@@ -1,29 +1,33 @@
 ## pad_sequences
 
 ```python
-keras.preprocessing.sequence.pad_sequences(sequences, maxlen=None, dtype='int32')
+keras.preprocessing.sequence.pad_sequences(sequences, maxlen=None, dtype='int32',
+    padding='pre', truncating='pre', value=0.)
 ```
 
-Transform a list of `nb_samples sequences` (lists of scalars) into a 2D numpy array of shape `(nb_samples, nb_timesteps)`. `nb_timesteps` is either the `maxlen` argument if provided, or the length of the longest sequence otherwise. Sequences that are shorter than `nb_timesteps` are padded with zeros at the end.
+Transform a list of `num_samples` sequences (lists of scalars) into a 2D Numpy array of shape `(num_samples, num_timesteps)`. `num_timesteps` is either the `maxlen` argument if provided, or the length of the longest sequence otherwise. Sequences that are shorter than `num_timesteps` are padded with `value` at the end. Sequences longer than `num_timesteps` are truncated so that it fits the desired length. Position where padding or truncation happens is determined by `padding` or `truncating`, respectively.
 
-- __Return__: 2D numpy array of shape `(nb_samples, nb_timesteps)`.
+- __Return__: 2D Numpy array of shape `(num_samples, num_timesteps)`.
 
 - __Arguments__:
     - __sequences__: List of lists of int or float.
     - __maxlen__: None or int. Maximum sequence length, longer sequences are truncated and shorter sequences are padded with zeros at the end.
-    - __dtype__: datatype of the numpy array returned.
+    - __dtype__: datatype of the Numpy array returned.
+    - __padding__: 'pre' or 'post', pad either before or after each sequence.
+    - __truncating__: 'pre' or 'post', remove values from sequences larger than maxlen either in the beginning or in the end of the sequence
+    - __value__: float, value to pad the sequences to the desired value.
 
 ---
 
 ## skipgrams
 
 ```python
-keras.preprocessing.sequence.skipgrams(sequence, vocabulary_size, 
-    window_size=4, negative_samples=1., shuffle=True, 
+keras.preprocessing.sequence.skipgrams(sequence, vocabulary_size,
+    window_size=4, negative_samples=1., shuffle=True,
     categorical=False, sampling_table=None)
 ```
 
-Transforms a sequence of word indexes (list of int) into couples of the form: 
+Transforms a sequence of word indexes (list of int) into couples of the form:
 
 - (word, word in the same window), with label 1 (positive samples).
 - (word, random word from the vocabulary), with label 0 (negative samples).
@@ -31,8 +35,8 @@ Transforms a sequence of word indexes (list of int) into couples of the form:
 Read more about Skipgram in this gnomic paper by Mikolov et al.: [Efficient Estimation of Word Representations in
 Vector Space](http://arxiv.org/pdf/1301.3781v3.pdf)
 
-- __Return__: tuple `(couples, labels)`. 
-    - `couples` is a list of 2-elements lists of int: `[word_index, other_word_index]`. 
+- __Return__: tuple `(couples, labels)`.
+    - `couples` is a list of 2-elements lists of int: `[word_index, other_word_index]`.
     - `labels` is a list of 0 and 1, where 1 indicates that `other_word_index` was found in the same window as `word_index`, and 0 indicates that `other_word_index` was random.
     - if categorical is set to True, the labels are categorical, ie. 1 becomes [0,1], and 0 becomes [1, 0].
 
@@ -43,7 +47,7 @@ Vector Space](http://arxiv.org/pdf/1301.3781v3.pdf)
     - __negative_samples__: float >= 0. 0 for no negative (=random) samples. 1 for same number as positive samples. etc.
     - __shuffle__: boolean. Whether to shuffle the samples.
     - __categorical__: boolean. Whether to make the returned labels categorical.
-    - __sampling_table__: numpy array of shape `(vocabulary_size,)` where `sampling_table[i]` is the probability of sampling the word with index i (assumed to be i-th most common word in the dataset).
+    - __sampling_table__: Numpy array of shape `(vocabulary_size,)` where `sampling_table[i]` is the probability of sampling the word with index i (assumed to be i-th most common word in the dataset).
 
 
 ---
@@ -56,7 +60,7 @@ keras.preprocessing.sequence.make_sampling_table(size, sampling_factor=1e-5)
 
 Used for generating the `sampling_table` argument for `skipgrams`. `sampling_table[i]` is the probability of sampling the word i-th most common word in a dataset (more common words should be sampled less frequently, for balance).
 
-- __Return__: numpy array of shape `(size,)`.
+- __Return__: Numpy array of shape `(size,)`.
 
 - __Arguments__:
     - __size__: size of the vocabulary considered.

docs/templates/preprocessing/text.md

@@ -0,0 +1,129 @@
+
+## text_to_word_sequence
+
+```python
+keras.preprocessing.text.text_to_word_sequence(text,
+                                               filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
+                                               lower=True,
+                                               split=" ")
+```
+
+Split a sentence into a list of words.
+
+- __Return__: List of words (str).
+
+- __Arguments__:
+    - __text__: str.
+    - __filters__: list (or concatenation) of characters to filter out, such as
+         punctuation. Default: '!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n' , includes
+         basic punctuation, tabs, and newlines.
+    - __lower__: boolean. Whether to set the text to lowercase.
+    - __split__: str. Separator for word splitting.
+
+## one_hot
+
+```python
+keras.preprocessing.text.one_hot(text,
+                                 n,
+                                 filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
+                                 lower=True,
+                                 split=" ")
+```
+
+One-hot encodes a text into a list of word indexes in a vocabulary of size n.
+
+This is a wrapper to the `hashing_trick` function using `hash` as the hashing function.
+
+- __Return__: List of integers in [1, n]. Each integer encodes a word (unicity non-guaranteed).
+
+- __Arguments__:
+    - __text__: str.
+    - __n__: int. Size of vocabulary.
+    - __filters__: list (or concatenation) of characters to filter out, such as
+         punctuation. Default: '!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n' , includes
+         basic punctuation, tabs, and newlines.
+    - __lower__: boolean. Whether to set the text to lowercase.
+    - __split__: str. Separator for word splitting.
+    
+## hashing_trick
+
+```python
+keras.preprocessing.text.hashing_trick(text, 
+                                       n,
+                                       hash_function=None,
+                                       filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
+                                       lower=True,
+                                       split=' ')
+```
+
+Converts a text to a sequence of indices in a fixed-size hashing space
+
+- __Return__:
+        A list of integer word indices (unicity non-guaranteed).
+- __Arguments__:
+    - __text__: str.
+    - __n__: Dimension of the hashing space.
+    - __hash_function__: defaults to python `hash` function, can be 'md5' or
+            any function that takes in input a string and returns a int.
+            Note that 'hash' is not a stable hashing function, so
+            it is not consistent across different runs, while 'md5'
+            is a stable hashing function.
+    - __filters__: list (or concatenation) of characters to filter out, such as
+         punctuation. Default: '!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n' , includes
+         basic punctuation, tabs, and newlines.
+    - __lower__: boolean. Whether to set the text to lowercase.
+    - __split__: str. Separator for word splitting.
+
+## Tokenizer
+
+```python
+keras.preprocessing.text.Tokenizer(num_words=None,
+                                   filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
+                                   lower=True,
+                                   split=" ",
+                                   char_level=False)
+```
+
+Class for vectorizing texts, or/and turning texts into sequences (=list of word indexes, where the word of rank i in the dataset (starting at 1) has index i).
+
+- __Arguments__: Same as `text_to_word_sequence` above.
+    - __num_words__: None or int. Maximum number of words to work with (if set, tokenization will be restricted to the top num_words most common words in the dataset).
+    - __char_level__: if True, every character will be treated as a token.
+
+- __Methods__:
+
+    - __fit_on_texts(texts)__: 
+        - __Arguments__:
+            - __texts__: list of texts to train on.
+
+    - __texts_to_sequences(texts)__
+        - __Arguments__: 
+            - __texts__: list of texts to turn to sequences.
+        - __Return__: list of sequences (one per text input).
+
+    - __texts_to_sequences_generator(texts)__: generator version of the above. 
+        - __Return__: yield one sequence per input text.
+
+    - __texts_to_matrix(texts)__:
+        - __Return__: numpy array of shape `(len(texts), num_words)`.
+        - __Arguments__:
+            - __texts__: list of texts to vectorize.
+            - __mode__: one of "binary", "count", "tfidf", "freq" (default: "binary").
+
+    - __fit_on_sequences(sequences)__: 
+        - __Arguments__:
+            - __sequences__: list of sequences to train on. 
+
+    - __sequences_to_matrix(sequences)__:
+        - __Return__: numpy array of shape `(len(sequences), num_words)`.
+        - __Arguments__:
+            - __sequences__: list of sequences to vectorize.
+            - __mode__: one of "binary", "count", "tfidf", "freq" (default: "binary").
+
+- __Attributes__:
+    - __word_counts__: dictionary mapping words (str) to the number of times they appeared on during fit. Only set after fit_on_texts was called. 
+    - __word_docs__: dictionary mapping words (str) to the number of documents/texts they appeared on during fit. Only set after fit_on_texts was called.
+    - __word_index__: dictionary mapping words (str) to their rank/index (int). Only set after fit_on_texts was called.
+    - __document_count__: int. Number of documents (texts/sequences) the tokenizer was trained on. Only set after fit_on_texts or fit_on_sequences was called.
+
+

docs/templates/regularizers.md

@@ -0,0 +1,46 @@
+## Usage of regularizers
+
+Regularizers allow to apply penalties on layer parameters or layer activity during optimization. These penalties are incorporated in the loss function that the network optimizes.
+
+The penalties are applied on a per-layer basis. The exact API will depend on the layer, but the layers `Dense`, `Conv1D`, `Conv2D` and `Conv3D` have a unified API.
+
+These layers expose 3 keyword arguments:
+
+- `kernel_regularizer`: instance of `keras.regularizers.Regularizer`
+- `bias_regularizer`: instance of `keras.regularizers.Regularizer`
+- `activity_regularizer`: instance of `keras.regularizers.Regularizer`
+
+
+## Example
+
+```python
+from keras import regularizers
+model.add(Dense(64, input_dim=64,
+                kernel_regularizer=regularizers.l2(0.01),
+                activity_regularizer=regularizers.l1(0.01)))
+```
+
+## Available penalties
+
+```python
+keras.regularizers.l1(0.)
+keras.regularizers.l2(0.)
+keras.regularizers.l1_l2(0.)
+```
+
+## Developing new regularizers
+
+Any function that takes in a weight matrix and returns a loss contribution tensor can be used as a regularizer, e.g.:
+
+```python
+from keras import backend as K
+
+def l1_reg(weight_matrix):
+    return 0.01 * K.sum(K.abs(weight_matrix))
+
+model.add(Dense(64, input_dim=64,
+                kernel_regularizer=l1_reg)
+```
+
+Alternatively, you can write your regularizers in an object-oriented way;
+see the [keras/regularizers.py](https://github.com/fchollet/keras/blob/master/keras/regularizers.py) module for examples.

docs/templates/scikit-learn-api.md

@@ -0,0 +1,45 @@
+# Wrappers for the Scikit-Learn API
+
+You can use `Sequential` Keras models (single-input only) as part of your Scikit-Learn workflow via the wrappers found at `keras.wrappers.scikit_learn.py`.
+
+There are two wrappers available:
+
+`keras.wrappers.scikit_learn.KerasClassifier(build_fn=None, **sk_params)`, which implements the Scikit-Learn classifier interface,
+
+`keras.wrappers.scikit_learn.KerasRegressor(build_fn=None, **sk_params)`, which implements the Scikit-Learn regressor interface.
+
+### Arguments
+
+- __build_fn__: callable function or class instance
+- __sk_params__: model parameters & fitting parameters
+
+`build_fn` should construct, compile and return a Keras model, which
+will then be used to fit/predict. One of the following
+three values could be passed to build_fn:
+
+1. A function
+2. An instance of a class that implements the __call__ method
+3. None. This means you implement a class that inherits from either
+`KerasClassifier` or `KerasRegressor`. The __call__ method of the
+present class will then be treated as the default build_fn.
+
+`sk_params` takes both model parameters and fitting parameters. Legal model
+parameters are the arguments of `build_fn`. Note that like all other
+estimators in scikit-learn, 'build_fn' should provide default values for
+its arguments, so that you could create the estimator without passing any
+values to `sk_params`.
+
+`sk_params` could also accept parameters for calling `fit`, `predict`,
+`predict_proba`, and `score` methods (e.g., `epochs`, `batch_size`).
+fitting (predicting) parameters are selected in the following order:
+
+1. Values passed to the dictionary arguments of
+`fit`, `predict`, `predict_proba`, and `score` methods
+2. Values passed to `sk_params`
+3. The default values of the `keras.models.Sequential`
+`fit`, `predict`, `predict_proba` and `score` methods
+
+When using scikit-learn's `grid_search` API, legal tunable parameters are
+those you could pass to `sk_params`, including fitting parameters.
+In other words, you could use `grid_search` to search for the best
+`batch_size` or `epochs` as well as the model parameters.

docs/templates/visualization.md

@@ -0,0 +1,25 @@
+
+## Model visualization
+
+The `keras.utils.vis_utils` module provides utility functions to plot
+a Keras model (using `graphviz`).
+
+This will plot a graph of the model and save it to a file:
+```python
+from keras.utils import plot_model
+plot_model(model, to_file='model.png')
+```
+
+`plot_model` takes two optional arguments:
+
+- `show_shapes` (defaults to False) controls whether output shapes are shown in the graph.
+- `show_layer_names` (defaults to True) controls whether layer names are shown in the graph.
+
+You can also directly obtain the `pydot.Graph` object and render it yourself,
+for example to show it in an ipython notebook :
+```python
+from IPython.display import SVG
+from keras.utils.vis_utils import model_to_dot
+
+SVG(model_to_dot(model).create(prog='dot', format='svg'))
+```

examples/README.md

@@ -0,0 +1,100 @@
+# Keras examples directory
+
+[addition_rnn.py](addition_rnn.py)
+Implementation of sequence to sequence learning for performing addition of two numbers (as strings).
+
+[antirectifier.py](antirectifier.py)
+Demonstrates how to write custom layers for Keras.
+
+[babi_memnn.py](babi_memnn.py)
+Trains a memory network on the bAbI dataset for reading comprehension.
+
+[babi_rnn.py](babi_rnn.py)
+Trains a two-branch recurrent network on the bAbI dataset for reading comprehension.
+
+[cifar10_cnn.py](cifar10_cnn.py)
+Trains a simple deep CNN on the CIFAR10 small images dataset.
+
+[conv_filter_visualization.py](conv_filter_visualization.py)
+Visualization of the filters of VGG16, via gradient ascent in input space.
+
+[conv_lstm.py](conv_lstm.py)
+Demonstrates the use of a convolutional LSTM network.
+
+[deep_dream.py](deep_dream.py)
+Deep Dreams in Keras.
+
+[image_ocr.py](image_ocr.py)
+Trains a convolutional stack followed by a recurrent stack and a CTC logloss function to perform optical character recognition (OCR).
+
+[imdb_bidirectional_lstm.py](imdb_bidirectional_lstm.py)
+Trains a Bidirectional LSTM on the IMDB sentiment classification task.
+
+[imdb_cnn.py](imdb_cnn.py)
+Demonstrates the use of Convolution1D for text classification.
+
+[imdb_cnn_lstm.py](imdb_cnn_lstm.py)
+Trains a convolutional stack followed by a recurrent stack network on the IMDB sentiment classification task.
+
+[imdb_fasttext.py](imdb_fasttext.py)
+Trains a FastText model on the IMDB sentiment classification task.
+
+[imdb_lstm.py](imdb_lstm.py)
+Trains a LSTM on the IMDB sentiment classification task.
+
+[lstm_benchmark.py](lstm_benchmark.py)
+Compares different LSTM implementations on the IMDB sentiment classification task.
+
+[lstm_text_generation.py](lstm_text_generation.py)
+Generates text from Nietzsche's writings.
+
+[mnist_acgan.py](mnist_acgan.py)
+Implementation of AC-GAN ( Auxiliary Classifier GAN ) on the MNIST dataset
+
+[mnist_cnn.py](mnist_cnn.py)
+Trains a simple convnet on the MNIST dataset.
+
+[mnist_hierarchical_rnn.py](mnist_hierarchical_rnn.py)
+Trains a Hierarchical RNN (HRNN) to classify MNIST digits.
+
+[mnist_irnn.py](mnist_irnn.py)
+Reproduction of the IRNN experiment with pixel-by-pixel sequential MNIST in "A Simple Way to Initialize Recurrent Networks of Rectified Linear Units" by Le et al.
+
+[mnist_mlp.py](mnist_mlp.py)
+Trains a simple deep multi-layer perceptron on the MNIST dataset.
+
+[mnist_net2net.py](mnist_net2net.py)
+Reproduction of the Net2Net experiment with MNIST in "Net2Net: Accelerating Learning via Knowledge Transfer".
+
+[mnist_siamese_graph.py](mnist_siamese_graph.py)
+Trains a Siamese multi-layer perceptron on pairs of digits from the MNIST dataset.
+
+[mnist_sklearn_wrapper.py](mnist_sklearn_wrapper.py)
+Demonstrates how to use the sklearn wrapper.
+
+[mnist_swwae.py](mnist_swwae.py)
+Trains a Stacked What-Where AutoEncoder built on residual blocks on the MNIST dataset.
+
+[mnist_transfer_cnn.py](mnist_transfer_cnn.py)
+Transfer learning toy example.
+
+[neural_doodle.py](neural_doodle.py)
+Neural doodle.
+
+[neural_style_transfer.py](neural_style_transfer.py)
+Neural style transfer.
+
+[pretrained_word_embeddings.py](pretrained_word_embeddings.py)
+Loads pre-trained word embeddings (GloVe embeddings) into a frozen Keras Embedding layer, and uses it to train a text classification model on the 20 Newsgroup dataset.
+
+[reuters_mlp.py](reuters_mlp.py)
+Trains and evaluate a simple MLP on the Reuters newswire topic classification task.
+
+[stateful_lstm.py](stateful_lstm.py)
+Demonstrates how to use stateful RNNs to model long sequences efficiently.
+
+[variational_autoencoder.py](variational_autoencoder.py)
+Demonstrates how to build a variational autoencoder.
+
+[variational_autoencoder_deconv.py](variational_autoencoder_deconv.py)
+Demonstrates how to build a variational autoencoder with Keras using deconvolution layers.

examples/addition_rnn.py

@@ -0,0 +1,202 @@
+# -*- coding: utf-8 -*-
+'''An implementation of sequence to sequence learning for performing addition
+Input: "535+61"
+Output: "596"
+Padding is handled by using a repeated sentinel character (space)
+
+Input may optionally be inverted, shown to increase performance in many tasks in:
+"Learning to Execute"
+http://arxiv.org/abs/1410.4615
+and
+"Sequence to Sequence Learning with Neural Networks"
+http://papers.nips.cc/paper/5346-sequence-to-sequence-learning-with-neural-networks.pdf
+Theoretically it introduces shorter term dependencies between source and target.
+
+Two digits inverted:
++ One layer LSTM (128 HN), 5k training examples = 99% train/test accuracy in 55 epochs
+
+Three digits inverted:
++ One layer LSTM (128 HN), 50k training examples = 99% train/test accuracy in 100 epochs
+
+Four digits inverted:
++ One layer LSTM (128 HN), 400k training examples = 99% train/test accuracy in 20 epochs
+
+Five digits inverted:
++ One layer LSTM (128 HN), 550k training examples = 99% train/test accuracy in 30 epochs
+'''
+
+from __future__ import print_function
+from keras.models import Sequential
+from keras import layers
+import numpy as np
+from six.moves import range
+
+
+class CharacterTable(object):
+    """Given a set of characters:
+    + Encode them to a one hot integer representation
+    + Decode the one hot integer representation to their character output
+    + Decode a vector of probabilities to their character output
+    """
+    def __init__(self, chars):
+        """Initialize character table.
+
+        # Arguments
+            chars: Characters that can appear in the input.
+        """
+        self.chars = sorted(set(chars))
+        self.char_indices = dict((c, i) for i, c in enumerate(self.chars))
+        self.indices_char = dict((i, c) for i, c in enumerate(self.chars))
+
+    def encode(self, C, num_rows):
+        """One hot encode given string C.
+
+        # Arguments
+            num_rows: Number of rows in the returned one hot encoding. This is
+                used to keep the # of rows for each data the same.
+        """
+        x = np.zeros((num_rows, len(self.chars)))
+        for i, c in enumerate(C):
+            x[i, self.char_indices[c]] = 1
+        return x
+
+    def decode(self, x, calc_argmax=True):
+        if calc_argmax:
+            x = x.argmax(axis=-1)
+        return ''.join(self.indices_char[x] for x in x)
+
+
+class colors:
+    ok = '\033[92m'
+    fail = '\033[91m'
+    close = '\033[0m'
+
+# Parameters for the model and dataset.
+TRAINING_SIZE = 50000
+DIGITS = 3
+INVERT = True
+
+# Maximum length of input is 'int + int' (e.g., '345+678'). Maximum length of
+# int is DIGITS.
+MAXLEN = DIGITS + 1 + DIGITS
+
+# All the numbers, plus sign and space for padding.
+chars = '0123456789+ '
+ctable = CharacterTable(chars)
+
+questions = []
+expected = []
+seen = set()
+print('Generating data...')
+while len(questions) < TRAINING_SIZE:
+    f = lambda: int(''.join(np.random.choice(list('0123456789'))
+                    for i in range(np.random.randint(1, DIGITS + 1))))
+    a, b = f(), f()
+    # Skip any addition questions we've already seen
+    # Also skip any such that x+Y == Y+x (hence the sorting).
+    key = tuple(sorted((a, b)))
+    if key in seen:
+        continue
+    seen.add(key)
+    # Pad the data with spaces such that it is always MAXLEN.
+    q = '{}+{}'.format(a, b)
+    query = q + ' ' * (MAXLEN - len(q))
+    ans = str(a + b)
+    # Answers can be of maximum size DIGITS + 1.
+    ans += ' ' * (DIGITS + 1 - len(ans))
+    if INVERT:
+        # Reverse the query, e.g., '12+345  ' becomes '  543+21'. (Note the
+        # space used for padding.)
+        query = query[::-1]
+    questions.append(query)
+    expected.append(ans)
+print('Total addition questions:', len(questions))
+
+print('Vectorization...')
+x = np.zeros((len(questions), MAXLEN, len(chars)), dtype=np.bool)
+y = np.zeros((len(questions), DIGITS + 1, len(chars)), dtype=np.bool)
+for i, sentence in enumerate(questions):
+    x[i] = ctable.encode(sentence, MAXLEN)
+for i, sentence in enumerate(expected):
+    y[i] = ctable.encode(sentence, DIGITS + 1)
+
+# Shuffle (x, y) in unison as the later parts of x will almost all be larger
+# digits.
+indices = np.arange(len(y))
+np.random.shuffle(indices)
+x = x[indices]
+y = y[indices]
+
+# Explicitly set apart 10% for validation data that we never train over.
+split_at = len(x) - len(x) // 10
+(x_train, x_val) = x[:split_at], x[split_at:]
+(y_train, y_val) = y[:split_at], y[split_at:]
+
+print('Training Data:')
+print(x_train.shape)
+print(y_train.shape)
+
+print('Validation Data:')
+print(x_val.shape)
+print(y_val.shape)
+
+# Try replacing GRU, or SimpleRNN.
+RNN = layers.LSTM
+HIDDEN_SIZE = 128
+BATCH_SIZE = 128
+LAYERS = 1
+
+print('Build model...')
+model = Sequential()
+# "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE.
+# Note: In a situation where your input sequences have a variable length,
+# use input_shape=(None, num_feature).
+model.add(RNN(HIDDEN_SIZE, input_shape=(MAXLEN, len(chars))))
+# As the decoder RNN's input, repeatedly provide with the last hidden state of
+# RNN for each time step. Repeat 'DIGITS + 1' times as that's the maximum
+# length of output, e.g., when DIGITS=3, max output is 999+999=1998.
+model.add(layers.RepeatVector(DIGITS + 1))
+# The decoder RNN could be multiple layers stacked or a single layer.
+for _ in range(LAYERS):
+    # By setting return_sequences to True, return not only the last output but
+    # all the outputs so far in the form of (num_samples, timesteps,
+    # output_dim). This is necessary as TimeDistributed in the below expects
+    # the first dimension to be the timesteps.
+    model.add(RNN(HIDDEN_SIZE, return_sequences=True))
+
+# Apply a dense layer to the every temporal slice of an input. For each of step
+# of the output sequence, decide which character should be chosen.
+model.add(layers.TimeDistributed(layers.Dense(len(chars))))
+model.add(layers.Activation('softmax'))
+model.compile(loss='categorical_crossentropy',
+              optimizer='adam',
+              metrics=['accuracy'])
+model.summary()
+
+# Train the model each generation and show predictions against the validation
+# dataset.
+for iteration in range(1, 200):
+    print()
+    print('-' * 50)
+    print('Iteration', iteration)
+    model.fit(x_train, y_train,
+              batch_size=BATCH_SIZE,
+              epochs=1,
+              validation_data=(x_val, y_val))
+    # Select 10 samples from the validation set at random so we can visualize
+    # errors.
+    for i in range(10):
+        ind = np.random.randint(0, len(x_val))
+        rowx, rowy = x_val[np.array([ind])], y_val[np.array([ind])]
+        preds = model.predict_classes(rowx, verbose=0)
+        q = ctable.decode(rowx[0])
+        correct = ctable.decode(rowy[0])
+        guess = ctable.decode(preds[0], calc_argmax=False)
+        print('Q', q[::-1] if INVERT else q)
+        print('T', correct)
+        if correct == guess:
+            print(colors.ok + '☑' + colors.close, end=" ")
+        else:
+            print(colors.fail + '☒' + colors.close, end=" ")
+        print(guess)
+        print('---')

examples/antirectifier.py

@@ -0,0 +1,107 @@
+'''The example demonstrates how to write custom layers for Keras.
+
+We build a custom activation layer called 'Antirectifier',
+which modifies the shape of the tensor that passes through it.
+We need to specify two methods: `compute_output_shape` and `call`.
+
+Note that the same result can also be achieved via a Lambda layer.
+
+Because our custom layer is written with primitives from the Keras
+backend (`K`), our code can run both on TensorFlow and Theano.
+'''
+
+from __future__ import print_function
+import keras
+from keras.models import Sequential
+from keras import layers
+from keras.datasets import mnist
+from keras import backend as K
+
+
+class Antirectifier(layers.Layer):
+    '''This is the combination of a sample-wise
+    L2 normalization with the concatenation of the
+    positive part of the input with the negative part
+    of the input. The result is a tensor of samples that are
+    twice as large as the input samples.
+
+    It can be used in place of a ReLU.
+
+    # Input shape
+        2D tensor of shape (samples, n)
+
+    # Output shape
+        2D tensor of shape (samples, 2*n)
+
+    # Theoretical justification
+        When applying ReLU, assuming that the distribution
+        of the previous output is approximately centered around 0.,
+        you are discarding half of your input. This is inefficient.
+
+        Antirectifier allows to return all-positive outputs like ReLU,
+        without discarding any data.
+
+        Tests on MNIST show that Antirectifier allows to train networks
+        with twice less parameters yet with comparable
+        classification accuracy as an equivalent ReLU-based network.
+    '''
+
+    def compute_output_shape(self, input_shape):
+        shape = list(input_shape)
+        assert len(shape) == 2  # only valid for 2D tensors
+        shape[-1] *= 2
+        return tuple(shape)
+
+    def call(self, inputs):
+        inputs -= K.mean(inputs, axis=1, keepdims=True)
+        inputs = K.l2_normalize(inputs, axis=1)
+        pos = K.relu(inputs)
+        neg = K.relu(-inputs)
+        return K.concatenate([pos, neg], axis=1)
+
+# global parameters
+batch_size = 128
+num_classes = 10
+epochs = 40
+
+# the data, shuffled and split between train and test sets
+(x_train, y_train), (x_test, y_test) = mnist.load_data()
+
+x_train = x_train.reshape(60000, 784)
+x_test = x_test.reshape(10000, 784)
+x_train = x_train.astype('float32')
+x_test = x_test.astype('float32')
+x_train /= 255
+x_test /= 255
+print(x_train.shape[0], 'train samples')
+print(x_test.shape[0], 'test samples')
+
+# convert class vectors to binary class matrices
+y_train = keras.utils.to_categorical(y_train, num_classes)
+y_test = keras.utils.to_categorical(y_test, num_classes)
+
+# build the model
+model = Sequential()
+model.add(layers.Dense(256, input_shape=(784,)))
+model.add(Antirectifier())
+model.add(layers.Dropout(0.1))
+model.add(layers.Dense(256))
+model.add(Antirectifier())
+model.add(layers.Dropout(0.1))
+model.add(layers.Dense(10))
+model.add(layers.Activation('softmax'))
+
+# compile the model
+model.compile(loss='categorical_crossentropy',
+              optimizer='rmsprop',
+              metrics=['accuracy'])
+
+# train the model
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=epochs,
+          verbose=1,
+          validation_data=(x_test, y_test))
+
+# next, compare with an equivalent network
+# with2x bigger Dense layers and ReLU

examples/babi_memnn.py

@@ -0,0 +1,235 @@
+'''Trains a memory network on the bAbI dataset.
+
+References:
+- Jason Weston, Antoine Bordes, Sumit Chopra, Tomas Mikolov, Alexander M. Rush,
+  "Towards AI-Complete Question Answering: A Set of Prerequisite Toy Tasks",
+  http://arxiv.org/abs/1502.05698
+
+- Sainbayar Sukhbaatar, Arthur Szlam, Jason Weston, Rob Fergus,
+  "End-To-End Memory Networks",
+  http://arxiv.org/abs/1503.08895
+
+Reaches 98.6% accuracy on task 'single_supporting_fact_10k' after 120 epochs.
+Time per epoch: 3s on CPU (core i7).
+'''
+from __future__ import print_function
+
+from keras.models import Sequential, Model
+from keras.layers.embeddings import Embedding
+from keras.layers import Input, Activation, Dense, Permute, Dropout, add, dot, concatenate
+from keras.layers import LSTM
+from keras.utils.data_utils import get_file
+from keras.preprocessing.sequence import pad_sequences
+from functools import reduce
+import tarfile
+import numpy as np
+import re
+
+
+def tokenize(sent):
+    '''Return the tokens of a sentence including punctuation.
+
+    >>> tokenize('Bob dropped the apple. Where is the apple?')
+    ['Bob', 'dropped', 'the', 'apple', '.', 'Where', 'is', 'the', 'apple', '?']
+    '''
+    return [x.strip() for x in re.split('(\W+)?', sent) if x.strip()]
+
+
+def parse_stories(lines, only_supporting=False):
+    '''Parse stories provided in the bAbi tasks format
+
+    If only_supporting is true, only the sentences
+    that support the answer are kept.
+    '''
+    data = []
+    story = []
+    for line in lines:
+        line = line.decode('utf-8').strip()
+        nid, line = line.split(' ', 1)
+        nid = int(nid)
+        if nid == 1:
+            story = []
+        if '\t' in line:
+            q, a, supporting = line.split('\t')
+            q = tokenize(q)
+            substory = None
+            if only_supporting:
+                # Only select the related substory
+                supporting = map(int, supporting.split())
+                substory = [story[i - 1] for i in supporting]
+            else:
+                # Provide all the substories
+                substory = [x for x in story if x]
+            data.append((substory, q, a))
+            story.append('')
+        else:
+            sent = tokenize(line)
+            story.append(sent)
+    return data
+
+
+def get_stories(f, only_supporting=False, max_length=None):
+    '''Given a file name, read the file,
+    retrieve the stories,
+    and then convert the sentences into a single story.
+
+    If max_length is supplied,
+    any stories longer than max_length tokens will be discarded.
+    '''
+    data = parse_stories(f.readlines(), only_supporting=only_supporting)
+    flatten = lambda data: reduce(lambda x, y: x + y, data)
+    data = [(flatten(story), q, answer) for story, q, answer in data if not max_length or len(flatten(story)) < max_length]
+    return data
+
+
+def vectorize_stories(data, word_idx, story_maxlen, query_maxlen):
+    X = []
+    Xq = []
+    Y = []
+    for story, query, answer in data:
+        x = [word_idx[w] for w in story]
+        xq = [word_idx[w] for w in query]
+        # let's not forget that index 0 is reserved
+        y = np.zeros(len(word_idx) + 1)
+        y[word_idx[answer]] = 1
+        X.append(x)
+        Xq.append(xq)
+        Y.append(y)
+    return (pad_sequences(X, maxlen=story_maxlen),
+            pad_sequences(Xq, maxlen=query_maxlen), np.array(Y))
+
+try:
+    path = get_file('babi-tasks-v1-2.tar.gz', origin='https://s3.amazonaws.com/text-datasets/babi_tasks_1-20_v1-2.tar.gz')
+except:
+    print('Error downloading dataset, please download it manually:\n'
+          '$ wget http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz\n'
+          '$ mv tasks_1-20_v1-2.tar.gz ~/.keras/datasets/babi-tasks-v1-2.tar.gz')
+    raise
+tar = tarfile.open(path)
+
+challenges = {
+    # QA1 with 10,000 samples
+    'single_supporting_fact_10k': 'tasks_1-20_v1-2/en-10k/qa1_single-supporting-fact_{}.txt',
+    # QA2 with 10,000 samples
+    'two_supporting_facts_10k': 'tasks_1-20_v1-2/en-10k/qa2_two-supporting-facts_{}.txt',
+}
+challenge_type = 'single_supporting_fact_10k'
+challenge = challenges[challenge_type]
+
+print('Extracting stories for the challenge:', challenge_type)
+train_stories = get_stories(tar.extractfile(challenge.format('train')))
+test_stories = get_stories(tar.extractfile(challenge.format('test')))
+
+vocab = set()
+for story, q, answer in train_stories + test_stories:
+    vocab |= set(story + q + [answer])
+vocab = sorted(vocab)
+
+# Reserve 0 for masking via pad_sequences
+vocab_size = len(vocab) + 1
+story_maxlen = max(map(len, (x for x, _, _ in train_stories + test_stories)))
+query_maxlen = max(map(len, (x for _, x, _ in train_stories + test_stories)))
+
+print('-')
+print('Vocab size:', vocab_size, 'unique words')
+print('Story max length:', story_maxlen, 'words')
+print('Query max length:', query_maxlen, 'words')
+print('Number of training stories:', len(train_stories))
+print('Number of test stories:', len(test_stories))
+print('-')
+print('Here\'s what a "story" tuple looks like (input, query, answer):')
+print(train_stories[0])
+print('-')
+print('Vectorizing the word sequences...')
+
+word_idx = dict((c, i + 1) for i, c in enumerate(vocab))
+inputs_train, queries_train, answers_train = vectorize_stories(train_stories,
+                                                               word_idx,
+                                                               story_maxlen,
+                                                               query_maxlen)
+inputs_test, queries_test, answers_test = vectorize_stories(test_stories,
+                                                            word_idx,
+                                                            story_maxlen,
+                                                            query_maxlen)
+
+print('-')
+print('inputs: integer tensor of shape (samples, max_length)')
+print('inputs_train shape:', inputs_train.shape)
+print('inputs_test shape:', inputs_test.shape)
+print('-')
+print('queries: integer tensor of shape (samples, max_length)')
+print('queries_train shape:', queries_train.shape)
+print('queries_test shape:', queries_test.shape)
+print('-')
+print('answers: binary (1 or 0) tensor of shape (samples, vocab_size)')
+print('answers_train shape:', answers_train.shape)
+print('answers_test shape:', answers_test.shape)
+print('-')
+print('Compiling...')
+
+# placeholders
+input_sequence = Input((story_maxlen,))
+question = Input((query_maxlen,))
+
+# encoders
+# embed the input sequence into a sequence of vectors
+input_encoder_m = Sequential()
+input_encoder_m.add(Embedding(input_dim=vocab_size,
+                              output_dim=64))
+input_encoder_m.add(Dropout(0.3))
+# output: (samples, story_maxlen, embedding_dim)
+
+# embed the input into a sequence of vectors of size query_maxlen
+input_encoder_c = Sequential()
+input_encoder_c.add(Embedding(input_dim=vocab_size,
+                              output_dim=query_maxlen))
+input_encoder_c.add(Dropout(0.3))
+# output: (samples, story_maxlen, query_maxlen)
+
+# embed the question into a sequence of vectors
+question_encoder = Sequential()
+question_encoder.add(Embedding(input_dim=vocab_size,
+                               output_dim=64,
+                               input_length=query_maxlen))
+question_encoder.add(Dropout(0.3))
+# output: (samples, query_maxlen, embedding_dim)
+
+# encode input sequence and questions (which are indices)
+# to sequences of dense vectors
+input_encoded_m = input_encoder_m(input_sequence)
+input_encoded_c = input_encoder_c(input_sequence)
+question_encoded = question_encoder(question)
+
+# compute a 'match' between the first input vector sequence
+# and the question vector sequence
+# shape: `(samples, story_maxlen, query_maxlen)`
+match = dot([input_encoded_m, question_encoded], axes=(2, 2))
+match = Activation('softmax')(match)
+
+# add the match matrix with the second input vector sequence
+response = add([match, input_encoded_c])  # (samples, story_maxlen, query_maxlen)
+response = Permute((2, 1))(response)  # (samples, query_maxlen, story_maxlen)
+
+# concatenate the match matrix with the question vector sequence
+answer = concatenate([response, question_encoded])
+
+# the original paper uses a matrix multiplication for this reduction step.
+# we choose to use a RNN instead.
+answer = LSTM(32)(answer)  # (samples, 32)
+
+# one regularization layer -- more would probably be needed.
+answer = Dropout(0.3)(answer)
+answer = Dense(vocab_size)(answer)  # (samples, vocab_size)
+# we output a probability distribution over the vocabulary
+answer = Activation('softmax')(answer)
+
+# build the final model
+model = Model([input_sequence, question], answer)
+model.compile(optimizer='rmsprop', loss='categorical_crossentropy',
+              metrics=['accuracy'])
+
+# train
+model.fit([inputs_train, queries_train], answers_train,
+          batch_size=32,
+          epochs=120,
+          validation_data=([inputs_test, queries_test], answers_test))

examples/babi_rnn.py

@@ -0,0 +1,223 @@
+'''Trains two recurrent neural networks based upon a story and a question.
+The resulting merged vector is then queried to answer a range of bAbI tasks.
+
+The results are comparable to those for an LSTM model provided in Weston et al.:
+"Towards AI-Complete Question Answering: A Set of Prerequisite Toy Tasks"
+http://arxiv.org/abs/1502.05698
+
+Task Number                  | FB LSTM Baseline | Keras QA
+---                          | ---              | ---
+QA1 - Single Supporting Fact | 50               | 100.0
+QA2 - Two Supporting Facts   | 20               | 50.0
+QA3 - Three Supporting Facts | 20               | 20.5
+QA4 - Two Arg. Relations     | 61               | 62.9
+QA5 - Three Arg. Relations   | 70               | 61.9
+QA6 - yes/No Questions       | 48               | 50.7
+QA7 - Counting               | 49               | 78.9
+QA8 - Lists/Sets             | 45               | 77.2
+QA9 - Simple Negation        | 64               | 64.0
+QA10 - Indefinite Knowledge  | 44               | 47.7
+QA11 - Basic Coreference     | 72               | 74.9
+QA12 - Conjunction           | 74               | 76.4
+QA13 - Compound Coreference  | 94               | 94.4
+QA14 - Time Reasoning        | 27               | 34.8
+QA15 - Basic Deduction       | 21               | 32.4
+QA16 - Basic Induction       | 23               | 50.6
+QA17 - Positional Reasoning  | 51               | 49.1
+QA18 - Size Reasoning        | 52               | 90.8
+QA19 - Path Finding          | 8                | 9.0
+QA20 - Agent's Motivations   | 91               | 90.7
+
+For the resources related to the bAbI project, refer to:
+https://research.facebook.com/researchers/1543934539189348
+
+Notes:
+
+- With default word, sentence, and query vector sizes, the GRU model achieves:
+  - 100% test accuracy on QA1 in 20 epochs (2 seconds per epoch on CPU)
+  - 50% test accuracy on QA2 in 20 epochs (16 seconds per epoch on CPU)
+In comparison, the Facebook paper achieves 50% and 20% for the LSTM baseline.
+
+- The task does not traditionally parse the question separately. This likely
+improves accuracy and is a good example of merging two RNNs.
+
+- The word vector embeddings are not shared between the story and question RNNs.
+
+- See how the accuracy changes given 10,000 training samples (en-10k) instead
+of only 1000. 1000 was used in order to be comparable to the original paper.
+
+- Experiment with GRU, LSTM, and JZS1-3 as they give subtly different results.
+
+- The length and noise (i.e. 'useless' story components) impact the ability for
+LSTMs / GRUs to provide the correct answer. Given only the supporting facts,
+these RNNs can achieve 100% accuracy on many tasks. Memory networks and neural
+networks that use attentional processes can efficiently search through this
+noise to find the relevant statements, improving performance substantially.
+This becomes especially obvious on QA2 and QA3, both far longer than QA1.
+'''
+
+from __future__ import print_function
+from functools import reduce
+import re
+import tarfile
+
+import numpy as np
+
+from keras.utils.data_utils import get_file
+from keras.layers.embeddings import Embedding
+from keras import layers
+from keras.layers import recurrent
+from keras.models import Model
+from keras.preprocessing.sequence import pad_sequences
+
+
+def tokenize(sent):
+    '''Return the tokens of a sentence including punctuation.
+
+    >>> tokenize('Bob dropped the apple. Where is the apple?')
+    ['Bob', 'dropped', 'the', 'apple', '.', 'Where', 'is', 'the', 'apple', '?']
+    '''
+    return [x.strip() for x in re.split('(\W+)?', sent) if x.strip()]
+
+
+def parse_stories(lines, only_supporting=False):
+    '''Parse stories provided in the bAbi tasks format
+
+    If only_supporting is true,
+    only the sentences that support the answer are kept.
+    '''
+    data = []
+    story = []
+    for line in lines:
+        line = line.decode('utf-8').strip()
+        nid, line = line.split(' ', 1)
+        nid = int(nid)
+        if nid == 1:
+            story = []
+        if '\t' in line:
+            q, a, supporting = line.split('\t')
+            q = tokenize(q)
+            substory = None
+            if only_supporting:
+                # Only select the related substory
+                supporting = map(int, supporting.split())
+                substory = [story[i - 1] for i in supporting]
+            else:
+                # Provide all the substories
+                substory = [x for x in story if x]
+            data.append((substory, q, a))
+            story.append('')
+        else:
+            sent = tokenize(line)
+            story.append(sent)
+    return data
+
+
+def get_stories(f, only_supporting=False, max_length=None):
+    '''Given a file name, read the file, retrieve the stories,
+    and then convert the sentences into a single story.
+
+    If max_length is supplied,
+    any stories longer than max_length tokens will be discarded.
+    '''
+    data = parse_stories(f.readlines(), only_supporting=only_supporting)
+    flatten = lambda data: reduce(lambda x, y: x + y, data)
+    data = [(flatten(story), q, answer) for story, q, answer in data if not max_length or len(flatten(story)) < max_length]
+    return data
+
+
+def vectorize_stories(data, word_idx, story_maxlen, query_maxlen):
+    xs = []
+    xqs = []
+    ys = []
+    for story, query, answer in data:
+        x = [word_idx[w] for w in story]
+        xq = [word_idx[w] for w in query]
+        # let's not forget that index 0 is reserved
+        y = np.zeros(len(word_idx) + 1)
+        y[word_idx[answer]] = 1
+        xs.append(x)
+        xqs.append(xq)
+        ys.append(y)
+    return pad_sequences(xs, maxlen=story_maxlen), pad_sequences(xqs, maxlen=query_maxlen), np.array(ys)
+
+RNN = recurrent.LSTM
+EMBED_HIDDEN_SIZE = 50
+SENT_HIDDEN_SIZE = 100
+QUERY_HIDDEN_SIZE = 100
+BATCH_SIZE = 32
+EPOCHS = 40
+print('RNN / Embed / Sent / Query = {}, {}, {}, {}'.format(RNN,
+                                                           EMBED_HIDDEN_SIZE,
+                                                           SENT_HIDDEN_SIZE,
+                                                           QUERY_HIDDEN_SIZE))
+
+try:
+    path = get_file('babi-tasks-v1-2.tar.gz', origin='https://s3.amazonaws.com/text-datasets/babi_tasks_1-20_v1-2.tar.gz')
+except:
+    print('Error downloading dataset, please download it manually:\n'
+          '$ wget http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz\n'
+          '$ mv tasks_1-20_v1-2.tar.gz ~/.keras/datasets/babi-tasks-v1-2.tar.gz')
+    raise
+tar = tarfile.open(path)
+# Default QA1 with 1000 samples
+# challenge = 'tasks_1-20_v1-2/en/qa1_single-supporting-fact_{}.txt'
+# QA1 with 10,000 samples
+# challenge = 'tasks_1-20_v1-2/en-10k/qa1_single-supporting-fact_{}.txt'
+# QA2 with 1000 samples
+challenge = 'tasks_1-20_v1-2/en/qa2_two-supporting-facts_{}.txt'
+# QA2 with 10,000 samples
+# challenge = 'tasks_1-20_v1-2/en-10k/qa2_two-supporting-facts_{}.txt'
+train = get_stories(tar.extractfile(challenge.format('train')))
+test = get_stories(tar.extractfile(challenge.format('test')))
+
+vocab = set()
+for story, q, answer in train + test:
+    vocab |= set(story + q + [answer])
+vocab = sorted(vocab)
+
+# Reserve 0 for masking via pad_sequences
+vocab_size = len(vocab) + 1
+word_idx = dict((c, i + 1) for i, c in enumerate(vocab))
+story_maxlen = max(map(len, (x for x, _, _ in train + test)))
+query_maxlen = max(map(len, (x for _, x, _ in train + test)))
+
+x, xq, y = vectorize_stories(train, word_idx, story_maxlen, query_maxlen)
+tx, txq, ty = vectorize_stories(test, word_idx, story_maxlen, query_maxlen)
+
+print('vocab = {}'.format(vocab))
+print('x.shape = {}'.format(x.shape))
+print('xq.shape = {}'.format(xq.shape))
+print('y.shape = {}'.format(y.shape))
+print('story_maxlen, query_maxlen = {}, {}'.format(story_maxlen, query_maxlen))
+
+print('Build model...')
+
+sentence = layers.Input(shape=(story_maxlen,), dtype='int32')
+encoded_sentence = layers.Embedding(vocab_size, EMBED_HIDDEN_SIZE)(sentence)
+encoded_sentence = layers.Dropout(0.3)(encoded_sentence)
+
+question = layers.Input(shape=(query_maxlen,), dtype='int32')
+encoded_question = layers.Embedding(vocab_size, EMBED_HIDDEN_SIZE)(question)
+encoded_question = layers.Dropout(0.3)(encoded_question)
+encoded_question = RNN(EMBED_HIDDEN_SIZE)(encoded_question)
+encoded_question = layers.RepeatVector(story_maxlen)(encoded_question)
+
+merged = layers.add([encoded_sentence, encoded_question])
+merged = RNN(EMBED_HIDDEN_SIZE)(merged)
+merged = layers.Dropout(0.3)(merged)
+preds = layers.Dense(vocab_size, activation='softmax')(merged)
+
+model = Model([sentence, question], preds)
+model.compile(optimizer='adam',
+              loss='categorical_crossentropy',
+              metrics=['accuracy'])
+
+print('Training')
+model.fit([x, xq], y,
+          batch_size=BATCH_SIZE,
+          epochs=EPOCHS,
+          validation_split=0.05)
+loss, acc = model.evaluate([tx, txq], ty,
+                           batch_size=BATCH_SIZE)
+print('Test loss / test accuracy = {:.4f} / {:.4f}'.format(loss, acc))

examples/cifar10_cnn.py

@@ -1,123 +1,101 @@
-from __future__ import absolute_import
+'''Train a simple deep CNN on the CIFAR10 small images dataset.
+
+GPU run command with Theano backend (with TensorFlow, the GPU is automatically used):
+    THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatx=float32 python cifar10_cnn.py
+
+It gets down to 0.65 test logloss in 25 epochs, and down to 0.55 after 50 epochs.
+(it's still underfitting at that point, though).
+'''
+
 from __future__ import print_function
+import keras
 from keras.datasets import cifar10
 from keras.preprocessing.image import ImageDataGenerator
 from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation, Flatten
-from keras.layers.convolutional import Convolution2D, MaxPooling2D
-from keras.optimizers import SGD, Adadelta, Adagrad
-from keras.utils import np_utils, generic_utils
-from six.moves import range
-
-'''
-    Train a (fairly simple) deep CNN on the CIFAR10 small images dataset.
-
-    GPU run command:
-        THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python cifar10_cnn.py
-
-    It gets down to 0.65 test logloss in 25 epochs, and down to 0.55 after 50 epochs.
-    (it's still underfitting at that point, though).
-
-    Note: the data was pickled with Python 2, and some encoding issues might prevent you
-    from loading it in Python 3. You might have to load it in Python 2, 
-    save it in a different format, load it in Python 3 and repickle it.
-'''
+from keras.layers import Dense, Dropout, Activation, Flatten
+from keras.layers import Conv2D, MaxPooling2D
 
 batch_size = 32
-nb_classes = 10
-nb_epoch = 200
+num_classes = 10
+epochs = 200
 data_augmentation = True
 
-# the data, shuffled and split between tran and test sets
-(X_train, y_train), (X_test, y_test) = cifar10.load_data()
-print(X_train.shape[0], 'train samples')
-print(X_test.shape[0], 'test samples')
+# The data, shuffled and split between train and test sets:
+(x_train, y_train), (x_test, y_test) = cifar10.load_data()
+print('x_train shape:', x_train.shape)
+print(x_train.shape[0], 'train samples')
+print(x_test.shape[0], 'test samples')
 
-# convert class vectors to binary class matrices
-Y_train = np_utils.to_categorical(y_train, nb_classes)
-Y_test = np_utils.to_categorical(y_test, nb_classes)
+# Convert class vectors to binary class matrices.
+y_train = keras.utils.to_categorical(y_train, num_classes)
+y_test = keras.utils.to_categorical(y_test, num_classes)
 
 model = Sequential()
 
-model.add(Convolution2D(32, 3, 3, 3, border_mode='full')) 
+model.add(Conv2D(32, (3, 3), padding='same',
+                 input_shape=x_train.shape[1:]))
 model.add(Activation('relu'))
-model.add(Convolution2D(32, 32, 3, 3))
+model.add(Conv2D(32, (3, 3)))
 model.add(Activation('relu'))
-model.add(MaxPooling2D(poolsize=(2, 2)))
+model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
 
-model.add(Convolution2D(64, 32, 3, 3, border_mode='full')) 
+model.add(Conv2D(64, (3, 3), padding='same'))
 model.add(Activation('relu'))
-model.add(Convolution2D(64, 64, 3, 3)) 
+model.add(Conv2D(64, (3, 3)))
 model.add(Activation('relu'))
-model.add(MaxPooling2D(poolsize=(2, 2)))
+model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
 
 model.add(Flatten())
-model.add(Dense(64*8*8, 512, init='normal'))
+model.add(Dense(512))
 model.add(Activation('relu'))
 model.add(Dropout(0.5))
-
-model.add(Dense(512, nb_classes, init='normal'))
+model.add(Dense(num_classes))
 model.add(Activation('softmax'))
 
-# let's train the model using SGD + momentum (how original).
-sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
-model.compile(loss='categorical_crossentropy', optimizer=sgd)
+# initiate RMSprop optimizer
+opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6)
+
+# Let's train the model using RMSprop
+model.compile(loss='categorical_crossentropy',
+              optimizer=opt,
+              metrics=['accuracy'])
+
+x_train = x_train.astype('float32')
+x_test = x_test.astype('float32')
+x_train /= 255
+x_test /= 255
 
 if not data_augmentation:
-    print("Not using data augmentation or normalization")
-
-    X_train = X_train.astype("float32")
-    X_test = X_test.astype("float32")
-    X_train /= 255
-    X_test /= 255
-    model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=10)
-    score = model.evaluate(X_test, Y_test, batch_size=batch_size)
-    print('Test score:', score)
-
+    print('Not using data augmentation.')
+    model.fit(x_train, y_train,
+              batch_size=batch_size,
+              epochs=epochs,
+              validation_data=(x_test, y_test),
+              shuffle=True)
 else:
-    print("Using real time data augmentation")
-
-    # this will do preprocessing and realtime data augmentation
+    print('Using real-time data augmentation.')
+    # This will do preprocessing and realtime data augmentation:
     datagen = ImageDataGenerator(
-        featurewise_center=True, # set input mean to 0 over the dataset
-        samplewise_center=False, # set each sample mean to 0
-        featurewise_std_normalization=True, # divide inputs by std of the dataset
-        samplewise_std_normalization=False, # divide each input by its std
-        zca_whitening=False, # apply ZCA whitening
-        rotation_range=20, # randomly rotate images in the range (degrees, 0 to 180)
-        width_shift_range=0.2, # randomly shift images horizontally (fraction of total width)
-        height_shift_range=0.2, # randomly shift images vertically (fraction of total height)
-        horizontal_flip=True, # randomly flip images
-        vertical_flip=False) # randomly flip images
-
-    # compute quantities required for featurewise normalization 
-    # (std, mean, and principal components if ZCA whitening is applied)
-    datagen.fit(X_train)
-
-    for e in range(nb_epoch):
-        print('-'*40)
-        print('Epoch', e)
-        print('-'*40)
-        print("Training...")
-        # batch train with realtime data augmentation
-        progbar = generic_utils.Progbar(X_train.shape[0])
-        for X_batch, Y_batch in datagen.flow(X_train, Y_train):
-            loss = model.train(X_batch, Y_batch)
-            progbar.add(X_batch.shape[0], values=[("train loss", loss)])
-
-        print("Testing...")
-        # test time!
-        progbar = generic_utils.Progbar(X_test.shape[0])
-        for X_batch, Y_batch in datagen.flow(X_test, Y_test):
-            score = model.test(X_batch, Y_batch)
-            progbar.add(X_batch.shape[0], values=[("test loss", score)])
-
-            
-
-
-
-
+        featurewise_center=False,  # set input mean to 0 over the dataset
+        samplewise_center=False,  # set each sample mean to 0
+        featurewise_std_normalization=False,  # divide inputs by std of the dataset
+        samplewise_std_normalization=False,  # divide each input by its std
+        zca_whitening=False,  # apply ZCA whitening
+        rotation_range=0,  # randomly rotate images in the range (degrees, 0 to 180)
+        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)
+        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)
+        horizontal_flip=True,  # randomly flip images
+        vertical_flip=False)  # randomly flip images
 
+    # Compute quantities required for feature-wise normalization
+    # (std, mean, and principal components if ZCA whitening is applied).
+    datagen.fit(x_train)
 
+    # Fit the model on the batches generated by datagen.flow().
+    model.fit_generator(datagen.flow(x_train, y_train,
+                                     batch_size=batch_size),
+                        steps_per_epoch=x_train.shape[0] // batch_size,
+                        epochs=epochs,
+                        validation_data=(x_test, y_test))

examples/conv_filter_visualization.py

@@ -0,0 +1,135 @@
+'''Visualization of the filters of VGG16, via gradient ascent in input space.
+
+This script can run on CPU in a few minutes (with the TensorFlow backend).
+
+Results example: http://i.imgur.com/4nj4KjN.jpg
+'''
+from __future__ import print_function
+
+from scipy.misc import imsave
+import numpy as np
+import time
+from keras.applications import vgg16
+from keras import backend as K
+
+# dimensions of the generated pictures for each filter.
+img_width = 128
+img_height = 128
+
+# the name of the layer we want to visualize
+# (see model definition at keras/applications/vgg16.py)
+layer_name = 'block5_conv1'
+
+# util function to convert a tensor into a valid image
+
+
+def deprocess_image(x):
+    # normalize tensor: center on 0., ensure std is 0.1
+    x -= x.mean()
+    x /= (x.std() + 1e-5)
+    x *= 0.1
+
+    # clip to [0, 1]
+    x += 0.5
+    x = np.clip(x, 0, 1)
+
+    # convert to RGB array
+    x *= 255
+    if K.image_data_format() == 'channels_first':
+        x = x.transpose((1, 2, 0))
+    x = np.clip(x, 0, 255).astype('uint8')
+    return x
+
+# build the VGG16 network with ImageNet weights
+model = vgg16.VGG16(weights='imagenet', include_top=False)
+print('Model loaded.')
+
+model.summary()
+
+# this is the placeholder for the input images
+input_img = model.input
+
+# get the symbolic outputs of each "key" layer (we gave them unique names).
+layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
+
+
+def normalize(x):
+    # utility function to normalize a tensor by its L2 norm
+    return x / (K.sqrt(K.mean(K.square(x))) + 1e-5)
+
+
+kept_filters = []
+for filter_index in range(0, 200):
+    # we only scan through the first 200 filters,
+    # but there are actually 512 of them
+    print('Processing filter %d' % filter_index)
+    start_time = time.time()
+
+    # we build a loss function that maximizes the activation
+    # of the nth filter of the layer considered
+    layer_output = layer_dict[layer_name].output
+    if K.image_data_format() == 'channels_first':
+        loss = K.mean(layer_output[:, filter_index, :, :])
+    else:
+        loss = K.mean(layer_output[:, :, :, filter_index])
+
+    # we compute the gradient of the input picture wrt this loss
+    grads = K.gradients(loss, input_img)[0]
+
+    # normalization trick: we normalize the gradient
+    grads = normalize(grads)
+
+    # this function returns the loss and grads given the input picture
+    iterate = K.function([input_img], [loss, grads])
+
+    # step size for gradient ascent
+    step = 1.
+
+    # we start from a gray image with some random noise
+    if K.image_data_format() == 'channels_first':
+        input_img_data = np.random.random((1, 3, img_width, img_height))
+    else:
+        input_img_data = np.random.random((1, img_width, img_height, 3))
+    input_img_data = (input_img_data - 0.5) * 20 + 128
+
+    # we run gradient ascent for 20 steps
+    for i in range(20):
+        loss_value, grads_value = iterate([input_img_data])
+        input_img_data += grads_value * step
+
+        print('Current loss value:', loss_value)
+        if loss_value <= 0.:
+            # some filters get stuck to 0, we can skip them
+            break
+
+    # decode the resulting input image
+    if loss_value > 0:
+        img = deprocess_image(input_img_data[0])
+        kept_filters.append((img, loss_value))
+    end_time = time.time()
+    print('Filter %d processed in %ds' % (filter_index, end_time - start_time))
+
+# we will stich the best 64 filters on a 8 x 8 grid.
+n = 8
+
+# the filters that have the highest loss are assumed to be better-looking.
+# we will only keep the top 64 filters.
+kept_filters.sort(key=lambda x: x[1], reverse=True)
+kept_filters = kept_filters[:n * n]
+
+# build a black picture with enough space for
+# our 8 x 8 filters of size 128 x 128, with a 5px margin in between
+margin = 5
+width = n * img_width + (n - 1) * margin
+height = n * img_height + (n - 1) * margin
+stitched_filters = np.zeros((width, height, 3))
+
+# fill the picture with our saved filters
+for i in range(n):
+    for j in range(n):
+        img, loss = kept_filters[i * n + j]
+        stitched_filters[(img_width + margin) * i: (img_width + margin) * i + img_width,
+                         (img_height + margin) * j: (img_height + margin) * j + img_height, :] = img
+
+# save the result to disk
+imsave('stitched_filters_%dx%d.png' % (n, n), stitched_filters)

examples/conv_lstm.py

@@ -0,0 +1,141 @@
+""" This script demonstrates the use of a convolutional LSTM network.
+This network is used to predict the next frame of an artificially
+generated movie which contains moving squares.
+"""
+from keras.models import Sequential
+from keras.layers.convolutional import Conv3D
+from keras.layers.convolutional_recurrent import ConvLSTM2D
+from keras.layers.normalization import BatchNormalization
+import numpy as np
+import pylab as plt
+
+# We create a layer which take as input movies of shape
+# (n_frames, width, height, channels) and returns a movie
+# of identical shape.
+
+seq = Sequential()
+seq.add(ConvLSTM2D(filters=40, kernel_size=(3, 3),
+                   input_shape=(None, 40, 40, 1),
+                   padding='same', return_sequences=True))
+seq.add(BatchNormalization())
+
+seq.add(ConvLSTM2D(filters=40, kernel_size=(3, 3),
+                   padding='same', return_sequences=True))
+seq.add(BatchNormalization())
+
+seq.add(ConvLSTM2D(filters=40, kernel_size=(3, 3),
+                   padding='same', return_sequences=True))
+seq.add(BatchNormalization())
+
+seq.add(ConvLSTM2D(filters=40, kernel_size=(3, 3),
+                   padding='same', return_sequences=True))
+seq.add(BatchNormalization())
+
+seq.add(Conv3D(filters=1, kernel_size=(3, 3, 3),
+               activation='sigmoid',
+               padding='same', data_format='channels_last'))
+seq.compile(loss='binary_crossentropy', optimizer='adadelta')
+
+
+# Artificial data generation:
+# Generate movies with 3 to 7 moving squares inside.
+# The squares are of shape 1x1 or 2x2 pixels,
+# which move linearly over time.
+# For convenience we first create movies with bigger width and height (80x80)
+# and at the end we select a 40x40 window.
+
+def generate_movies(n_samples=1200, n_frames=15):
+    row = 80
+    col = 80
+    noisy_movies = np.zeros((n_samples, n_frames, row, col, 1), dtype=np.float)
+    shifted_movies = np.zeros((n_samples, n_frames, row, col, 1),
+                              dtype=np.float)
+
+    for i in range(n_samples):
+        # Add 3 to 7 moving squares
+        n = np.random.randint(3, 8)
+
+        for j in range(n):
+            # Initial position
+            xstart = np.random.randint(20, 60)
+            ystart = np.random.randint(20, 60)
+            # Direction of motion
+            directionx = np.random.randint(0, 3) - 1
+            directiony = np.random.randint(0, 3) - 1
+
+            # Size of the square
+            w = np.random.randint(2, 4)
+
+            for t in range(n_frames):
+                x_shift = xstart + directionx * t
+                y_shift = ystart + directiony * t
+                noisy_movies[i, t, x_shift - w: x_shift + w,
+                             y_shift - w: y_shift + w, 0] += 1
+
+                # Make it more robust by adding noise.
+                # The idea is that if during inference,
+                # the value of the pixel is not exactly one,
+                # we need to train the network to be robust and still
+                # consider it as a pixel belonging to a square.
+                if np.random.randint(0, 2):
+                    noise_f = (-1)**np.random.randint(0, 2)
+                    noisy_movies[i, t,
+                                 x_shift - w - 1: x_shift + w + 1,
+                                 y_shift - w - 1: y_shift + w + 1,
+                                 0] += noise_f * 0.1
+
+                # Shift the ground truth by 1
+                x_shift = xstart + directionx * (t + 1)
+                y_shift = ystart + directiony * (t + 1)
+                shifted_movies[i, t, x_shift - w: x_shift + w,
+                               y_shift - w: y_shift + w, 0] += 1
+
+    # Cut to a 40x40 window
+    noisy_movies = noisy_movies[::, ::, 20:60, 20:60, ::]
+    shifted_movies = shifted_movies[::, ::, 20:60, 20:60, ::]
+    noisy_movies[noisy_movies >= 1] = 1
+    shifted_movies[shifted_movies >= 1] = 1
+    return noisy_movies, shifted_movies
+
+# Train the network
+noisy_movies, shifted_movies = generate_movies(n_samples=1200)
+seq.fit(noisy_movies[:1000], shifted_movies[:1000], batch_size=10,
+        epochs=300, validation_split=0.05)
+
+# Testing the network on one movie
+# feed it with the first 7 positions and then
+# predict the new positions
+which = 1004
+track = noisy_movies[which][:7, ::, ::, ::]
+
+for j in range(16):
+    new_pos = seq.predict(track[np.newaxis, ::, ::, ::, ::])
+    new = new_pos[::, -1, ::, ::, ::]
+    track = np.concatenate((track, new), axis=0)
+
+
+# And then compare the predictions
+# to the ground truth
+track2 = noisy_movies[which][::, ::, ::, ::]
+for i in range(15):
+    fig = plt.figure(figsize=(10, 5))
+
+    ax = fig.add_subplot(121)
+
+    if i >= 7:
+        ax.text(1, 3, 'Predictions !', fontsize=20, color='w')
+    else:
+        ax.text(1, 3, 'Inital trajectory', fontsize=20)
+
+    toplot = track[i, ::, ::, 0]
+
+    plt.imshow(toplot)
+    ax = fig.add_subplot(122)
+    plt.text(1, 3, 'Ground truth', fontsize=20)
+
+    toplot = track2[i, ::, ::, 0]
+    if i >= 2:
+        toplot = shifted_movies[which][i - 1, ::, ::, 0]
+
+    plt.imshow(toplot)
+    plt.savefig('%i_animate.png' % (i + 1))

examples/deep_dream.py

@@ -0,0 +1,195 @@
+'''Deep Dreaming in Keras.
+
+Run the script with:
+```
+python deep_dream.py path_to_your_base_image.jpg prefix_for_results
+```
+e.g.:
+```
+python deep_dream.py img/mypic.jpg results/dream
+```
+'''
+from __future__ import print_function
+
+from keras.preprocessing.image import load_img, img_to_array
+import numpy as np
+import scipy
+import argparse
+
+from keras.applications import inception_v3
+from keras import backend as K
+
+parser = argparse.ArgumentParser(description='Deep Dreams with Keras.')
+parser.add_argument('base_image_path', metavar='base', type=str,
+                    help='Path to the image to transform.')
+parser.add_argument('result_prefix', metavar='res_prefix', type=str,
+                    help='Prefix for the saved results.')
+
+args = parser.parse_args()
+base_image_path = args.base_image_path
+result_prefix = args.result_prefix
+
+# These are the names of the layers
+# for which we try to maximize activation,
+# as well as their weight in the final loss
+# we try to maximize.
+# You can tweak these setting to obtain new visual effects.
+settings = {
+    'features': {
+        'mixed2': 0.2,
+        'mixed3': 0.5,
+        'mixed4': 2.,
+        'mixed5': 1.5,
+    },
+}
+
+
+def preprocess_image(image_path):
+    # Util function to open, resize and format pictures
+    # into appropriate tensors.
+    img = load_img(image_path)
+    img = img_to_array(img)
+    img = np.expand_dims(img, axis=0)
+    img = inception_v3.preprocess_input(img)
+    return img
+
+
+def deprocess_image(x):
+    # Util function to convert a tensor into a valid image.
+    if K.image_data_format() == 'channels_first':
+        x = x.reshape((3, x.shape[2], x.shape[3]))
+        x = x.transpose((1, 2, 0))
+    else:
+        x = x.reshape((x.shape[1], x.shape[2], 3))
+    x /= 2.
+    x += 0.5
+    x *= 255.
+    x = np.clip(x, 0, 255).astype('uint8')
+    return x
+
+K.set_learning_phase(0)
+
+# Build the InceptionV3 network with our placeholder.
+# The model will be loaded with pre-trained ImageNet weights.
+model = inception_v3.InceptionV3(weights='imagenet',
+                                 include_top=False)
+dream = model.input
+print('Model loaded.')
+
+# Get the symbolic outputs of each "key" layer (we gave them unique names).
+layer_dict = dict([(layer.name, layer) for layer in model.layers])
+
+# Define the loss.
+loss = K.variable(0.)
+for layer_name in settings['features']:
+    # Add the L2 norm of the features of a layer to the loss.
+    assert layer_name in layer_dict.keys(), 'Layer ' + layer_name + ' not found in model.'
+    coeff = settings['features'][layer_name]
+    x = layer_dict[layer_name].output
+    # We avoid border artifacts by only involving non-border pixels in the loss.
+    scaling = K.prod(K.cast(K.shape(x), 'float32'))
+    if K.image_data_format() == 'channels_first':
+        loss += coeff * K.sum(K.square(x[:, :, 2: -2, 2: -2])) / scaling
+    else:
+        loss += coeff * K.sum(K.square(x[:, 2: -2, 2: -2, :])) / scaling
+
+# Compute the gradients of the dream wrt the loss.
+grads = K.gradients(loss, dream)[0]
+# Normalize gradients.
+grads /= K.maximum(K.mean(K.abs(grads)), 1e-7)
+
+# Set up function to retrieve the value
+# of the loss and gradients given an input image.
+outputs = [loss, grads]
+fetch_loss_and_grads = K.function([dream], outputs)
+
+
+def eval_loss_and_grads(x):
+    outs = fetch_loss_and_grads([x])
+    loss_value = outs[0]
+    grad_values = outs[1]
+    return loss_value, grad_values
+
+
+def resize_img(img, size):
+    img = np.copy(img)
+    if K.image_data_format() == 'channels_first':
+        factors = (1, 1,
+                   float(size[0]) / img.shape[2],
+                   float(size[1]) / img.shape[3])
+    else:
+        factors = (1,
+                   float(size[0]) / img.shape[1],
+                   float(size[1]) / img.shape[2],
+                   1)
+    return scipy.ndimage.zoom(img, factors, order=1)
+
+
+def gradient_ascent(x, iterations, step, max_loss=None):
+    for i in range(iterations):
+        loss_value, grad_values = eval_loss_and_grads(x)
+        if max_loss is not None and loss_value > max_loss:
+            break
+        print('..Loss value at', i, ':', loss_value)
+        x += step * grad_values
+    return x
+
+
+def save_img(img, fname):
+    pil_img = deprocess_image(np.copy(img))
+    scipy.misc.imsave(fname, pil_img)
+
+
+"""Process:
+
+- Load the original image.
+- Define a number of processing scales (i.e. image shapes),
+    from smallest to largest.
+- Resize the original image to the smallest scale.
+- For every scale, starting with the smallest (i.e. current one):
+    - Run gradient ascent
+    - Upscale image to the next scale
+    - Reinject the detail that was lost at upscaling time
+- Stop when we are back to the original size.
+
+To obtain the detail lost during upscaling, we simply
+take the original image, shrink it down, upscale it,
+and compare the result to the (resized) original image.
+"""
+
+
+# Playing with these hyperparameters will also allow you to achieve new effects
+step = 0.01  # Gradient ascent step size
+num_octave = 3  # Number of scales at which to run gradient ascent
+octave_scale = 1.4  # Size ratio between scales
+iterations = 20  # Number of ascent steps per scale
+max_loss = 10.
+
+img = preprocess_image(base_image_path)
+if K.image_data_format() == 'channels_first':
+    original_shape = img.shape[2:]
+else:
+    original_shape = img.shape[1:3]
+successive_shapes = [original_shape]
+for i in range(1, num_octave):
+    shape = tuple([int(dim / (octave_scale ** i)) for dim in original_shape])
+    successive_shapes.append(shape)
+successive_shapes = successive_shapes[::-1]
+original_img = np.copy(img)
+shrunk_original_img = resize_img(img, successive_shapes[0])
+
+for shape in successive_shapes:
+    print('Processing image shape', shape)
+    img = resize_img(img, shape)
+    img = gradient_ascent(img,
+                          iterations=iterations,
+                          step=step,
+                          max_loss=max_loss)
+    upscaled_shrunk_original_img = resize_img(shrunk_original_img, shape)
+    same_size_original = resize_img(original_img, shape)
+    lost_detail = same_size_original - upscaled_shrunk_original_img
+
+    img += lost_detail
+    shrunk_original_img = resize_img(original_img, shape)
+
+save_img(img, fname=result_prefix + '.png')

examples/image_ocr.py

@@ -0,0 +1,491 @@
+'''This example uses a convolutional stack followed by a recurrent stack
+and a CTC logloss function to perform optical character recognition
+of generated text images. I have no evidence of whether it actually
+learns general shapes of text, or just is able to recognize all
+the different fonts thrown at it...the purpose is more to demonstrate CTC
+inside of Keras.  Note that the font list may need to be updated
+for the particular OS in use.
+
+This starts off with 4 letter words.  For the first 12 epochs, the
+difficulty is gradually increased using the TextImageGenerator class
+which is both a generator class for test/train data and a Keras
+callback class. After 20 epochs, longer sequences are thrown at it
+by recompiling the model to handle a wider image and rebuilding
+the word list to include two words separated by a space.
+
+The table below shows normalized edit distance values. Theano uses
+a slightly different CTC implementation, hence the different results.
+
+            Norm. ED
+Epoch |   TF   |   TH
+------------------------
+    10   0.027   0.064
+    15   0.038   0.035
+    20   0.043   0.045
+    25   0.014   0.019
+
+This requires cairo and editdistance packages:
+pip install cairocffi
+pip install editdistance
+
+Created by Mike Henry
+https://github.com/mbhenry/
+'''
+import os
+import itertools
+import re
+import datetime
+import cairocffi as cairo
+import editdistance
+import numpy as np
+from scipy import ndimage
+import pylab
+from keras import backend as K
+from keras.layers.convolutional import Conv2D, MaxPooling2D
+from keras.layers import Input, Dense, Activation
+from keras.layers import Reshape, Lambda
+from keras.layers.merge import add, concatenate
+from keras.models import Model
+from keras.layers.recurrent import GRU
+from keras.optimizers import SGD
+from keras.utils.data_utils import get_file
+from keras.preprocessing import image
+import keras.callbacks
+
+
+OUTPUT_DIR = 'image_ocr'
+
+np.random.seed(55)
+
+
+# this creates larger "blotches" of noise which look
+# more realistic than just adding gaussian noise
+# assumes greyscale with pixels ranging from 0 to 1
+
+def speckle(img):
+    severity = np.random.uniform(0, 0.6)
+    blur = ndimage.gaussian_filter(np.random.randn(*img.shape) * severity, 1)
+    img_speck = (img + blur)
+    img_speck[img_speck > 1] = 1
+    img_speck[img_speck <= 0] = 0
+    return img_speck
+
+
+# paints the string in a random location the bounding box
+# also uses a random font, a slight random rotation,
+# and a random amount of speckle noise
+
+def paint_text(text, w, h, rotate=False, ud=False, multi_fonts=False):
+    surface = cairo.ImageSurface(cairo.FORMAT_RGB24, w, h)
+    with cairo.Context(surface) as context:
+        context.set_source_rgb(1, 1, 1)  # White
+        context.paint()
+        # this font list works in Centos 7
+        if multi_fonts:
+            fonts = ['Century Schoolbook', 'Courier', 'STIX', 'URW Chancery L', 'FreeMono']
+            context.select_font_face(np.random.choice(fonts), cairo.FONT_SLANT_NORMAL,
+                                     np.random.choice([cairo.FONT_WEIGHT_BOLD, cairo.FONT_WEIGHT_NORMAL]))
+        else:
+            context.select_font_face('Courier', cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_BOLD)
+        context.set_font_size(25)
+        box = context.text_extents(text)
+        border_w_h = (4, 4)
+        if box[2] > (w - 2 * border_w_h[1]) or box[3] > (h - 2 * border_w_h[0]):
+            raise IOError('Could not fit string into image. Max char count is too large for given image width.')
+
+        # teach the RNN translational invariance by
+        # fitting text box randomly on canvas, with some room to rotate
+        max_shift_x = w - box[2] - border_w_h[0]
+        max_shift_y = h - box[3] - border_w_h[1]
+        top_left_x = np.random.randint(0, int(max_shift_x))
+        if ud:
+            top_left_y = np.random.randint(0, int(max_shift_y))
+        else:
+            top_left_y = h // 2
+        context.move_to(top_left_x - int(box[0]), top_left_y - int(box[1]))
+        context.set_source_rgb(0, 0, 0)
+        context.show_text(text)
+
+    buf = surface.get_data()
+    a = np.frombuffer(buf, np.uint8)
+    a.shape = (h, w, 4)
+    a = a[:, :, 0]  # grab single channel
+    a = a.astype(np.float32) / 255
+    a = np.expand_dims(a, 0)
+    if rotate:
+        a = image.random_rotation(a, 3 * (w - top_left_x) / w + 1)
+    a = speckle(a)
+
+    return a
+
+
+def shuffle_mats_or_lists(matrix_list, stop_ind=None):
+    ret = []
+    assert all([len(i) == len(matrix_list[0]) for i in matrix_list])
+    len_val = len(matrix_list[0])
+    if stop_ind is None:
+        stop_ind = len_val
+    assert stop_ind <= len_val
+
+    a = list(range(stop_ind))
+    np.random.shuffle(a)
+    a += list(range(stop_ind, len_val))
+    for mat in matrix_list:
+        if isinstance(mat, np.ndarray):
+            ret.append(mat[a])
+        elif isinstance(mat, list):
+            ret.append([mat[i] for i in a])
+        else:
+            raise TypeError('`shuffle_mats_or_lists` only supports '
+                            'numpy.array and list objects.')
+    return ret
+
+
+def text_to_labels(text, num_classes):
+    ret = []
+    for char in text:
+        if char >= 'a' and char <= 'z':
+            ret.append(ord(char) - ord('a'))
+        elif char == ' ':
+            ret.append(26)
+    return ret
+
+
+# only a-z and space..probably not to difficult
+# to expand to uppercase and symbols
+
+def is_valid_str(in_str):
+    search = re.compile(r'[^a-z\ ]').search
+    return not bool(search(in_str))
+
+
+# Uses generator functions to supply train/test with
+# data. Image renderings are text are created on the fly
+# each time with random perturbations
+
+class TextImageGenerator(keras.callbacks.Callback):
+
+    def __init__(self, monogram_file, bigram_file, minibatch_size,
+                 img_w, img_h, downsample_factor, val_split,
+                 absolute_max_string_len=16):
+
+        self.minibatch_size = minibatch_size
+        self.img_w = img_w
+        self.img_h = img_h
+        self.monogram_file = monogram_file
+        self.bigram_file = bigram_file
+        self.downsample_factor = downsample_factor
+        self.val_split = val_split
+        self.blank_label = self.get_output_size() - 1
+        self.absolute_max_string_len = absolute_max_string_len
+
+    def get_output_size(self):
+        return 28
+
+    # num_words can be independent of the epoch size due to the use of generators
+    # as max_string_len grows, num_words can grow
+    def build_word_list(self, num_words, max_string_len=None, mono_fraction=0.5):
+        assert max_string_len <= self.absolute_max_string_len
+        assert num_words % self.minibatch_size == 0
+        assert (self.val_split * num_words) % self.minibatch_size == 0
+        self.num_words = num_words
+        self.string_list = [''] * self.num_words
+        tmp_string_list = []
+        self.max_string_len = max_string_len
+        self.Y_data = np.ones([self.num_words, self.absolute_max_string_len]) * -1
+        self.X_text = []
+        self.Y_len = [0] * self.num_words
+
+        # monogram file is sorted by frequency in english speech
+        with open(self.monogram_file, 'rt') as f:
+            for line in f:
+                if len(tmp_string_list) == int(self.num_words * mono_fraction):
+                    break
+                word = line.rstrip()
+                if max_string_len == -1 or max_string_len is None or len(word) <= max_string_len:
+                    tmp_string_list.append(word)
+
+        # bigram file contains common word pairings in english speech
+        with open(self.bigram_file, 'rt') as f:
+            lines = f.readlines()
+            for line in lines:
+                if len(tmp_string_list) == self.num_words:
+                    break
+                columns = line.lower().split()
+                word = columns[0] + ' ' + columns[1]
+                if is_valid_str(word) and \
+                        (max_string_len == -1 or max_string_len is None or len(word) <= max_string_len):
+                    tmp_string_list.append(word)
+        if len(tmp_string_list) != self.num_words:
+            raise IOError('Could not pull enough words from supplied monogram and bigram files. ')
+        # interlace to mix up the easy and hard words
+        self.string_list[::2] = tmp_string_list[:self.num_words // 2]
+        self.string_list[1::2] = tmp_string_list[self.num_words // 2:]
+
+        for i, word in enumerate(self.string_list):
+            self.Y_len[i] = len(word)
+            self.Y_data[i, 0:len(word)] = text_to_labels(word, self.get_output_size())
+            self.X_text.append(word)
+        self.Y_len = np.expand_dims(np.array(self.Y_len), 1)
+
+        self.cur_val_index = self.val_split
+        self.cur_train_index = 0
+
+    # each time an image is requested from train/val/test, a new random
+    # painting of the text is performed
+    def get_batch(self, index, size, train):
+        # width and height are backwards from typical Keras convention
+        # because width is the time dimension when it gets fed into the RNN
+        if K.image_data_format() == 'channels_first':
+            X_data = np.ones([size, 1, self.img_w, self.img_h])
+        else:
+            X_data = np.ones([size, self.img_w, self.img_h, 1])
+
+        labels = np.ones([size, self.absolute_max_string_len])
+        input_length = np.zeros([size, 1])
+        label_length = np.zeros([size, 1])
+        source_str = []
+        for i in range(0, size):
+            # Mix in some blank inputs.  This seems to be important for
+            # achieving translational invariance
+            if train and i > size - 4:
+                if K.image_data_format() == 'channels_first':
+                    X_data[i, 0, 0:self.img_w, :] = self.paint_func('')[0, :, :].T
+                else:
+                    X_data[i, 0:self.img_w, :, 0] = self.paint_func('',)[0, :, :].T
+                labels[i, 0] = self.blank_label
+                input_length[i] = self.img_w // self.downsample_factor - 2
+                label_length[i] = 1
+                source_str.append('')
+            else:
+                if K.image_data_format() == 'channels_first':
+                    X_data[i, 0, 0:self.img_w, :] = self.paint_func(self.X_text[index + i])[0, :, :].T
+                else:
+                    X_data[i, 0:self.img_w, :, 0] = self.paint_func(self.X_text[index + i])[0, :, :].T
+                labels[i, :] = self.Y_data[index + i]
+                input_length[i] = self.img_w // self.downsample_factor - 2
+                label_length[i] = self.Y_len[index + i]
+                source_str.append(self.X_text[index + i])
+        inputs = {'the_input': X_data,
+                  'the_labels': labels,
+                  'input_length': input_length,
+                  'label_length': label_length,
+                  'source_str': source_str  # used for visualization only
+                  }
+        outputs = {'ctc': np.zeros([size])}  # dummy data for dummy loss function
+        return (inputs, outputs)
+
+    def next_train(self):
+        while 1:
+            ret = self.get_batch(self.cur_train_index, self.minibatch_size, train=True)
+            self.cur_train_index += self.minibatch_size
+            if self.cur_train_index >= self.val_split:
+                self.cur_train_index = self.cur_train_index % 32
+                (self.X_text, self.Y_data, self.Y_len) = shuffle_mats_or_lists(
+                    [self.X_text, self.Y_data, self.Y_len], self.val_split)
+            yield ret
+
+    def next_val(self):
+        while 1:
+            ret = self.get_batch(self.cur_val_index, self.minibatch_size, train=False)
+            self.cur_val_index += self.minibatch_size
+            if self.cur_val_index >= self.num_words:
+                self.cur_val_index = self.val_split + self.cur_val_index % 32
+            yield ret
+
+    def on_train_begin(self, logs={}):
+        self.build_word_list(16000, 4, 1)
+        self.paint_func = lambda text: paint_text(text, self.img_w, self.img_h,
+                                                  rotate=False, ud=False, multi_fonts=False)
+
+    def on_epoch_begin(self, epoch, logs={}):
+        # rebind the paint function to implement curriculum learning
+        if epoch >= 3 and epoch < 6:
+            self.paint_func = lambda text: paint_text(text, self.img_w, self.img_h,
+                                                      rotate=False, ud=True, multi_fonts=False)
+        elif epoch >= 6 and epoch < 9:
+            self.paint_func = lambda text: paint_text(text, self.img_w, self.img_h,
+                                                      rotate=False, ud=True, multi_fonts=True)
+        elif epoch >= 9:
+            self.paint_func = lambda text: paint_text(text, self.img_w, self.img_h,
+                                                      rotate=True, ud=True, multi_fonts=True)
+        if epoch >= 21 and self.max_string_len < 12:
+            self.build_word_list(32000, 12, 0.5)
+
+
+# the actual loss calc occurs here despite it not being
+# an internal Keras loss function
+
+def ctc_lambda_func(args):
+    y_pred, labels, input_length, label_length = args
+    # the 2 is critical here since the first couple outputs of the RNN
+    # tend to be garbage:
+    y_pred = y_pred[:, 2:, :]
+    return K.ctc_batch_cost(labels, y_pred, input_length, label_length)
+
+
+# For a real OCR application, this should be beam search with a dictionary
+# and language model.  For this example, best path is sufficient.
+
+def decode_batch(test_func, word_batch):
+    out = test_func([word_batch])[0]
+    ret = []
+    for j in range(out.shape[0]):
+        out_best = list(np.argmax(out[j, 2:], 1))
+        out_best = [k for k, g in itertools.groupby(out_best)]
+        # 26 is space, 27 is CTC blank char
+        outstr = ''
+        for c in out_best:
+            if c >= 0 and c < 26:
+                outstr += chr(c + ord('a'))
+            elif c == 26:
+                outstr += ' '
+        ret.append(outstr)
+    return ret
+
+
+class VizCallback(keras.callbacks.Callback):
+
+    def __init__(self, run_name, test_func, text_img_gen, num_display_words=6):
+        self.test_func = test_func
+        self.output_dir = os.path.join(
+            OUTPUT_DIR, run_name)
+        self.text_img_gen = text_img_gen
+        self.num_display_words = num_display_words
+        if not os.path.exists(self.output_dir):
+            os.makedirs(self.output_dir)
+
+    def show_edit_distance(self, num):
+        num_left = num
+        mean_norm_ed = 0.0
+        mean_ed = 0.0
+        while num_left > 0:
+            word_batch = next(self.text_img_gen)[0]
+            num_proc = min(word_batch['the_input'].shape[0], num_left)
+            decoded_res = decode_batch(self.test_func, word_batch['the_input'][0:num_proc])
+            for j in range(0, num_proc):
+                edit_dist = editdistance.eval(decoded_res[j], word_batch['source_str'][j])
+                mean_ed += float(edit_dist)
+                mean_norm_ed += float(edit_dist) / len(word_batch['source_str'][j])
+            num_left -= num_proc
+        mean_norm_ed = mean_norm_ed / num
+        mean_ed = mean_ed / num
+        print('\nOut of %d samples:  Mean edit distance: %.3f Mean normalized edit distance: %0.3f'
+              % (num, mean_ed, mean_norm_ed))
+
+    def on_epoch_end(self, epoch, logs={}):
+        self.model.save_weights(os.path.join(self.output_dir, 'weights%02d.h5' % (epoch)))
+        self.show_edit_distance(256)
+        word_batch = next(self.text_img_gen)[0]
+        res = decode_batch(self.test_func, word_batch['the_input'][0:self.num_display_words])
+        if word_batch['the_input'][0].shape[0] < 256:
+            cols = 2
+        else:
+            cols = 1
+        for i in range(self.num_display_words):
+            pylab.subplot(self.num_display_words // cols, cols, i + 1)
+            if K.image_data_format() == 'channels_first':
+                the_input = word_batch['the_input'][i, 0, :, :]
+            else:
+                the_input = word_batch['the_input'][i, :, :, 0]
+            pylab.imshow(the_input.T, cmap='Greys_r')
+            pylab.xlabel('Truth = \'%s\'\nDecoded = \'%s\'' % (word_batch['source_str'][i], res[i]))
+        fig = pylab.gcf()
+        fig.set_size_inches(10, 13)
+        pylab.savefig(os.path.join(self.output_dir, 'e%02d.png' % (epoch)))
+        pylab.close()
+
+
+def train(run_name, start_epoch, stop_epoch, img_w):
+    # Input Parameters
+    img_h = 64
+    words_per_epoch = 16000
+    val_split = 0.2
+    val_words = int(words_per_epoch * (val_split))
+
+    # Network parameters
+    conv_filters = 16
+    kernel_size = (3, 3)
+    pool_size = 2
+    time_dense_size = 32
+    rnn_size = 512
+
+    if K.image_data_format() == 'channels_first':
+        input_shape = (1, img_w, img_h)
+    else:
+        input_shape = (img_w, img_h, 1)
+
+    fdir = os.path.dirname(get_file('wordlists.tgz',
+                                    origin='http://www.mythic-ai.com/datasets/wordlists.tgz', untar=True))
+
+    img_gen = TextImageGenerator(monogram_file=os.path.join(fdir, 'wordlist_mono_clean.txt'),
+                                 bigram_file=os.path.join(fdir, 'wordlist_bi_clean.txt'),
+                                 minibatch_size=32,
+                                 img_w=img_w,
+                                 img_h=img_h,
+                                 downsample_factor=(pool_size ** 2),
+                                 val_split=words_per_epoch - val_words
+                                 )
+    act = 'relu'
+    input_data = Input(name='the_input', shape=input_shape, dtype='float32')
+    inner = Conv2D(conv_filters, kernel_size, padding='same',
+                   activation=act, kernel_initializer='he_normal',
+                   name='conv1')(input_data)
+    inner = MaxPooling2D(pool_size=(pool_size, pool_size), name='max1')(inner)
+    inner = Conv2D(conv_filters, kernel_size, padding='same',
+                   activation=act, kernel_initializer='he_normal',
+                   name='conv2')(inner)
+    inner = MaxPooling2D(pool_size=(pool_size, pool_size), name='max2')(inner)
+
+    conv_to_rnn_dims = (img_w // (pool_size ** 2), (img_h // (pool_size ** 2)) * conv_filters)
+    inner = Reshape(target_shape=conv_to_rnn_dims, name='reshape')(inner)
+
+    # cuts down input size going into RNN:
+    inner = Dense(time_dense_size, activation=act, name='dense1')(inner)
+
+    # Two layers of bidirectional GRUs
+    # GRU seems to work as well, if not better than LSTM:
+    gru_1 = GRU(rnn_size, return_sequences=True, kernel_initializer='he_normal', name='gru1')(inner)
+    gru_1b = GRU(rnn_size, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='gru1_b')(inner)
+    gru1_merged = add([gru_1, gru_1b])
+    gru_2 = GRU(rnn_size, return_sequences=True, kernel_initializer='he_normal', name='gru2')(gru1_merged)
+    gru_2b = GRU(rnn_size, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='gru2_b')(gru1_merged)
+
+    # transforms RNN output to character activations:
+    inner = Dense(img_gen.get_output_size(), kernel_initializer='he_normal',
+                  name='dense2')(concatenate([gru_2, gru_2b]))
+    y_pred = Activation('softmax', name='softmax')(inner)
+    Model(inputs=input_data, outputs=y_pred).summary()
+
+    labels = Input(name='the_labels', shape=[img_gen.absolute_max_string_len], dtype='float32')
+    input_length = Input(name='input_length', shape=[1], dtype='int64')
+    label_length = Input(name='label_length', shape=[1], dtype='int64')
+    # Keras doesn't currently support loss funcs with extra parameters
+    # so CTC loss is implemented in a lambda layer
+    loss_out = Lambda(ctc_lambda_func, output_shape=(1,), name='ctc')([y_pred, labels, input_length, label_length])
+
+    # clipnorm seems to speeds up convergence
+    sgd = SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5)
+
+    model = Model(inputs=[input_data, labels, input_length, label_length], outputs=loss_out)
+
+    # the loss calc occurs elsewhere, so use a dummy lambda func for the loss
+    model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd)
+    if start_epoch > 0:
+        weight_file = os.path.join(OUTPUT_DIR, os.path.join(run_name, 'weights%02d.h5' % (start_epoch - 1)))
+        model.load_weights(weight_file)
+    # captures output of softmax so we can decode the output during visualization
+    test_func = K.function([input_data], [y_pred])
+
+    viz_cb = VizCallback(run_name, test_func, img_gen.next_val())
+
+    model.fit_generator(generator=img_gen.next_train(), steps_per_epoch=(words_per_epoch - val_words),
+                        epochs=stop_epoch, validation_data=img_gen.next_val(), validation_steps=val_words,
+                        callbacks=[viz_cb, img_gen], initial_epoch=start_epoch)
+
+
+if __name__ == '__main__':
+    run_name = datetime.datetime.now().strftime('%Y:%m:%d:%H:%M:%S')
+    train(run_name, 0, 20, 128)
+    # increase to wider images and start at epoch 20. The learned weights are reloaded
+    train(run_name, 20, 25, 512)

examples/imdb_bidirectional_lstm.py

@@ -0,0 +1,48 @@
+'''Train a Bidirectional LSTM on the IMDB sentiment classification task.
+
+Output after 4 epochs on CPU: ~0.8146
+Time per epoch on CPU (Core i7): ~150s.
+'''
+
+from __future__ import print_function
+import numpy as np
+
+from keras.preprocessing import sequence
+from keras.models import Sequential
+from keras.layers import Dense, Dropout, Embedding, LSTM, Bidirectional
+from keras.datasets import imdb
+
+
+max_features = 20000
+# cut texts after this number of words
+# (among top max_features most common words)
+maxlen = 100
+batch_size = 32
+
+print('Loading data...')
+(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)
+print(len(x_train), 'train sequences')
+print(len(x_test), 'test sequences')
+
+print('Pad sequences (samples x time)')
+x_train = sequence.pad_sequences(x_train, maxlen=maxlen)
+x_test = sequence.pad_sequences(x_test, maxlen=maxlen)
+print('x_train shape:', x_train.shape)
+print('x_test shape:', x_test.shape)
+y_train = np.array(y_train)
+y_test = np.array(y_test)
+
+model = Sequential()
+model.add(Embedding(max_features, 128, input_length=maxlen))
+model.add(Bidirectional(LSTM(64)))
+model.add(Dropout(0.5))
+model.add(Dense(1, activation='sigmoid'))
+
+# try using different optimizers and different optimizer configs
+model.compile('adam', 'binary_crossentropy', metrics=['accuracy'])
+
+print('Train...')
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=4,
+          validation_data=[x_test, y_test])

examples/imdb_cnn.py

@@ -0,0 +1,74 @@
+'''This example demonstrates the use of Convolution1D for text classification.
+
+Gets to 0.89 test accuracy after 2 epochs.
+90s/epoch on Intel i5 2.4Ghz CPU.
+10s/epoch on Tesla K40 GPU.
+
+'''
+
+from __future__ import print_function
+
+from keras.preprocessing import sequence
+from keras.models import Sequential
+from keras.layers import Dense, Dropout, Activation
+from keras.layers import Embedding
+from keras.layers import Conv1D, GlobalMaxPooling1D
+from keras.datasets import imdb
+
+# set parameters:
+max_features = 5000
+maxlen = 400
+batch_size = 32
+embedding_dims = 50
+filters = 250
+kernel_size = 3
+hidden_dims = 250
+epochs = 2
+
+print('Loading data...')
+(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)
+print(len(x_train), 'train sequences')
+print(len(x_test), 'test sequences')
+
+print('Pad sequences (samples x time)')
+x_train = sequence.pad_sequences(x_train, maxlen=maxlen)
+x_test = sequence.pad_sequences(x_test, maxlen=maxlen)
+print('x_train shape:', x_train.shape)
+print('x_test shape:', x_test.shape)
+
+print('Build model...')
+model = Sequential()
+
+# we start off with an efficient embedding layer which maps
+# our vocab indices into embedding_dims dimensions
+model.add(Embedding(max_features,
+                    embedding_dims,
+                    input_length=maxlen))
+model.add(Dropout(0.2))
+
+# we add a Convolution1D, which will learn filters
+# word group filters of size filter_length:
+model.add(Conv1D(filters,
+                 kernel_size,
+                 padding='valid',
+                 activation='relu',
+                 strides=1))
+# we use max pooling:
+model.add(GlobalMaxPooling1D())
+
+# We add a vanilla hidden layer:
+model.add(Dense(hidden_dims))
+model.add(Dropout(0.2))
+model.add(Activation('relu'))
+
+# We project onto a single unit output layer, and squash it with a sigmoid:
+model.add(Dense(1))
+model.add(Activation('sigmoid'))
+
+model.compile(loss='binary_crossentropy',
+              optimizer='adam',
+              metrics=['accuracy'])
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=epochs,
+          validation_data=(x_test, y_test))

examples/imdb_cnn_lstm.py

@@ -0,0 +1,76 @@
+'''Train a recurrent convolutional network on the IMDB sentiment
+classification task.
+
+Gets to 0.8498 test accuracy after 2 epochs. 41s/epoch on K520 GPU.
+'''
+from __future__ import print_function
+
+from keras.preprocessing import sequence
+from keras.models import Sequential
+from keras.layers import Dense, Dropout, Activation
+from keras.layers import Embedding
+from keras.layers import LSTM
+from keras.layers import Conv1D, MaxPooling1D
+from keras.datasets import imdb
+
+# Embedding
+max_features = 20000
+maxlen = 100
+embedding_size = 128
+
+# Convolution
+kernel_size = 5
+filters = 64
+pool_size = 4
+
+# LSTM
+lstm_output_size = 70
+
+# Training
+batch_size = 30
+epochs = 2
+
+'''
+Note:
+batch_size is highly sensitive.
+Only 2 epochs are needed as the dataset is very small.
+'''
+
+print('Loading data...')
+(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)
+print(len(x_train), 'train sequences')
+print(len(x_test), 'test sequences')
+
+print('Pad sequences (samples x time)')
+x_train = sequence.pad_sequences(x_train, maxlen=maxlen)
+x_test = sequence.pad_sequences(x_test, maxlen=maxlen)
+print('x_train shape:', x_train.shape)
+print('x_test shape:', x_test.shape)
+
+print('Build model...')
+
+model = Sequential()
+model.add(Embedding(max_features, embedding_size, input_length=maxlen))
+model.add(Dropout(0.25))
+model.add(Conv1D(filters,
+                 kernel_size,
+                 padding='valid',
+                 activation='relu',
+                 strides=1))
+model.add(MaxPooling1D(pool_size=pool_size))
+model.add(LSTM(lstm_output_size))
+model.add(Dense(1))
+model.add(Activation('sigmoid'))
+
+model.compile(loss='binary_crossentropy',
+              optimizer='adam',
+              metrics=['accuracy'])
+
+print('Train...')
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=epochs,
+          validation_data=(x_test, y_test))
+score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
+print('Test score:', score)
+print('Test accuracy:', acc)

examples/imdb_fasttext.py

@@ -0,0 +1,135 @@
+'''This example demonstrates the use of fasttext for text classification
+
+Based on Joulin et al's paper:
+
+Bags of Tricks for Efficient Text Classification
+https://arxiv.org/abs/1607.01759
+
+Results on IMDB datasets with uni and bi-gram embeddings:
+    Uni-gram: 0.8813 test accuracy after 5 epochs. 8s/epoch on i7 cpu.
+    Bi-gram : 0.9056 test accuracy after 5 epochs. 2s/epoch on GTx 980M gpu.
+'''
+
+from __future__ import print_function
+import numpy as np
+
+from keras.preprocessing import sequence
+from keras.models import Sequential
+from keras.layers import Dense
+from keras.layers import Embedding
+from keras.layers import GlobalAveragePooling1D
+from keras.datasets import imdb
+
+
+def create_ngram_set(input_list, ngram_value=2):
+    """
+    Extract a set of n-grams from a list of integers.
+
+    >>> create_ngram_set([1, 4, 9, 4, 1, 4], ngram_value=2)
+    {(4, 9), (4, 1), (1, 4), (9, 4)}
+
+    >>> create_ngram_set([1, 4, 9, 4, 1, 4], ngram_value=3)
+    [(1, 4, 9), (4, 9, 4), (9, 4, 1), (4, 1, 4)]
+    """
+    return set(zip(*[input_list[i:] for i in range(ngram_value)]))
+
+
+def add_ngram(sequences, token_indice, ngram_range=2):
+    """
+    Augment the input list of list (sequences) by appending n-grams values.
+
+    Example: adding bi-gram
+    >>> sequences = [[1, 3, 4, 5], [1, 3, 7, 9, 2]]
+    >>> token_indice = {(1, 3): 1337, (9, 2): 42, (4, 5): 2017}
+    >>> add_ngram(sequences, token_indice, ngram_range=2)
+    [[1, 3, 4, 5, 1337, 2017], [1, 3, 7, 9, 2, 1337, 42]]
+
+    Example: adding tri-gram
+    >>> sequences = [[1, 3, 4, 5], [1, 3, 7, 9, 2]]
+    >>> token_indice = {(1, 3): 1337, (9, 2): 42, (4, 5): 2017, (7, 9, 2): 2018}
+    >>> add_ngram(sequences, token_indice, ngram_range=3)
+    [[1, 3, 4, 5, 1337], [1, 3, 7, 9, 2, 1337, 2018]]
+    """
+    new_sequences = []
+    for input_list in sequences:
+        new_list = input_list[:]
+        for i in range(len(new_list) - ngram_range + 1):
+            for ngram_value in range(2, ngram_range + 1):
+                ngram = tuple(new_list[i:i + ngram_value])
+                if ngram in token_indice:
+                    new_list.append(token_indice[ngram])
+        new_sequences.append(new_list)
+
+    return new_sequences
+
+# Set parameters:
+# ngram_range = 2 will add bi-grams features
+ngram_range = 1
+max_features = 20000
+maxlen = 400
+batch_size = 32
+embedding_dims = 50
+epochs = 5
+
+print('Loading data...')
+(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)
+print(len(x_train), 'train sequences')
+print(len(x_test), 'test sequences')
+print('Average train sequence length: {}'.format(np.mean(list(map(len, x_train)), dtype=int)))
+print('Average test sequence length: {}'.format(np.mean(list(map(len, x_test)), dtype=int)))
+
+if ngram_range > 1:
+    print('Adding {}-gram features'.format(ngram_range))
+    # Create set of unique n-gram from the training set.
+    ngram_set = set()
+    for input_list in x_train:
+        for i in range(2, ngram_range + 1):
+            set_of_ngram = create_ngram_set(input_list, ngram_value=i)
+            ngram_set.update(set_of_ngram)
+
+    # Dictionary mapping n-gram token to a unique integer.
+    # Integer values are greater than max_features in order
+    # to avoid collision with existing features.
+    start_index = max_features + 1
+    token_indice = {v: k + start_index for k, v in enumerate(ngram_set)}
+    indice_token = {token_indice[k]: k for k in token_indice}
+
+    # max_features is the highest integer that could be found in the dataset.
+    max_features = np.max(list(indice_token.keys())) + 1
+
+    # Augmenting x_train and x_test with n-grams features
+    x_train = add_ngram(x_train, token_indice, ngram_range)
+    x_test = add_ngram(x_test, token_indice, ngram_range)
+    print('Average train sequence length: {}'.format(np.mean(list(map(len, x_train)), dtype=int)))
+    print('Average test sequence length: {}'.format(np.mean(list(map(len, x_test)), dtype=int)))
+
+print('Pad sequences (samples x time)')
+x_train = sequence.pad_sequences(x_train, maxlen=maxlen)
+x_test = sequence.pad_sequences(x_test, maxlen=maxlen)
+print('x_train shape:', x_train.shape)
+print('x_test shape:', x_test.shape)
+
+print('Build model...')
+model = Sequential()
+
+# we start off with an efficient embedding layer which maps
+# our vocab indices into embedding_dims dimensions
+model.add(Embedding(max_features,
+                    embedding_dims,
+                    input_length=maxlen))
+
+# we add a GlobalAveragePooling1D, which will average the embeddings
+# of all words in the document
+model.add(GlobalAveragePooling1D())
+
+# We project onto a single unit output layer, and squash it with a sigmoid:
+model.add(Dense(1, activation='sigmoid'))
+
+model.compile(loss='binary_crossentropy',
+              optimizer='adam',
+              metrics=['accuracy'])
+
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=epochs,
+          validation_data=(x_test, y_test))

examples/imdb_lstm.py

@@ -1,70 +1,55 @@
-from __future__ import absolute_import
+'''Trains a LSTM on the IMDB sentiment classification task.
+The dataset is actually too small for LSTM to be of any advantage
+compared to simpler, much faster methods such as TF-IDF + LogReg.
+Notes:
+
+- RNNs are tricky. Choice of batch size is important,
+choice of loss and optimizer is critical, etc.
+Some configurations won't converge.
+
+- LSTM loss decrease patterns during training can be quite different
+from what you see with CNNs/MLPs/etc.
+'''
 from __future__ import print_function
-import numpy as np
 
 from keras.preprocessing import sequence
-from keras.optimizers import SGD, RMSprop, Adagrad
-from keras.utils import np_utils
 from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation
-from keras.layers.embeddings import Embedding
-from keras.layers.recurrent import LSTM, GRU
+from keras.layers import Dense, Embedding
+from keras.layers import LSTM
 from keras.datasets import imdb
 
-'''
-    Train a LSTM on the IMDB sentiment classification task.
+max_features = 20000
+maxlen = 80  # cut texts after this number of words (among top max_features most common words)
+batch_size = 32
 
-    The dataset is actually too small for LSTM to be of any advantage 
-    compared to simpler, much faster methods such as TF-IDF+LogReg.
+print('Loading data...')
+(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)
+print(len(x_train), 'train sequences')
+print(len(x_test), 'test sequences')
 
-    Notes: 
-
-    - RNNs are tricky. Choice of batch size is important, 
-    choice of loss and optimizer is critical, etc. 
-    Most configurations won't converge.
-
-    - LSTM loss decrease during training can be quite different 
-    from what you see with CNNs/MLPs/etc. It's more or less a sigmoid
-    instead of an inverse exponential.
-
-    GPU command:
-        THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python imdb_lstm.py
-
-    250s/epoch on GPU (GT 650M), vs. 400s/epoch on CPU (2.4Ghz Core i7).
-'''
-
-max_features=20000
-maxlen = 100 # cut texts after this number of words (among top max_features most common words)
-batch_size = 16
-
-print("Loading data...")
-(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features, test_split=0.2)
-print(len(X_train), 'train sequences')
-print(len(X_test), 'test sequences')
-
-print("Pad sequences (samples x time)")
-X_train = sequence.pad_sequences(X_train, maxlen=maxlen)
-X_test = sequence.pad_sequences(X_test, maxlen=maxlen)
-print('X_train shape:', X_train.shape)
-print('X_test shape:', X_test.shape)
+print('Pad sequences (samples x time)')
+x_train = sequence.pad_sequences(x_train, maxlen=maxlen)
+x_test = sequence.pad_sequences(x_test, maxlen=maxlen)
+print('x_train shape:', x_train.shape)
+print('x_test shape:', x_test.shape)
 
 print('Build model...')
 model = Sequential()
-model.add(Embedding(max_features, 256))
-model.add(LSTM(256, 128)) # try using a GRU instead, for fun
-model.add(Dropout(0.5))
-model.add(Dense(128, 1))
-model.add(Activation('sigmoid'))
+model.add(Embedding(max_features, 128))
+model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2))
+model.add(Dense(1, activation='sigmoid'))
 
 # try using different optimizers and different optimizer configs
-model.compile(loss='binary_crossentropy', optimizer='adam', class_mode="binary")
+model.compile(loss='binary_crossentropy',
+              optimizer='adam',
+              metrics=['accuracy'])
 
-print("Train...")
-model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=5, validation_split=0.1, show_accuracy=True)
-score = model.evaluate(X_test, y_test, batch_size=batch_size)
+print('Train...')
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=15,
+          validation_data=(x_test, y_test))
+score, acc = model.evaluate(x_test, y_test,
+                            batch_size=batch_size)
 print('Test score:', score)
-
-classes = model.predict_classes(X_test, batch_size=batch_size)
-acc = np_utils.accuracy(classes, y_test)
 print('Test accuracy:', acc)
-

examples/kaggle_otto_nn.py

@@ -1,124 +0,0 @@
-from __future__ import absolute_import
-from __future__ import print_function
-
-import numpy as np
-import pandas as pd
-
-from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation
-from keras.layers.normalization import BatchNormalization
-from keras.layers.advanced_activations import PReLU
-from keras.utils import np_utils, generic_utils
-
-from sklearn.preprocessing import LabelEncoder
-from sklearn.preprocessing import StandardScaler
-
-'''
-    This demonstrates how to reach a score of 0.4890 (local validation)
-    on the Kaggle Otto challenge, with a deep net using Keras.
-
-    Compatible Python 2.7-3.4 
-
-    Recommended to run on GPU: 
-        Command: THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python kaggle_otto_nn.py
-        On EC2 g2.2xlarge instance: 19s/epoch. 6-7 minutes total training time.
-
-    Best validation score at epoch 21: 0.4881 
-
-    Try it at home:
-        - with/without BatchNormalization (BatchNormalization helps!)
-        - with ReLU or with PReLU (PReLU helps!)
-        - with smaller layers, largers layers
-        - with more layers, less layers
-        - with different optimizers (SGD+momentum+decay is probably better than Adam!)
-
-    Get the data from Kaggle: https://www.kaggle.com/c/otto-group-product-classification-challenge/data
-'''
-
-np.random.seed(1337) # for reproducibility
-
-def load_data(path, train=True):
-    df = pd.read_csv(path)
-    X = df.values.copy()
-    if train:
-        np.random.shuffle(X) # https://youtu.be/uyUXoap67N8
-        X, labels = X[:, 1:-1].astype(np.float32), X[:, -1]
-        return X, labels
-    else:
-        X, ids = X[:, 1:].astype(np.float32), X[:, 0].astype(str)
-        return X, ids
-
-def preprocess_data(X, scaler=None):
-    if not scaler:
-        scaler = StandardScaler()
-        scaler.fit(X)
-    X = scaler.transform(X)
-    return X, scaler
-
-def preprocess_labels(labels, encoder=None, categorical=True):
-    if not encoder:
-        encoder = LabelEncoder()
-        encoder.fit(labels)
-    y = encoder.transform(labels).astype(np.int32)
-    if categorical:
-        y = np_utils.to_categorical(y)
-    return y, encoder
-
-def make_submission(y_prob, ids, encoder, fname):
-    with open(fname, 'w') as f:
-        f.write('id,')
-        f.write(','.join([str(i) for i in encoder.classes_]))
-        f.write('\n')
-        for i, probs in zip(ids, y_prob):
-            probas = ','.join([i] + [str(p) for p in probs.tolist()])
-            f.write(probas)
-            f.write('\n')
-    print("Wrote submission to file {}.".format(fname))
-
-
-print("Loading data...")
-X, labels = load_data('train.csv', train=True)
-X, scaler = preprocess_data(X)
-y, encoder = preprocess_labels(labels)
-
-X_test, ids = load_data('test.csv', train=False)
-X_test, _ = preprocess_data(X_test, scaler)
-
-nb_classes = y.shape[1]
-print(nb_classes, 'classes')
-
-dims = X.shape[1]
-print(dims, 'dims')
-
-print("Building model...")
-
-model = Sequential()
-model.add(Dense(dims, 512, init='glorot_uniform'))
-model.add(PReLU((512,)))
-model.add(BatchNormalization((512,)))
-model.add(Dropout(0.5))
-
-model.add(Dense(512, 512, init='glorot_uniform'))
-model.add(PReLU((512,)))
-model.add(BatchNormalization((512,)))
-model.add(Dropout(0.5))
-
-model.add(Dense(512, 512, init='glorot_uniform'))
-model.add(PReLU((512,)))
-model.add(BatchNormalization((512,)))
-model.add(Dropout(0.5))
-
-model.add(Dense(512, nb_classes, init='glorot_uniform'))
-model.add(Activation('softmax'))
-
-model.compile(loss='categorical_crossentropy', optimizer="adam")
-
-print("Training model...")
-
-model.fit(X, y, nb_epoch=20, batch_size=128, validation_split=0.15)
-
-print("Generating submission...")
-
-proba = model.predict_proba(X_test)
-make_submission(proba, ids, encoder, fname='keras-otto.csv')
-

examples/lstm_benchmark.py

@@ -0,0 +1,88 @@
+'''Compare LSTM implementations on the IMDB sentiment classification task.
+
+implementation=0 preprocesses input to the LSTM which typically results in
+faster computations at the expense of increased peak memory usage as the
+preprocessed input must be kept in memory.
+
+implementation=1 does away with the preprocessing, meaning that it might take
+a little longer, but should require less peak memory.
+
+implementation=2 concatenates the input, output and forget gate's weights
+into one, large matrix, resulting in faster computation time as the GPU can
+utilize more cores, at the expense of reduced regularization because the same
+dropout is shared across the gates.
+
+Note that the relative performance of the different implementations can
+vary depending on your device, your model and the size of your data.
+'''
+
+import time
+import numpy as np
+import matplotlib.pyplot as plt
+
+from keras.preprocessing import sequence
+from keras.models import Sequential
+from keras.layers import Embedding, Dense, LSTM, Dropout
+from keras.datasets import imdb
+
+max_features = 20000
+max_length = 80
+embedding_dim = 256
+batch_size = 128
+epochs = 10
+modes = [0, 1, 2]
+
+print('Loading data...')
+(X_train, y_train), (X_test, y_test) = imdb.load_data(num_words=max_features)
+X_train = sequence.pad_sequences(X_train, max_length)
+X_test = sequence.pad_sequences(X_test, max_length)
+
+# Compile and train different models while measuring performance.
+results = []
+for mode in modes:
+    print('Testing mode: implementation={}'.format(mode))
+
+    model = Sequential()
+    model.add(Embedding(max_features, embedding_dim,
+                        input_length=max_length))
+    model.add(Dropout(0.2))
+    model.add(LSTM(embedding_dim,
+                   dropout=0.2,
+                   recurrent_dropout=0.2,
+                   implementation=mode))
+    model.add(Dense(1, activation='sigmoid'))
+    model.compile(loss='binary_crossentropy',
+                  optimizer='adam',
+                  metrics=['accuracy'])
+
+    start_time = time.time()
+    history = model.fit(X_train, y_train,
+                        batch_size=batch_size,
+                        epochs=epochs,
+                        validation_data=(X_test, y_test))
+    average_time_per_epoch = (time.time() - start_time) / epochs
+
+    results.append((history, average_time_per_epoch))
+
+# Compare models' accuracy, loss and elapsed time per epoch.
+plt.style.use('ggplot')
+ax1 = plt.subplot2grid((2, 2), (0, 0))
+ax1.set_title('Accuracy')
+ax1.set_ylabel('Validation Accuracy')
+ax1.set_xlabel('Epochs')
+ax2 = plt.subplot2grid((2, 2), (1, 0))
+ax2.set_title('Loss')
+ax2.set_ylabel('Validation Loss')
+ax2.set_xlabel('Epochs')
+ax3 = plt.subplot2grid((2, 2), (0, 1), rowspan=2)
+ax3.set_title('Time')
+ax3.set_ylabel('Seconds')
+for mode, result in zip(modes, results):
+    ax1.plot(result[0].epoch, result[0].history['val_acc'], label=mode)
+    ax2.plot(result[0].epoch, result[0].history['val_loss'], label=mode)
+ax1.legend()
+ax2.legend()
+ax3.bar(np.arange(len(results)), [x[1] for x in results],
+        tick_label=modes, align='center')
+plt.tight_layout()
+plt.show()

examples/lstm_text_generation.py

@@ -0,0 +1,106 @@
+'''Example script to generate text from Nietzsche's writings.
+
+At least 20 epochs are required before the generated text
+starts sounding coherent.
+
+It is recommended to run this script on GPU, as recurrent
+networks are quite computationally intensive.
+
+If you try this script on new data, make sure your corpus
+has at least ~100k characters. ~1M is better.
+'''
+
+from __future__ import print_function
+from keras.models import Sequential
+from keras.layers import Dense, Activation
+from keras.layers import LSTM
+from keras.optimizers import RMSprop
+from keras.utils.data_utils import get_file
+import numpy as np
+import random
+import sys
+
+path = get_file('nietzsche.txt', origin='https://s3.amazonaws.com/text-datasets/nietzsche.txt')
+text = open(path).read().lower()
+print('corpus length:', len(text))
+
+chars = sorted(list(set(text)))
+print('total chars:', len(chars))
+char_indices = dict((c, i) for i, c in enumerate(chars))
+indices_char = dict((i, c) for i, c in enumerate(chars))
+
+# cut the text in semi-redundant sequences of maxlen characters
+maxlen = 40
+step = 3
+sentences = []
+next_chars = []
+for i in range(0, len(text) - maxlen, step):
+    sentences.append(text[i: i + maxlen])
+    next_chars.append(text[i + maxlen])
+print('nb sequences:', len(sentences))
+
+print('Vectorization...')
+X = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool)
+y = np.zeros((len(sentences), len(chars)), dtype=np.bool)
+for i, sentence in enumerate(sentences):
+    for t, char in enumerate(sentence):
+        X[i, t, char_indices[char]] = 1
+    y[i, char_indices[next_chars[i]]] = 1
+
+
+# build the model: a single LSTM
+print('Build model...')
+model = Sequential()
+model.add(LSTM(128, input_shape=(maxlen, len(chars))))
+model.add(Dense(len(chars)))
+model.add(Activation('softmax'))
+
+optimizer = RMSprop(lr=0.01)
+model.compile(loss='categorical_crossentropy', optimizer=optimizer)
+
+
+def sample(preds, temperature=1.0):
+    # helper function to sample an index from a probability array
+    preds = np.asarray(preds).astype('float64')
+    preds = np.log(preds) / temperature
+    exp_preds = np.exp(preds)
+    preds = exp_preds / np.sum(exp_preds)
+    probas = np.random.multinomial(1, preds, 1)
+    return np.argmax(probas)
+
+# train the model, output generated text after each iteration
+for iteration in range(1, 60):
+    print()
+    print('-' * 50)
+    print('Iteration', iteration)
+    model.fit(X, y,
+              batch_size=128,
+              epochs=1)
+
+    start_index = random.randint(0, len(text) - maxlen - 1)
+
+    for diversity in [0.2, 0.5, 1.0, 1.2]:
+        print()
+        print('----- diversity:', diversity)
+
+        generated = ''
+        sentence = text[start_index: start_index + maxlen]
+        generated += sentence
+        print('----- Generating with seed: "' + sentence + '"')
+        sys.stdout.write(generated)
+
+        for i in range(400):
+            x = np.zeros((1, maxlen, len(chars)))
+            for t, char in enumerate(sentence):
+                x[0, t, char_indices[char]] = 1.
+
+            preds = model.predict(x, verbose=0)[0]
+            next_index = sample(preds, diversity)
+            next_char = indices_char[next_index]
+
+            generated += next_char
+            sentence = sentence[1:] + next_char
+
+            sys.stdout.write(next_char)
+            sys.stdout.flush()
+        print()

examples/mnist_acgan.py

@@ -0,0 +1,315 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Train an Auxiliary Classifier Generative Adversarial Network (ACGAN) on the
+MNIST dataset. See https://arxiv.org/abs/1610.09585 for more details.
+
+You should start to see reasonable images after ~5 epochs, and good images
+by ~15 epochs. You should use a GPU, as the convolution-heavy operations are
+very slow on the CPU. Prefer the TensorFlow backend if you plan on iterating,
+as the compilation time can be a blocker using Theano.
+
+Timings:
+
+Hardware           | Backend | Time / Epoch
+-------------------------------------------
+ CPU               | TF      | 3 hrs
+ Titan X (maxwell) | TF      | 4 min
+ Titan X (maxwell) | TH      | 7 min
+
+Consult https://github.com/lukedeo/keras-acgan for more information and
+example output
+"""
+from __future__ import print_function
+
+from collections import defaultdict
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+from PIL import Image
+
+from six.moves import range
+
+import keras.backend as K
+from keras.datasets import mnist
+from keras import layers
+from keras.layers import Input, Dense, Reshape, Flatten, Embedding, Dropout
+from keras.layers.advanced_activations import LeakyReLU
+from keras.layers.convolutional import UpSampling2D, Conv2D
+from keras.models import Sequential, Model
+from keras.optimizers import Adam
+from keras.utils.generic_utils import Progbar
+import numpy as np
+
+np.random.seed(1337)
+
+K.set_image_data_format('channels_first')
+
+
+def build_generator(latent_size):
+    # we will map a pair of (z, L), where z is a latent vector and L is a
+    # label drawn from P_c, to image space (..., 1, 28, 28)
+    cnn = Sequential()
+
+    cnn.add(Dense(1024, input_dim=latent_size, activation='relu'))
+    cnn.add(Dense(128 * 7 * 7, activation='relu'))
+    cnn.add(Reshape((128, 7, 7)))
+
+    # upsample to (..., 14, 14)
+    cnn.add(UpSampling2D(size=(2, 2)))
+    cnn.add(Conv2D(256, 5, padding='same',
+                   activation='relu',
+                   kernel_initializer='glorot_normal'))
+
+    # upsample to (..., 28, 28)
+    cnn.add(UpSampling2D(size=(2, 2)))
+    cnn.add(Conv2D(128, 5, padding='same',
+                   activation='relu',
+                   kernel_initializer='glorot_normal'))
+
+    # take a channel axis reduction
+    cnn.add(Conv2D(1, 2, padding='same',
+                   activation='tanh',
+                   kernel_initializer='glorot_normal'))
+
+    # this is the z space commonly refered to in GAN papers
+    latent = Input(shape=(latent_size, ))
+
+    # this will be our label
+    image_class = Input(shape=(1,), dtype='int32')
+
+    # 10 classes in MNIST
+    cls = Flatten()(Embedding(10, latent_size,
+                              embeddings_initializer='glorot_normal')(image_class))
+
+    # hadamard product between z-space and a class conditional embedding
+    h = layers.multiply([latent, cls])
+
+    fake_image = cnn(h)
+
+    return Model([latent, image_class], fake_image)
+
+
+def build_discriminator():
+    # build a relatively standard conv net, with LeakyReLUs as suggested in
+    # the reference paper
+    cnn = Sequential()
+
+    cnn.add(Conv2D(32, 3, padding='same', strides=2,
+                   input_shape=(1, 28, 28)))
+    cnn.add(LeakyReLU())
+    cnn.add(Dropout(0.3))
+
+    cnn.add(Conv2D(64, 3, padding='same', strides=1))
+    cnn.add(LeakyReLU())
+    cnn.add(Dropout(0.3))
+
+    cnn.add(Conv2D(128, 3, padding='same', strides=2))
+    cnn.add(LeakyReLU())
+    cnn.add(Dropout(0.3))
+
+    cnn.add(Conv2D(256, 3, padding='same', strides=1))
+    cnn.add(LeakyReLU())
+    cnn.add(Dropout(0.3))
+
+    cnn.add(Flatten())
+
+    image = Input(shape=(1, 28, 28))
+
+    features = cnn(image)
+
+    # first output (name=generation) is whether or not the discriminator
+    # thinks the image that is being shown is fake, and the second output
+    # (name=auxiliary) is the class that the discriminator thinks the image
+    # belongs to.
+    fake = Dense(1, activation='sigmoid', name='generation')(features)
+    aux = Dense(10, activation='softmax', name='auxiliary')(features)
+
+    return Model(image, [fake, aux])
+
+if __name__ == '__main__':
+
+    # batch and latent size taken from the paper
+    epochs = 50
+    batch_size = 100
+    latent_size = 100
+
+    # Adam parameters suggested in https://arxiv.org/abs/1511.06434
+    adam_lr = 0.0002
+    adam_beta_1 = 0.5
+
+    # build the discriminator
+    discriminator = build_discriminator()
+    discriminator.compile(
+        optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1),
+        loss=['binary_crossentropy', 'sparse_categorical_crossentropy']
+    )
+
+    # build the generator
+    generator = build_generator(latent_size)
+    generator.compile(optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1),
+                      loss='binary_crossentropy')
+
+    latent = Input(shape=(latent_size, ))
+    image_class = Input(shape=(1,), dtype='int32')
+
+    # get a fake image
+    fake = generator([latent, image_class])
+
+    # we only want to be able to train generation for the combined model
+    discriminator.trainable = False
+    fake, aux = discriminator(fake)
+    combined = Model([latent, image_class], [fake, aux])
+
+    combined.compile(
+        optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1),
+        loss=['binary_crossentropy', 'sparse_categorical_crossentropy']
+    )
+
+    # get our mnist data, and force it to be of shape (..., 1, 28, 28) with
+    # range [-1, 1]
+    (X_train, y_train), (X_test, y_test) = mnist.load_data()
+    X_train = (X_train.astype(np.float32) - 127.5) / 127.5
+    X_train = np.expand_dims(X_train, axis=1)
+
+    X_test = (X_test.astype(np.float32) - 127.5) / 127.5
+    X_test = np.expand_dims(X_test, axis=1)
+
+    num_train, num_test = X_train.shape[0], X_test.shape[0]
+
+    train_history = defaultdict(list)
+    test_history = defaultdict(list)
+
+    for epoch in range(epochs):
+        print('Epoch {} of {}'.format(epoch + 1, epochs))
+
+        num_batches = int(X_train.shape[0] / batch_size)
+        progress_bar = Progbar(target=num_batches)
+
+        epoch_gen_loss = []
+        epoch_disc_loss = []
+
+        for index in range(num_batches):
+            progress_bar.update(index)
+            # generate a new batch of noise
+            noise = np.random.uniform(-1, 1, (batch_size, latent_size))
+
+            # get a batch of real images
+            image_batch = X_train[index * batch_size:(index + 1) * batch_size]
+            label_batch = y_train[index * batch_size:(index + 1) * batch_size]
+
+            # sample some labels from p_c
+            sampled_labels = np.random.randint(0, 10, batch_size)
+
+            # generate a batch of fake images, using the generated labels as a
+            # conditioner. We reshape the sampled labels to be
+            # (batch_size, 1) so that we can feed them into the embedding
+            # layer as a length one sequence
+            generated_images = generator.predict(
+                [noise, sampled_labels.reshape((-1, 1))], verbose=0)
+
+            X = np.concatenate((image_batch, generated_images))
+            y = np.array([1] * batch_size + [0] * batch_size)
+            aux_y = np.concatenate((label_batch, sampled_labels), axis=0)
+
+            # see if the discriminator can figure itself out...
+            epoch_disc_loss.append(discriminator.train_on_batch(X, [y, aux_y]))
+
+            # make new noise. we generate 2 * batch size here such that we have
+            # the generator optimize over an identical number of images as the
+            # discriminator
+            noise = np.random.uniform(-1, 1, (2 * batch_size, latent_size))
+            sampled_labels = np.random.randint(0, 10, 2 * batch_size)
+
+            # we want to train the generator to trick the discriminator
+            # For the generator, we want all the {fake, not-fake} labels to say
+            # not-fake
+            trick = np.ones(2 * batch_size)
+
+            epoch_gen_loss.append(combined.train_on_batch(
+                [noise, sampled_labels.reshape((-1, 1))],
+                [trick, sampled_labels]))
+
+        print('\nTesting for epoch {}:'.format(epoch + 1))
+
+        # evaluate the testing loss here
+
+        # generate a new batch of noise
+        noise = np.random.uniform(-1, 1, (num_test, latent_size))
+
+        # sample some labels from p_c and generate images from them
+        sampled_labels = np.random.randint(0, 10, num_test)
+        generated_images = generator.predict(
+            [noise, sampled_labels.reshape((-1, 1))], verbose=False)
+
+        X = np.concatenate((X_test, generated_images))
+        y = np.array([1] * num_test + [0] * num_test)
+        aux_y = np.concatenate((y_test, sampled_labels), axis=0)
+
+        # see if the discriminator can figure itself out...
+        discriminator_test_loss = discriminator.evaluate(
+            X, [y, aux_y], verbose=False)
+
+        discriminator_train_loss = np.mean(np.array(epoch_disc_loss), axis=0)
+
+        # make new noise
+        noise = np.random.uniform(-1, 1, (2 * num_test, latent_size))
+        sampled_labels = np.random.randint(0, 10, 2 * num_test)
+
+        trick = np.ones(2 * num_test)
+
+        generator_test_loss = combined.evaluate(
+            [noise, sampled_labels.reshape((-1, 1))],
+            [trick, sampled_labels], verbose=False)
+
+        generator_train_loss = np.mean(np.array(epoch_gen_loss), axis=0)
+
+        # generate an epoch report on performance
+        train_history['generator'].append(generator_train_loss)
+        train_history['discriminator'].append(discriminator_train_loss)
+
+        test_history['generator'].append(generator_test_loss)
+        test_history['discriminator'].append(discriminator_test_loss)
+
+        print('{0:<22s} | {1:4s} | {2:15s} | {3:5s}'.format(
+            'component', *discriminator.metrics_names))
+        print('-' * 65)
+
+        ROW_FMT = '{0:<22s} | {1:<4.2f} | {2:<15.2f} | {3:<5.2f}'
+        print(ROW_FMT.format('generator (train)',
+                             *train_history['generator'][-1]))
+        print(ROW_FMT.format('generator (test)',
+                             *test_history['generator'][-1]))
+        print(ROW_FMT.format('discriminator (train)',
+                             *train_history['discriminator'][-1]))
+        print(ROW_FMT.format('discriminator (test)',
+                             *test_history['discriminator'][-1]))
+
+        # save weights every epoch
+        generator.save_weights(
+            'params_generator_epoch_{0:03d}.hdf5'.format(epoch), True)
+        discriminator.save_weights(
+            'params_discriminator_epoch_{0:03d}.hdf5'.format(epoch), True)
+
+        # generate some digits to display
+        noise = np.random.uniform(-1, 1, (100, latent_size))
+
+        sampled_labels = np.array([
+            [i] * 10 for i in range(10)
+        ]).reshape(-1, 1)
+
+        # get a batch to display
+        generated_images = generator.predict(
+            [noise, sampled_labels], verbose=0)
+
+        # arrange them into a grid
+        img = (np.concatenate([r.reshape(-1, 28)
+                               for r in np.split(generated_images, 10)
+                               ], axis=-1) * 127.5 + 127.5).astype(np.uint8)
+
+        Image.fromarray(img).save(
+            'plot_epoch_{0:03d}_generated.png'.format(epoch))
+
+    pickle.dump({'train': train_history, 'test': test_history},
+                open('acgan-history.pkl', 'wb'))

examples/mnist_cnn.py

@@ -0,0 +1,70 @@
+'''Trains a simple convnet on the MNIST dataset.
+
+Gets to 99.25% test accuracy after 12 epochs
+(there is still a lot of margin for parameter tuning).
+16 seconds per epoch on a GRID K520 GPU.
+'''
+
+from __future__ import print_function
+import keras
+from keras.datasets import mnist
+from keras.models import Sequential
+from keras.layers import Dense, Dropout, Flatten
+from keras.layers import Conv2D, MaxPooling2D
+from keras import backend as K
+
+batch_size = 128
+num_classes = 10
+epochs = 12
+
+# input image dimensions
+img_rows, img_cols = 28, 28
+
+# the data, shuffled and split between train and test sets
+(x_train, y_train), (x_test, y_test) = mnist.load_data()
+
+if K.image_data_format() == 'channels_first':
+    x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
+    x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)
+    input_shape = (1, img_rows, img_cols)
+else:
+    x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
+    x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)
+    input_shape = (img_rows, img_cols, 1)
+
+x_train = x_train.astype('float32')
+x_test = x_test.astype('float32')
+x_train /= 255
+x_test /= 255
+print('x_train shape:', x_train.shape)
+print(x_train.shape[0], 'train samples')
+print(x_test.shape[0], 'test samples')
+
+# convert class vectors to binary class matrices
+y_train = keras.utils.to_categorical(y_train, num_classes)
+y_test = keras.utils.to_categorical(y_test, num_classes)
+
+model = Sequential()
+model.add(Conv2D(32, kernel_size=(3, 3),
+                 activation='relu',
+                 input_shape=input_shape))
+model.add(Conv2D(64, (3, 3), activation='relu'))
+model.add(MaxPooling2D(pool_size=(2, 2)))
+model.add(Dropout(0.25))
+model.add(Flatten())
+model.add(Dense(128, activation='relu'))
+model.add(Dropout(0.5))
+model.add(Dense(num_classes, activation='softmax'))
+
+model.compile(loss=keras.losses.categorical_crossentropy,
+              optimizer=keras.optimizers.Adadelta(),
+              metrics=['accuracy'])
+
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=epochs,
+          verbose=1,
+          validation_data=(x_test, y_test))
+score = model.evaluate(x_test, y_test, verbose=0)
+print('Test loss:', score[0])
+print('Test accuracy:', score[1])

examples/mnist_hierarchical_rnn.py

@@ -0,0 +1,90 @@
+"""This is an example of using Hierarchical RNN (HRNN) to classify MNIST digits.
+
+HRNNs can learn across multiple levels of temporal hiearchy over a complex sequence.
+Usually, the first recurrent layer of an HRNN encodes a sentence (e.g. of word vectors)
+into a  sentence vector. The second recurrent layer then encodes a sequence of
+such vectors (encoded by the first layer) into a document vector. This
+document vector is considered to preserve both the word-level and
+sentence-level structure of the context.
+
+# References
+    - [A Hierarchical Neural Autoencoder for Paragraphs and Documents](https://arxiv.org/abs/1506.01057)
+        Encodes paragraphs and documents with HRNN.
+        Results have shown that HRNN outperforms standard
+        RNNs and may play some role in more sophisticated generation tasks like
+        summarization or question answering.
+    - [Hierarchical recurrent neural network for skeleton based action recognition](http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=7298714)
+        Achieved state-of-the-art results on skeleton based action recognition with 3 levels
+        of bidirectional HRNN combined with fully connected layers.
+
+In the below MNIST example the first LSTM layer first encodes every
+column of pixels of shape (28, 1) to a column vector of shape (128,). The second LSTM
+layer encodes then these 28 column vectors of shape (28, 128) to a image vector
+representing the whole image. A final Dense layer is added for prediction.
+
+After 5 epochs: train acc: 0.9858, val acc: 0.9864
+"""
+from __future__ import print_function
+
+import keras
+from keras.datasets import mnist
+from keras.models import Model
+from keras.layers import Input, Dense, TimeDistributed
+from keras.layers import LSTM
+
+# Training parameters.
+batch_size = 32
+num_classes = 10
+epochs = 5
+
+# Embedding dimensions.
+row_hidden = 128
+col_hidden = 128
+
+# The data, shuffled and split between train and test sets.
+(x_train, y_train), (x_test, y_test) = mnist.load_data()
+
+# Reshapes data to 4D for Hierarchical RNN.
+x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)
+x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)
+x_train = x_train.astype('float32')
+x_test = x_test.astype('float32')
+x_train /= 255
+x_test /= 255
+print('x_train shape:', x_train.shape)
+print(x_train.shape[0], 'train samples')
+print(x_test.shape[0], 'test samples')
+
+# Converts class vectors to binary class matrices.
+y_train = keras.utils.to_categorical(y_train, num_classes)
+y_test = keras.utils.to_categorical(y_test, num_classes)
+
+row, col, pixel = x_train.shape[1:]
+
+# 4D input.
+x = Input(shape=(row, col, pixel))
+
+# Encodes a row of pixels using TimeDistributed Wrapper.
+encoded_rows = TimeDistributed(LSTM(row_hidden))(x)
+
+# Encodes columns of encoded rows.
+encoded_columns = LSTM(col_hidden)(encoded_rows)
+
+# Final predictions and model.
+prediction = Dense(num_classes, activation='softmax')(encoded_columns)
+model = Model(x, prediction)
+model.compile(loss='categorical_crossentropy',
+              optimizer='rmsprop',
+              metrics=['accuracy'])
+
+# Training.
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=epochs,
+          verbose=1,
+          validation_data=(x_test, y_test))
+
+# Evaluation.
+scores = model.evaluate(x_test, y_test, verbose=0)
+print('Test loss:', scores[0])
+print('Test accuracy:', scores[1])

examples/mnist_irnn.py

@@ -0,0 +1,73 @@
+'''This is a reproduction of the IRNN experiment
+with pixel-by-pixel sequential MNIST in
+"A Simple Way to Initialize Recurrent Networks of Rectified Linear Units"
+by Quoc V. Le, Navdeep Jaitly, Geoffrey E. Hinton
+
+arxiv:1504.00941v2 [cs.NE] 7 Apr 2015
+http://arxiv.org/pdf/1504.00941v2.pdf
+
+Optimizer is replaced with RMSprop which yields more stable and steady
+improvement.
+
+Reaches 0.93 train/test accuracy after 900 epochs
+(which roughly corresponds to 1687500 steps in the original paper.)
+'''
+
+from __future__ import print_function
+
+import keras
+from keras.datasets import mnist
+from keras.models import Sequential
+from keras.layers import Dense, Activation
+from keras.layers import SimpleRNN
+from keras import initializers
+from keras.optimizers import RMSprop
+
+batch_size = 32
+num_classes = 10
+epochs = 200
+hidden_units = 100
+
+learning_rate = 1e-6
+clip_norm = 1.0
+
+# the data, shuffled and split between train and test sets
+(x_train, y_train), (x_test, y_test) = mnist.load_data()
+
+x_train = x_train.reshape(x_train.shape[0], -1, 1)
+x_test = x_test.reshape(x_test.shape[0], -1, 1)
+x_train = x_train.astype('float32')
+x_test = x_test.astype('float32')
+x_train /= 255
+x_test /= 255
+print('x_train shape:', x_train.shape)
+print(x_train.shape[0], 'train samples')
+print(x_test.shape[0], 'test samples')
+
+# convert class vectors to binary class matrices
+y_train = keras.utils.to_categorical(y_train, num_classes)
+y_test = keras.utils.to_categorical(y_test, num_classes)
+
+print('Evaluate IRNN...')
+model = Sequential()
+model.add(SimpleRNN(hidden_units,
+                    kernel_initializer=initializers.RandomNormal(stddev=0.001),
+                    recurrent_initializer=initializers.Identity(gain=1.0),
+                    activation='relu',
+                    input_shape=x_train.shape[1:]))
+model.add(Dense(num_classes))
+model.add(Activation('softmax'))
+rmsprop = RMSprop(lr=learning_rate)
+model.compile(loss='categorical_crossentropy',
+              optimizer=rmsprop,
+              metrics=['accuracy'])
+
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=epochs,
+          verbose=1,
+          validation_data=(x_test, y_test))
+
+scores = model.evaluate(x_test, y_test, verbose=0)
+print('IRNN test score:', scores[0])
+print('IRNN test accuracy:', scores[1])

examples/mnist_mlp.py

@@ -0,0 +1,57 @@
+'''Trains a simple deep NN on the MNIST dataset.
+
+Gets to 98.40% test accuracy after 20 epochs
+(there is *a lot* of margin for parameter tuning).
+2 seconds per epoch on a K520 GPU.
+'''
+
+from __future__ import print_function
+
+import keras
+from keras.datasets import mnist
+from keras.models import Sequential
+from keras.layers import Dense, Dropout
+from keras.optimizers import RMSprop
+
+
+batch_size = 128
+num_classes = 10
+epochs = 20
+
+# the data, shuffled and split between train and test sets
+(x_train, y_train), (x_test, y_test) = mnist.load_data()
+
+x_train = x_train.reshape(60000, 784)
+x_test = x_test.reshape(10000, 784)
+x_train = x_train.astype('float32')
+x_test = x_test.astype('float32')
+x_train /= 255
+x_test /= 255
+print(x_train.shape[0], 'train samples')
+print(x_test.shape[0], 'test samples')
+
+# convert class vectors to binary class matrices
+y_train = keras.utils.to_categorical(y_train, num_classes)
+y_test = keras.utils.to_categorical(y_test, num_classes)
+
+model = Sequential()
+model.add(Dense(512, activation='relu', input_shape=(784,)))
+model.add(Dropout(0.2))
+model.add(Dense(512, activation='relu'))
+model.add(Dropout(0.2))
+model.add(Dense(10, activation='softmax'))
+
+model.summary()
+
+model.compile(loss='categorical_crossentropy',
+              optimizer=RMSprop(),
+              metrics=['accuracy'])
+
+history = model.fit(x_train, y_train,
+                    batch_size=batch_size,
+                    epochs=epochs,
+                    verbose=1,
+                    validation_data=(x_test, y_test))
+score = model.evaluate(x_test, y_test, verbose=0)
+print('Test loss:', score[0])
+print('Test accuracy:', score[1])

examples/mnist_net2net.py

@@ -0,0 +1,389 @@
+'''This is an implementation of Net2Net experiment with MNIST in
+'Net2Net: Accelerating Learning via Knowledge Transfer'
+by Tianqi Chen, Ian Goodfellow, and Jonathon Shlens
+
+arXiv:1511.05641v4 [cs.LG] 23 Apr 2016
+http://arxiv.org/abs/1511.05641
+
+Notes
+- What:
+  + Net2Net is a group of methods to transfer knowledge from a teacher neural
+    net to a student net,so that the student net can be trained faster than
+    from scratch.
+  + The paper discussed two specific methods of Net2Net, i.e. Net2WiderNet
+    and Net2DeeperNet.
+  + Net2WiderNet replaces a model with an equivalent wider model that has
+    more units in each hidden layer.
+  + Net2DeeperNet replaces a model with an equivalent deeper model.
+  + Both are based on the idea of 'function-preserving transformations of
+    neural nets'.
+- Why:
+  + Enable fast exploration of multiple neural nets in experimentation and
+    design process,by creating a series of wider and deeper models with
+    transferable knowledge.
+  + Enable 'lifelong learning system' by gradually adjusting model complexity
+    to data availability,and reusing transferable knowledge.
+
+Experiments
+- Teacher model: a basic CNN model trained on MNIST for 3 epochs.
+- Net2WiderNet experiment:
+  + Student model has a wider Conv2D layer and a wider FC layer.
+  + Comparison of 'random-padding' vs 'net2wider' weight initialization.
+  + With both methods, student model should immediately perform as well as
+    teacher model, but 'net2wider' is slightly better.
+- Net2DeeperNet experiment:
+  + Student model has an extra Conv2D layer and an extra FC layer.
+  + Comparison of 'random-init' vs 'net2deeper' weight initialization.
+  + Starting performance of 'net2deeper' is better than 'random-init'.
+- Hyper-parameters:
+  + SGD with momentum=0.9 is used for training teacher and student models.
+  + Learning rate adjustment: it's suggested to reduce learning rate
+    to 1/10 for student model.
+  + Addition of noise in 'net2wider' is used to break weight symmetry
+    and thus enable full capacity of student models. It is optional
+    when a Dropout layer is used.
+
+Results
+- Tested with 'Theano' backend and 'channels_first' image_data_format.
+- Running on GPU GeForce GTX 980M
+- Performance Comparisons - validation loss values during first 3 epochs:
+(1) teacher_model:             0.075    0.041    0.041
+(2) wider_random_pad:          0.036    0.034    0.032
+(3) wider_net2wider:           0.032    0.030    0.030
+(4) deeper_random_init:        0.061    0.043    0.041
+(5) deeper_net2deeper:         0.032    0.031    0.029
+'''
+
+from __future__ import print_function
+from six.moves import xrange
+import numpy as np
+import keras
+from keras.models import Sequential
+from keras.layers import Conv2D, MaxPooling2D, Dense, Flatten
+from keras.optimizers import SGD
+from keras.datasets import mnist
+
+if keras.backend.image_data_format() == 'channels_first':
+    input_shape = (1, 28, 28)  # image shape
+else:
+    input_shape = (28, 28, 1)  # image shape
+num_class = 10  # number of class
+
+
+# load and pre-process data
+def preprocess_input(x):
+    return x.reshape((-1, ) + input_shape) / 255.
+
+
+def preprocess_output(y):
+    return keras.utils.to_categorical(y)
+
+(train_x, train_y), (validation_x, validation_y) = mnist.load_data()
+train_x, validation_x = map(preprocess_input, [train_x, validation_x])
+train_y, validation_y = map(preprocess_output, [train_y, validation_y])
+print('Loading MNIST data...')
+print('train_x shape:', train_x.shape, 'train_y shape:', train_y.shape)
+print('validation_x shape:', validation_x.shape,
+      'validation_y shape', validation_y.shape)
+
+
+# knowledge transfer algorithms
+def wider2net_conv2d(teacher_w1, teacher_b1, teacher_w2, new_width, init):
+    '''Get initial weights for a wider conv2d layer with a bigger filters,
+    by 'random-padding' or 'net2wider'.
+
+    # Arguments
+        teacher_w1: `weight` of conv2d layer to become wider,
+          of shape (filters1, num_channel1, kh1, kw1)
+        teacher_b1: `bias` of conv2d layer to become wider,
+          of shape (filters1, )
+        teacher_w2: `weight` of next connected conv2d layer,
+          of shape (filters2, num_channel2, kh2, kw2)
+        new_width: new `filters` for the wider conv2d layer
+        init: initialization algorithm for new weights,
+          either 'random-pad' or 'net2wider'
+    '''
+    assert teacher_w1.shape[0] == teacher_w2.shape[1], (
+        'successive layers from teacher model should have compatible shapes')
+    assert teacher_w1.shape[0] == teacher_b1.shape[0], (
+        'weight and bias from same layer should have compatible shapes')
+    assert new_width > teacher_w1.shape[0], (
+        'new width (filters) should be bigger than the existing one')
+
+    n = new_width - teacher_w1.shape[0]
+    if init == 'random-pad':
+        new_w1 = np.random.normal(0, 0.1, size=(n, ) + teacher_w1.shape[1:])
+        new_b1 = np.ones(n) * 0.1
+        new_w2 = np.random.normal(0, 0.1, size=(
+            teacher_w2.shape[0], n) + teacher_w2.shape[2:])
+    elif init == 'net2wider':
+        index = np.random.randint(teacher_w1.shape[0], size=n)
+        factors = np.bincount(index)[index] + 1.
+        new_w1 = teacher_w1[index, :, :, :]
+        new_b1 = teacher_b1[index]
+        new_w2 = teacher_w2[:, index, :, :] / factors.reshape((1, -1, 1, 1))
+    else:
+        raise ValueError('Unsupported weight initializer: %s' % init)
+
+    student_w1 = np.concatenate((teacher_w1, new_w1), axis=0)
+    if init == 'random-pad':
+        student_w2 = np.concatenate((teacher_w2, new_w2), axis=1)
+    elif init == 'net2wider':
+        # add small noise to break symmetry, so that student model will have
+        # full capacity later
+        noise = np.random.normal(0, 5e-2 * new_w2.std(), size=new_w2.shape)
+        student_w2 = np.concatenate((teacher_w2, new_w2 + noise), axis=1)
+        student_w2[:, index, :, :] = new_w2
+    student_b1 = np.concatenate((teacher_b1, new_b1), axis=0)
+
+    return student_w1, student_b1, student_w2
+
+
+def wider2net_fc(teacher_w1, teacher_b1, teacher_w2, new_width, init):
+    '''Get initial weights for a wider fully connected (dense) layer
+       with a bigger nout, by 'random-padding' or 'net2wider'.
+
+    # Arguments
+        teacher_w1: `weight` of fc layer to become wider,
+          of shape (nin1, nout1)
+        teacher_b1: `bias` of fc layer to become wider,
+          of shape (nout1, )
+        teacher_w2: `weight` of next connected fc layer,
+          of shape (nin2, nout2)
+        new_width: new `nout` for the wider fc layer
+        init: initialization algorithm for new weights,
+          either 'random-pad' or 'net2wider'
+    '''
+    assert teacher_w1.shape[1] == teacher_w2.shape[0], (
+        'successive layers from teacher model should have compatible shapes')
+    assert teacher_w1.shape[1] == teacher_b1.shape[0], (
+        'weight and bias from same layer should have compatible shapes')
+    assert new_width > teacher_w1.shape[1], (
+        'new width (nout) should be bigger than the existing one')
+
+    n = new_width - teacher_w1.shape[1]
+    if init == 'random-pad':
+        new_w1 = np.random.normal(0, 0.1, size=(teacher_w1.shape[0], n))
+        new_b1 = np.ones(n) * 0.1
+        new_w2 = np.random.normal(0, 0.1, size=(n, teacher_w2.shape[1]))
+    elif init == 'net2wider':
+        index = np.random.randint(teacher_w1.shape[1], size=n)
+        factors = np.bincount(index)[index] + 1.
+        new_w1 = teacher_w1[:, index]
+        new_b1 = teacher_b1[index]
+        new_w2 = teacher_w2[index, :] / factors[:, np.newaxis]
+    else:
+        raise ValueError('Unsupported weight initializer: %s' % init)
+
+    student_w1 = np.concatenate((teacher_w1, new_w1), axis=1)
+    if init == 'random-pad':
+        student_w2 = np.concatenate((teacher_w2, new_w2), axis=0)
+    elif init == 'net2wider':
+        # add small noise to break symmetry, so that student model will have
+        # full capacity later
+        noise = np.random.normal(0, 5e-2 * new_w2.std(), size=new_w2.shape)
+        student_w2 = np.concatenate((teacher_w2, new_w2 + noise), axis=0)
+        student_w2[index, :] = new_w2
+    student_b1 = np.concatenate((teacher_b1, new_b1), axis=0)
+
+    return student_w1, student_b1, student_w2
+
+
+def deeper2net_conv2d(teacher_w):
+    '''Get initial weights for a deeper conv2d layer by net2deeper'.
+
+    # Arguments
+        teacher_w: `weight` of previous conv2d layer,
+          of shape (filters, num_channel, kh, kw)
+    '''
+    filters, num_channel, kh, kw = teacher_w.shape
+    student_w = np.zeros((filters, filters, kh, kw))
+    for i in xrange(filters):
+        student_w[i, i, (kh - 1) / 2, (kw - 1) / 2] = 1.
+    student_b = np.zeros(filters)
+    return student_w, student_b
+
+
+def copy_weights(teacher_model, student_model, layer_names):
+    '''Copy weights from teacher_model to student_model,
+     for layers with names listed in layer_names
+    '''
+    for name in layer_names:
+        weights = teacher_model.get_layer(name=name).get_weights()
+        student_model.get_layer(name=name).set_weights(weights)
+
+
+# methods to construct teacher_model and student_models
+def make_teacher_model(train_data, validation_data, epochs=3):
+    '''Train a simple CNN as teacher model.
+    '''
+    model = Sequential()
+    model.add(Conv2D(64, 3, input_shape=input_shape,
+                     padding='same', name='conv1'))
+    model.add(MaxPooling2D(2, name='pool1'))
+    model.add(Conv2D(64, 3, padding='same', name='conv2'))
+    model.add(MaxPooling2D(2, name='pool2'))
+    model.add(Flatten(name='flatten'))
+    model.add(Dense(64, activation='relu', name='fc1'))
+    model.add(Dense(num_class, activation='softmax', name='fc2'))
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=SGD(lr=0.01, momentum=0.9),
+                  metrics=['accuracy'])
+
+    train_x, train_y = train_data
+    history = model.fit(train_x, train_y,
+                        epochs=epochs,
+                        validation_data=validation_data)
+    return model, history
+
+
+def make_wider_student_model(teacher_model, train_data,
+                             validation_data, init, epochs=3):
+    '''Train a wider student model based on teacher_model,
+       with either 'random-pad' (baseline) or 'net2wider'
+    '''
+    new_conv1_width = 128
+    new_fc1_width = 128
+
+    model = Sequential()
+    # a wider conv1 compared to teacher_model
+    model.add(Conv2D(new_conv1_width, 3, input_shape=input_shape,
+                     padding='same', name='conv1'))
+    model.add(MaxPooling2D(2, name='pool1'))
+    model.add(Conv2D(64, 3, padding='same', name='conv2'))
+    model.add(MaxPooling2D(2, name='pool2'))
+    model.add(Flatten(name='flatten'))
+    # a wider fc1 compared to teacher model
+    model.add(Dense(new_fc1_width, activation='relu', name='fc1'))
+    model.add(Dense(num_class, activation='softmax', name='fc2'))
+
+    # The weights for other layers need to be copied from teacher_model
+    # to student_model, except for widened layers
+    # and their immediate downstreams, which will be initialized separately.
+    # For this example there are no other layers that need to be copied.
+
+    w_conv1, b_conv1 = teacher_model.get_layer('conv1').get_weights()
+    w_conv2, b_conv2 = teacher_model.get_layer('conv2').get_weights()
+    new_w_conv1, new_b_conv1, new_w_conv2 = wider2net_conv2d(
+        w_conv1, b_conv1, w_conv2, new_conv1_width, init)
+    model.get_layer('conv1').set_weights([new_w_conv1, new_b_conv1])
+    model.get_layer('conv2').set_weights([new_w_conv2, b_conv2])
+
+    w_fc1, b_fc1 = teacher_model.get_layer('fc1').get_weights()
+    w_fc2, b_fc2 = teacher_model.get_layer('fc2').get_weights()
+    new_w_fc1, new_b_fc1, new_w_fc2 = wider2net_fc(
+        w_fc1, b_fc1, w_fc2, new_fc1_width, init)
+    model.get_layer('fc1').set_weights([new_w_fc1, new_b_fc1])
+    model.get_layer('fc2').set_weights([new_w_fc2, b_fc2])
+
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=SGD(lr=0.001, momentum=0.9),
+                  metrics=['accuracy'])
+
+    train_x, train_y = train_data
+    history = model.fit(train_x, train_y,
+                        epochs=epochs,
+                        validation_data=validation_data)
+    return model, history
+
+
+def make_deeper_student_model(teacher_model, train_data,
+                              validation_data, init, epochs=3):
+    '''Train a deeper student model based on teacher_model,
+       with either 'random-init' (baseline) or 'net2deeper'
+    '''
+    model = Sequential()
+    model.add(Conv2D(64, 3, input_shape=input_shape,
+                     padding='same', name='conv1'))
+    model.add(MaxPooling2D(2, name='pool1'))
+    model.add(Conv2D(64, 3, padding='same', name='conv2'))
+    # add another conv2d layer to make original conv2 deeper
+    if init == 'net2deeper':
+        prev_w, _ = model.get_layer('conv2').get_weights()
+        new_weights = deeper2net_conv2d(prev_w)
+        model.add(Conv2D(64, 3, padding='same',
+                         name='conv2-deeper', weights=new_weights))
+    elif init == 'random-init':
+        model.add(Conv2D(64, 3, padding='same', name='conv2-deeper'))
+    else:
+        raise ValueError('Unsupported weight initializer: %s' % init)
+    model.add(MaxPooling2D(2, name='pool2'))
+    model.add(Flatten(name='flatten'))
+    model.add(Dense(64, activation='relu', name='fc1'))
+    # add another fc layer to make original fc1 deeper
+    if init == 'net2deeper':
+        # net2deeper for fc layer with relu, is just an identity initializer
+        model.add(Dense(64, kernel_initializer='identity',
+                        activation='relu', name='fc1-deeper'))
+    elif init == 'random-init':
+        model.add(Dense(64, activation='relu', name='fc1-deeper'))
+    else:
+        raise ValueError('Unsupported weight initializer: %s' % init)
+    model.add(Dense(num_class, activation='softmax', name='fc2'))
+
+    # copy weights for other layers
+    copy_weights(teacher_model, model, layer_names=[
+                 'conv1', 'conv2', 'fc1', 'fc2'])
+
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=SGD(lr=0.001, momentum=0.9),
+                  metrics=['accuracy'])
+
+    train_x, train_y = train_data
+    history = model.fit(train_x, train_y,
+                        epochs=epochs,
+                        validation_data=validation_data)
+    return model, history
+
+
+# experiments setup
+def net2wider_experiment():
+    '''Benchmark performances of
+    (1) a teacher model,
+    (2) a wider student model with `random_pad` initializer
+    (3) a wider student model with `Net2WiderNet` initializer
+    '''
+    train_data = (train_x, train_y)
+    validation_data = (validation_x, validation_y)
+    print('\nExperiment of Net2WiderNet ...')
+    print('\nbuilding teacher model ...')
+    teacher_model, _ = make_teacher_model(train_data,
+                                          validation_data,
+                                          epochs=3)
+
+    print('\nbuilding wider student model by random padding ...')
+    make_wider_student_model(teacher_model, train_data,
+                             validation_data, 'random-pad',
+                             epochs=3)
+    print('\nbuilding wider student model by net2wider ...')
+    make_wider_student_model(teacher_model, train_data,
+                             validation_data, 'net2wider',
+                             epochs=3)
+
+
+def net2deeper_experiment():
+    '''Benchmark performances of
+    (1) a teacher model,
+    (2) a deeper student model with `random_init` initializer
+    (3) a deeper student model with `Net2DeeperNet` initializer
+    '''
+    train_data = (train_x, train_y)
+    validation_data = (validation_x, validation_y)
+    print('\nExperiment of Net2DeeperNet ...')
+    print('\nbuilding teacher model ...')
+    teacher_model, _ = make_teacher_model(train_data,
+                                          validation_data,
+                                          epochs=3)
+
+    print('\nbuilding deeper student model by random init ...')
+    make_deeper_student_model(teacher_model, train_data,
+                              validation_data, 'random-init',
+                              epochs=3)
+    print('\nbuilding deeper student model by net2deeper ...')
+    make_deeper_student_model(teacher_model, train_data,
+                              validation_data, 'net2deeper',
+                              epochs=3)
+
+# run the experiments
+net2wider_experiment()
+net2deeper_experiment()

examples/mnist_nn.py

@@ -1,54 +0,0 @@
-from __future__ import absolute_import
-from __future__ import print_function
-from keras.datasets import mnist
-from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation
-from keras.regularizers import l2, l1
-from keras.constraints import maxnorm
-from keras.optimizers import SGD, Adam, RMSprop
-from keras.utils import np_utils
-import numpy as np
-
-'''
-    Train a simple deep NN on the MNIST dataset.
-'''
-
-batch_size = 64
-nb_classes = 10
-nb_epoch = 20
-
-np.random.seed(1337) # for reproducibility
-
-# the data, shuffled and split between tran and test sets
-(X_train, y_train), (X_test, y_test) = mnist.load_data()
-
-X_train = X_train.reshape(60000,784)
-X_test = X_test.reshape(10000,784)
-X_train = X_train.astype("float32")
-X_test = X_test.astype("float32")
-X_train /= 255
-X_test /= 255
-print(X_train.shape[0], 'train samples')
-print(X_test.shape[0], 'test samples')
-
-# convert class vectors to binary class matrices
-Y_train = np_utils.to_categorical(y_train, nb_classes)
-Y_test = np_utils.to_categorical(y_test, nb_classes)
-
-model = Sequential()
-model.add(Dense(784, 128))
-model.add(Activation('relu'))
-model.add(Dropout(0.2))
-model.add(Dense(128, 128))
-model.add(Activation('relu'))
-model.add(Dropout(0.2))
-model.add(Dense(128, 10))
-model.add(Activation('softmax'))
-
-rms = RMSprop()
-model.compile(loss='categorical_crossentropy', optimizer=rms)
-
-model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, verbose=2, validation_data=(X_test, Y_test))
-score = model.evaluate(X_test, Y_test, show_accuracy=True, verbose=0)
-print('Test score:', score[0])
-print('Test accuracy:', score[1])

examples/mnist_siamese_graph.py

@@ -0,0 +1,131 @@
+'''Train a Siamese MLP on pairs of digits from the MNIST dataset.
+
+It follows Hadsell-et-al.'06 [1] by computing the Euclidean distance on the
+output of the shared network and by optimizing the contrastive loss (see paper
+for mode details).
+
+[1] "Dimensionality Reduction by Learning an Invariant Mapping"
+    http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
+
+Gets to 99.5% test accuracy after 20 epochs.
+3 seconds per epoch on a Titan X GPU
+'''
+from __future__ import absolute_import
+from __future__ import print_function
+import numpy as np
+
+import random
+from keras.datasets import mnist
+from keras.models import Sequential, Model
+from keras.layers import Dense, Dropout, Input, Lambda
+from keras.optimizers import RMSprop
+from keras import backend as K
+
+
+def euclidean_distance(vects):
+    x, y = vects
+    return K.sqrt(K.maximum(K.sum(K.square(x - y), axis=1, keepdims=True), K.epsilon()))
+
+
+def eucl_dist_output_shape(shapes):
+    shape1, shape2 = shapes
+    return (shape1[0], 1)
+
+
+def contrastive_loss(y_true, y_pred):
+    '''Contrastive loss from Hadsell-et-al.'06
+    http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
+    '''
+    margin = 1
+    return K.mean(y_true * K.square(y_pred) +
+                  (1 - y_true) * K.square(K.maximum(margin - y_pred, 0)))
+
+
+def create_pairs(x, digit_indices):
+    '''Positive and negative pair creation.
+    Alternates between positive and negative pairs.
+    '''
+    pairs = []
+    labels = []
+    n = min([len(digit_indices[d]) for d in range(10)]) - 1
+    for d in range(10):
+        for i in range(n):
+            z1, z2 = digit_indices[d][i], digit_indices[d][i + 1]
+            pairs += [[x[z1], x[z2]]]
+            inc = random.randrange(1, 10)
+            dn = (d + inc) % 10
+            z1, z2 = digit_indices[d][i], digit_indices[dn][i]
+            pairs += [[x[z1], x[z2]]]
+            labels += [1, 0]
+    return np.array(pairs), np.array(labels)
+
+
+def create_base_network(input_dim):
+    '''Base network to be shared (eq. to feature extraction).
+    '''
+    seq = Sequential()
+    seq.add(Dense(128, input_shape=(input_dim,), activation='relu'))
+    seq.add(Dropout(0.1))
+    seq.add(Dense(128, activation='relu'))
+    seq.add(Dropout(0.1))
+    seq.add(Dense(128, activation='relu'))
+    return seq
+
+
+def compute_accuracy(predictions, labels):
+    '''Compute classification accuracy with a fixed threshold on distances.
+    '''
+    return labels[predictions.ravel() < 0.5].mean()
+
+
+# the data, shuffled and split between train and test sets
+(x_train, y_train), (x_test, y_test) = mnist.load_data()
+x_train = x_train.reshape(60000, 784)
+x_test = x_test.reshape(10000, 784)
+x_train = x_train.astype('float32')
+x_test = x_test.astype('float32')
+x_train /= 255
+x_test /= 255
+input_dim = 784
+epochs = 20
+
+# create training+test positive and negative pairs
+digit_indices = [np.where(y_train == i)[0] for i in range(10)]
+tr_pairs, tr_y = create_pairs(x_train, digit_indices)
+
+digit_indices = [np.where(y_test == i)[0] for i in range(10)]
+te_pairs, te_y = create_pairs(x_test, digit_indices)
+
+# network definition
+base_network = create_base_network(input_dim)
+
+input_a = Input(shape=(input_dim,))
+input_b = Input(shape=(input_dim,))
+
+# because we re-use the same instance `base_network`,
+# the weights of the network
+# will be shared across the two branches
+processed_a = base_network(input_a)
+processed_b = base_network(input_b)
+
+distance = Lambda(euclidean_distance,
+                  output_shape=eucl_dist_output_shape)([processed_a, processed_b])
+
+model = Model([input_a, input_b], distance)
+
+# train
+rms = RMSprop()
+model.compile(loss=contrastive_loss, optimizer=rms)
+model.fit([tr_pairs[:, 0], tr_pairs[:, 1]], tr_y,
+          batch_size=128,
+          epochs=epochs,
+          validation_data=([te_pairs[:, 0], te_pairs[:, 1]], te_y))
+
+# compute final accuracy on training and test sets
+pred = model.predict([tr_pairs[:, 0], tr_pairs[:, 1]])
+tr_acc = compute_accuracy(pred, tr_y)
+pred = model.predict([te_pairs[:, 0], te_pairs[:, 1]])
+te_acc = compute_accuracy(pred, te_y)
+
+print('* Accuracy on training set: %0.2f%%' % (100 * tr_acc))
+print('* Accuracy on test set: %0.2f%%' % (100 * te_acc))

examples/mnist_sklearn_wrapper.py

@@ -0,0 +1,102 @@
+'''Example of how to use sklearn wrapper
+
+Builds simple CNN models on MNIST and uses sklearn's GridSearchCV to find best model
+'''
+
+from __future__ import print_function
+
+import keras
+from keras.datasets import mnist
+from keras.models import Sequential
+from keras.layers import Dense, Dropout, Activation, Flatten
+from keras.layers import Conv2D, MaxPooling2D
+from keras.wrappers.scikit_learn import KerasClassifier
+from keras import backend as K
+from sklearn.grid_search import GridSearchCV
+
+
+num_classes = 10
+
+# input image dimensions
+img_rows, img_cols = 28, 28
+
+# load training data and do basic data normalization
+(x_train, y_train), (x_test, y_test) = mnist.load_data()
+
+if K.image_data_format() == 'channels_first':
+    x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
+    x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)
+    input_shape = (1, img_rows, img_cols)
+else:
+    x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
+    x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)
+    input_shape = (img_rows, img_cols, 1)
+
+x_train = x_train.astype('float32')
+x_test = x_test.astype('float32')
+x_train /= 255
+x_test /= 255
+
+# convert class vectors to binary class matrices
+y_train = keras.utils.to_categorical(y_train, num_classes)
+y_test = keras.utils.to_categorical(y_test, num_classes)
+
+
+def make_model(dense_layer_sizes, filters, kernel_size, pool_size):
+    '''Creates model comprised of 2 convolutional layers followed by dense layers
+
+    dense_layer_sizes: List of layer sizes.
+        This list has one number for each layer
+    filters: Number of convolutional filters in each convolutional layer
+    kernel_size: Convolutional kernel size
+    pool_size: Size of pooling area for max pooling
+    '''
+
+    model = Sequential()
+    model.add(Conv2D(filters, kernel_size,
+                     padding='valid',
+                     input_shape=input_shape))
+    model.add(Activation('relu'))
+    model.add(Conv2D(filters, kernel_size))
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D(pool_size=pool_size))
+    model.add(Dropout(0.25))
+
+    model.add(Flatten())
+    for layer_size in dense_layer_sizes:
+        model.add(Dense(layer_size))
+    model.add(Activation('relu'))
+    model.add(Dropout(0.5))
+    model.add(Dense(num_classes))
+    model.add(Activation('softmax'))
+
+    model.compile(loss='categorical_crossentropy',
+                  optimizer='adadelta',
+                  metrics=['accuracy'])
+
+    return model
+
+dense_size_candidates = [[32], [64], [32, 32], [64, 64]]
+my_classifier = KerasClassifier(make_model, batch_size=32)
+validator = GridSearchCV(my_classifier,
+                         param_grid={'dense_layer_sizes': dense_size_candidates,
+                                     # epochs is avail for tuning even when not
+                                     # an argument to model building function
+                                     'epochs': [3, 6],
+                                     'filters': [8],
+                                     'kernel_size': [3],
+                                     'pool_size': [2]},
+                         scoring='neg_log_loss',
+                         n_jobs=1)
+validator.fit(x_train, y_train)
+
+print('The parameters of the best model are: ')
+print(validator.best_params_)
+
+# validator.best_estimator_ returns sklearn-wrapped version of best model.
+# validator.best_estimator_.model returns the (unwrapped) keras model
+best_model = validator.best_estimator_.model
+metric_names = best_model.metrics_names
+metric_values = best_model.evaluate(x_test, y_test)
+for metric, value in zip(metric_names, metric_values):
+    print(metric, ': ', value)

examples/mnist_swwae.py

@@ -0,0 +1,203 @@
+'''Trains a stacked what-where autoencoder built on residual blocks on the
+MNIST dataset.  It exemplifies two influential methods that have been developed
+in the past few years.
+
+The first is the idea of properly 'unpooling.' During any max pool, the
+exact location (the 'where') of the maximal value in a pooled receptive field
+is lost, however it can be very useful in the overall reconstruction of an
+input image.  Therefore, if the 'where' is handed from the encoder
+to the corresponding decoder layer, features being decoded can be 'placed' in
+the right location, allowing for reconstructions of much higher fidelity.
+
+References:
+[1]
+'Visualizing and Understanding Convolutional Networks'
+Matthew D Zeiler, Rob Fergus
+https://arxiv.org/abs/1311.2901v3
+
+[2]
+'Stacked What-Where Auto-encoders'
+Junbo Zhao, Michael Mathieu, Ross Goroshin, Yann LeCun
+https://arxiv.org/abs/1506.02351v8
+
+The second idea exploited here is that of residual learning.  Residual blocks
+ease the training process by allowing skip connections that give the network
+the ability to be as linear (or non-linear) as the data sees fit.  This allows
+for much deep networks to be easily trained.  The residual element seems to
+be advantageous in the context of this example as it allows a nice symmetry
+between the encoder and decoder.  Normally, in the decoder, the final
+projection to the space where the image is reconstructed is linear, however
+this does not have to be the case for a residual block as the degree to which
+its output is linear or non-linear is determined by the data it is fed.
+However, in order to cap the reconstruction in this example, a hard softmax is
+applied as a bias because we know the MNIST digits are mapped to [0,1].
+
+References:
+[3]
+'Deep Residual Learning for Image Recognition'
+Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun
+https://arxiv.org/abs/1512.03385v1
+
+[4]
+'Identity Mappings in Deep Residual Networks'
+Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun
+https://arxiv.org/abs/1603.05027v3
+
+'''
+from __future__ import print_function
+import numpy as np
+
+from keras.datasets import mnist
+from keras.models import Model
+from keras.layers import Activation
+from keras.layers import UpSampling2D, Conv2D, MaxPooling2D
+from keras.layers import Input, BatchNormalization, ELU
+import matplotlib.pyplot as plt
+import keras.backend as K
+from keras import layers
+
+
+def convresblock(x, nfeats=8, ksize=3, nskipped=2, elu=True):
+    """The proposed residual block from [4].
+
+    Running with elu=True will use ELU nonlinearity and running with
+    elu=False will use BatchNorm + RELU nonlinearity.  While ELU's are fast
+    due to the fact they do not suffer from BatchNorm overhead, they may
+    overfit because they do not offer the stochastic element of the batch
+    formation process of BatchNorm, which acts as a good regularizer.
+
+    # Arguments
+        x: 4D tensor, the tensor to feed through the block
+        nfeats: Integer, number of feature maps for conv layers.
+        ksize: Integer, width and height of conv kernels in first convolution.
+        nskipped: Integer, number of conv layers for the residual function.
+        elu: Boolean, whether to use ELU or BN+RELU.
+
+    # Input shape
+        4D tensor with shape:
+        `(batch, channels, rows, cols)`
+
+    # Output shape
+        4D tensor with shape:
+        `(batch, filters, rows, cols)`
+    """
+    y0 = Conv2D(nfeats, ksize, padding='same')(x)
+    y = y0
+    for i in range(nskipped):
+        if elu:
+            y = ELU()(y)
+        else:
+            y = BatchNormalization(axis=1)(y)
+            y = Activation('relu')(y)
+        y = Conv2D(nfeats, 1, padding='same')(y)
+    return layers.add([y0, y])
+
+
+def getwhere(x):
+    ''' Calculate the 'where' mask that contains switches indicating which
+    index contained the max value when MaxPool2D was applied.  Using the
+    gradient of the sum is a nice trick to keep everything high level.'''
+    y_prepool, y_postpool = x
+    return K.gradients(K.sum(y_postpool), y_prepool)
+
+if K.backend() == 'tensorflow':
+    raise RuntimeError('This example can only run with the '
+                       'Theano backend for the time being, '
+                       'because it requires taking the gradient '
+                       'of a gradient, which isn\'t '
+                       'supported for all TF ops.')
+
+# This example assume 'channels_first' data format.
+K.set_image_data_format('channels_first')
+
+# input image dimensions
+img_rows, img_cols = 28, 28
+
+# the data, shuffled and split between train and test sets
+(x_train, _), (x_test, _) = mnist.load_data()
+
+x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
+x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)
+x_train = x_train.astype('float32')
+x_test = x_test.astype('float32')
+x_train /= 255
+x_test /= 255
+print('x_train shape:', x_train.shape)
+print(x_train.shape[0], 'train samples')
+print(x_test.shape[0], 'test samples')
+
+# The size of the kernel used for the MaxPooling2D
+pool_size = 2
+# The total number of feature maps at each layer
+nfeats = [8, 16, 32, 64, 128]
+# The sizes of the pooling kernel at each layer
+pool_sizes = np.array([1, 1, 1, 1, 1]) * pool_size
+# The convolution kernel size
+ksize = 3
+# Number of epochs to train for
+epochs = 5
+# Batch size during training
+batch_size = 128
+
+if pool_size == 2:
+    # if using a 5 layer net of pool_size = 2
+    x_train = np.pad(x_train, [[0, 0], [0, 0], [2, 2], [2, 2]],
+                     mode='constant')
+    x_test = np.pad(x_test, [[0, 0], [0, 0], [2, 2], [2, 2]], mode='constant')
+    nlayers = 5
+elif pool_size == 3:
+    # if using a 3 layer net of pool_size = 3
+    x_train = x_train[:, :, :-1, :-1]
+    x_test = x_test[:, :, :-1, :-1]
+    nlayers = 3
+else:
+    import sys
+    sys.exit('Script supports pool_size of 2 and 3.')
+
+# Shape of input to train on (note that model is fully convolutional however)
+input_shape = x_train.shape[1:]
+# The final list of the size of axis=1 for all layers, including input
+nfeats_all = [input_shape[0]] + nfeats
+
+# First build the encoder, all the while keeping track of the 'where' masks
+img_input = Input(shape=input_shape)
+
+# We push the 'where' masks to the following list
+wheres = [None] * nlayers
+y = img_input
+for i in range(nlayers):
+    y_prepool = convresblock(y, nfeats=nfeats_all[i + 1], ksize=ksize)
+    y = MaxPooling2D(pool_size=(pool_sizes[i], pool_sizes[i]))(y_prepool)
+    wheres[i] = layers.Lambda(
+        getwhere, output_shape=lambda x: x[0])([y_prepool, y])
+
+# Now build the decoder, and use the stored 'where' masks to place the features
+for i in range(nlayers):
+    ind = nlayers - 1 - i
+    y = UpSampling2D(size=(pool_sizes[ind], pool_sizes[ind]))(y)
+    y = layers.multiply([y, wheres[ind]])
+    y = convresblock(y, nfeats=nfeats_all[ind], ksize=ksize)
+
+# Use hard_simgoid to clip range of reconstruction
+y = Activation('hard_sigmoid')(y)
+
+# Define the model and it's mean square error loss, and compile it with Adam
+model = Model(img_input, y)
+model.compile('adam', 'mse')
+
+# Fit the model
+model.fit(x_train, x_train,
+          batch_size=batch_size,
+          epochs=epochs,
+          validation_data=(x_test, x_test))
+
+# Plot
+x_recon = model.predict(x_test[:25])
+x_plot = np.concatenate((x_test[:25], x_recon), axis=1)
+x_plot = x_plot.reshape((5, 10, input_shape[-2], input_shape[-1]))
+x_plot = np.vstack([np.hstack(x) for x in x_plot])
+plt.figure()
+plt.axis('off')
+plt.title('Test Samples: Originals/Reconstructions')
+plt.imshow(x_plot, interpolation='none', cmap='gray')
+plt.savefig('reconstructions.png')

examples/mnist_transfer_cnn.py

@@ -0,0 +1,126 @@
+'''Transfer learning toy example:
+
+1- Train a simple convnet on the MNIST dataset the first 5 digits [0..4].
+2- Freeze convolutional layers and fine-tune dense layers
+   for the classification of digits [5..9].
+
+Run on GPU: THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python mnist_transfer_cnn.py
+
+Get to 99.8% test accuracy after 5 epochs
+for the first five digits classifier
+and 99.2% for the last five digits after transfer + fine-tuning.
+'''
+
+from __future__ import print_function
+
+import datetime
+import keras
+from keras.datasets import mnist
+from keras.models import Sequential
+from keras.layers import Dense, Dropout, Activation, Flatten
+from keras.layers import Conv2D, MaxPooling2D
+from keras import backend as K
+
+now = datetime.datetime.now
+
+batch_size = 128
+num_classes = 5
+epochs = 5
+
+# input image dimensions
+img_rows, img_cols = 28, 28
+# number of convolutional filters to use
+filters = 32
+# size of pooling area for max pooling
+pool_size = 2
+# convolution kernel size
+kernel_size = 3
+
+if K.image_data_format() == 'channels_first':
+    input_shape = (1, img_rows, img_cols)
+else:
+    input_shape = (img_rows, img_cols, 1)
+
+
+def train_model(model, train, test, num_classes):
+    x_train = train[0].reshape((train[0].shape[0],) + input_shape)
+    x_test = test[0].reshape((test[0].shape[0],) + input_shape)
+    x_train = x_train.astype('float32')
+    x_test = x_test.astype('float32')
+    x_train /= 255
+    x_test /= 255
+    print('x_train shape:', x_train.shape)
+    print(x_train.shape[0], 'train samples')
+    print(x_test.shape[0], 'test samples')
+
+    # convert class vectors to binary class matrices
+    y_train = keras.utils.to_categorical(train[1], num_classes)
+    y_test = keras.utils.to_categorical(test[1], num_classes)
+
+    model.compile(loss='categorical_crossentropy',
+                  optimizer='adadelta',
+                  metrics=['accuracy'])
+
+    t = now()
+    model.fit(x_train, y_train,
+              batch_size=batch_size,
+              epochs=epochs,
+              verbose=1,
+              validation_data=(x_test, y_test))
+    print('Training time: %s' % (now() - t))
+    score = model.evaluate(x_test, y_test, verbose=0)
+    print('Test score:', score[0])
+    print('Test accuracy:', score[1])
+
+
+# the data, shuffled and split between train and test sets
+(x_train, y_train), (x_test, y_test) = mnist.load_data()
+
+# create two datasets one with digits below 5 and one with 5 and above
+x_train_lt5 = x_train[y_train < 5]
+y_train_lt5 = y_train[y_train < 5]
+x_test_lt5 = x_test[y_test < 5]
+y_test_lt5 = y_test[y_test < 5]
+
+x_train_gte5 = x_train[y_train >= 5]
+y_train_gte5 = y_train[y_train >= 5] - 5
+x_test_gte5 = x_test[y_test >= 5]
+y_test_gte5 = y_test[y_test >= 5] - 5
+
+# define two groups of layers: feature (convolutions) and classification (dense)
+feature_layers = [
+    Conv2D(filters, kernel_size,
+           padding='valid',
+           input_shape=input_shape),
+    Activation('relu'),
+    Conv2D(filters, kernel_size),
+    Activation('relu'),
+    MaxPooling2D(pool_size=pool_size),
+    Dropout(0.25),
+    Flatten(),
+]
+
+classification_layers = [
+    Dense(128),
+    Activation('relu'),
+    Dropout(0.5),
+    Dense(num_classes),
+    Activation('softmax')
+]
+
+# create complete model
+model = Sequential(feature_layers + classification_layers)
+
+# train model for 5-digit classification [0..4]
+train_model(model,
+            (x_train_lt5, y_train_lt5),
+            (x_test_lt5, y_test_lt5), num_classes)
+
+# freeze feature layers and rebuild model
+for l in feature_layers:
+    l.trainable = False
+
+# transfer: train dense layers for new classification task [5..9]
+train_model(model,
+            (x_train_gte5, y_train_gte5),
+            (x_test_gte5, y_test_gte5), num_classes)

examples/neural_doodle.py

@@ -0,0 +1,367 @@
+'''Neural doodle with Keras
+
+Script Usage:
+    # Arguments:
+    ```
+    --nlabels:              # of regions (colors) in mask images
+    --style-image:          image to learn style from
+    --style-mask:           semantic labels for style image
+    --target-mask:          semantic labels for target image (your doodle)
+    --content-image:        optional image to learn content from
+    --target-image-prefix:  path prefix for generated target images
+    ```
+
+    # Example 1: doodle using a style image, style mask
+    and target mask.
+    ```
+    python neural_doodle.py --nlabels 4 --style-image Monet/style.png \
+    --style-mask Monet/style_mask.png --target-mask Monet/target_mask.png \
+    --target-image-prefix generated/monet
+    ```
+
+    # Example 2: doodle using a style image, style mask,
+    target mask and an optional content image.
+    ```
+    python neural_doodle.py --nlabels 4 --style-image Renoir/style.png \
+    --style-mask Renoir/style_mask.png --target-mask Renoir/target_mask.png \
+    --content-image Renoir/creek.jpg \
+    --target-image-prefix generated/renoir
+    ```
+
+References:
+[Dmitry Ulyanov's blog on fast-neural-doodle](http://dmitryulyanov.github.io/feed-forward-neural-doodle/)
+[Torch code for fast-neural-doodle](https://github.com/DmitryUlyanov/fast-neural-doodle)
+[Torch code for online-neural-doodle](https://github.com/DmitryUlyanov/online-neural-doodle)
+[Paper Texture Networks: Feed-forward Synthesis of Textures and Stylized Images](http://arxiv.org/abs/1603.03417)
+[Discussion on parameter tuning](https://github.com/fchollet/keras/issues/3705)
+
+Resources:
+Example images can be downloaded from
+https://github.com/DmitryUlyanov/fast-neural-doodle/tree/master/data
+'''
+from __future__ import print_function
+import time
+import argparse
+import numpy as np
+from scipy.optimize import fmin_l_bfgs_b
+from scipy.misc import imread, imsave
+
+from keras import backend as K
+from keras.layers import Input, AveragePooling2D
+from keras.models import Model
+from keras.preprocessing.image import load_img, img_to_array
+from keras.applications import vgg19
+
+# Command line arguments
+parser = argparse.ArgumentParser(description='Keras neural doodle example')
+parser.add_argument('--nlabels', type=int,
+                    help='number of semantic labels'
+                    ' (regions in differnet colors)'
+                    ' in style_mask/target_mask')
+parser.add_argument('--style-image', type=str,
+                    help='path to image to learn style from')
+parser.add_argument('--style-mask', type=str,
+                    help='path to semantic mask of style image')
+parser.add_argument('--target-mask', type=str,
+                    help='path to semantic mask of target image')
+parser.add_argument('--content-image', type=str, default=None,
+                    help='path to optional content image')
+parser.add_argument('--target-image-prefix', type=str,
+                    help='path prefix for generated results')
+args = parser.parse_args()
+
+style_img_path = args.style_image
+style_mask_path = args.style_mask
+target_mask_path = args.target_mask
+content_img_path = args.content_image
+target_img_prefix = args.target_image_prefix
+use_content_img = content_img_path is not None
+
+num_labels = args.nlabels
+num_colors = 3  # RGB
+# determine image sizes based on target_mask
+ref_img = imread(target_mask_path)
+img_nrows, img_ncols = ref_img.shape[:2]
+
+total_variation_weight = 50.
+style_weight = 1.
+content_weight = 0.1 if use_content_img else 0
+
+content_feature_layers = ['block5_conv2']
+# To get better generation qualities, use more conv layers for style features
+style_feature_layers = ['block1_conv1', 'block2_conv1', 'block3_conv1',
+                        'block4_conv1', 'block5_conv1']
+
+
+# helper functions for reading/processing images
+def preprocess_image(image_path):
+    img = load_img(image_path, target_size=(img_nrows, img_ncols))
+    img = img_to_array(img)
+    img = np.expand_dims(img, axis=0)
+    img = vgg19.preprocess_input(img)
+    return img
+
+
+def deprocess_image(x):
+    if K.image_data_format() == 'channels_first':
+        x = x.reshape((3, img_nrows, img_ncols))
+        x = x.transpose((1, 2, 0))
+    else:
+        x = x.reshape((img_nrows, img_ncols, 3))
+    # Remove zero-center by mean pixel
+    x[:, :, 0] += 103.939
+    x[:, :, 1] += 116.779
+    x[:, :, 2] += 123.68
+    # 'BGR'->'RGB'
+    x = x[:, :, ::-1]
+    x = np.clip(x, 0, 255).astype('uint8')
+    return x
+
+
+def kmeans(xs, k):
+    assert xs.ndim == 2
+    try:
+        from sklearn.cluster import k_means
+        _, labels, _ = k_means(xs.astype('float64'), k)
+    except ImportError:
+        from scipy.cluster.vq import kmeans2
+        _, labels = kmeans2(xs, k, missing='raise')
+    return labels
+
+
+def load_mask_labels():
+    '''Load both target and style masks.
+    A mask image (nr x nc) with m labels/colors will be loaded
+    as a 4D boolean tensor: (1, m, nr, nc) for 'channels_first' or (1, nr, nc, m) for 'channels_last'
+    '''
+    target_mask_img = load_img(target_mask_path,
+                               target_size=(img_nrows, img_ncols))
+    target_mask_img = img_to_array(target_mask_img)
+    style_mask_img = load_img(style_mask_path,
+                              target_size=(img_nrows, img_ncols))
+    style_mask_img = img_to_array(style_mask_img)
+    if K.image_data_format() == 'channels_first':
+        mask_vecs = np.vstack([style_mask_img.reshape((3, -1)).T,
+                               target_mask_img.reshape((3, -1)).T])
+    else:
+        mask_vecs = np.vstack([style_mask_img.reshape((-1, 3)),
+                               target_mask_img.reshape((-1, 3))])
+
+    labels = kmeans(mask_vecs, num_labels)
+    style_mask_label = labels[:img_nrows *
+                              img_ncols].reshape((img_nrows, img_ncols))
+    target_mask_label = labels[img_nrows *
+                               img_ncols:].reshape((img_nrows, img_ncols))
+
+    stack_axis = 0 if K.image_data_format() == 'channels_first' else -1
+    style_mask = np.stack([style_mask_label == r for r in xrange(num_labels)],
+                          axis=stack_axis)
+    target_mask = np.stack([target_mask_label == r for r in xrange(num_labels)],
+                           axis=stack_axis)
+
+    return (np.expand_dims(style_mask, axis=0),
+            np.expand_dims(target_mask, axis=0))
+
+# Create tensor variables for images
+if K.image_data_format() == 'channels_first':
+    shape = (1, num_colors, img_nrows, img_ncols)
+else:
+    shape = (1, img_nrows, img_ncols, num_colors)
+
+style_image = K.variable(preprocess_image(style_img_path))
+target_image = K.placeholder(shape=shape)
+if use_content_img:
+    content_image = K.variable(preprocess_image(content_img_path))
+else:
+    content_image = K.zeros(shape=shape)
+
+images = K.concatenate([style_image, target_image, content_image], axis=0)
+
+# Create tensor variables for masks
+raw_style_mask, raw_target_mask = load_mask_labels()
+style_mask = K.variable(raw_style_mask.astype('float32'))
+target_mask = K.variable(raw_target_mask.astype('float32'))
+masks = K.concatenate([style_mask, target_mask], axis=0)
+
+# index constants for images and tasks variables
+STYLE, TARGET, CONTENT = 0, 1, 2
+
+# Build image model, mask model and use layer outputs as features
+# image model as VGG19
+image_model = vgg19.VGG19(include_top=False, input_tensor=images)
+
+# mask model as a series of pooling
+mask_input = Input(tensor=masks, shape=(None, None, None), name='mask_input')
+x = mask_input
+for layer in image_model.layers[1:]:
+    name = 'mask_%s' % layer.name
+    if 'conv' in layer.name:
+        x = AveragePooling2D((3, 3), padding='same', strides=(
+            1, 1), name=name)(x)
+    elif 'pool' in layer.name:
+        x = AveragePooling2D((2, 2), name=name)(x)
+mask_model = Model(mask_input, x)
+
+# Collect features from image_model and task_model
+image_features = {}
+mask_features = {}
+for img_layer, mask_layer in zip(image_model.layers, mask_model.layers):
+    if 'conv' in img_layer.name:
+        assert 'mask_' + img_layer.name == mask_layer.name
+        layer_name = img_layer.name
+        img_feat, mask_feat = img_layer.output, mask_layer.output
+        image_features[layer_name] = img_feat
+        mask_features[layer_name] = mask_feat
+
+
+# Define loss functions
+def gram_matrix(x):
+    assert K.ndim(x) == 3
+    features = K.batch_flatten(x)
+    gram = K.dot(features, K.transpose(features))
+    return gram
+
+
+def region_style_loss(style_image, target_image, style_mask, target_mask):
+    '''Calculate style loss between style_image and target_image,
+    for one common region specified by their (boolean) masks
+    '''
+    assert 3 == K.ndim(style_image) == K.ndim(target_image)
+    assert 2 == K.ndim(style_mask) == K.ndim(target_mask)
+    if K.image_data_format() == 'channels_first':
+        masked_style = style_image * style_mask
+        masked_target = target_image * target_mask
+        num_channels = K.shape(style_image)[0]
+    else:
+        masked_style = K.permute_dimensions(
+            style_image, (2, 0, 1)) * style_mask
+        masked_target = K.permute_dimensions(
+            target_image, (2, 0, 1)) * target_mask
+        num_channels = K.shape(style_image)[-1]
+    num_channels = K.cast(num_channels, dtype='float32')
+    s = gram_matrix(masked_style) / K.mean(style_mask) / num_channels
+    c = gram_matrix(masked_target) / K.mean(target_mask) / num_channels
+    return K.mean(K.square(s - c))
+
+
+def style_loss(style_image, target_image, style_masks, target_masks):
+    '''Calculate style loss between style_image and target_image,
+    in all regions.
+    '''
+    assert 3 == K.ndim(style_image) == K.ndim(target_image)
+    assert 3 == K.ndim(style_masks) == K.ndim(target_masks)
+    loss = K.variable(0)
+    for i in xrange(num_labels):
+        if K.image_data_format() == 'channels_first':
+            style_mask = style_masks[i, :, :]
+            target_mask = target_masks[i, :, :]
+        else:
+            style_mask = style_masks[:, :, i]
+            target_mask = target_masks[:, :, i]
+        loss += region_style_loss(style_image,
+                                  target_image, style_mask, target_mask)
+    return loss
+
+
+def content_loss(content_image, target_image):
+    return K.sum(K.square(target_image - content_image))
+
+
+def total_variation_loss(x):
+    assert 4 == K.ndim(x)
+    if K.image_data_format() == 'channels_first':
+        a = K.square(x[:, :, :img_nrows - 1, :img_ncols - 1] -
+                     x[:, :, 1:, :img_ncols - 1])
+        b = K.square(x[:, :, :img_nrows - 1, :img_ncols - 1] -
+                     x[:, :, :img_nrows - 1, 1:])
+    else:
+        a = K.square(x[:, :img_nrows - 1, :img_ncols - 1, :] -
+                     x[:, 1:, :img_ncols - 1, :])
+        b = K.square(x[:, :img_nrows - 1, :img_ncols - 1, :] -
+                     x[:, :img_nrows - 1, 1:, :])
+    return K.sum(K.pow(a + b, 1.25))
+
+# Overall loss is the weighted sum of content_loss, style_loss and tv_loss
+# Each individual loss uses features from image/mask models.
+loss = K.variable(0)
+for layer in content_feature_layers:
+    content_feat = image_features[layer][CONTENT, :, :, :]
+    target_feat = image_features[layer][TARGET, :, :, :]
+    loss += content_weight * content_loss(content_feat, target_feat)
+
+for layer in style_feature_layers:
+    style_feat = image_features[layer][STYLE, :, :, :]
+    target_feat = image_features[layer][TARGET, :, :, :]
+    style_masks = mask_features[layer][STYLE, :, :, :]
+    target_masks = mask_features[layer][TARGET, :, :, :]
+    sl = style_loss(style_feat, target_feat, style_masks, target_masks)
+    loss += (style_weight / len(style_feature_layers)) * sl
+
+loss += total_variation_weight * total_variation_loss(target_image)
+loss_grads = K.gradients(loss, target_image)
+
+# Evaluator class for computing efficiency
+outputs = [loss]
+if isinstance(loss_grads, (list, tuple)):
+    outputs += loss_grads
+else:
+    outputs.append(loss_grads)
+
+f_outputs = K.function([target_image], outputs)
+
+
+def eval_loss_and_grads(x):
+    if K.image_data_format() == 'channels_first':
+        x = x.reshape((1, 3, img_nrows, img_ncols))
+    else:
+        x = x.reshape((1, img_nrows, img_ncols, 3))
+    outs = f_outputs([x])
+    loss_value = outs[0]
+    if len(outs[1:]) == 1:
+        grad_values = outs[1].flatten().astype('float64')
+    else:
+        grad_values = np.array(outs[1:]).flatten().astype('float64')
+    return loss_value, grad_values
+
+
+class Evaluator(object):
+
+    def __init__(self):
+        self.loss_value = None
+        self.grads_values = None
+
+    def loss(self, x):
+        assert self.loss_value is None
+        loss_value, grad_values = eval_loss_and_grads(x)
+        self.loss_value = loss_value
+        self.grad_values = grad_values
+        return self.loss_value
+
+    def grads(self, x):
+        assert self.loss_value is not None
+        grad_values = np.copy(self.grad_values)
+        self.loss_value = None
+        self.grad_values = None
+        return grad_values
+
+evaluator = Evaluator()
+
+# Generate images by iterative optimization
+if K.image_data_format() == 'channels_first':
+    x = np.random.uniform(0, 255, (1, 3, img_nrows, img_ncols)) - 128.
+else:
+    x = np.random.uniform(0, 255, (1, img_nrows, img_ncols, 3)) - 128.
+
+for i in range(50):
+    print('Start of iteration', i)
+    start_time = time.time()
+    x, min_val, info = fmin_l_bfgs_b(evaluator.loss, x.flatten(),
+                                     fprime=evaluator.grads, maxfun=20)
+    print('Current loss value:', min_val)
+    # save current generated image
+    img = deprocess_image(x.copy())
+    fname = target_img_prefix + '_at_iteration_%d.png' % i
+    imsave(fname, img)
+    end_time = time.time()
+    print('Image saved as', fname)
+    print('Iteration %d completed in %ds' % (i, end_time - start_time))

examples/neural_style_transfer.py

@@ -0,0 +1,290 @@
+'''Neural style transfer with Keras.
+
+Run the script with:
+```
+python neural_style_transfer.py path_to_your_base_image.jpg path_to_your_reference.jpg prefix_for_results
+```
+e.g.:
+```
+python neural_style_transfer.py img/tuebingen.jpg img/starry_night.jpg results/my_result
+```
+Optional parameters:
+```
+--iter, To specify the number of iterations the style transfer takes place (Default is 10)
+--content_weight, The weight given to the content loss (Default is 0.025)
+--style_weight, The weight given to the style loss (Default is 1.0)
+--tv_weight, The weight given to the total variation loss (Default is 1.0)
+```
+
+It is preferable to run this script on GPU, for speed.
+
+Example result: https://twitter.com/fchollet/status/686631033085677568
+
+# Details
+
+Style transfer consists in generating an image
+with the same "content" as a base image, but with the
+"style" of a different picture (typically artistic).
+
+This is achieved through the optimization of a loss function
+that has 3 components: "style loss", "content loss",
+and "total variation loss":
+
+- The total variation loss imposes local spatial continuity between
+the pixels of the combination image, giving it visual coherence.
+
+- The style loss is where the deep learning keeps in --that one is defined
+using a deep convolutional neural network. Precisely, it consists in a sum of
+L2 distances between the Gram matrices of the representations of
+the base image and the style reference image, extracted from
+different layers of a convnet (trained on ImageNet). The general idea
+is to capture color/texture information at different spatial
+scales (fairly large scales --defined by the depth of the layer considered).
+
+ - The content loss is a L2 distance between the features of the base
+image (extracted from a deep layer) and the features of the combination image,
+keeping the generated image close enough to the original one.
+
+# References
+    - [A Neural Algorithm of Artistic Style](http://arxiv.org/abs/1508.06576)
+'''
+
+from __future__ import print_function
+from keras.preprocessing.image import load_img, img_to_array
+from scipy.misc import imsave
+import numpy as np
+from scipy.optimize import fmin_l_bfgs_b
+import time
+import argparse
+
+from keras.applications import vgg19
+from keras import backend as K
+
+parser = argparse.ArgumentParser(description='Neural style transfer with Keras.')
+parser.add_argument('base_image_path', metavar='base', type=str,
+                    help='Path to the image to transform.')
+parser.add_argument('style_reference_image_path', metavar='ref', type=str,
+                    help='Path to the style reference image.')
+parser.add_argument('result_prefix', metavar='res_prefix', type=str,
+                    help='Prefix for the saved results.')
+parser.add_argument('--iter', type=int, default=10, required=False,
+                    help='Number of iterations to run.')
+parser.add_argument('--content_weight', type=float, default=0.025, required=False,
+                    help='Content weight.')
+parser.add_argument('--style_weight', type=float, default=1.0, required=False,
+                    help='Style weight.')
+parser.add_argument('--tv_weight', type=float, default=1.0, required=False,
+                    help='Total Variation weight.')
+
+args = parser.parse_args()
+base_image_path = args.base_image_path
+style_reference_image_path = args.style_reference_image_path
+result_prefix = args.result_prefix
+iterations = args.iter
+
+# these are the weights of the different loss components
+total_variation_weight = args.tv_weight
+style_weight = args.style_weight
+content_weight = args.content_weight
+
+# dimensions of the generated picture.
+width, height = load_img(base_image_path).size
+img_nrows = 400
+img_ncols = int(width * img_nrows / height)
+
+# util function to open, resize and format pictures into appropriate tensors
+
+
+def preprocess_image(image_path):
+    img = load_img(image_path, target_size=(img_nrows, img_ncols))
+    img = img_to_array(img)
+    img = np.expand_dims(img, axis=0)
+    img = vgg19.preprocess_input(img)
+    return img
+
+# util function to convert a tensor into a valid image
+
+
+def deprocess_image(x):
+    if K.image_data_format() == 'channels_first':
+        x = x.reshape((3, img_nrows, img_ncols))
+        x = x.transpose((1, 2, 0))
+    else:
+        x = x.reshape((img_nrows, img_ncols, 3))
+    # Remove zero-center by mean pixel
+    x[:, :, 0] += 103.939
+    x[:, :, 1] += 116.779
+    x[:, :, 2] += 123.68
+    # 'BGR'->'RGB'
+    x = x[:, :, ::-1]
+    x = np.clip(x, 0, 255).astype('uint8')
+    return x
+
+# get tensor representations of our images
+base_image = K.variable(preprocess_image(base_image_path))
+style_reference_image = K.variable(preprocess_image(style_reference_image_path))
+
+# this will contain our generated image
+if K.image_data_format() == 'channels_first':
+    combination_image = K.placeholder((1, 3, img_nrows, img_ncols))
+else:
+    combination_image = K.placeholder((1, img_nrows, img_ncols, 3))
+
+# combine the 3 images into a single Keras tensor
+input_tensor = K.concatenate([base_image,
+                              style_reference_image,
+                              combination_image], axis=0)
+
+# build the VGG16 network with our 3 images as input
+# the model will be loaded with pre-trained ImageNet weights
+model = vgg19.VGG19(input_tensor=input_tensor,
+                    weights='imagenet', include_top=False)
+print('Model loaded.')
+
+# get the symbolic outputs of each "key" layer (we gave them unique names).
+outputs_dict = dict([(layer.name, layer.output) for layer in model.layers])
+
+# compute the neural style loss
+# first we need to define 4 util functions
+
+# the gram matrix of an image tensor (feature-wise outer product)
+
+
+def gram_matrix(x):
+    assert K.ndim(x) == 3
+    if K.image_data_format() == 'channels_first':
+        features = K.batch_flatten(x)
+    else:
+        features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1)))
+    gram = K.dot(features, K.transpose(features))
+    return gram
+
+# the "style loss" is designed to maintain
+# the style of the reference image in the generated image.
+# It is based on the gram matrices (which capture style) of
+# feature maps from the style reference image
+# and from the generated image
+
+
+def style_loss(style, combination):
+    assert K.ndim(style) == 3
+    assert K.ndim(combination) == 3
+    S = gram_matrix(style)
+    C = gram_matrix(combination)
+    channels = 3
+    size = img_nrows * img_ncols
+    return K.sum(K.square(S - C)) / (4. * (channels ** 2) * (size ** 2))
+
+# an auxiliary loss function
+# designed to maintain the "content" of the
+# base image in the generated image
+
+
+def content_loss(base, combination):
+    return K.sum(K.square(combination - base))
+
+# the 3rd loss function, total variation loss,
+# designed to keep the generated image locally coherent
+
+
+def total_variation_loss(x):
+    assert K.ndim(x) == 4
+    if K.image_data_format() == 'channels_first':
+        a = K.square(x[:, :, :img_nrows - 1, :img_ncols - 1] - x[:, :, 1:, :img_ncols - 1])
+        b = K.square(x[:, :, :img_nrows - 1, :img_ncols - 1] - x[:, :, :img_nrows - 1, 1:])
+    else:
+        a = K.square(x[:, :img_nrows - 1, :img_ncols - 1, :] - x[:, 1:, :img_ncols - 1, :])
+        b = K.square(x[:, :img_nrows - 1, :img_ncols - 1, :] - x[:, :img_nrows - 1, 1:, :])
+    return K.sum(K.pow(a + b, 1.25))
+
+# combine these loss functions into a single scalar
+loss = K.variable(0.)
+layer_features = outputs_dict['block5_conv2']
+base_image_features = layer_features[0, :, :, :]
+combination_features = layer_features[2, :, :, :]
+loss += content_weight * content_loss(base_image_features,
+                                      combination_features)
+
+feature_layers = ['block1_conv1', 'block2_conv1',
+                  'block3_conv1', 'block4_conv1',
+                  'block5_conv1']
+for layer_name in feature_layers:
+    layer_features = outputs_dict[layer_name]
+    style_reference_features = layer_features[1, :, :, :]
+    combination_features = layer_features[2, :, :, :]
+    sl = style_loss(style_reference_features, combination_features)
+    loss += (style_weight / len(feature_layers)) * sl
+loss += total_variation_weight * total_variation_loss(combination_image)
+
+# get the gradients of the generated image wrt the loss
+grads = K.gradients(loss, combination_image)
+
+outputs = [loss]
+if isinstance(grads, (list, tuple)):
+    outputs += grads
+else:
+    outputs.append(grads)
+
+f_outputs = K.function([combination_image], outputs)
+
+
+def eval_loss_and_grads(x):
+    if K.image_data_format() == 'channels_first':
+        x = x.reshape((1, 3, img_nrows, img_ncols))
+    else:
+        x = x.reshape((1, img_nrows, img_ncols, 3))
+    outs = f_outputs([x])
+    loss_value = outs[0]
+    if len(outs[1:]) == 1:
+        grad_values = outs[1].flatten().astype('float64')
+    else:
+        grad_values = np.array(outs[1:]).flatten().astype('float64')
+    return loss_value, grad_values
+
+# this Evaluator class makes it possible
+# to compute loss and gradients in one pass
+# while retrieving them via two separate functions,
+# "loss" and "grads". This is done because scipy.optimize
+# requires separate functions for loss and gradients,
+# but computing them separately would be inefficient.
+
+
+class Evaluator(object):
+
+    def __init__(self):
+        self.loss_value = None
+        self.grads_values = None
+
+    def loss(self, x):
+        assert self.loss_value is None
+        loss_value, grad_values = eval_loss_and_grads(x)
+        self.loss_value = loss_value
+        self.grad_values = grad_values
+        return self.loss_value
+
+    def grads(self, x):
+        assert self.loss_value is not None
+        grad_values = np.copy(self.grad_values)
+        self.loss_value = None
+        self.grad_values = None
+        return grad_values
+
+evaluator = Evaluator()
+
+# run scipy-based optimization (L-BFGS) over the pixels of the generated image
+# so as to minimize the neural style loss
+x = preprocess_image(base_image_path)
+
+for i in range(iterations):
+    print('Start of iteration', i)
+    start_time = time.time()
+    x, min_val, info = fmin_l_bfgs_b(evaluator.loss, x.flatten(),
+                                     fprime=evaluator.grads, maxfun=20)
+    print('Current loss value:', min_val)
+    # save current generated image
+    img = deprocess_image(x.copy())
+    fname = result_prefix + '_at_iteration_%d.png' % i
+    imsave(fname, img)
+    end_time = time.time()
+    print('Image saved as', fname)
+    print('Iteration %d completed in %ds' % (i, end_time - start_time))

examples/pretrained_word_embeddings.py

@@ -0,0 +1,149 @@
+'''This script loads pre-trained word embeddings (GloVe embeddings)
+into a frozen Keras Embedding layer, and uses it to
+train a text classification model on the 20 Newsgroup dataset
+(classication of newsgroup messages into 20 different categories).
+
+GloVe embedding data can be found at:
+http://nlp.stanford.edu/data/glove.6B.zip
+(source page: http://nlp.stanford.edu/projects/glove/)
+
+20 Newsgroup data can be found at:
+http://www.cs.cmu.edu/afs/cs.cmu.edu/project/theo-20/www/data/news20.html
+'''
+
+from __future__ import print_function
+
+import os
+import sys
+import numpy as np
+from keras.preprocessing.text import Tokenizer
+from keras.preprocessing.sequence import pad_sequences
+from keras.utils import to_categorical
+from keras.layers import Dense, Input, Flatten
+from keras.layers import Conv1D, MaxPooling1D, Embedding
+from keras.models import Model
+
+
+BASE_DIR = ''
+GLOVE_DIR = BASE_DIR + '/glove.6B/'
+TEXT_DATA_DIR = BASE_DIR + '/20_newsgroup/'
+MAX_SEQUENCE_LENGTH = 1000
+MAX_NB_WORDS = 20000
+EMBEDDING_DIM = 100
+VALIDATION_SPLIT = 0.2
+
+# first, build index mapping words in the embeddings set
+# to their embedding vector
+
+print('Indexing word vectors.')
+
+embeddings_index = {}
+f = open(os.path.join(GLOVE_DIR, 'glove.6B.100d.txt'))
+for line in f:
+    values = line.split()
+    word = values[0]
+    coefs = np.asarray(values[1:], dtype='float32')
+    embeddings_index[word] = coefs
+f.close()
+
+print('Found %s word vectors.' % len(embeddings_index))
+
+# second, prepare text samples and their labels
+print('Processing text dataset')
+
+texts = []  # list of text samples
+labels_index = {}  # dictionary mapping label name to numeric id
+labels = []  # list of label ids
+for name in sorted(os.listdir(TEXT_DATA_DIR)):
+    path = os.path.join(TEXT_DATA_DIR, name)
+    if os.path.isdir(path):
+        label_id = len(labels_index)
+        labels_index[name] = label_id
+        for fname in sorted(os.listdir(path)):
+            if fname.isdigit():
+                fpath = os.path.join(path, fname)
+                if sys.version_info < (3,):
+                    f = open(fpath)
+                else:
+                    f = open(fpath, encoding='latin-1')
+                t = f.read()
+                i = t.find('\n\n')  # skip header
+                if 0 < i:
+                    t = t[i:]
+                texts.append(t)
+                f.close()
+                labels.append(label_id)
+
+print('Found %s texts.' % len(texts))
+
+# finally, vectorize the text samples into a 2D integer tensor
+tokenizer = Tokenizer(num_words=MAX_NB_WORDS)
+tokenizer.fit_on_texts(texts)
+sequences = tokenizer.texts_to_sequences(texts)
+
+word_index = tokenizer.word_index
+print('Found %s unique tokens.' % len(word_index))
+
+data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)
+
+labels = to_categorical(np.asarray(labels))
+print('Shape of data tensor:', data.shape)
+print('Shape of label tensor:', labels.shape)
+
+# split the data into a training set and a validation set
+indices = np.arange(data.shape[0])
+np.random.shuffle(indices)
+data = data[indices]
+labels = labels[indices]
+num_validation_samples = int(VALIDATION_SPLIT * data.shape[0])
+
+x_train = data[:-num_validation_samples]
+y_train = labels[:-num_validation_samples]
+x_val = data[-num_validation_samples:]
+y_val = labels[-num_validation_samples:]
+
+print('Preparing embedding matrix.')
+
+# prepare embedding matrix
+num_words = min(MAX_NB_WORDS, len(word_index))
+embedding_matrix = np.zeros((num_words, EMBEDDING_DIM))
+for word, i in word_index.items():
+    if i >= MAX_NB_WORDS:
+        continue
+    embedding_vector = embeddings_index.get(word)
+    if embedding_vector is not None:
+        # words not found in embedding index will be all-zeros.
+        embedding_matrix[i] = embedding_vector
+
+# load pre-trained word embeddings into an Embedding layer
+# note that we set trainable = False so as to keep the embeddings fixed
+embedding_layer = Embedding(num_words,
+                            EMBEDDING_DIM,
+                            weights=[embedding_matrix],
+                            input_length=MAX_SEQUENCE_LENGTH,
+                            trainable=False)
+
+print('Training model.')
+
+# train a 1D convnet with global maxpooling
+sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
+embedded_sequences = embedding_layer(sequence_input)
+x = Conv1D(128, 5, activation='relu')(embedded_sequences)
+x = MaxPooling1D(5)(x)
+x = Conv1D(128, 5, activation='relu')(x)
+x = MaxPooling1D(5)(x)
+x = Conv1D(128, 5, activation='relu')(x)
+x = MaxPooling1D(35)(x)
+x = Flatten()(x)
+x = Dense(128, activation='relu')(x)
+preds = Dense(len(labels_index), activation='softmax')(x)
+
+model = Model(sequence_input, preds)
+model.compile(loss='categorical_crossentropy',
+              optimizer='rmsprop',
+              metrics=['acc'])
+
+model.fit(x_train, y_train,
+          batch_size=128,
+          epochs=10,
+          validation_data=(x_val, y_val))

examples/reuters_mlp.py

@@ -1,57 +1,60 @@
-from __future__ import absolute_import
+'''Trains and evaluate a simple MLP
+on the Reuters newswire topic classification task.
+'''
 from __future__ import print_function
-import numpy as np
 
+import numpy as np
+import keras
 from keras.datasets import reuters
 from keras.models import Sequential
-from keras.layers.core import Dense, Dropout, Activation
-from keras.layers.normalization import BatchNormalization
-from keras.utils import np_utils
+from keras.layers import Dense, Dropout, Activation
 from keras.preprocessing.text import Tokenizer
 
-'''
-    Train and evaluate a simple MLP on the Reuters newswire topic classification task.
-    GPU run command:
-        THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python examples/reuters_mlp.py
-    CPU run command:
-        python examples/reuters_mlp.py
-'''
-
 max_words = 1000
 batch_size = 32
+epochs = 5
 
-print("Loading data...")
-(X_train, y_train), (X_test, y_test) = reuters.load_data(nb_words=max_words, test_split=0.2)
-print(len(X_train), 'train sequences')
-print(len(X_test), 'test sequences')
+print('Loading data...')
+(x_train, y_train), (x_test, y_test) = reuters.load_data(num_words=max_words,
+                                                         test_split=0.2)
+print(len(x_train), 'train sequences')
+print(len(x_test), 'test sequences')
 
-nb_classes = np.max(y_train)+1
-print(nb_classes, 'classes')
+num_classes = np.max(y_train) + 1
+print(num_classes, 'classes')
 
-print("Vectorizing sequence data...")
-tokenizer = Tokenizer(nb_words=max_words)
-X_train = tokenizer.sequences_to_matrix(X_train, mode="binary")
-X_test = tokenizer.sequences_to_matrix(X_test, mode="binary")
-print('X_train shape:', X_train.shape)
-print('X_test shape:', X_test.shape)
+print('Vectorizing sequence data...')
+tokenizer = Tokenizer(num_words=max_words)
+x_train = tokenizer.sequences_to_matrix(x_train, mode='binary')
+x_test = tokenizer.sequences_to_matrix(x_test, mode='binary')
+print('x_train shape:', x_train.shape)
+print('x_test shape:', x_test.shape)
 
-print("Convert class vector to binary class matrix (for use with categorical_crossentropy)")
-Y_train = np_utils.to_categorical(y_train, nb_classes)
-Y_test = np_utils.to_categorical(y_test, nb_classes)
-print('Y_train shape:', Y_train.shape)
-print('Y_test shape:', Y_test.shape)
+print('Convert class vector to binary class matrix '
+      '(for use with categorical_crossentropy)')
+y_train = keras.utils.to_categorical(y_train, num_classes)
+y_test = keras.utils.to_categorical(y_test, num_classes)
+print('y_train shape:', y_train.shape)
+print('y_test shape:', y_test.shape)
 
-print("Building model...")
+print('Building model...')
 model = Sequential()
-model.add(Dense(max_words, 256, init='normal'))
+model.add(Dense(512, input_shape=(max_words,)))
 model.add(Activation('relu'))
 model.add(Dropout(0.5))
-model.add(Dense(256, nb_classes, init='normal'))
+model.add(Dense(num_classes))
 model.add(Activation('softmax'))
 
-model.compile(loss='categorical_crossentropy', optimizer='adam')
+model.compile(loss='categorical_crossentropy',
+              optimizer='adam',
+              metrics=['accuracy'])
 
-history = model.fit(X_train, Y_train, nb_epoch=4, batch_size=batch_size, verbose=1, show_accuracy=True, validation_split=0.1)
-score = model.evaluate(X_test, Y_test, batch_size=batch_size, verbose=1, show_accuracy=True)
+history = model.fit(x_train, y_train,
+                    batch_size=batch_size,
+                    epochs=epochs,
+                    verbose=1,
+                    validation_split=0.1)
+score = model.evaluate(x_test, y_test,
+                       batch_size=batch_size, verbose=1)
 print('Test score:', score[0])
 print('Test accuracy:', score[1])

examples/reuters_mlp_relu_vs_selu.py

@@ -0,0 +1,174 @@
+'''Compares self-normalizing MLPs with regular MLPs.
+
+Compares the performance of a simple MLP using two
+different activation functions: RELU and SELU
+on the Reuters newswire topic classification task.
+
+# Reference:
+  Klambauer, G., Unterthiner, T., Mayr, A., & Hochreiter, S. (2017).
+  Self-Normalizing Neural Networks. arXiv preprint arXiv:1706.02515.
+  https://arxiv.org/abs/1706.02515
+'''
+from __future__ import print_function
+
+import numpy as np
+import matplotlib.pyplot as plt
+import keras
+from keras.datasets import reuters
+from keras.models import Sequential
+from keras.layers import Dense, Activation, Dropout
+from keras.layers.noise import AlphaDropout
+from keras.preprocessing.text import Tokenizer
+
+max_words = 1000
+batch_size = 16
+epochs = 40
+plot = True
+
+
+def create_network(n_dense=6,
+                   dense_units=16,
+                   activation='selu',
+                   dropout=AlphaDropout,
+                   dropout_rate=0.1,
+                   kernel_initializer='lecun_normal',
+                   optimizer='adam',
+                   num_classes=1,
+                   max_words=max_words):
+    """Generic function to create a fully-connected neural network.
+
+    # Arguments
+        n_dense: int > 0. Number of dense layers.
+        dense_units: int > 0. Number of dense units per layer.
+        dropout: keras.layers.Layer. A dropout layer to apply.
+        dropout_rate: 0 <= float <= 1. The rate of dropout.
+        kernel_initializer: str. The initializer for the weights.
+        optimizer: str/keras.optimizers.Optimizer. The optimizer to use.
+        num_classes: int > 0. The number of classes to predict.
+        max_words: int > 0. The maximum number of words per data point.
+
+    # Returns
+        A Keras model instance (compiled).
+    """
+    model = Sequential()
+    model.add(Dense(dense_units, input_shape=(max_words,),
+                    kernel_initializer=kernel_initializer))
+    model.add(Activation(activation))
+    model.add(dropout(dropout_rate))
+
+    for i in range(n_dense - 1):
+        model.add(Dense(dense_units, kernel_initializer=kernel_initializer))
+        model.add(Activation(activation))
+        model.add(dropout(dropout_rate))
+
+    model.add(Dense(num_classes))
+    model.add(Activation('softmax'))
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=optimizer,
+                  metrics=['accuracy'])
+    return model
+
+
+network1 = {
+    'n_dense': 6,
+    'dense_units': 16,
+    'activation': 'relu',
+    'dropout': Dropout,
+    'dropout_rate': 0.5,
+    'kernel_initializer': 'glorot_uniform',
+    'optimizer': 'sgd'
+}
+
+network2 = {
+    'n_dense': 6,
+    'dense_units': 16,
+    'activation': 'selu',
+    'dropout': AlphaDropout,
+    'dropout_rate': 0.1,
+    'kernel_initializer': 'lecun_normal',
+    'optimizer': 'sgd'
+}
+
+print('Loading data...')
+(x_train, y_train), (x_test, y_test) = reuters.load_data(num_words=max_words,
+                                                         test_split=0.2)
+print(len(x_train), 'train sequences')
+print(len(x_test), 'test sequences')
+
+num_classes = np.max(y_train) + 1
+print(num_classes, 'classes')
+
+print('Vectorizing sequence data...')
+tokenizer = Tokenizer(num_words=max_words)
+x_train = tokenizer.sequences_to_matrix(x_train, mode='binary')
+x_test = tokenizer.sequences_to_matrix(x_test, mode='binary')
+print('x_train shape:', x_train.shape)
+print('x_test shape:', x_test.shape)
+
+print('Convert class vector to binary class matrix '
+      '(for use with categorical_crossentropy)')
+y_train = keras.utils.to_categorical(y_train, num_classes)
+y_test = keras.utils.to_categorical(y_test, num_classes)
+print('y_train shape:', y_train.shape)
+print('y_test shape:', y_test.shape)
+
+print('\nBuilding network 1...')
+
+model1 = create_network(num_classes=num_classes, **network1)
+history_model1 = model1.fit(x_train,
+                            y_train,
+                            batch_size=batch_size,
+                            epochs=epochs,
+                            verbose=1,
+                            validation_split=0.1)
+
+score_model1 = model1.evaluate(x_test,
+                               y_test,
+                               batch_size=batch_size,
+                               verbose=1)
+
+
+print('\nBuilding network 2...')
+model2 = create_network(num_classes=num_classes, **network2)
+
+history_model2 = model2.fit(x_train,
+                            y_train,
+                            batch_size=batch_size,
+                            epochs=epochs,
+                            verbose=1,
+                            validation_split=0.1)
+
+score_model2 = model2.evaluate(x_test,
+                               y_test,
+                               batch_size=batch_size,
+                               verbose=1)
+
+print('\nNetwork 1 results')
+print('Hyperparameters:', network1)
+print('Test score:', score_model1[0])
+print('Test accuracy:', score_model1[1])
+print('Network 2 results')
+print('Hyperparameters:', network2)
+print('Test score:', score_model2[0])
+print('Test accuracy:', score_model2[1])
+
+plt.plot(range(epochs),
+         history_model1.history['val_loss'],
+         'g-',
+         label='Network 1 Val Loss')
+plt.plot(range(epochs),
+         history_model2.history['val_loss'],
+         'r-',
+         label='Network 2 Val Loss')
+plt.plot(range(epochs),
+         history_model1.history['loss'],
+         'g--',
+         label='Network 1 Loss')
+plt.plot(range(epochs),
+         history_model2.history['loss'],
+         'r--',
+         label='Network 2 Loss')
+plt.xlabel('Epochs')
+plt.ylabel('Loss')
+plt.legend()
+plt.savefig('comparison_of_networks.png')

examples/skipgram_word_embeddings.py

@@ -1,215 +0,0 @@
-
-'''
-    We loop over words in a dataset, and for each word, we look at a context window around the word. 
-    We generate pairs of (pivot_word, other_word_from_same_context) with label 1,
-    and pairs of (pivot_word, random_word) with label 0 (skip-gram method).
-
-    We use the layer WordContextProduct to learn embeddings for the word couples,
-    and compute a proximity score between the embeddings (= p(context|word)),
-    trained with our positive and negative labels.
-
-    We then use the weights computed by WordContextProduct to encode words 
-    and demonstrate that the geometry of the embedding space 
-    captures certain useful semantic properties.
-
-    Read more about skip-gram in this particularly gnomic paper by Mikolov et al.: 
-        http://arxiv.org/pdf/1301.3781v3.pdf
-
-    Note: you should run this on GPU, otherwise training will be quite slow. 
-    On a EC2 GPU instance, expect 3 hours per 10e6 comments (~10e8 words) per epoch with dim_proj=256.
-    Should be much faster on a modern GPU.
-
-    GPU command:
-        THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python skipgram_word_embeddings.py
-
-    Dataset: 5,845,908 Hacker News comments. 
-    Obtain the dataset at: 
-        https://mega.co.nz/#F!YohlwD7R!wec0yNO86SeaNGIYQBOR0A 
-        (HNCommentsAll.1perline.json.bz2)
-'''
-from __future__ import absolute_import
-from __future__ import print_function
-
-import numpy as np
-import theano
-import six.moves.cPickle
-import os, re, json
-
-from keras.preprocessing import sequence, text
-from keras.optimizers import SGD, RMSprop, Adagrad
-from keras.utils import np_utils, generic_utils
-from keras.models import Sequential
-from keras.layers.embeddings import WordContextProduct, Embedding
-from six.moves import range
-from six.moves import zip
-
-max_features = 50000 # vocabulary size: top 50,000 most common words in data
-skip_top = 100 # ignore top 100 most common words
-nb_epoch = 1
-dim_proj = 256 # embedding space dimension
-
-save = True
-load_model = False
-load_tokenizer = False
-train_model = True
-save_dir = os.path.expanduser("~/.keras/models")
-model_load_fname = "HN_skipgram_model.pkl"
-model_save_fname = "HN_skipgram_model.pkl"
-tokenizer_fname = "HN_tokenizer.pkl"
-
-data_path = os.path.expanduser("~/")+"HNCommentsAll.1perline.json"
-
-# text preprocessing utils
-html_tags = re.compile(r'<.*?>')
-to_replace = [('&#x27;', "'")]
-hex_tags = re.compile(r'&.*?;')
-
-def clean_comment(comment):
-    c = str(comment.encode("utf-8"))
-    c = html_tags.sub(' ', c)
-    for tag, char in to_replace:
-        c = c.replace(tag, char)
-    c = hex_tags.sub(' ', c)
-    return c
-
-def text_generator(path=data_path):
-    f = open(path)
-    for i, l in enumerate(f):
-        comment_data = json.loads(l)
-        comment_text = comment_data["comment_text"]
-        comment_text = clean_comment(comment_text)
-        if i % 10000 == 0:
-            print(i)
-        yield comment_text
-    f.close()
-
-# model management
-if load_tokenizer:
-    print('Load tokenizer...')
-    tokenizer = six.moves.cPickle.load(open(os.path.join(save_dir, tokenizer_fname), 'rb'))
-else:
-    print("Fit tokenizer...")
-    tokenizer = text.Tokenizer(nb_words=max_features)
-    tokenizer.fit_on_texts(text_generator())
-    if save:
-        print("Save tokenizer...")
-        if not os.path.exists(save_dir):
-            os.makedirs(save_dir)
-        six.moves.cPickle.dump(tokenizer, open(os.path.join(save_dir, tokenizer_fname), "wb"))
-
-# training process
-if train_model:
-    if load_model:
-        print('Load model...')
-        model = six.moves.cPickle.load(open(os.path.join(save_dir, model_load_fname), 'rb'))
-    else:
-        print('Build model...')
-        model = Sequential()
-        model.add(WordContextProduct(max_features, proj_dim=dim_proj, init="uniform"))
-        model.compile(loss='mse', optimizer='rmsprop')
-
-    sampling_table = sequence.make_sampling_table(max_features)
-
-    for e in range(nb_epoch):
-        print('-'*40)
-        print('Epoch', e)
-        print('-'*40)
-
-        progbar = generic_utils.Progbar(tokenizer.document_count)
-        samples_seen = 0
-        losses = []
-        
-        for i, seq in enumerate(tokenizer.texts_to_sequences_generator(text_generator())):
-            # get skipgram couples for one text in the dataset
-            couples, labels = sequence.skipgrams(seq, max_features, window_size=4, negative_samples=1., sampling_table=sampling_table)
-            if couples:
-                # one gradient update per sentence (one sentence = a few 1000s of word couples)
-                X = np.array(couples, dtype="int32")
-                loss = model.train(X, labels)
-                losses.append(loss)
-                if len(losses) % 100 == 0:
-                    progbar.update(i, values=[("loss", np.mean(losses))])
-                    losses = []
-                samples_seen += len(labels)
-        print('Samples seen:', samples_seen)
-    print("Training completed!")
-
-    if save:
-        print("Saving model...")
-        if not os.path.exists(save_dir):
-            os.makedirs(save_dir)
-        six.moves.cPickle.dump(model, open(os.path.join(save_dir, model_save_fname), "wb"))
-
-
-print("It's test time!")
-
-# recover the embedding weights trained with skipgram:
-weights = model.layers[0].get_weights()[0]
-
-# we no longer need this
-del model
-
-weights[:skip_top] = np.zeros((skip_top, dim_proj))
-norm_weights = np_utils.normalize(weights)
-
-word_index = tokenizer.word_index
-reverse_word_index = dict([(v, k) for k, v in list(word_index.items())])
-word_index = tokenizer.word_index
-
-def embed_word(w):
-    i = word_index.get(w)
-    if (not i) or (i<skip_top) or (i>=max_features):
-        return None
-    return norm_weights[i]
-
-def closest_to_point(point, nb_closest=10):
-    proximities = np.dot(norm_weights, point)
-    tups = list(zip(list(range(len(proximities))), proximities))
-    tups.sort(key=lambda x: x[1], reverse=True)
-    return [(reverse_word_index.get(t[0]), t[1]) for t in tups[:nb_closest]]  
-
-def closest_to_word(w, nb_closest=10):
-    i = word_index.get(w)
-    if (not i) or (i<skip_top) or (i>=max_features):
-        return []
-    return closest_to_point(norm_weights[i].T, nb_closest)
-
-
-''' the resuls in comments below were for: 
-    5.8M HN comments
-    dim_proj = 256
-    nb_epoch = 2
-    optimizer = rmsprop
-    loss = mse
-    max_features = 50000
-    skip_top = 100
-    negative_samples = 1.
-    window_size = 4
-    and frequency subsampling of factor 10e-5. 
-'''
-
-words = ["article", # post, story, hn, read, comments
-"3", # 6, 4, 5, 2
-"two", # three, few, several, each
-"great", # love, nice, working, looking
-"data", # information, memory, database
-"money", # company, pay, customers, spend
-"years", # ago, year, months, hours, week, days
-"android", # ios, release, os, mobile, beta
-"javascript", # js, css, compiler, library, jquery, ruby
-"look", # looks, looking
-"business", # industry, professional, customers
-"company", # companies, startup, founders, startups
-"after", # before, once, until
-"own", # personal, our, having
-"us", # united, country, american, tech, diversity, usa, china, sv
-"using", # javascript, js, tools (lol)
-"here", # hn, post, comments
-]
-
-for w in words:
-    res = closest_to_word(w)
-    print('====', w)
-    for r in res:
-        print(r)
-

examples/stateful_lstm.py

@@ -0,0 +1,90 @@
+'''Example script showing how to use stateful RNNs
+to model long sequences efficiently.
+'''
+from __future__ import print_function
+import numpy as np
+import matplotlib.pyplot as plt
+from keras.models import Sequential
+from keras.layers import Dense, LSTM
+
+
+# since we are using stateful rnn tsteps can be set to 1
+tsteps = 1
+batch_size = 25
+epochs = 25
+# number of elements ahead that are used to make the prediction
+lahead = 1
+
+
+def gen_cosine_amp(amp=100, period=1000, x0=0, xn=50000, step=1, k=0.0001):
+    """Generates an absolute cosine time series with the amplitude
+    exponentially decreasing
+
+    Arguments:
+        amp: amplitude of the cosine function
+        period: period of the cosine function
+        x0: initial x of the time series
+        xn: final x of the time series
+        step: step of the time series discretization
+        k: exponential rate
+    """
+    cos = np.zeros(((xn - x0) * step, 1, 1))
+    for i in range(len(cos)):
+        idx = x0 + i * step
+        cos[i, 0, 0] = amp * np.cos(2 * np.pi * idx / period)
+        cos[i, 0, 0] = cos[i, 0, 0] * np.exp(-k * idx)
+    return cos
+
+
+print('Generating Data...')
+cos = gen_cosine_amp()
+print('Input shape:', cos.shape)
+
+expected_output = np.zeros((len(cos), 1))
+for i in range(len(cos) - lahead):
+    expected_output[i, 0] = np.mean(cos[i + 1:i + lahead + 1])
+
+print('Output shape:', expected_output.shape)
+
+print('Creating Model...')
+model = Sequential()
+model.add(LSTM(50,
+               input_shape=(tsteps, 1),
+               batch_size=batch_size,
+               return_sequences=True,
+               stateful=True))
+model.add(LSTM(50,
+               return_sequences=False,
+               stateful=True))
+model.add(Dense(1))
+model.compile(loss='mse', optimizer='rmsprop')
+
+print('Training')
+for i in range(epochs):
+    print('Epoch', i, '/', epochs)
+
+    # Note that the last state for sample i in a batch will
+    # be used as initial state for sample i in the next batch.
+    # Thus we are simultaneously training on batch_size series with
+    # lower resolution than the original series contained in cos.
+    # Each of these series are offset by one step and can be
+    # extracted with cos[i::batch_size].
+
+    model.fit(cos, expected_output,
+              batch_size=batch_size,
+              epochs=1,
+              verbose=1,
+              shuffle=False)
+    model.reset_states()
+
+print('Predicting')
+predicted_output = model.predict(cos, batch_size=batch_size)
+
+print('Plotting Results')
+plt.subplot(2, 1, 1)
+plt.plot(expected_output)
+plt.title('Expected')
+plt.subplot(2, 1, 2)
+plt.plot(predicted_output)
+plt.title('Predicted')
+plt.show()

examples/variational_autoencoder.py

@@ -0,0 +1,118 @@
+'''This script demonstrates how to build a variational autoencoder with Keras.
+
+Reference: "Auto-Encoding Variational Bayes" https://arxiv.org/abs/1312.6114
+'''
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.stats import norm
+
+from keras.layers import Input, Dense, Lambda, Layer
+from keras.models import Model
+from keras import backend as K
+from keras import metrics
+from keras.datasets import mnist
+
+batch_size = 100
+original_dim = 784
+latent_dim = 2
+intermediate_dim = 256
+epochs = 50
+epsilon_std = 1.0
+
+
+x = Input(batch_shape=(batch_size, original_dim))
+h = Dense(intermediate_dim, activation='relu')(x)
+z_mean = Dense(latent_dim)(h)
+z_log_var = Dense(latent_dim)(h)
+
+
+def sampling(args):
+    z_mean, z_log_var = args
+    epsilon = K.random_normal(shape=(batch_size, latent_dim), mean=0.,
+                              stddev=epsilon_std)
+    return z_mean + K.exp(z_log_var / 2) * epsilon
+
+# note that "output_shape" isn't necessary with the TensorFlow backend
+z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_var])
+
+# we instantiate these layers separately so as to reuse them later
+decoder_h = Dense(intermediate_dim, activation='relu')
+decoder_mean = Dense(original_dim, activation='sigmoid')
+h_decoded = decoder_h(z)
+x_decoded_mean = decoder_mean(h_decoded)
+
+
+# Custom loss layer
+class CustomVariationalLayer(Layer):
+    def __init__(self, **kwargs):
+        self.is_placeholder = True
+        super(CustomVariationalLayer, self).__init__(**kwargs)
+
+    def vae_loss(self, x, x_decoded_mean):
+        xent_loss = original_dim * metrics.binary_crossentropy(x, x_decoded_mean)
+        kl_loss = - 0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
+        return K.mean(xent_loss + kl_loss)
+
+    def call(self, inputs):
+        x = inputs[0]
+        x_decoded_mean = inputs[1]
+        loss = self.vae_loss(x, x_decoded_mean)
+        self.add_loss(loss, inputs=inputs)
+        # We won't actually use the output.
+        return x
+
+y = CustomVariationalLayer()([x, x_decoded_mean])
+vae = Model(x, y)
+vae.compile(optimizer='rmsprop', loss=None)
+
+
+# train the VAE on MNIST digits
+(x_train, y_train), (x_test, y_test) = mnist.load_data()
+
+x_train = x_train.astype('float32') / 255.
+x_test = x_test.astype('float32') / 255.
+x_train = x_train.reshape((len(x_train), np.prod(x_train.shape[1:])))
+x_test = x_test.reshape((len(x_test), np.prod(x_test.shape[1:])))
+
+vae.fit(x_train,
+        shuffle=True,
+        epochs=epochs,
+        batch_size=batch_size,
+        validation_data=(x_test, x_test))
+
+# build a model to project inputs on the latent space
+encoder = Model(x, z_mean)
+
+# display a 2D plot of the digit classes in the latent space
+x_test_encoded = encoder.predict(x_test, batch_size=batch_size)
+plt.figure(figsize=(6, 6))
+plt.scatter(x_test_encoded[:, 0], x_test_encoded[:, 1], c=y_test)
+plt.colorbar()
+plt.show()
+
+# build a digit generator that can sample from the learned distribution
+decoder_input = Input(shape=(latent_dim,))
+_h_decoded = decoder_h(decoder_input)
+_x_decoded_mean = decoder_mean(_h_decoded)
+generator = Model(decoder_input, _x_decoded_mean)
+
+# display a 2D manifold of the digits
+n = 15  # figure with 15x15 digits
+digit_size = 28
+figure = np.zeros((digit_size * n, digit_size * n))
+# linearly spaced coordinates on the unit square were transformed through the inverse CDF (ppf) of the Gaussian
+# to produce values of the latent variables z, since the prior of the latent space is Gaussian
+grid_x = norm.ppf(np.linspace(0.05, 0.95, n))
+grid_y = norm.ppf(np.linspace(0.05, 0.95, n))
+
+for i, yi in enumerate(grid_x):
+    for j, xi in enumerate(grid_y):
+        z_sample = np.array([[xi, yi]])
+        x_decoded = generator.predict(z_sample)
+        digit = x_decoded[0].reshape(digit_size, digit_size)
+        figure[i * digit_size: (i + 1) * digit_size,
+               j * digit_size: (j + 1) * digit_size] = digit
+
+plt.figure(figsize=(10, 10))
+plt.imshow(figure, cmap='Greys_r')
+plt.show()

examples/variational_autoencoder_deconv.py

@@ -0,0 +1,194 @@
+'''This script demonstrates how to build a variational autoencoder
+with Keras and deconvolution layers.
+
+Reference: "Auto-Encoding Variational Bayes" https://arxiv.org/abs/1312.6114
+'''
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.stats import norm
+
+from keras.layers import Input, Dense, Lambda, Flatten, Reshape, Layer
+from keras.layers import Conv2D, Conv2DTranspose
+from keras.models import Model
+from keras import backend as K
+from keras import metrics
+from keras.datasets import mnist
+
+# input image dimensions
+img_rows, img_cols, img_chns = 28, 28, 1
+# number of convolutional filters to use
+filters = 64
+# convolution kernel size
+num_conv = 3
+
+batch_size = 100
+if K.image_data_format() == 'channels_first':
+    original_img_size = (img_chns, img_rows, img_cols)
+else:
+    original_img_size = (img_rows, img_cols, img_chns)
+latent_dim = 2
+intermediate_dim = 128
+epsilon_std = 1.0
+epochs = 5
+
+x = Input(batch_shape=(batch_size,) + original_img_size)
+conv_1 = Conv2D(img_chns,
+                kernel_size=(2, 2),
+                padding='same', activation='relu')(x)
+conv_2 = Conv2D(filters,
+                kernel_size=(2, 2),
+                padding='same', activation='relu',
+                strides=(2, 2))(conv_1)
+conv_3 = Conv2D(filters,
+                kernel_size=num_conv,
+                padding='same', activation='relu',
+                strides=1)(conv_2)
+conv_4 = Conv2D(filters,
+                kernel_size=num_conv,
+                padding='same', activation='relu',
+                strides=1)(conv_3)
+flat = Flatten()(conv_4)
+hidden = Dense(intermediate_dim, activation='relu')(flat)
+
+z_mean = Dense(latent_dim)(hidden)
+z_log_var = Dense(latent_dim)(hidden)
+
+
+def sampling(args):
+    z_mean, z_log_var = args
+    epsilon = K.random_normal(shape=(batch_size, latent_dim),
+                              mean=0., stddev=epsilon_std)
+    return z_mean + K.exp(z_log_var) * epsilon
+
+# note that "output_shape" isn't necessary with the TensorFlow backend
+# so you could write `Lambda(sampling)([z_mean, z_log_var])`
+z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_var])
+
+# we instantiate these layers separately so as to reuse them later
+decoder_hid = Dense(intermediate_dim, activation='relu')
+decoder_upsample = Dense(filters * 14 * 14, activation='relu')
+
+if K.image_data_format() == 'channels_first':
+    output_shape = (batch_size, filters, 14, 14)
+else:
+    output_shape = (batch_size, 14, 14, filters)
+
+decoder_reshape = Reshape(output_shape[1:])
+decoder_deconv_1 = Conv2DTranspose(filters,
+                                   kernel_size=num_conv,
+                                   padding='same',
+                                   strides=1,
+                                   activation='relu')
+decoder_deconv_2 = Conv2DTranspose(filters,
+                                   kernel_size=num_conv,
+                                   padding='same',
+                                   strides=1,
+                                   activation='relu')
+if K.image_data_format() == 'channels_first':
+    output_shape = (batch_size, filters, 29, 29)
+else:
+    output_shape = (batch_size, 29, 29, filters)
+decoder_deconv_3_upsamp = Conv2DTranspose(filters,
+                                          kernel_size=(3, 3),
+                                          strides=(2, 2),
+                                          padding='valid',
+                                          activation='relu')
+decoder_mean_squash = Conv2D(img_chns,
+                             kernel_size=2,
+                             padding='valid',
+                             activation='sigmoid')
+
+hid_decoded = decoder_hid(z)
+up_decoded = decoder_upsample(hid_decoded)
+reshape_decoded = decoder_reshape(up_decoded)
+deconv_1_decoded = decoder_deconv_1(reshape_decoded)
+deconv_2_decoded = decoder_deconv_2(deconv_1_decoded)
+x_decoded_relu = decoder_deconv_3_upsamp(deconv_2_decoded)
+x_decoded_mean_squash = decoder_mean_squash(x_decoded_relu)
+
+
+# Custom loss layer
+class CustomVariationalLayer(Layer):
+    def __init__(self, **kwargs):
+        self.is_placeholder = True
+        super(CustomVariationalLayer, self).__init__(**kwargs)
+
+    def vae_loss(self, x, x_decoded_mean_squash):
+        x = K.flatten(x)
+        x_decoded_mean_squash = K.flatten(x_decoded_mean_squash)
+        xent_loss = img_rows * img_cols * metrics.binary_crossentropy(x, x_decoded_mean_squash)
+        kl_loss = - 0.5 * K.mean(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
+        return K.mean(xent_loss + kl_loss)
+
+    def call(self, inputs):
+        x = inputs[0]
+        x_decoded_mean_squash = inputs[1]
+        loss = self.vae_loss(x, x_decoded_mean_squash)
+        self.add_loss(loss, inputs=inputs)
+        # We don't use this output.
+        return x
+
+
+y = CustomVariationalLayer()([x, x_decoded_mean_squash])
+vae = Model(x, y)
+vae.compile(optimizer='rmsprop', loss=None)
+vae.summary()
+
+# train the VAE on MNIST digits
+(x_train, _), (x_test, y_test) = mnist.load_data()
+
+x_train = x_train.astype('float32') / 255.
+x_train = x_train.reshape((x_train.shape[0],) + original_img_size)
+x_test = x_test.astype('float32') / 255.
+x_test = x_test.reshape((x_test.shape[0],) + original_img_size)
+
+print('x_train.shape:', x_train.shape)
+
+vae.fit(x_train,
+        shuffle=True,
+        epochs=epochs,
+        batch_size=batch_size,
+        validation_data=(x_test, x_test))
+
+# build a model to project inputs on the latent space
+encoder = Model(x, z_mean)
+
+# display a 2D plot of the digit classes in the latent space
+x_test_encoded = encoder.predict(x_test, batch_size=batch_size)
+plt.figure(figsize=(6, 6))
+plt.scatter(x_test_encoded[:, 0], x_test_encoded[:, 1], c=y_test)
+plt.colorbar()
+plt.show()
+
+# build a digit generator that can sample from the learned distribution
+decoder_input = Input(shape=(latent_dim,))
+_hid_decoded = decoder_hid(decoder_input)
+_up_decoded = decoder_upsample(_hid_decoded)
+_reshape_decoded = decoder_reshape(_up_decoded)
+_deconv_1_decoded = decoder_deconv_1(_reshape_decoded)
+_deconv_2_decoded = decoder_deconv_2(_deconv_1_decoded)
+_x_decoded_relu = decoder_deconv_3_upsamp(_deconv_2_decoded)
+_x_decoded_mean_squash = decoder_mean_squash(_x_decoded_relu)
+generator = Model(decoder_input, _x_decoded_mean_squash)
+
+# display a 2D manifold of the digits
+n = 15  # figure with 15x15 digits
+digit_size = 28
+figure = np.zeros((digit_size * n, digit_size * n))
+# linearly spaced coordinates on the unit square were transformed through the inverse CDF (ppf) of the Gaussian
+# to produce values of the latent variables z, since the prior of the latent space is Gaussian
+grid_x = norm.ppf(np.linspace(0.05, 0.95, n))
+grid_y = norm.ppf(np.linspace(0.05, 0.95, n))
+
+for i, yi in enumerate(grid_x):
+    for j, xi in enumerate(grid_y):
+        z_sample = np.array([[xi, yi]])
+        z_sample = np.tile(z_sample, batch_size).reshape(batch_size, 2)
+        x_decoded = generator.predict(z_sample, batch_size=batch_size)
+        digit = x_decoded[0].reshape(digit_size, digit_size)
+        figure[i * digit_size: (i + 1) * digit_size,
+               j * digit_size: (j + 1) * digit_size] = digit
+
+plt.figure(figsize=(10, 10))
+plt.imshow(figure, cmap='Greys_r')
+plt.show()

keras/__init__.py

@@ -0,0 +1,23 @@
+from __future__ import absolute_import
+
+from . import utils
+from . import activations
+from . import applications
+from . import backend
+from . import datasets
+from . import engine
+from . import layers
+from . import preprocessing
+from . import wrappers
+from . import callbacks
+from . import constraints
+from . import initializers
+from . import metrics
+from . import models
+from . import losses
+from . import optimizers
+from . import regularizers
+# Importable from root because it's technically not a layer
+from .layers import Input
+
+__version__ = '2.0.5'