Prepare new PyPI release.

* Fix CSV formatting for windows with python 2

* Fix pep8 whitespace

* Fix quote style

.gitignore

@@ -11,6 +11,7 @@ docs/theme/*
 docs/sources/*
 tags
 Keras.egg-info
+examples/img/*
 
 # test-related
 .coverage

.travis.yml

@@ -7,6 +7,8 @@ matrix:
           env: KERAS_BACKEND=tensorflow TEST_MODE=PEP8
         - python: 2.7
           env: KERAS_BACKEND=tensorflow TEST_MODE=INTEGRATION_TESTS
+        - python: 3.5
+          env: KERAS_BACKEND=tensorflow TEST_MODE=DOC
         - python: 2.7
           env: KERAS_BACKEND=tensorflow
         - python: 3.5
@@ -34,7 +36,7 @@ install:
 
   - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION numpy scipy matplotlib pandas pytest h5py
   - source activate test-environment
-  - pip install git+git://github.com/Theano/Theano.git
+  - pip install theano
 
   # install PIL for preprocessing tests
   - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
@@ -45,7 +47,7 @@ install:
 
   - pip install -e .[tests]
 
-  # install TensorFlow
+  # install TensorFlow (CPU version).
   - pip install tensorflow
 
 # command to run tests
@@ -61,8 +63,8 @@ script:
        PYTHONPATH=$PWD:$PYTHONPATH py.test tests/integration_tests;
     elif [[ "$TEST_MODE" == "PEP8" ]]; then
        PYTHONPATH=$PWD:$PYTHONPATH py.test --pep8 -m pep8 -n0;
+    elif [[ "$TEST_MODE" == "DOC" ]]; then
+        PYTHONPATH=$PWD:$PYTHONPATH py.test tests/test_documentation.py;
     else
-       PYTHONPATH=$PWD:$PYTHONPATH py.test tests/ --ignore=tests/integration_tests;
+       PYTHONPATH=$PWD:$PYTHONPATH py.test tests/ --ignore=tests/integration_tests --ignore=tests/test_documentation.py --cov=keras tests/ --cov-fail-under 78 --cov-report term-missing;
     fi
-after_success:
-  - coveralls

docker/Dockerfile

@@ -7,10 +7,10 @@ RUN mkdir -p $CONDA_DIR && \
     echo export PATH=$CONDA_DIR/bin:'$PATH' > /etc/profile.d/conda.sh && \
     apt-get update && \
     apt-get install -y wget git libhdf5-dev g++ graphviz && \
-    wget --quiet https://repo.continuum.io/miniconda/Miniconda3-3.9.1-Linux-x86_64.sh && \
-    echo "6c6b44acdd0bc4229377ee10d52c8ac6160c336d9cdd669db7371aa9344e1ac3 *Miniconda3-3.9.1-Linux-x86_64.sh" | sha256sum -c - && \
-    /bin/bash /Miniconda3-3.9.1-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
-    rm Miniconda3-3.9.1-Linux-x86_64.sh
+    wget --quiet https://repo.continuum.io/miniconda/Miniconda3-4.2.12-Linux-x86_64.sh && \
+    echo "c59b3dd3cad550ac7596e0d599b91e75d88826db132e4146030ef471bb434e9a *Miniconda3-4.2.12-Linux-x86_64.sh" | sha256sum -c - && \
+    /bin/bash /Miniconda3-4.2.12-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
+    rm Miniconda3-4.2.12-Linux-x86_64.sh
 
 ENV NB_USER keras
 ENV NB_UID 1000
@@ -24,13 +24,14 @@ RUN useradd -m -s /bin/bash -N -u $NB_UID $NB_USER && \
 USER keras
 
 # Python
-ARG python_version=3.5.2
-ARG tensorflow_version=0.12.0rc0-cp35-cp35m
+ARG python_version=3.5
+
 RUN conda install -y python=${python_version} && \
-    pip install https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-${tensorflow_version}-linux_x86_64.whl && \
-    pip install git+git://github.com/Theano/Theano.git && \
-    pip install ipdb pytest pytest-cov python-coveralls coverage==3.7.1 pytest-xdist pep8 pytest-pep8 pydot_ng && \
-    conda install Pillow scikit-learn notebook pandas matplotlib nose pyyaml six h5py && \
+    pip install --upgrade pip && \
+    pip install tensorflow-gpu && \
+    conda install Pillow scikit-learn notebook pandas matplotlib mkl nose pyyaml six h5py && \
+    conda install theano pygpu && \
+    git clone git://github.com/fchollet/keras.git /src && pip install -e /src[tests] && \
     pip install git+git://github.com/fchollet/keras.git && \
     conda clean -yt
 

docker/theanorc

@@ -1,5 +1,5 @@
 [global]
 floatX = float32
 optimizer=None
-device = gpu
+device = cuda
 

docs/autogen.py

@@ -8,9 +8,7 @@ Index
 - Getting started
     Getting started with the sequential model
     Getting started with the functional api
-    Examples
     FAQ
-    Installation guide
 
 - Models
     About Keras models
@@ -26,18 +24,23 @@ Index
         explain common layer functions: get_weights, set_weights, get_config
         explain input_shape
         explain usage on non-Keras tensors
-    Core layers
-    Convolutional
-    Recurrent
-    Embeddings
-    Normalization
-    Advanced activations
-    Noise
+    Core Layers
+    Convolutional Layers
+    Pooling Layers
+    Locally-connected Layers
+    Recurrent Layers
+    Embedding Layers
+    Merge Layers
+    Advanced Activations Layers
+    Normalization Layers
+    Noise Layers
+    Layer Wrappers
+    Writing your own Keras layers
 
 - Preprocessing
-    Image preprocessing
-    Text preprocessing
-    Sequence preprocessing
+    Sequence Preprocessing
+    Text Preprocessing
+    Image Preprocessing
 
 Losses
 Metrics
@@ -45,12 +48,15 @@ Optimizers
 Activations
 Callbacks
 Datasets
+Applications
 Backend
-Initializations
+Initializers
 Regularizers
 Constraints
 Visualization
 Scikit-learn API
+Utils
+Contributing
 
 '''
 from __future__ import print_function
@@ -315,7 +321,9 @@ def get_classes_ancestors(classes):
 
 
 def get_function_signature(function, method=True):
-    signature = inspect.getargspec(function)
+    signature = getattr(function, '_legacy_support_signature', None)
+    if signature is None:
+        signature = inspect.getargspec(function)
     defaults = signature.defaults
     if method:
         args = signature.args[1:]
@@ -507,3 +515,5 @@ for page_data in PAGES:
     if not os.path.exists(subdir):
         os.makedirs(subdir)
     open(path, 'w').write(mkdown)
+
+shutil.copyfile('../CONTRIBUTING.md', 'sources/contributing.md')

docs/mkdocs.yml

@@ -1,6 +1,6 @@
 site_name: Keras Documentation
 theme: readthedocs
-theme_dir: theme
+#theme_dir: theme
 docs_dir: sources
 repo_url: http://github.com/fchollet/keras
 site_url: http://keras.io/
@@ -51,3 +51,4 @@ pages:
 - Visualization: visualization.md
 - Scikit-learn API: scikit-learn-api.md
 - Utils: utils.md
+- Contributing: contributing.md

docs/templates/applications.md

@@ -15,7 +15,7 @@ Weights are downloaded automatically when instantiating a model. They are stored
 - [ResNet50](#resnet50)
 - [InceptionV3](#inceptionv3)
 
-All of these architectures (except Xception) are compatible with both TensorFlow and Theano, and upon instantiation the models will be built according to the image data format set in your Keras configuration file at `~/.keras/keras.json`. For instance, if you have set `image_data_format=tf`, then any model loaded from this repository will get built according to the TensorFlow data format convention, "Width-Height-Depth".
+All of these architectures (except Xception) are compatible with both TensorFlow and Theano, and upon instantiation the models will be built according to the image data format set in your Keras configuration file at `~/.keras/keras.json`. For instance, if you have set `image_data_format=channels_last`, then any model loaded from this repository will get built according to the TensorFlow data format convention, "Width-Height-Depth".
 
 The Xception model is only available for TensorFlow, due to its reliance on `SeparableConvolution` layers.
 
@@ -75,7 +75,7 @@ from keras.models import Model
 import numpy as np
 
 base_model = VGG19(weights='imagenet')
-model = Model(input=base_model.input, output=base_model.get_layer('block4_pool').output)
+model = Model(inputs=base_model.input, outputs=base_model.get_layer('block4_pool').output)
 
 img_path = 'elephant.jpg'
 img = image.load_img(img_path, target_size=(224, 224))
@@ -107,7 +107,7 @@ x = Dense(1024, activation='relu')(x)
 predictions = Dense(200, activation='softmax')(x)
 
 # this is the model we will train
-model = Model(input=base_model.input, output=predictions)
+model = Model(inputs=base_model.input, outputs=predictions)
 
 # first: train only the top layers (which were randomly initialized)
 # i.e. freeze all convolutional InceptionV3 layers

docs/templates/backend.md

@@ -75,7 +75,7 @@ If you want the Keras modules you write to be compatible with both Theano (`th`)
 
 You can import the backend module via:
 ```python
-*from keras import backend as K*
+from keras import backend as K
 ```
 
 The code below instantiates an input placeholder. It's equivalent to `tf.placeholder()` or `th.tensor.matrix()`, `th.tensor.tensor3()`, etc.

docs/templates/callbacks.md

@@ -36,7 +36,7 @@ class LossHistory(keras.callbacks.Callback):
         self.losses.append(logs.get('loss'))
 
 model = Sequential()
-model.add(Dense(10, input_dim=784, init='uniform'))
+model.add(Dense(10, input_dim=784, kernel_initializer='uniform'))
 model.add(Activation('softmax'))
 model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
 
@@ -58,7 +58,7 @@ print history.losses
 from keras.callbacks import ModelCheckpoint
 
 model = Sequential()
-model.add(Dense(10, input_dim=784, init='uniform'))
+model.add(Dense(10, input_dim=784, kernel_initializer='uniform'))
 model.add(Activation('softmax'))
 model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
 

docs/templates/constraints.md

@@ -2,7 +2,7 @@
 
 Functions from the `constraints` module allow setting constraints (eg. non-negativity) on network parameters during optimization.
 
-The penalties are applied on a per-layer basis. The exact API will depend on the layer, but the layers `Dense`, `Convolution1D`, `Convolution2D` and `Convolution3D` have a unified API.
+The penalties are applied on a per-layer basis. The exact API will depend on the layer, but the layers `Dense`, `Conv1D`, `Conv2D` and `Conv3D` have a unified API.
 
 These layers expose 2 keyword arguments:
 

docs/templates/datasets.md

@@ -55,7 +55,7 @@ As a convention, "0" does not stand for a specific word, but instead is used to
 ```python
 from keras.datasets import imdb
 
-(x_train, y_train), (x_test, y_test) = imdb.load_data(path="imdb_full.pkl",
+(x_train, y_train), (x_test, y_test) = imdb.load_data(path="imdb.npz",
                                                       num_words=None,
                                                       skip_top=0,
                                                       maxlen=None,
@@ -72,13 +72,13 @@ from keras.datasets import imdb
 - __Arguments:__
 
     - __path__: if you do not have the data locally (at `'~/.keras/datasets/' + path`), it will be downloaded to this location.
-    - __num_words__: integer or None. Top most frequent words to consider. Any less frequent word will appear as 0 in the sequence data.
-    - __skip_top__: integer. Top most frequent words to ignore (they will appear as 0s in the sequence data).
+    - __num_words__: integer or None. Top most frequent words to consider. Any less frequent word will appear as `oov_char` value in the sequence data.
+    - __skip_top__: integer. Top most frequent words to ignore (they will appear as `oov_char` value in the sequence data).
     - __maxlen__: int. Maximum sequence length. Any longer sequence will be truncated.
     - __seed__: int. Seed for reproducible data shuffling.
-    - __start_char__: char. The start of a sequence will be marked with this character.
+    - __start_char__: int. The start of a sequence will be marked with this character.
         Set to 1 because 0 is usually the padding character.
-    - __oov_char__: char. words that were cut out because of the `num_words`
+    - __oov_char__: int. words that were cut out because of the `num_words`
         or `skip_top` limit will be replaced with this character.
     - __index_from__: int. Index actual words with this index and higher.
 
@@ -94,7 +94,7 @@ Dataset of 11,228 newswires from Reuters, labeled over 46 topics. As with the IM
 ```python
 from keras.datasets import reuters
 
-(x_train, y_train), (x_test, y_test) = reuters.load_data(path="reuters.pkl",
+(x_train, y_train), (x_test, y_test) = reuters.load_data(path="reuters.npz",
                                                          num_words=None,
                                                          skip_top=0,
                                                          maxlen=None,
@@ -107,12 +107,12 @@ from keras.datasets import reuters
 
 The specifications are the same as that of the IMDB dataset, with the addition of:
 
-    - __test_split__: float. Fraction of the dataset to be used as test data.
+- __test_split__: float. Fraction of the dataset to be used as test data.
 
 This dataset also makes available the word index used for encoding the sequences:
 
 ```python
-word_index = reuters.get_word_index(path="reuters_word_index.pkl")
+word_index = reuters.get_word_index(path="reuters_word_index.json")
 ```
 
 - __Returns:__ A dictionary where key are words (str) and values are indexes (integer). eg. `word_index["giraffe"]` might return `1234`. 

docs/templates/getting-started/faq.md

@@ -2,7 +2,7 @@
 
 - [How should I cite Keras?](#how-should-i-cite-keras)
 - [How can I run Keras on GPU?](#how-can-i-run-keras-on-gpu)
-- [What does \["sample", "batch", "epoch"\] mean?](#what-does-sample-batch-epoch-mean)
+- [What does "sample", "batch", "epoch" mean?](#what-does-sample-batch-epoch-mean)
 - [How can I save a Keras model?](#how-can-i-save-a-keras-model)
 - [Why is the training loss much higher than the testing loss?](#why-is-the-training-loss-much-higher-than-the-testing-loss)
 - [How can I obtain the output of an intermediate layer?](#how-can-i-obtain-the-output-of-an-intermediate-layer)
@@ -16,6 +16,7 @@
 - [How can I remove a layer from a Sequential model?](#how-can-i-remove-a-layer-from-a-sequential-model)
 - [How can I use pre-trained models in Keras?](#how-can-i-use-pre-trained-models-in-keras)
 - [How can I use HDF5 inputs with Keras?](#how-can-i-use-hdf5-inputs-with-keras)
+- [Where is the Keras configuration filed stored?](#where-is-the-keras-configuration-filed-stored)
 
 ---
 
@@ -26,7 +27,7 @@ Please cite Keras in your publications if it helps your research. Here is an exa
 ```
 @misc{chollet2015keras,
   title={Keras},
-  author={Chollet, Fran\c{c}ois},
+  author={Chollet, Fran\c{c}ois and others},
   year={2015},
   publisher={GitHub},
   howpublished={\url{https://github.com/fchollet/keras}},
@@ -59,7 +60,7 @@ theano.config.floatX = 'float32'
 
 ---
 
-### What does \["sample", "batch", "epoch"\] mean?
+### What does "sample", "batch", "epoch" mean?
 
 Below are some common definitions that are necessary to know and understand to correctly utilize Keras:
 
@@ -225,7 +226,7 @@ layer_output = get_3rd_layer_output([X, 1])[0]
 
 You can do batch training using `model.train_on_batch(X, y)` and `model.test_on_batch(X, y)`. See the [models documentation](/models/sequential).
 
-Alternatively, you can write a generator that yields batches of training data and use the method `model.fit_generator(data_generator, samples_per_epoch, epochs)`.
+Alternatively, you can write a generator that yields batches of training data and use the method `model.fit_generator(data_generator, steps_per_epoch, epochs)`.
 
 You can see batch training in action in our [CIFAR10 example](https://github.com/fchollet/keras/blob/master/examples/cifar10_cnn.py).
 
@@ -410,7 +411,7 @@ The VGG16 model is also the basis for several Keras example scripts:
 
 ### How can I use HDF5 inputs with Keras?
 
-You can use the `HDF5Matrix` class from `keras.utils.io_utils`. See [the documentation](/io_utils/#HDF5Matrix) for details.
+You can use the `HDF5Matrix` class from `keras.utils.io_utils`. See [the HDF5Matrix documentation](/utils/#hdf5matrix) for details.
 
 You can also directly use a HDF5 dataset:
 
@@ -420,3 +421,36 @@ with h5py.File('input/file.hdf5', 'r') as f:
     X_data = f['X_data']
     model.predict(X_data)
 ```
+
+---
+
+### Where is the Keras configuration filed stored?
+
+The default directory where all Keras data is stored is:
+
+```bash
+$HOME/.keras/
+```
+
+Note that Windows users should replace `$HOME` with `%USERPROFILE%`.
+In case Keras cannot create the above directory (e.g. due to permission issues), `/tmp/.keras/` is used as a backup.
+
+The Keras configuration file is a JSON file stored at `$HOME/.keras/keras.json`. The default configuration file looks like this:
+
+```
+{
+    "image_data_format": "channels_last",
+    "epsilon": 1e-07,
+    "floatx": "float32",
+    "backend": "tensorflow"
+}
+```
+
+It contains the following fields:
+
+- The image data format to be used as default by image processing layers and utilities (either `channels_last` or `channels_first`).
+- The `epsilon` numerical fuzz factor to be used to prevent division by zero in some operations.
+- The default float data type.
+- The default backend. See the [backend documentation](/backend).
+
+Likewise, cached dataset files, such as those downloaded with [`get_file()`](/utils/#get_file), are stored by default in `$HOME/.keras/datasets/`.

docs/templates/getting-started/sequential-model-guide.md

@@ -9,7 +9,7 @@ from keras.models import Sequential
 from keras.layers import Dense, Activation
 
 model = Sequential([
-    Dense(32, input_dim=784),
+    Dense(32, input_shape=(784,)),
     Activation('relu'),
     Dense(10),
     Activation('softmax'),
@@ -121,10 +121,10 @@ data = np.random.random((1000, 100))
 labels = np.random.randint(10, size=(1000, 1))
 
 # Convert labels to categorical one-hot encoding
-binary_labels = keras.utils.to_categorical(labels, num_classes=10)
+one_hot_labels = keras.utils.to_categorical(labels, num_classes=10)
 
 # Train the model, iterating on the data in batches of 32 samples
-model.fit(data, binary_labels, epochs=10, batch_size=32)
+model.fit(data, one_hot_labels, epochs=10, batch_size=32)
 ```
 
 ----
@@ -152,6 +152,13 @@ from keras.models import Sequential
 from keras.layers import Dense, Dropout, Activation
 from keras.optimizers import SGD
 
+# Generate dummy data
+import numpy as np
+x_train = np.random.random((1000, 20))
+y_train = keras.utils.to_categorical(np.random.randint(10, size=(1000, 1)), num_classes=10)
+x_test = np.random.random((100, 20))
+y_test = keras.utils.to_categorical(np.random.randint(10, size=(100, 1)), num_classes=10)
+
 model = Sequential()
 # Dense(64) is a fully-connected layer with 64 hidden units.
 # in the first layer, you must specify the expected input data shape:
@@ -177,6 +184,16 @@ score = model.evaluate(x_test, y_test, batch_size=128)
 ### MLP for binary classification:
 
 ```python
+import numpy as np
+from keras.models import Sequential
+from keras.layers import Dense, Dropout
+
+# Generate dummy data
+x_train = np.random.random((1000, 20))
+y_train = np.random.randint(2, size=(1000, 1))
+x_test = np.random.random((100, 20))
+y_test = np.random.randint(2, size=(100, 1))
+
 model = Sequential()
 model.add(Dense(64, input_dim=20, activation='relu'))
 model.add(Dropout(0.5))
@@ -187,17 +204,30 @@ model.add(Dense(1, activation='sigmoid'))
 model.compile(loss='binary_crossentropy',
               optimizer='rmsprop',
               metrics=['accuracy'])
+
+model.fit(x_train, y_train,
+          epochs=20,
+          batch_size=128)
+score = model.evaluate(x_test, y_test, batch_size=128)
 ```
 
 
 ### VGG-like convnet:
 
 ```python
+import numpy as np
+import keras
 from keras.models import Sequential
 from keras.layers import Dense, Dropout, Flatten
 from keras.layers import Conv2D, MaxPooling2D
 from keras.optimizers import SGD
 
+# Generate dummy data
+x_train = np.random.random((100, 100, 100, 3))
+y_train = keras.utils.to_categorical(np.random.randint(10, size=(100, 1)), num_classes=10)
+x_test = np.random.random((20, 100, 100, 3))
+y_test = keras.utils.to_categorical(np.random.randint(10, size=(20, 1)), num_classes=10)
+
 model = Sequential()
 # input: 100x100 images with 3 channels -> (100, 100, 3) tensors.
 # this applies 32 convolution filters of size 3x3 each.
@@ -220,6 +250,7 @@ sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
 model.compile(loss='categorical_crossentropy', optimizer=sgd)
 
 model.fit(x_train, y_train, batch_size=32, epochs=10)
+score = model.evaluate(x_test, y_test, batch_size=32)
 ```
 
 
@@ -323,7 +354,7 @@ A stateful recurrent model is one for which the internal states (memories) obtai
 of samples are reused as initial states for the samples of the next batch. This allows to process longer sequences
 while keeping computational complexity manageable.
 
-[You can read more about stateful RNNs in the FAQ.](/faq/#how-can-i-use-stateful-rnns)
+[You can read more about stateful RNNs in the FAQ.](/getting-started/faq/#how-can-i-use-stateful-rnns)
 
 ```python
 from keras.models import Sequential

docs/templates/initializers.md

@@ -39,5 +39,5 @@ from keras import backend as K
 def my_init(shape, dtype=None):
     return K.random_normal(shape, dtype=dtype)
 
-model.add(Dense(64, init=my_init))
+model.add(Dense(64, kernel_initializer=my_init))
 ```

docs/templates/metrics.md

@@ -17,7 +17,7 @@ model.compile(loss='mean_squared_error',
               metrics=[metrics.mae, metrics.categorical_accuracy])
 ```
 
-A metric function is similar to an [objective function](/objectives), except that the results from evaluating a metric are not used when training the model.
+A metric function is similar to an [loss function](/losses), except that the results from evaluating a metric are not used when training the model.
 
 You can either pass the name of an existing metric, or pass a Theano/TensorFlow symbolic function (see [Custom metrics](#custom-metrics)).
 

docs/templates/preprocessing/image.md

@@ -18,6 +18,7 @@ keras.preprocessing.image.ImageDataGenerator(featurewise_center=False,
     horizontal_flip=False,
     vertical_flip=False,
     rescale=None,
+    preprocessing_function=None,
     data_format=K.image_data_format())
 ```
 
@@ -42,6 +43,11 @@ Generate batches of tensor image data with real-time data augmentation. The data
     - __rescale__: rescaling factor. Defaults to None. If None or 0, no rescaling is applied,
             otherwise we multiply the data by the value provided (before applying
             any other transformation).
+    - __preprocessing_function__: function that will be implied on each input.
+            The function will run before any other modification on it.
+            The function should take one argument:
+            one image (Numpy tensor with rank 3),
+            and should output a Numpy tensor with the same shape.
     - _data_format_: One of {"channels_first", "channels_last"}.
         "channels_last" mode means that the images should have shape `(samples, height, width, channels)`,
         "channels_first" mode means that the images should have shape `(samples, channels, height, width)`.
@@ -50,10 +56,10 @@ Generate batches of tensor image data with real-time data augmentation. The data
         If you never set it, then it will be "channels_last".
 
 - __Methods__:
-    - __fit(X)__: Compute the internal data stats related to the data-dependent transformations, based on an array of sample data.
+    - __fit(x)__: Compute the internal data stats related to the data-dependent transformations, based on an array of sample data.
         Only required if featurewise_center or featurewise_std_normalization or zca_whitening.
         - __Arguments__:
-            - __X__: sample data. Should have rank 4.
+            - __x__: sample data. Should have rank 4.
                 In case of grayscale data,
                 the channels axis should have value 1, and in case
                 of RGB data, it should have value 3.
@@ -62,7 +68,7 @@ Generate batches of tensor image data with real-time data augmentation. The data
             - __seed__: int (default: None). Random seed.
     - __flow(X, y)__: Takes numpy data & label arrays, and generates batches of augmented/normalized data. Yields batches indefinitely, in an infinite loop.
         - __Arguments__:
-            - __X__: data. Should have rank 4.
+            - __x__: data. Should have rank 4.
                 In case of grayscale data,
                 the channels axis should have value 1, and in case
                 of RGB data, it should have value 3.
@@ -82,8 +88,8 @@ Generate batches of tensor image data with real-time data augmentation. The data
                 See [this script](https://gist.github.com/fchollet/0830affa1f7f19fd47b06d4cf89ed44d) for more details.
             - __target_size__: tuple of integers, default: `(256, 256)`. The dimensions to which all images found will be resized.
             - __color_mode__: one of "grayscale", "rbg". Default: "rgb". Whether the images will be converted to have 1 or 3 color channels.
-            - __classes__: optional list of class subdirectories (e.g. `['dogs', 'cats']`). Default: None. If not provided, the list of classes will be automatically inferred (and the order of the classes, which will map to the label indices, will be alphanumeric).
-            - __class_mode__: one of "categorical", "binary", "sparse" or None. Default: "categorical". Determines the type of label arrays that are returned: "categorical" will be 2D one-hot encoded labels, "binary" will be 1D binary labels, "sparse" will be 1D integer labels. If None, no labels are returned (the generator will only yield batches of image data, which is useful to use `model.predict_generator()`, `model.evaluate_generator()`, etc.).
+            - __classes__: optional list of class subdirectories (e.g. `['dogs', 'cats']`). Default: None. If not provided, the list of classes will be automatically inferred from the subdirectory names/structure under `directory`, where each subdirectory will be treated as a different class (and the order of the classes, which will map to the label indices, will be alphanumeric). The dictionary containing the mapping from class names to class indices can be obtained via the attribute `class_indices`.
+            - __class_mode__: one of "categorical", "binary", "sparse" or None. Default: "categorical". Determines the type of label arrays that are returned: "categorical" will be 2D one-hot encoded labels, "binary" will be 1D binary labels, "sparse" will be 1D integer labels. If None, no labels are returned (the generator will only yield batches of image data, which is useful to use `model.predict_generator()`, `model.evaluate_generator()`, etc.). Please note that in case of class_mode None, the data still needs to reside in a subdirectory of `directory` for it to work correctly.
             - __batch_size__: size of the batches of data (default: 32).
             - __shuffle__: whether to shuffle the data (default: True)
             - __seed__: optional random seed for shuffling and transformations.
@@ -95,12 +101,12 @@ Generate batches of tensor image data with real-time data augmentation. The data
 
 - __Examples__:
 
-Example of using `.flow(X, y)`:
+Example of using `.flow(x, y)`:
 
 ```python
-(X_train, y_train), (X_test, y_test) = cifar10.load_data()
-Y_train = np_utils.to_categorical(y_train, num_classes)
-Y_test = np_utils.to_categorical(y_test, num_classes)
+(x_train, y_train), (x_test, y_test) = cifar10.load_data()
+y_train = np_utils.to_categorical(y_train, num_classes)
+y_test = np_utils.to_categorical(y_test, num_classes)
 
 datagen = ImageDataGenerator(
     featurewise_center=True,
@@ -112,20 +118,20 @@ datagen = ImageDataGenerator(
 
 # compute quantities required for featurewise normalization
 # (std, mean, and principal components if ZCA whitening is applied)
-datagen.fit(X_train)
+datagen.fit(x_train)
 
 # fits the model on batches with real-time data augmentation:
-model.fit_generator(datagen.flow(X_train, Y_train, batch_size=32),
-                    samples_per_epoch=len(X_train), epochs=epochs)
+model.fit_generator(datagen.flow(x_train, y_train, batch_size=32),
+                    steps_per_epoch=len(x_train) / 32, epochs=epochs)
 
 # here's a more "manual" example
 for e in range(epochs):
-    print 'Epoch', e
+    print('Epoch', e)
     batches = 0
-    for X_batch, Y_batch in datagen.flow(X_train, Y_train, batch_size=32):
-        loss = model.train(X_batch, Y_batch)
+    for x_batch, y_batch in datagen.flow(x_train, y_train, batch_size=32):
+        model.fit(x_batch, y_batch)
         batches += 1
-        if batches >= len(X_train) / 32:
+        if batches >= len(x_train) / 32:
             # we need to break the loop by hand because
             # the generator loops indefinitely
             break
@@ -156,10 +162,10 @@ validation_generator = test_datagen.flow_from_directory(
 
 model.fit_generator(
         train_generator,
-        samples_per_epoch=2000,
+        steps_per_epoch=2000,
         epochs=50,
         validation_data=validation_generator,
-        num_val_samples=800)
+        validation_steps=800)
 ```
 
 Example of transforming images and masks together.
@@ -195,6 +201,6 @@ train_generator = zip(image_generator, mask_generator)
 
 model.fit_generator(
     train_generator,
-    samples_per_epoch=2000,
+    steps_per_epoch=2000,
     epochs=50)
 ```

docs/templates/visualization.md

@@ -19,7 +19,7 @@ You can also directly obtain the `pydot.Graph` object and render it yourself,
 for example to show it in an ipython notebook :
 ```python
 from IPython.display import SVG
-from keras.utils.visualize_util import model_to_dot
+from keras.utils.vis_utils import model_to_dot
 
 SVG(model_to_dot(model).create(prog='dot', format='svg'))
 ```

examples/addition_rnn.py

@@ -78,7 +78,7 @@ INVERT = True
 
 # Maximum length of input is 'int + int' (e.g., '345+678'). Maximum length of
 # int is DIGITS.
-MAxLEN = DIGITS + 1 + DIGITS
+MAXLEN = DIGITS + 1 + DIGITS
 
 # All the numbers, plus sign and space for padding.
 chars = '0123456789+ '
@@ -98,9 +98,9 @@ while len(questions) < TRAINING_SIZE:
     if key in seen:
         continue
     seen.add(key)
-    # Pad the data with spaces such that it is always MAxLEN.
+    # Pad the data with spaces such that it is always MAXLEN.
     q = '{}+{}'.format(a, b)
-    query = q + ' ' * (MAxLEN - len(q))
+    query = q + ' ' * (MAXLEN - len(q))
     ans = str(a + b)
     # Answers can be of maximum size DIGITS + 1.
     ans += ' ' * (DIGITS + 1 - len(ans))
@@ -113,10 +113,10 @@ while len(questions) < TRAINING_SIZE:
 print('Total addition questions:', len(questions))
 
 print('Vectorization...')
-x = np.zeros((len(questions), MAxLEN, len(chars)), dtype=np.bool)
+x = np.zeros((len(questions), MAXLEN, len(chars)), dtype=np.bool)
 y = np.zeros((len(questions), DIGITS + 1, len(chars)), dtype=np.bool)
 for i, sentence in enumerate(questions):
-    x[i] = ctable.encode(sentence, MAxLEN)
+    x[i] = ctable.encode(sentence, MAXLEN)
 for i, sentence in enumerate(expected):
     y[i] = ctable.encode(sentence, DIGITS + 1)
 
@@ -151,7 +151,7 @@ model = Sequential()
 # "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE.
 # Note: In a situation where your input sequences have a variable length,
 # use input_shape=(None, num_feature).
-model.add(RNN(HIDDEN_SIZE, input_shape=(MAxLEN, len(chars))))
+model.add(RNN(HIDDEN_SIZE, input_shape=(MAXLEN, len(chars))))
 # As the decoder RNN's input, repeatedly provide with the last hidden state of
 # RNN for each time step. Repeat 'DIGITS + 1' times as that's the maximum
 # length of output, e.g., when DIGITS=3, max output is 999+999=1998.
@@ -179,7 +179,9 @@ for iteration in range(1, 200):
     print()
     print('-' * 50)
     print('Iteration', iteration)
-    model.fit(x_train, y_train, batch_size=BATCH_SIZE, epochs=1,
+    model.fit(x_train, y_train,
+              batch_size=BATCH_SIZE,
+              epochs=1,
               validation_data=(x_val, y_val))
     # Select 10 samples from the validation set at random so we can visualize
     # errors.

examples/antirectifier.py

@@ -2,7 +2,7 @@
 
 We build a custom activation layer called 'Antirectifier',
 which modifies the shape of the tensor that passes through it.
-We need to specify two methods: `get_output_shape_for` and `call`.
+We need to specify two methods: `compute_output_shape` and `call`.
 
 Note that the same result can also be achieved via a Lambda layer.
 
@@ -98,8 +98,10 @@ model.compile(loss='categorical_crossentropy',
 
 # train the model
 model.fit(x_train, y_train,
-          batch_size=batch_size, epochs=epochs,
-          verbose=1, validation_data=(x_test, y_test))
+          batch_size=batch_size,
+          epochs=epochs,
+          verbose=1,
+          validation_data=(x_test, y_test))
 
 # next, compare with an equivalent network
 # with2x bigger Dense layers and ReLU

examples/cifar10_cnn.py

@@ -20,11 +20,6 @@ num_classes = 10
 epochs = 200
 data_augmentation = True
 
-# input image dimensions
-img_rows, img_cols = 32, 32
-# The CIFAR10 images are RGB.
-img_channels = 3
-
 # The data, shuffled and split between train and test sets:
 (x_train, y_train), (x_test, y_test) = cifar10.load_data()
 print('x_train shape:', x_train.shape)
@@ -59,9 +54,12 @@ model.add(Dropout(0.5))
 model.add(Dense(num_classes))
 model.add(Activation('softmax'))
 
+# initiate RMSprop optimizer
+opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6)
+
 # Let's train the model using RMSprop
 model.compile(loss='categorical_crossentropy',
-              optimizer='rmsprop',
+              optimizer=opt,
               metrics=['accuracy'])
 
 x_train = x_train.astype('float32')
@@ -91,7 +89,7 @@ else:
         horizontal_flip=True,  # randomly flip images
         vertical_flip=False)  # randomly flip images
 
-    # Compute quantities required for featurewise normalization
+    # Compute quantities required for feature-wise normalization
     # (std, mean, and principal components if ZCA whitening is applied).
     datagen.fit(x_train)
 

examples/deep_dream.py

@@ -8,24 +8,16 @@ e.g.:
 ```
 python deep_dream.py img/mypic.jpg results/dream
 ```
-
-It is preferable to run this script on GPU, for speed.
-If running on CPU, prefer the TensorFlow backend (much faster).
-
-Example results: http://i.imgur.com/FX6ROg9.jpg
 '''
 from __future__ import print_function
 
 from keras.preprocessing.image import load_img, img_to_array
 import numpy as np
-from scipy.misc import imsave
-from scipy.optimize import fmin_l_bfgs_b
-import time
+import scipy
 import argparse
 
-from keras.applications import vgg16
+from keras.applications import inception_v3
 from keras import backend as K
-from keras.layers import Input
 
 parser = argparse.ArgumentParser(description='Deep Dreams with Keras.')
 parser.add_argument('base_image_path', metavar='base', type=str,
@@ -37,183 +29,167 @@ args = parser.parse_args()
 base_image_path = args.base_image_path
 result_prefix = args.result_prefix
 
-# dimensions of the generated picture.
-img_height = 600
-img_width = 600
-
-# some settings we found interesting
-saved_settings = {
-    'bad_trip': {'features': {'block4_conv1': 0.05,
-                              'block4_conv2': 0.01,
-                              'block4_conv3': 0.01},
-                 'continuity': 0.1,
-                 'dream_l2': 0.8,
-                 'jitter': 5},
-    'dreamy': {'features': {'block5_conv1': 0.05,
-                            'block5_conv2': 0.02},
-               'continuity': 0.1,
-               'dream_l2': 0.02,
-               'jitter': 0},
+# These are the names of the layers
+# for which we try to maximize activation,
+# as well as their weight in the final loss
+# we try to maximize.
+# You can tweak these setting to obtain new visual effects.
+settings = {
+    'features': {
+        'mixed2': 0.2,
+        'mixed3': 0.5,
+        'mixed4': 2.,
+        'mixed5': 1.5,
+    },
 }
-# the settings we will use in this experiment
-settings = saved_settings['dreamy']
 
 
 def preprocess_image(image_path):
-    # util function to open, resize and format pictures
-    # into appropriate tensors
-    img = load_img(image_path, target_size=(img_height, img_width))
+    # Util function to open, resize and format pictures
+    # into appropriate tensors.
+    img = load_img(image_path)
     img = img_to_array(img)
     img = np.expand_dims(img, axis=0)
-    img = vgg16.preprocess_input(img)
+    img = inception_v3.preprocess_input(img)
     return img
 
 
 def deprocess_image(x):
-    # util function to convert a tensor into a valid image
+    # Util function to convert a tensor into a valid image.
     if K.image_data_format() == 'channels_first':
-        x = x.reshape((3, img_height, img_width))
+        x = x.reshape((3, x.shape[2], x.shape[3]))
         x = x.transpose((1, 2, 0))
     else:
-        x = x.reshape((img_height, img_width, 3))
-    # Remove zero-center by mean pixel
-    x[:, :, 0] += 103.939
-    x[:, :, 1] += 116.779
-    x[:, :, 2] += 123.68
-    # 'BGR'->'RGB'
-    x = x[:, :, ::-1]
+        x = x.reshape((x.shape[1], x.shape[2], 3))
+    x /= 2.
+    x += 0.5
+    x *= 255.
     x = np.clip(x, 0, 255).astype('uint8')
     return x
 
-if K.image_data_format() == 'channels_first':
-    img_size = (3, img_height, img_width)
-else:
-    img_size = (img_height, img_width, 3)
-# this will contain our generated image
-dream = Input(batch_shape=(1,) + img_size)
+K.set_learning_phase(0)
 
-# build the VGG16 network with our placeholder
-# the model will be loaded with pre-trained ImageNet weights
-model = vgg16.VGG16(input_tensor=dream,
-                    weights='imagenet', include_top=False)
+# Build the InceptionV3 network with our placeholder.
+# The model will be loaded with pre-trained ImageNet weights.
+model = inception_v3.InceptionV3(weights='imagenet',
+                                 include_top=False)
+dream = model.input
 print('Model loaded.')
 
-# get the symbolic outputs of each "key" layer (we gave them unique names).
+# Get the symbolic outputs of each "key" layer (we gave them unique names).
 layer_dict = dict([(layer.name, layer) for layer in model.layers])
 
-
-def continuity_loss(x):
-    # continuity loss util function
-    assert K.ndim(x) == 4
-    if K.image_data_format() == 'channels_first':
-        a = K.square(x[:, :, :img_height - 1, :img_width - 1] -
-                     x[:, :, 1:, :img_width - 1])
-        b = K.square(x[:, :, :img_height - 1, :img_width - 1] -
-                     x[:, :, :img_height - 1, 1:])
-    else:
-        a = K.square(x[:, :img_height - 1, :img_width - 1, :] -
-                     x[:, 1:, :img_width - 1, :])
-        b = K.square(x[:, :img_height - 1, :img_width - 1, :] -
-                     x[:, :img_height - 1, 1:, :])
-    return K.sum(K.pow(a + b, 1.25))
-
-# define the loss
+# Define the loss.
 loss = K.variable(0.)
 for layer_name in settings['features']:
-    # add the L2 norm of the features of a layer to the loss
+    # Add the L2 norm of the features of a layer to the loss.
     assert layer_name in layer_dict.keys(), 'Layer ' + layer_name + ' not found in model.'
     coeff = settings['features'][layer_name]
     x = layer_dict[layer_name].output
-    shape = layer_dict[layer_name].output_shape
-    # we avoid border artifacts by only involving non-border pixels in the loss
+    # We avoid border artifacts by only involving non-border pixels in the loss.
+    scaling = K.prod(K.cast(K.shape(x), 'float32'))
     if K.image_data_format() == 'channels_first':
-        loss -= coeff * K.sum(K.square(x[:, :, 2: shape[2] - 2, 2: shape[3] - 2])) / np.prod(shape[1:])
+        loss += coeff * K.sum(K.square(x[:, :, 2: -2, 2: -2])) / scaling
     else:
-        loss -= coeff * K.sum(K.square(x[:, 2: shape[1] - 2, 2: shape[2] - 2, :])) / np.prod(shape[1:])
+        loss += coeff * K.sum(K.square(x[:, 2: -2, 2: -2, :])) / scaling
 
-# add continuity loss (gives image local coherence, can result in an artful blur)
-loss += settings['continuity'] * continuity_loss(dream) / np.prod(img_size)
-# add image L2 norm to loss (prevents pixels from taking very high values, makes image darker)
-loss += settings['dream_l2'] * K.sum(K.square(dream)) / np.prod(img_size)
+# Compute the gradients of the dream wrt the loss.
+grads = K.gradients(loss, dream)[0]
+# Normalize gradients.
+grads /= K.maximum(K.mean(K.abs(grads)), 1e-7)
 
-# feel free to further modify the loss as you see fit, to achieve new effects...
-
-# compute the gradients of the dream wrt the loss
-grads = K.gradients(loss, dream)
-
-outputs = [loss]
-if isinstance(grads, (list, tuple)):
-    outputs += grads
-else:
-    outputs.append(grads)
-
-f_outputs = K.function([dream], outputs)
+# Set up function to retrieve the value
+# of the loss and gradients given an input image.
+outputs = [loss, grads]
+fetch_loss_and_grads = K.function([dream], outputs)
 
 
 def eval_loss_and_grads(x):
-    x = x.reshape((1,) + img_size)
-    outs = f_outputs([x])
+    outs = fetch_loss_and_grads([x])
     loss_value = outs[0]
-    if len(outs[1:]) == 1:
-        grad_values = outs[1].flatten().astype('float64')
-    else:
-        grad_values = np.array(outs[1:]).flatten().astype('float64')
+    grad_values = outs[1]
     return loss_value, grad_values
 
 
-class Evaluator(object):
-    """Loss and gradients evaluator.
+def resize_img(img, size):
+    img = np.copy(img)
+    if K.image_data_format() == 'channels_first':
+        factors = (1, 1,
+                   float(size[0]) / img.shape[2],
+                   float(size[1]) / img.shape[3])
+    else:
+        factors = (1,
+                   float(size[0]) / img.shape[1],
+                   float(size[1]) / img.shape[2],
+                   1)
+    return scipy.ndimage.zoom(img, factors, order=1)
 
-    This Evaluator class makes it possible
-    to compute loss and gradients in one pass
-    while retrieving them via two separate functions,
-    "loss" and "grads". This is done because scipy.optimize
-    requires separate functions for loss and gradients,
-    but computing them separately would be inefficient.
-    """
 
-    def __init__(self):
-        self.loss_value = None
-        self.grad_values = None
-
-    def loss(self, x):
-        assert self.loss_value is None
+def gradient_ascent(x, iterations, step, max_loss=None):
+    for i in range(iterations):
         loss_value, grad_values = eval_loss_and_grads(x)
-        self.loss_value = loss_value
-        self.grad_values = grad_values
-        return self.loss_value
+        if max_loss is not None and loss_value > max_loss:
+            break
+        print('..Loss value at', i, ':', loss_value)
+        x += step * grad_values
+    return x
 
-    def grads(self, x):
-        assert self.loss_value is not None
-        grad_values = np.copy(self.grad_values)
-        self.loss_value = None
-        self.grad_values = None
-        return grad_values
 
-evaluator = Evaluator()
+def save_img(img, fname):
+    pil_img = deprocess_image(np.copy(img))
+    scipy.misc.imsave(fname, pil_img)
 
-# Run scipy-based optimization (L-BFGS) over the pixels of the generated image
-# so as to minimize the loss
-x = preprocess_image(base_image_path)
-for i in range(5):
-    print('Start of iteration', i)
-    start_time = time.time()
 
-    # Add a random jitter to the initial image.
-    # This will be reverted at decoding time
-    random_jitter = (settings['jitter'] * 2) * (np.random.random(img_size) - 0.5)
-    x += random_jitter
+"""Process:
 
-    # Run L-BFGS for 7 steps
-    x, min_val, info = fmin_l_bfgs_b(evaluator.loss, x.flatten(),
-                                     fprime=evaluator.grads, maxfun=7)
-    print('Current loss value:', min_val)
-    # Decode the dream and save it
-    x = x.reshape(img_size)
-    x -= random_jitter
-    img = deprocess_image(np.copy(x))
-    fname = result_prefix + '_at_iteration_%d.png' % i
-    imsave(fname, img)
-    end_time = time.time()
-    print('Image saved as', fname)
-    print('Iteration %d completed in %ds' % (i, end_time - start_time))
+- Load the original image.
+- Define a number of processing scales (i.e. image shapes),
+    from smallest to largest.
+- Resize the original image to the smallest scale.
+- For every scale, starting with the smallest (i.e. current one):
+    - Run gradient ascent
+    - Upscale image to the next scale
+    - Reinject the detail that was lost at upscaling time
+- Stop when we are back to the original size.
+
+To obtain the detail lost during upscaling, we simply
+take the original image, shrink it down, upscale it,
+and compare the result to the (resized) original image.
+"""
+
+
+# Playing with these hyperparameters will also allow you to achieve new effects
+step = 0.01  # Gradient ascent step size
+num_octave = 3  # Number of scales at which to run gradient ascent
+octave_scale = 1.4  # Size ratio between scales
+iterations = 20  # Number of ascent steps per scale
+max_loss = 10.
+
+img = preprocess_image(base_image_path)
+if K.image_data_format() == 'channels_first':
+    original_shape = img.shape[2:]
+else:
+    original_shape = img.shape[1:3]
+successive_shapes = [original_shape]
+for i in range(1, num_octave):
+    shape = tuple([int(dim / (octave_scale ** i)) for dim in original_shape])
+    successive_shapes.append(shape)
+successive_shapes = successive_shapes[::-1]
+original_img = np.copy(img)
+shrunk_original_img = resize_img(img, successive_shapes[0])
+
+for shape in successive_shapes:
+    print('Processing image shape', shape)
+    img = resize_img(img, shape)
+    img = gradient_ascent(img,
+                          iterations=iterations,
+                          step=step,
+                          max_loss=max_loss)
+    upscaled_shrunk_original_img = resize_img(shrunk_original_img, shape)
+    same_size_original = resize_img(original_img, shape)
+    lost_detail = same_size_original - upscaled_shrunk_original_img
+
+    img += lost_detail
+    shrunk_original_img = resize_img(original_img, shape)
+
+save_img(img, fname=result_prefix + '.png')

examples/image_ocr.py

@@ -41,9 +41,10 @@ import numpy as np
 from scipy import ndimage
 import pylab
 from keras import backend as K
-from keras.layers.convolutional import Convolution2D, MaxPooling2D
+from keras.layers.convolutional import Conv2D, MaxPooling2D
 from keras.layers import Input, Dense, Activation
-from keras.layers import Reshape, Lambda, merge
+from keras.layers import Reshape, Lambda
+from keras.layers.merge import add, concatenate
 from keras.models import Model
 from keras.layers.recurrent import GRU
 from keras.optimizers import SGD
@@ -126,9 +127,9 @@ def shuffle_mats_or_lists(matrix_list, stop_ind=None):
         stop_ind = len_val
     assert stop_ind <= len_val
 
-    a = range(stop_ind)
+    a = list(range(stop_ind))
     np.random.shuffle(a)
-    a += range(stop_ind, len_val)
+    a += list(range(stop_ind, len_val))
     for mat in matrix_list:
         if isinstance(mat, np.ndarray):
             ret.append(mat[a])
@@ -403,8 +404,8 @@ def train(run_name, start_epoch, stop_epoch, img_w):
     val_words = int(words_per_epoch * (val_split))
 
     # Network parameters
-    conv_filterss = 16
-    filter_size = 3
+    conv_filters = 16
+    kernel_size = (3, 3)
     pool_size = 2
     time_dense_size = 32
     rnn_size = 512
@@ -415,7 +416,7 @@ def train(run_name, start_epoch, stop_epoch, img_w):
         input_shape = (img_w, img_h, 1)
 
     fdir = os.path.dirname(get_file('wordlists.tgz',
-                                    origin='http://www.isosemi.com/datasets/wordlists.tgz', untar=True))
+                                    origin='http://www.mythic-ai.com/datasets/wordlists.tgz', untar=True))
 
     img_gen = TextImageGenerator(monogram_file=os.path.join(fdir, 'wordlist_mono_clean.txt'),
                                  bigram_file=os.path.join(fdir, 'wordlist_bi_clean.txt'),
@@ -427,14 +428,16 @@ def train(run_name, start_epoch, stop_epoch, img_w):
                                  )
     act = 'relu'
     input_data = Input(name='the_input', shape=input_shape, dtype='float32')
-    inner = Convolution2D(conv_filterss, filter_size, filter_size, border_mode='same',
-                          activation=act, init='he_normal', name='conv1')(input_data)
+    inner = Conv2D(conv_filters, kernel_size, padding='same',
+                   activation=act, kernel_initializer='he_normal',
+                   name='conv1')(input_data)
     inner = MaxPooling2D(pool_size=(pool_size, pool_size), name='max1')(inner)
-    inner = Convolution2D(conv_filterss, filter_size, filter_size, border_mode='same',
-                          activation=act, init='he_normal', name='conv2')(inner)
+    inner = Conv2D(conv_filters, kernel_size, padding='same',
+                   activation=act, kernel_initializer='he_normal',
+                   name='conv2')(inner)
     inner = MaxPooling2D(pool_size=(pool_size, pool_size), name='max2')(inner)
 
-    conv_to_rnn_dims = (img_w // (pool_size ** 2), (img_h // (pool_size ** 2)) * conv_filterss)
+    conv_to_rnn_dims = (img_w // (pool_size ** 2), (img_h // (pool_size ** 2)) * conv_filters)
     inner = Reshape(target_shape=conv_to_rnn_dims, name='reshape')(inner)
 
     # cuts down input size going into RNN:
@@ -442,17 +445,17 @@ def train(run_name, start_epoch, stop_epoch, img_w):
 
     # Two layers of bidirecitonal GRUs
     # GRU seems to work as well, if not better than LSTM:
-    gru_1 = GRU(rnn_size, return_sequences=True, init='he_normal', name='gru1')(inner)
-    gru_1b = GRU(rnn_size, return_sequences=True, go_backwards=True, init='he_normal', name='gru1_b')(inner)
-    gru1_merged = merge([gru_1, gru_1b], mode='sum')
-    gru_2 = GRU(rnn_size, return_sequences=True, init='he_normal', name='gru2')(gru1_merged)
-    gru_2b = GRU(rnn_size, return_sequences=True, go_backwards=True, init='he_normal', name='gru2_b')(gru1_merged)
+    gru_1 = GRU(rnn_size, return_sequences=True, kernel_initializer='he_normal', name='gru1')(inner)
+    gru_1b = GRU(rnn_size, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='gru1_b')(inner)
+    gru1_merged = add([gru_1, gru_1b])
+    gru_2 = GRU(rnn_size, return_sequences=True, kernel_initializer='he_normal', name='gru2')(gru1_merged)
+    gru_2b = GRU(rnn_size, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='gru2_b')(gru1_merged)
 
     # transforms RNN output to character activations:
-    inner = Dense(img_gen.get_output_size(), init='he_normal',
-                  name='dense2')(merge([gru_2, gru_2b], mode='concat'))
+    inner = Dense(img_gen.get_output_size(), kernel_initializer='he_normal',
+                  name='dense2')(concatenate([gru_2, gru_2b]))
     y_pred = Activation('softmax', name='softmax')(inner)
-    Model(input=[input_data], output=y_pred).summary()
+    Model(inputs=input_data, outputs=y_pred).summary()
 
     labels = Input(name='the_labels', shape=[img_gen.absolute_max_string_len], dtype='float32')
     input_length = Input(name='input_length', shape=[1], dtype='int64')
@@ -464,7 +467,7 @@ def train(run_name, start_epoch, stop_epoch, img_w):
     # clipnorm seems to speeds up convergence
     sgd = SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5)
 
-    model = Model(input=[input_data, labels, input_length, label_length], output=[loss_out])
+    model = Model(inputs=[input_data, labels, input_length, label_length], outputs=loss_out)
 
     # the loss calc occurs elsewhere, so use a dummy lambda func for the loss
     model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd)
@@ -476,8 +479,8 @@ def train(run_name, start_epoch, stop_epoch, img_w):
 
     viz_cb = VizCallback(run_name, test_func, img_gen.next_val())
 
-    model.fit_generator(generator=img_gen.next_train(), samples_per_epoch=(words_per_epoch - val_words),
-                        epochs=stop_epoch, validation_data=img_gen.next_val(), num_val_samples=val_words,
+    model.fit_generator(generator=img_gen.next_train(), steps_per_epoch=(words_per_epoch - val_words),
+                        epochs=stop_epoch, validation_data=img_gen.next_val(), validation_steps=val_words,
                         callbacks=[viz_cb, img_gen], initial_epoch=start_epoch)
 
 

examples/imdb_cnn_lstm.py

@@ -67,7 +67,9 @@ model.compile(loss='binary_crossentropy',
               metrics=['accuracy'])
 
 print('Train...')
-model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs,
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=epochs,
           validation_data=(x_test, y_test))
 score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
 print('Test score:', score)

examples/imdb_lstm.py

@@ -45,7 +45,9 @@ model.compile(loss='binary_crossentropy',
               metrics=['accuracy'])
 
 print('Train...')
-model.fit(x_train, y_train, batch_size=batch_size, epochs=15,
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=15,
           validation_data=(x_test, y_test))
 score, acc = model.evaluate(x_test, y_test,
                             batch_size=batch_size)

examples/lstm_benchmark.py

@@ -12,8 +12,8 @@ into one, large matrix, resulting in faster computation time as the GPU can
 utilize more cores, at the expense of reduced regularization because the same
 dropout is shared across the gates.
 
-Note that the relative performance of the different `consume_less` modes
-can vary depending on your device, your model and the size of your data.
+Note that the relative performance of the different implementations can
+vary depending on your device, your model and the size of your data.
 '''
 
 import time

examples/lstm_text_generation.py

@@ -73,7 +73,9 @@ for iteration in range(1, 60):
     print()
     print('-' * 50)
     print('Iteration', iteration)
-    model.fit(X, y, batch_size=128, epochs=1)
+    model.fit(X, y,
+              batch_size=128,
+              epochs=1)
 
     start_index = random.randint(0, len(text) - maxlen - 1)
 

examples/mnist_acgan.py

@@ -101,7 +101,7 @@ def build_discriminator():
     cnn.add(LeakyReLU())
     cnn.add(Dropout(0.3))
 
-    cnn.add(Conv2D(64, 3, padding='same', strides=2))
+    cnn.add(Conv2D(64, 3, padding='same', strides=1))
     cnn.add(LeakyReLU())
     cnn.add(Dropout(0.3))
 
@@ -222,7 +222,7 @@ if __name__ == '__main__':
             noise = np.random.uniform(-1, 1, (2 * batch_size, latent_size))
             sampled_labels = np.random.randint(0, 10, 2 * batch_size)
 
-            # we want to train the genrator to trick the discriminator
+            # we want to train the generator to trick the discriminator
             # For the generator, we want all the {fake, not-fake} labels to say
             # not-fake
             trick = np.ones(2 * batch_size)

examples/mnist_cnn.py

@@ -60,8 +60,11 @@ model.compile(loss=keras.losses.categorical_crossentropy,
               optimizer=keras.optimizers.Adadelta(),
               metrics=['accuracy'])
 
-model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs,
-          verbose=1, validation_data=(x_test, y_test))
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=epochs,
+          verbose=1,
+          validation_data=(x_test, y_test))
 score = model.evaluate(x_test, y_test, verbose=0)
 print('Test loss:', score[0])
 print('Test accuracy:', score[1])

examples/mnist_hierarchical_rnn.py

@@ -79,8 +79,10 @@ model.compile(loss='categorical_crossentropy',
 
 # Training.
 model.fit(x_train, y_train,
-          batch_size=batch_size, epochs=epochs,
-          verbose=1, validation_data=(x_test, y_test))
+          batch_size=batch_size,
+          epochs=epochs,
+          verbose=1,
+          validation_data=(x_test, y_test))
 
 # Evaluation.
 scores = model.evaluate(x_test, y_test, verbose=0)

examples/mnist_irnn.py

@@ -62,8 +62,11 @@ model.compile(loss='categorical_crossentropy',
               optimizer=rmsprop,
               metrics=['accuracy'])
 
-model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs,
-          verbose=1, validation_data=(x_test, y_test))
+model.fit(x_train, y_train,
+          batch_size=batch_size,
+          epochs=epochs,
+          verbose=1,
+          validation_data=(x_test, y_test))
 
 scores = model.evaluate(x_test, y_test, verbose=0)
 print('IRNN test score:', scores[0])

examples/mnist_mlp.py

@@ -48,8 +48,10 @@ model.compile(loss='categorical_crossentropy',
               metrics=['accuracy'])
 
 history = model.fit(x_train, y_train,
-                    batch_size=batch_size, epochs=epochs,
-                    verbose=1, validation_data=(x_test, y_test))
+                    batch_size=batch_size,
+                    epochs=epochs,
+                    verbose=1,
+                    validation_data=(x_test, y_test))
 score = model.evaluate(x_test, y_test, verbose=0)
 print('Test loss:', score[0])
 print('Test accuracy:', score[1])

examples/mnist_net2net.py

@@ -26,7 +26,7 @@ Notes
 
 Experiments
 - Teacher model: a basic CNN model trained on MNIST for 3 epochs.
-- Net2WiderNet exepriment:
+- Net2WiderNet experiment:
   + Student model has a wider Conv2D layer and a wider FC layer.
   + Comparison of 'random-padding' vs 'net2wider' weight initialization.
   + With both methods, student model should immediately perform as well as
@@ -231,7 +231,8 @@ def make_teacher_model(train_data, validation_data, epochs=3):
                   metrics=['accuracy'])
 
     train_x, train_y = train_data
-    history = model.fit(train_x, train_y, epochs=epochs,
+    history = model.fit(train_x, train_y,
+                        epochs=epochs,
                         validation_data=validation_data)
     return model, history
 
@@ -280,7 +281,8 @@ def make_wider_student_model(teacher_model, train_data,
                   metrics=['accuracy'])
 
     train_x, train_y = train_data
-    history = model.fit(train_x, train_y, epochs=epochs,
+    history = model.fit(train_x, train_y,
+                        epochs=epochs,
                         validation_data=validation_data)
     return model, history
 
@@ -328,7 +330,8 @@ def make_deeper_student_model(teacher_model, train_data,
                   metrics=['accuracy'])
 
     train_x, train_y = train_data
-    history = model.fit(train_x, train_y, epochs=epochs,
+    history = model.fit(train_x, train_y,
+                        epochs=epochs,
                         validation_data=validation_data)
     return model, history
 

examples/mnist_siamese_graph.py

@@ -24,7 +24,7 @@ from keras import backend as K
 
 def euclidean_distance(vects):
     x, y = vects
-    return K.sqrt(K.sum(K.square(x - y), axis=1, keepdims=True))
+    return K.sqrt(K.maximum(K.sum(K.square(x - y), axis=1, keepdims=True), K.epsilon()))
 
 
 def eucl_dist_output_shape(shapes):
@@ -117,9 +117,9 @@ model = Model([input_a, input_b], distance)
 rms = RMSprop()
 model.compile(loss=contrastive_loss, optimizer=rms)
 model.fit([tr_pairs[:, 0], tr_pairs[:, 1]], tr_y,
-          validation_data=([te_pairs[:, 0], te_pairs[:, 1]], te_y),
           batch_size=128,
-          epochs=epochs)
+          epochs=epochs,
+          validation_data=([te_pairs[:, 0], te_pairs[:, 1]], te_y))
 
 # compute final accuracy on training and test sets
 pred = model.predict([tr_pairs[:, 0], tr_pairs[:, 1]])

examples/mnist_swwae.py

@@ -35,12 +35,12 @@ applied as a bias because we know the MNIST digits are mapped to [0,1].
 References:
 [3]
 'Deep Residual Learning for Image Recognition'
-Kaiming He, xiangyu Zhang, Shaoqing Ren, Jian Sun
+Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun
 https://arxiv.org/abs/1512.03385v1
 
 [4]
 'Identity Mappings in Deep Residual Networks'
-Kaiming He, xiangyu Zhang, Shaoqing Ren, Jian Sun
+Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun
 https://arxiv.org/abs/1603.05027v3
 
 '''
@@ -186,8 +186,10 @@ model = Model(img_input, y)
 model.compile('adam', 'mse')
 
 # Fit the model
-model.fit(x_train, x_train, validation_data=(x_test, x_test),
-          batch_size=batch_size, epochs=epochs)
+model.fit(x_train, x_train,
+          batch_size=batch_size,
+          epochs=epochs,
+          validation_data=(x_test, x_test))
 
 # Plot
 x_recon = model.predict(x_test[:25])

examples/mnist_transfer_cnn.py

@@ -63,7 +63,8 @@ def train_model(model, train, test, num_classes):
 
     t = now()
     model.fit(x_train, y_train,
-              batch_size=batch_size, epochs=epochs,
+              batch_size=batch_size,
+              epochs=epochs,
               verbose=1,
               validation_data=(x_test, y_test))
     print('Training time: %s' % (now() - t))

examples/pretrained_word_embeddings.py

@@ -143,6 +143,7 @@ model.compile(loss='categorical_crossentropy',
               optimizer='rmsprop',
               metrics=['acc'])
 
-# happy learning!
-model.fit(x_train, y_train, validation_data=(x_val, y_val),
-          epochs=10, batch_size=128)
+model.fit(x_train, y_train,
+          batch_size=128,
+          epochs=10,
+          validation_data=(x_val, y_val))

examples/reuters_mlp.py

@@ -50,8 +50,10 @@ model.compile(loss='categorical_crossentropy',
               metrics=['accuracy'])
 
 history = model.fit(x_train, y_train,
-                    epochs=epochs, batch_size=batch_size,
-                    verbose=1, validation_split=0.1)
+                    batch_size=batch_size,
+                    epochs=epochs,
+                    verbose=1,
+                    validation_split=0.1)
 score = model.evaluate(x_test, y_test,
                        batch_size=batch_size, verbose=1)
 print('Test score:', score[0])

examples/stateful_lstm.py

@@ -70,11 +70,10 @@ for i in range(epochs):
     # Each of these series are offset by one step and can be
     # extracted with cos[i::batch_size].
 
-    model.fit(cos,
-              expected_output,
+    model.fit(cos, expected_output,
               batch_size=batch_size,
-              verbose=1,
               epochs=1,
+              verbose=1,
               shuffle=False)
     model.reset_states()
 

examples/variational_autoencoder.py

@@ -6,7 +6,7 @@ import numpy as np
 import matplotlib.pyplot as plt
 from scipy.stats import norm
 
-from keras.layers import Input, Dense, Lambda
+from keras.layers import Input, Dense, Lambda, Layer
 from keras.models import Model
 from keras import backend as K
 from keras import metrics
@@ -19,6 +19,7 @@ intermediate_dim = 256
 epochs = 50
 epsilon_std = 1.0
 
+
 x = Input(batch_shape=(batch_size, original_dim))
 h = Dense(intermediate_dim, activation='relu')(x)
 z_mean = Dense(latent_dim)(h)
@@ -41,13 +42,29 @@ h_decoded = decoder_h(z)
 x_decoded_mean = decoder_mean(h_decoded)
 
 
-def vae_loss(x, x_decoded_mean):
-    xent_loss = original_dim * metrics.binary_crossentropy(x, x_decoded_mean)
-    kl_loss = - 0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
-    return xent_loss + kl_loss
+# Custom loss layer
+class CustomVariationalLayer(Layer):
+    def __init__(self, **kwargs):
+        self.is_placeholder = True
+        super(CustomVariationalLayer, self).__init__(**kwargs)
+
+    def vae_loss(self, x, x_decoded_mean):
+        xent_loss = original_dim * metrics.binary_crossentropy(x, x_decoded_mean)
+        kl_loss = - 0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
+        return K.mean(xent_loss + kl_loss)
+
+    def call(self, inputs):
+        x = inputs[0]
+        x_decoded_mean = inputs[1]
+        loss = self.vae_loss(x, x_decoded_mean)
+        self.add_loss(loss, inputs=inputs)
+        # We won't actually use the output.
+        return x
+
+y = CustomVariationalLayer()([x, x_decoded_mean])
+vae = Model(x, y)
+vae.compile(optimizer='rmsprop', loss=None)
 
-vae = Model(x, x_decoded_mean)
-vae.compile(optimizer='rmsprop', loss=vae_loss)
 
 # train the VAE on MNIST digits
 (x_train, y_train), (x_test, y_test) = mnist.load_data()
@@ -57,7 +74,7 @@ x_test = x_test.astype('float32') / 255.
 x_train = x_train.reshape((len(x_train), np.prod(x_train.shape[1:])))
 x_test = x_test.reshape((len(x_test), np.prod(x_test.shape[1:])))
 
-vae.fit(x_train, x_train,
+vae.fit(x_train,
         shuffle=True,
         epochs=epochs,
         batch_size=batch_size,

examples/variational_autoencoder_deconv.py

@@ -7,7 +7,7 @@ import numpy as np
 import matplotlib.pyplot as plt
 from scipy.stats import norm
 
-from keras.layers import Input, Dense, Lambda, Flatten, Reshape
+from keras.layers import Input, Dense, Lambda, Flatten, Reshape, Layer
 from keras.layers import Conv2D, Conv2DTranspose
 from keras.models import Model
 from keras import backend as K
@@ -106,17 +106,31 @@ x_decoded_relu = decoder_deconv_3_upsamp(deconv_2_decoded)
 x_decoded_mean_squash = decoder_mean_squash(x_decoded_relu)
 
 
-def vae_loss(x, x_decoded_mean):
-    # NOTE: binary_crossentropy expects a batch_size by dim
-    # for x and x_decoded_mean, so we MUST flatten these!
-    x = K.flatten(x)
-    x_decoded_mean = K.flatten(x_decoded_mean)
-    xent_loss = img_rows * img_cols * metrics.binary_crossentropy(x, x_decoded_mean)
-    kl_loss = - 0.5 * K.mean(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
-    return xent_loss + kl_loss
+# Custom loss layer
+class CustomVariationalLayer(Layer):
+    def __init__(self, **kwargs):
+        self.is_placeholder = True
+        super(CustomVariationalLayer, self).__init__(**kwargs)
 
-vae = Model(x, x_decoded_mean_squash)
-vae.compile(optimizer='rmsprop', loss=vae_loss)
+    def vae_loss(self, x, x_decoded_mean_squash):
+        x = K.flatten(x)
+        x_decoded_mean_squash = K.flatten(x_decoded_mean_squash)
+        xent_loss = img_rows * img_cols * metrics.binary_crossentropy(x, x_decoded_mean_squash)
+        kl_loss = - 0.5 * K.mean(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
+        return K.mean(xent_loss + kl_loss)
+
+    def call(self, inputs):
+        x = inputs[0]
+        x_decoded_mean_squash = inputs[1]
+        loss = self.vae_loss(x, x_decoded_mean_squash)
+        self.add_loss(loss, inputs=inputs)
+        # We don't use this output.
+        return x
+
+
+y = CustomVariationalLayer()([x, x_decoded_mean_squash])
+vae = Model(x, y)
+vae.compile(optimizer='rmsprop', loss=None)
 vae.summary()
 
 # train the VAE on MNIST digits
@@ -129,7 +143,7 @@ x_test = x_test.reshape((x_test.shape[0],) + original_img_size)
 
 print('x_train.shape:', x_train.shape)
 
-vae.fit(x_train, x_train,
+vae.fit(x_train,
         shuffle=True,
         epochs=epochs,
         batch_size=batch_size,

keras/__init__.py

@@ -17,5 +17,7 @@ from . import models
 from . import losses
 from . import optimizers
 from . import regularizers
+# Importable from root because it's technically not a layer
+from .layers import Input
 
-__version__ = '2.0.1'
+__version__ = '2.0.4'

keras/activations.py

@@ -1,21 +1,33 @@
 from __future__ import absolute_import
 import six
+import warnings
 from . import backend as K
 from .utils.generic_utils import deserialize_keras_object
+from .engine import Layer
 
 
-def softmax(x):
+def softmax(x, axis=-1):
+    """Softmax activation function.
+
+    # Arguments
+        x : Tensor.
+        axis: Integer, axis along which the softmax normalization is applied.
+
+    # Returns
+        Tensor, output of softmax transformation.
+
+    # Raises
+        ValueError: In case `dim(x) == 1`.
+    """
     ndim = K.ndim(x)
     if ndim == 2:
         return K.softmax(x)
-    elif ndim == 3:
-        e = K.exp(x - K.max(x, axis=-1, keepdims=True))
-        s = K.sum(e, axis=-1, keepdims=True)
+    elif ndim > 2:
+        e = K.exp(x - K.max(x, axis=axis, keepdims=True))
+        s = K.sum(e, axis=axis, keepdims=True)
         return e / s
     else:
-        raise ValueError('Cannot apply softmax to a tensor '
-                         'that is not 2D or 3D. '
-                         'Here, ndim=' + str(ndim))
+        raise ValueError('Cannot apply softmax to a tensor that is 1D')
 
 
 def elu(x, alpha=1.0):
@@ -68,6 +80,13 @@ def get(identifier):
         identifier = str(identifier)
         return deserialize(identifier)
     elif callable(identifier):
+        if isinstance(identifier, Layer):
+            warnings.warn((
+                'Do not pass a layer instance (such as {identifier}) as the '
+                'activation argument of another layer. Instead, advanced '
+                'activation layers should be used just like any other '
+                'layer in a model.'
+            ).format(identifier=identifier.__class__.__name__))
         return identifier
     else:
         raise ValueError('Could not interpret '

keras/applications/inception_v3.py

@@ -157,7 +157,10 @@ def InceptionV3(include_top=True,
     if input_tensor is None:
         img_input = Input(shape=input_shape)
     else:
-        img_input = Input(tensor=input_tensor, shape=input_shape)
+        if not K.is_keras_tensor(input_tensor):
+            img_input = Input(tensor=input_tensor, shape=input_shape)
+        else:
+            img_input = input_tensor
 
     if K.image_data_format() == 'channels_first':
         channel_axis = 1

keras/applications/resnet50.py

@@ -140,8 +140,8 @@ def ResNet50(include_top=True, weights='imagenet',
     specified in your Keras config file.
 
     # Arguments
-        include_top: whether to include the 3 fully-connected
-            layers at the top of the network.
+        include_top: whether to include the fully-connected
+            layer at the top of the network.
         weights: one of `None` (random initialization)
             or "imagenet" (pre-training on ImageNet).
         input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
@@ -149,7 +149,7 @@ def ResNet50(include_top=True, weights='imagenet',
         input_shape: optional shape tuple, only to be specified
             if `include_top` is False (otherwise the input shape
             has to be `(224, 224, 3)` (with `channels_last` data format)
-            or `(3, 224, 244)` (with `channels_first` data format).
+            or `(3, 224, 224)` (with `channels_first` data format).
             It should have exactly 3 inputs channels,
             and width and height should be no smaller than 197.
             E.g. `(200, 200, 3)` would be one valid value.

keras/backend/__init__.py

@@ -12,20 +12,22 @@ from .common import image_data_format
 from .common import set_image_data_format
 from .common import is_keras_tensor
 
+# Obtain Keras base dir path: either ~/.keras or /tmp.
 _keras_base_dir = os.path.expanduser('~')
 if not os.access(_keras_base_dir, os.W_OK):
     _keras_base_dir = '/tmp'
-
 _keras_dir = os.path.join(_keras_base_dir, '.keras')
-if not os.path.exists(_keras_dir):
-    os.makedirs(_keras_dir)
 
 # Default backend: TensorFlow.
 _BACKEND = 'tensorflow'
 
+# Attempt to read Keras config file.
 _config_path = os.path.expanduser(os.path.join(_keras_dir, 'keras.json'))
 if os.path.exists(_config_path):
-    _config = json.load(open(_config_path))
+    try:
+        _config = json.load(open(_config_path))
+    except ValueError:
+        _config = {}
     _floatx = _config.get('floatx', floatx())
     assert _floatx in {'float16', 'float32', 'float64'}
     _epsilon = _config.get('epsilon', epsilon())
@@ -41,21 +43,28 @@ if os.path.exists(_config_path):
     set_image_data_format(_image_data_format)
     _BACKEND = _backend
 
-# save config file
-if not os.path.exists(_config_path):
-    _config = {'floatx': floatx(),
-               'epsilon': epsilon(),
-               'backend': _BACKEND,
-               'image_data_format': image_data_format()}
-    with open(_config_path, 'w') as f:
-        f.write(json.dumps(_config, indent=4))
+# Save config file, if possible.
+if os.access(_keras_base_dir, os.W_OK):
+    if not os.path.exists(_keras_dir):
+        try:
+            os.makedirs(_keras_dir)
+        except OSError:
+            pass
+    if not os.path.exists(_config_path):
+        _config = {'floatx': floatx(),
+                   'epsilon': epsilon(),
+                   'backend': _BACKEND,
+                   'image_data_format': image_data_format()}
+        with open(_config_path, 'w') as f:
+            f.write(json.dumps(_config, indent=4))
 
+# Set backend based on KERAS_BACKEND flag, if applicable.
 if 'KERAS_BACKEND' in os.environ:
     _backend = os.environ['KERAS_BACKEND']
     assert _backend in {'theano', 'tensorflow'}
     _BACKEND = _backend
 
-# import backend
+# Import backend functions.
 if _BACKEND == 'theano':
     sys.stderr.write('Using Theano backend.\n')
     from .theano_backend import *

keras/backend/common.py

@@ -44,7 +44,7 @@ def set_epsilon(e):
 
 
 def floatx():
-    """Returns the default float type, as a string
+    """Returns the default float type, as a string.
     (e.g. 'float16', 'float32', 'float64').
 
     # Returns
@@ -109,8 +109,7 @@ def cast_to_floatx(x):
 
 
 def image_data_format():
-    """Returns the default image data format
-    convention ('channels_first' or 'channels_last').
+    """Returns the default image data format convention ('channels_first' or 'channels_last').
 
     # Returns
         A string, either `'channels_first'` or `'channels_last'`
@@ -181,7 +180,7 @@ def set_image_dim_ordering(dim_ordering):
     """Legacy setter for `image_data_format`.
 
     # Arguments
-        dim_ordering: string. `'tf'` or `'th'`.
+        dim_ordering: string. `tf` or `th`.
 
     # Example
     ```python
@@ -192,6 +191,9 @@ def set_image_dim_ordering(dim_ordering):
         >>> K.image_data_format()
         'channels_last'
     ```
+
+    # Raises
+        ValueError if invalid `dim_ordering`
     """
     global _IMAGE_DATA_FORMAT
     if dim_ordering not in {'tf', 'th'}:
@@ -205,6 +207,9 @@ def set_image_dim_ordering(dim_ordering):
 
 def image_dim_ordering():
     """Legacy getter for `image_data_format`.
+
+    # Returns
+        string, one of `'th'`, `'tf'`
     """
     if _IMAGE_DATA_FORMAT == 'channels_first':
         return 'th'

keras/backend/tensorflow_backend.py

@@ -43,6 +43,14 @@ _MANUAL_VAR_INIT = False
 
 
 def get_uid(prefix=''):
+    """Get the uid for the default graph.
+
+    # Arguments
+        prefix: An optional prefix of the graph.
+
+    # Returns
+        A unique identifier for the graph.
+    """
     global _GRAPH_UID_DICTS
     graph = tf.get_default_graph()
     if graph not in _GRAPH_UID_DICTS:
@@ -52,6 +60,7 @@ def get_uid(prefix=''):
 
 
 def reset_uids():
+    """Reset graph identifiers."""
     global _GRAPH_UID_DICTS
     _GRAPH_UID_DICTS = {}
 
@@ -67,6 +76,7 @@ def clear_session():
     reset_uids()
     _SESSION = None
     phase = tf.placeholder(dtype='bool', name='keras_learning_phase')
+    _GRAPH_LEARNING_PHASES = {}
     _GRAPH_LEARNING_PHASES[tf.get_default_graph()] = phase
 
 
@@ -150,7 +160,8 @@ def get_session():
             _SESSION = tf.Session(config=config)
         session = _SESSION
     if not _MANUAL_VAR_INIT:
-        _initialize_variables()
+        with session.graph.as_default():
+            _initialize_variables()
     return session
 
 
@@ -167,6 +178,17 @@ def set_session(session):
 # VARIABLE MANIPULATION
 
 def _convert_string_dtype(dtype):
+    """Get the type from a string.
+
+    # Arguments
+        dtype: A string representation of a type.
+
+    # Returns:
+        The type requested.
+
+    # Raises
+        ValueError if `dtype` is not supported
+    """
     if dtype == 'float16':
         return tf.float16
     if dtype == 'float32':
@@ -188,6 +210,15 @@ def _convert_string_dtype(dtype):
 
 
 def _to_tensor(x, dtype):
+    """Convert the input `x` to a tensor of type `dtype`.
+
+    # Arguments
+        x: An object to be converted (numpy array, list, tensors).
+        dtype: The destination type.
+
+    # Returns
+        A tensor.
+    """
     x = tf.convert_to_tensor(x)
     if x.dtype != dtype:
         x = tf.cast(x, dtype)
@@ -307,6 +338,17 @@ def _initialize_variables():
 
 
 def constant(value, dtype=None, shape=None, name=None):
+    """Creates a constant tensor.
+
+    # Arguments
+        value: A constant value (or list)
+        dtype: The type of the elements of the resulting tensor.
+        shape: Optional dimensions of resulting tensor.
+        name: Optional name for the tensor.
+
+    # Returns
+        A Constant Tensor.
+    """
     if dtype is None:
         dtype = floatx()
     return tf.constant(value, dtype=dtype, shape=shape, name=name)
@@ -624,6 +666,18 @@ def ones_like(x, dtype=None, name=None):
     return tf.ones_like(x, dtype=dtype, name=name)
 
 
+def identity(x):
+    """Returns a tensor with the same content as the input tensor.
+
+    # Arguments
+        x: The input tensor.
+
+    # Returns
+        A tensor of the same shape, type and content.
+    """
+    return tf.identity(x)
+
+
 def random_uniform_variable(shape, low, high, dtype=None,
                             name=None, seed=None):
     """Instantiates a variable with values drawn from a uniform distribution.
@@ -759,18 +813,54 @@ def cast(x, dtype):
 
 
 def update(x, new_x):
+    """Update the value of `x` to `new_x`.
+
+    # Arguments
+        x: A Variable.
+        new_x: A tensor of same shape as `x`.
+
+    # Returns
+        The variable `x` updated.
+    """
     return tf.assign(x, new_x)
 
 
 def update_add(x, increment):
+    """Update the value of `x` by adding `increment`.
+
+        # Arguments
+            x: A Variable.
+            increment: A tensor of same shape as `x`.
+
+        # Returns
+            The variable `x` updated.
+        """
     return tf.assign_add(x, increment)
 
 
 def update_sub(x, decrement):
+    """Update the value of `x` by subtracting `decrement`.
+
+        # Arguments
+            x: A Variable.
+            decrement: A tensor of same shape as `x`.
+
+        # Returns
+            The variable `x` updated.
+        """
     return tf.assign_sub(x, decrement)
 
 
 def moving_average_update(x, value, momentum):
+    """Compute the moving average of a variable.
+
+    # Arguments
+        x: A Variable.
+        value: A tensor with the same shape as `variable`.
+        momentum: The moving average momentum.
+
+    # Returns
+        An Operation to update the variable."""
     return moving_averages.assign_moving_average(
         x, value, momentum, zero_debias=False)
 
@@ -900,6 +990,16 @@ def batch_dot(x, y, axes=None):
     """
     if isinstance(axes, int):
         axes = (axes, axes)
+    x_ndim = ndim(x)
+    y_ndim = ndim(y)
+    if x_ndim > y_ndim:
+        diff = x_ndim - y_ndim
+        y = tf.reshape(y, tf.concat([tf.shape(y), [1] * (diff)], axis=0))
+    elif y_ndim > x_ndim:
+        diff = y_ndim - x_ndim
+        x = tf.reshape(x, tf.concat([tf.shape(x), [1] * (diff)], axis=0))
+    else:
+        diff = 0
     if ndim(x) == 2 and ndim(y) == 2:
         if axes[0] == axes[1]:
             out = tf.reduce_sum(tf.multiply(x, y), axes[0])
@@ -913,6 +1013,12 @@ def batch_dot(x, y, axes=None):
             adj_x = None
             adj_y = None
         out = tf.matmul(x, y, adjoint_a=adj_x, adjoint_b=adj_y)
+    if diff:
+        if x_ndim > y_ndim:
+            idx = x_ndim + y_ndim - 3
+        else:
+            idx = x_ndim - 1
+        out = tf.squeeze(out, list(range(idx, idx + diff)))
     if ndim(out) == 1:
         out = expand_dims(out, 1)
     return out
@@ -1062,6 +1168,34 @@ def prod(x, axis=None, keepdims=False):
     return tf.reduce_prod(x, reduction_indices=axis, keep_dims=keepdims)
 
 
+def cumsum(x, axis=0):
+    """Cumulative sum of the values in a tensor, alongside the specified axis.
+
+    # Arguments
+        x: A tensor or variable.
+        axis: An integer, the axis to compute the sum.
+
+    # Returns
+        A tensor of the cumulative sum of values of `x` along `axis`.
+    """
+    axis = _normalize_axis(axis, ndim(x))
+    return tf.cumsum(x, axis=axis)
+
+
+def cumprod(x, axis=0):
+    """Cumulative product of the values in a tensor, alongside the specified axis.
+
+    # Arguments
+        x: A tensor or variable.
+        axis: An integer, the axis to compute the product.
+
+    # Returns
+        A tensor of the cumulative product of values of `x` along `axis`.
+    """
+    axis = _normalize_axis(axis, ndim(x))
+    return tf.cumprod(x, axis=axis)
+
+
 def var(x, axis=None, keepdims=False):
     """Variance of a tensor, alongside the specified axis.
 
@@ -1136,8 +1270,7 @@ def any(x, axis=None, keepdims=False):
     """
     axis = _normalize_axis(axis, ndim(x))
     x = tf.cast(x, tf.bool)
-    x = tf.reduce_any(x, reduction_indices=axis, keep_dims=keepdims)
-    return tf.cast(x, tf.uint8)
+    return tf.reduce_any(x, reduction_indices=axis, keep_dims=keepdims)
 
 
 def all(x, axis=None, keepdims=False):
@@ -1153,8 +1286,7 @@ def all(x, axis=None, keepdims=False):
     """
     axis = _normalize_axis(axis, ndim(x))
     x = tf.cast(x, tf.bool)
-    x = tf.reduce_all(x, reduction_indices=axis, keep_dims=keepdims)
-    return tf.cast(x, tf.uint8)
+    return tf.reduce_all(x, reduction_indices=axis, keep_dims=keepdims)
 
 
 def argmax(x, axis=-1):
@@ -1248,6 +1380,28 @@ def log(x):
     return tf.log(x)
 
 
+def logsumexp(x, axis=None, keepdims=False):
+    """Computes log(sum(exp(elements across dimensions of a tensor))).
+
+    This function is more numerically stable than log(sum(exp(x))).
+    It avoids overflows caused by taking the exp of large inputs and
+    underflows caused by taking the log of small inputs.
+
+    # Arguments
+        x: A tensor or variable.
+        axis: An integer, the axis to reduce over.
+        keepdims: A boolean, whether to keep the dimensions or not.
+            If `keepdims` is `False`, the rank of the tensor is reduced
+            by 1. If `keepdims` is `True`, the reduced dimension is
+            retained with length 1.
+
+    # Returns
+        The reduced tensor.
+    """
+    axis = _normalize_axis(axis, ndim(x))
+    return tf.reduce_logsumexp(x, reduction_indices=axis, keep_dims=keepdims)
+
+
 def round(x):
     """Element-wise rounding to the closest integer.
 
@@ -1452,7 +1606,7 @@ def normalize_batch_in_training(x, gamma, beta,
     """
     mean, var = tf.nn.moments(x, reduction_axes,
                               shift=None, name=None, keep_dims=False)
-    if sorted(reduction_axes) == range(ndim(x))[:-1]:
+    if sorted(reduction_axes) == list(range(ndim(x)))[:-1]:
         normed = tf.nn.batch_normalization(x, mean, var,
                                            beta, gamma,
                                            epsilon)
@@ -2148,8 +2302,8 @@ def rnn(step_function, inputs, initial_states,
             (no time dimension),
             containing the initial values for the states used in
             the step function.
-        go_backwards: boolean. If True, do the iteration over
-            the time dimension in reverse order.
+        go_backwards: boolean. If True, do the iteration over the time
+            dimension in reverse order and return the reversed sequence.
         mask: binary tensor with shape `(samples, time, 1)`,
             with a zero for every element that is masked.
         constants: a list of constant values passed at each step.
@@ -2240,9 +2394,9 @@ def rnn(step_function, inputs, initial_states,
                 states = return_states
                 successive_outputs.append(output)
                 successive_states.append(states)
-                last_output = successive_outputs[-1]
-                new_states = successive_states[-1]
-                outputs = tf.stack(successive_outputs)
+            last_output = successive_outputs[-1]
+            new_states = successive_states[-1]
+            outputs = tf.stack(successive_outputs)
         else:
             for inp in input_list:
                 output, states = step_function(inp, states + constants)
@@ -2702,13 +2856,14 @@ def in_top_k(predictions, targets, k):
     """Returns whether the `targets` are in the top `k` `predictions`.
 
     # Arguments
-        predictions: A tensor of shape `batch_size` x classes and type `float32`.
-        targets: A tensor of shape batch_size and type `int32` or `int64`.
+        predictions: A tensor of shape `(batch_size, classes)` and type `float32`.
+        targets: A 1D tensor of length `batch_size` and type `int32` or `int64`.
         k: An `int`, number of top elements to consider.
 
     # Returns
-        A tensor of shape `batch_size` and type `bool`. `output_i` is `True` if
-        `targets_i` is within top-k values of `predictions_i`
+        A 1D tensor of length `batch_size` and type `bool`.
+        `output[i]` is `True` if `predictions[i, targets[i]]` is within top-`k`
+        values of `predictions[i]`.
     """
     return tf.nn.in_top_k(predictions, targets, k)
 
@@ -2716,6 +2871,16 @@ def in_top_k(predictions, targets, k):
 # CONVOLUTIONS
 
 def _preprocess_deconv_output_shape(x, shape, data_format):
+    """Get the output_shape for the deconvolution.
+
+    # Arguments
+        x: input tensor.
+        shape: output shape.
+        data_format: string, one of 'channels_last', 'channels_first'.
+
+    # Returns
+        The output shape.
+    """
     if data_format == 'channels_first':
         shape = (shape[0], shape[2], shape[3], shape[1])
 
@@ -2726,6 +2891,15 @@ def _preprocess_deconv_output_shape(x, shape, data_format):
 
 
 def _preprocess_conv2d_input(x, data_format):
+    """Transpose and cast the input before the conv2d.
+
+    # Arguments
+        x: input tensor.
+        data_format: string, one of 'channels_last', 'channels_first'.
+
+    # Returns
+        A tensor.
+    """
     if dtype(x) == 'float64':
         x = tf.cast(x, 'float32')
     if data_format == 'channels_first':
@@ -2738,6 +2912,15 @@ def _preprocess_conv2d_input(x, data_format):
 
 
 def _preprocess_conv3d_input(x, data_format):
+    """Transpose and cast the input before the conv3d.
+
+    # Arguments
+        x: input tensor.
+        data_format: string, one of 'channels_last', 'channels_first'.
+
+    # Returns
+        A tensor.
+    """
     if dtype(x) == 'float64':
         x = tf.cast(x, 'float32')
     if data_format == 'channels_first':
@@ -2746,6 +2929,15 @@ def _preprocess_conv3d_input(x, data_format):
 
 
 def _preprocess_conv2d_kernel(kernel, data_format):
+    """Transpose and cast the kernel before the conv2d.
+
+    # Arguments
+        kernel: kernel tensor.
+        data_format: string, one of 'channels_last', 'channels_first'.
+
+    # Returns
+        A tensor.
+    """
     if dtype(kernel) == 'float64':
         kernel = tf.cast(kernel, 'float32')
     if data_format == 'channels_first':
@@ -2754,6 +2946,15 @@ def _preprocess_conv2d_kernel(kernel, data_format):
 
 
 def _preprocess_conv3d_kernel(kernel, data_format):
+    """Transpose and cast the kernel before the conv3d.
+
+    # Arguments
+        kernel: kernel tensor.
+        data_format: string, one of 'channels_last', 'channels_first'.
+
+    # Returns
+        A tensor.
+    """
     if dtype(kernel) == 'float64':
         kernel = tf.cast(kernel, 'float32')
     if data_format == 'channels_first':
@@ -2762,16 +2963,37 @@ def _preprocess_conv3d_kernel(kernel, data_format):
 
 
 def _preprocess_padding(padding):
+    """Convert keras' padding to tensorflow's padding.
+
+    # Arguments
+        padding: string, one of 'same' , 'valid'
+
+    # Returns
+        a string, one of 'SAME', 'VALID'.
+
+    # Raises
+        ValueError if invalid `padding'`
+    """
     if padding == 'same':
         padding = 'SAME'
     elif padding == 'valid':
         padding = 'VALID'
     else:
-        raise ValueError('Invalid border mode:', padding)
+        raise ValueError('Invalid padding:', padding)
     return padding
 
 
 def _postprocess_conv2d_output(x, data_format):
+    """Transpose and cast the output from conv2d if needed.
+
+    # Arguments
+        x: A tensor.
+        data_format: string, one of "channels_last", "channels_first".
+
+    # Returns
+        A tensor.
+    """
+
     if data_format == 'channels_first':
         x = tf.transpose(x, (0, 3, 1, 2))
 
@@ -2781,6 +3003,15 @@ def _postprocess_conv2d_output(x, data_format):
 
 
 def _postprocess_conv3d_output(x, data_format):
+    """Transpose and cast the output from conv3d if needed.
+
+        # Arguments
+            x: A tensor.
+            data_format: string, one of "channels_last", "channels_first".
+
+        # Returns
+            A tensor.
+        """
     if data_format == 'channels_first':
         x = tf.transpose(x, (0, 4, 1, 2, 3))
 
@@ -3315,19 +3546,19 @@ def ctc_decode(y_pred, input_length, greedy=True, beam_width=100,
 
 # HIGH ORDER FUNCTIONS
 
-def map_fn(fn, elems, name=None):
+def map_fn(fn, elems, name=None, dtype=None):
     """Map the function fn over the elements elems and return the outputs.
 
     # Arguments
         fn: Callable that will be called upon each element in elems
         elems: tensor
         name: A string name for the map node in the graph
+        dtype: Output data type.
 
     # Returns
-        Tensor with first dimension equal to the elems and second depending on
-        fn
+        Tensor with dtype `dtype`.
     """
-    return tf.map_fn(fn, elems, name=name)
+    return tf.map_fn(fn, elems, name=name, dtype=dtype)
 
 
 def foldl(fn, elems, initializer=None, name=None):
@@ -3341,7 +3572,7 @@ def foldl(fn, elems, initializer=None, name=None):
         name: A string name for the foldl node in the graph
 
     # Returns
-        Same type and shape as initializer
+        Tensor with same type and shape as `initializer`.
     """
     return tf.foldl(fn, elems, initializer=initializer, name=name)
 

keras/backend/theano_backend.py

@@ -1,6 +1,7 @@
 from collections import defaultdict
 from contextlib import contextmanager
 import theano
+from theano import ifelse
 from theano import tensor as T
 from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
 from theano.tensor.signal import pool
@@ -258,6 +259,18 @@ def zeros_like(x, dtype=None, name=None):
     return T.zeros_like(x, dtype=dtype)
 
 
+def identity(x):
+    """Returns a tensor with the same content as the input tensor.
+
+    # Arguments
+        x: The input tensor.
+
+    # Returns
+        A tensor of the same shape, type and content.
+    """
+    return x.copy()
+
+
 def random_uniform_variable(shape, low, high, dtype=None, name=None):
     return variable(np.random.uniform(low=low, high=high, size=shape),
                     dtype=dtype, name=name)
@@ -403,8 +416,7 @@ def gather(reference, indices):
     """
     y = reference[indices]
     if hasattr(reference, '_keras_shape') and hasattr(indices, '_keras_shape'):
-        l = indices._keras_shape[0]
-        y._keras_shape = (l,) + reference._keras_shape[1:]
+        y._keras_shape = indices._keras_shape + reference._keras_shape[1:]
     return y
 
 
@@ -431,6 +443,32 @@ def prod(x, axis=None, keepdims=False):
     return T.prod(x, axis=axis, keepdims=keepdims)
 
 
+def cumsum(x, axis=0):
+    """Cumulative sum of the values in a tensor, alongside the specified axis.
+
+    # Arguments
+        x: A tensor or variable.
+        axis: An integer, the axis to compute the sum.
+
+    # Returns
+        A tensor of the cumulative sum of values of `x` along `axis`.
+    """
+    return T.extra_ops.cumsum(x, axis=axis)
+
+
+def cumprod(x, axis=0):
+    """Cumulative product of the values in a tensor, alongside the specified axis.
+
+    # Arguments
+        x: A tensor or variable.
+        axis: An integer, the axis to compute the product.
+
+    # Returns
+        A tensor of the cumulative product of values of `x` along `axis`.
+    """
+    return T.extra_ops.cumprod(x, axis=axis)
+
+
 def mean(x, axis=None, keepdims=False):
     """Mean of a tensor, alongside the specified axis.
     """
@@ -490,6 +528,29 @@ def log(x):
     return T.log(x)
 
 
+def logsumexp(x, axis=None, keepdims=False):
+    """Computes log(sum(exp(elements across dimensions of a tensor))).
+
+    This function is more numerically stable than log(sum(exp(x))).
+    It avoids overflows caused by taking the exp of large inputs and
+    underflows caused by taking the log of small inputs.
+
+    # Arguments
+        x: A tensor or variable.
+        axis: An integer, the axis to reduce over.
+        keepdims: A boolean, whether to keep the dimensions or not.
+            If `keepdims` is `False`, the rank of the tensor is reduced
+            by 1. If `keepdims` is `True`, the reduced dimension is
+            retained with length 1.
+
+    # Returns
+        The reduced tensor.
+    """
+    # Theano has a built-in optimization for logsumexp (see https://github.com/Theano/Theano/pull/4736)
+    # so we can just write the expression directly:
+    return T.log(T.sum(T.exp(x), axis=axis, keepdims=keepdims))
+
+
 def round(x):
     return T.round(x, mode='half_to_even')
 
@@ -590,7 +651,7 @@ def batch_normalization(x, mean, var, beta, gamma, epsilon=1e-3):
 
     if mean.ndim == 1:
         # based on TensorFlow's default: normalize along rightmost dimension
-        reduction_axes = range(x.ndim - 1)
+        reduction_axes = list(range(x.ndim - 1))
     else:
         reduction_axes = [i for i in range(x.ndim) if mean.broadcastable[i]]
 
@@ -713,6 +774,8 @@ def concatenate(tensors, axis=-1):
 def reshape(x, shape):
     y = T.reshape(x, shape)
     if _is_explicit_shape(shape):
+        if -1 in shape:
+            shape = tuple(x if x != -1 else None for x in shape)
         y._keras_shape = shape
         if hasattr(x, '_uses_learning_phase'):
             y._uses_learning_phase = x._uses_learning_phase
@@ -743,7 +806,9 @@ def repeat_elements(x, rep, axis):
     y = T.repeat(x, rep, axis=axis)
     if hasattr(x, '_keras_shape'):
         y._keras_shape = list(x._keras_shape)
-        y._keras_shape[axis] = x._keras_shape[axis] * rep
+        repeat_dim = x._keras_shape[axis]
+        if repeat_dim is not None:
+                y._keras_shape[axis] = repeat_dim * rep
         y._keras_shape = tuple(y._keras_shape)
     return y
 
@@ -821,22 +886,41 @@ def arange(start, stop=None, step=1, dtype='int32'):
 def tile(x, n):
     y = T.tile(x, n)
     if hasattr(x, '_keras_shape'):
-        xshape = np.asarray(x._keras_shape)
-        n = np.asarray(n)
-        diff = len(xshape) - len(n)
-        if diff > 0:
-            n = np.append([1] * diff, n)
+        if _is_explicit_shape(n):
+            output_shape = x._keras_shape[:-len(n)]
+            for i, j in zip(x._keras_shape, n):
+                if i is None:
+                    output_shape += (None,)
+                else:
+                    output_shape += (i * j,)
+        elif type(n) is int:
+            output_shape = x._keras_shape[:-1]
+            if x._keras_shape[-1] is None:
+                output_shape += (None,)
+            else:
+                output_shape += (x._keras_shape[-1] * n,)
         else:
-            xshape = np.append([1] * -diff, xshape)
-        y._keras_shape = tuple(xshape * n)
-
+            # symbolic n
+            if n.ndim == 0:
+                # n is a scalar
+                output_shape = x._keras_shape[:-1] + (None,)
+            elif hasattr(n, '_keras_shape'):
+                # n is a vector
+                n_size = n._keras_shape[0]
+                output_shape = x._keras_shape[:-n_size] + (None,) * n_size
+            else:
+                output_shape = (None,) * x.ndim
+        y._keras_shape = output_shape
     return y
 
 
 def flatten(x):
     y = T.flatten(x)
     if hasattr(x, '_keras_shape'):
-        y._keras_shape = (np.prod(x._keras_shape), )
+        if None in x._keras_shape:
+            y._keras_shape = (None,)
+        else:
+            y._keras_shape = (np.prod(x._keras_shape), )
     return y
 
 
@@ -846,7 +930,10 @@ def batch_flatten(x):
     """
     y = T.reshape(x, (x.shape[0], T.prod(x.shape[1:])))
     if hasattr(x, '_keras_shape'):
-        y._keras_shape = (x._keras_shape[0], np.prod(x._keras_shape[1:]))
+        if None in x._keras_shape[1:]:
+            y._keras_shape = (x._keras_shape[0], None)
+        else:
+            y._keras_shape = (x._keras_shape[0], np.prod(x._keras_shape[1:]))
     return y
 
 
@@ -1115,8 +1202,8 @@ def rnn(step_function, inputs, initial_states,
         initial_states: tensor with shape (samples, ...) (no time dimension),
             containing the initial values for the states used in
             the step function.
-        go_backwards: boolean. If True, do the iteration over
-            the time dimension in reverse order.
+        go_backwards: boolean. If True, do the iteration over the time
+            dimension in reverse order and return the reversed sequence.
         mask: binary tensor with shape (samples, time),
             with a zero for every element that is masked.
         constants: a list of constant values passed at each step.
@@ -1443,20 +1530,35 @@ def l2_normalize(x, axis):
 
 
 def in_top_k(predictions, targets, k):
-    """Returns whether the `targets` are in the top `k` `predictions`
+    """Returns whether the `targets` are in the top `k` `predictions`.
 
     # Arguments
-        predictions: A tensor of shape batch_size x classess and type float32.
-        targets: A tensor of shape batch_size and type int32 or int64.
-        k: An int, number of top elements to consider.
+        predictions: A tensor of shape `(batch_size, classes)` and type `float32`.
+        targets: A 1D tensor of length `batch_size` and type `int32` or `int64`.
+        k: An `int`, number of top elements to consider.
 
     # Returns
-        A tensor of shape batch_size and type int. output_i is 1 if
-        targets_i is within top-k values of predictions_i
+        A 1D tensor of length `batch_size` and type `bool`.
+        `output[i]` is `True` if `predictions[i, targets[i]]` is within top-`k`
+        values of `predictions[i]`.
     """
-    predictions_top_k = T.argsort(predictions)[:, -k:]
-    result, _ = theano.map(lambda prediction, target: any(equal(prediction, target)), sequences=[predictions_top_k, targets])
-    return result
+    # handle k < 1 and k >= predictions.shape[1] cases to match TF behavior
+    if k < 1:
+        # dtype='bool' is only available since Theano 0.9.0
+        try:
+            return T.zeros_like(targets, dtype='bool')
+        except TypeError:
+            return T.zeros_like(targets, dtype='int8')
+
+    if k >= int_shape(predictions)[1]:
+        try:
+            return T.ones_like(targets, dtype='bool')
+        except TypeError:
+            return T.ones_like(targets, dtype='int8')
+
+    predictions_k = T.sort(predictions)[:, -k]
+    targets_values = predictions[T.arange(targets.shape[0]), targets]
+    return T.ge(targets_values, predictions_k)
 
 
 # CONVOLUTIONS
@@ -1832,10 +1934,14 @@ def pool2d(x, pool_size, strides=(1, 1), padding='valid',
                                 pad=pad,
                                 mode='max')
     elif pool_mode == 'avg':
+        if padding == 'same':
+            th_avg_pool_mode = 'average_inc_pad'
+        elif padding == 'valid':
+            th_avg_pool_mode = 'average_exc_pad'
         pool_out = pool.pool_2d(x, ws=pool_size, stride=strides,
                                 ignore_border=True,
                                 pad=pad,
-                                mode='average_exc_pad')
+                                mode=th_avg_pool_mode)
     else:
         raise ValueError('Invalid pooling mode:', pool_mode)
     if padding == 'same':
@@ -2076,7 +2182,7 @@ def ctc_batch_cost(y_true, y_pred, input_length, label_length):
 
 # HIGH ORDER FUNCTIONS
 
-def map_fn(fn, elems, name=None):
+def map_fn(fn, elems, name=None, dtype=None):
     """Map the function fn over the elements elems and return the outputs.
 
     # Arguments

keras/callbacks.py

@@ -3,6 +3,7 @@ from __future__ import print_function
 
 import os
 import csv
+import six
 
 import numpy as np
 import time
@@ -22,6 +23,7 @@ except ImportError:
 
 if K.backend() == 'tensorflow':
     import tensorflow as tf
+    from tensorflow.contrib.tensorboard.plugins import projector
 
 
 class CallbackList(object):
@@ -502,8 +504,7 @@ class RemoteMonitor(Callback):
         field: String; JSON field under which the data will be stored.
         headers: Dictionary; optional custom HTTP headers.
             Defaults to:
-            `{'Accept': 'application/json',
-              'Content-Type': 'application/json'}`
+            `{'Accept': 'application/json', 'Content-Type': 'application/json'}`
     """
 
     def __init__(self,
@@ -577,7 +578,7 @@ class TensorBoard(Callback):
     tensorboard --logdir=/full_path_to_your_logs
     ```
     You can find more information about TensorBoard
-    [here](https://www.tensorflow.org/versions/master/how_tos/summaries_and_tensorboard/index.html).
+    [here](https://www.tensorflow.org/get_started/summaries_and_tensorboard).
 
     # Arguments
         log_dir: the path of the directory where to save the log
@@ -590,12 +591,24 @@ class TensorBoard(Callback):
             write_graph is set to True.
         write_images: whether to write model weights to visualize as
             image in Tensorboard.
+        embeddings_freq: frequency (in epochs) at which selected embedding
+            layers will be saved.
+        embeddings_layer_names: a list of names of layers to keep eye on. If
+            None or empty list all the embedding layer will be watched.
+        embeddings_metadata: a dictionary which maps layer name to a file name
+            in which metadata for this embedding layer is saved. See the
+            [details](https://www.tensorflow.org/how_tos/embedding_viz/#metadata_optional)
+            about metadata files format. In case if the same metadata file is
+            used for all embedding layers, string can be passed.
     """
 
     def __init__(self, log_dir='./logs',
                  histogram_freq=0,
                  write_graph=True,
-                 write_images=False):
+                 write_images=False,
+                 embeddings_freq=0,
+                 embeddings_layer_names=None,
+                 embeddings_metadata=None):
         super(TensorBoard, self).__init__()
         if K.backend() != 'tensorflow':
             raise RuntimeError('TensorBoard callback only works '
@@ -605,6 +618,9 @@ class TensorBoard(Callback):
         self.merged = None
         self.write_graph = write_graph
         self.write_images = write_images
+        self.embeddings_freq = embeddings_freq
+        self.embeddings_layer_names = embeddings_layer_names
+        self.embeddings_metadata = embeddings_metadata or {}
 
     def set_model(self, model):
         self.model = model
@@ -635,6 +651,42 @@ class TensorBoard(Callback):
         else:
             self.writer = tf.summary.FileWriter(self.log_dir)
 
+        if self.embeddings_freq:
+            self.saver = tf.train.Saver()
+
+            embeddings_layer_names = self.embeddings_layer_names
+
+            if not embeddings_layer_names:
+                embeddings_layer_names = [layer.name for layer in self.model.layers
+                                          if type(layer).__name__ == 'Embedding']
+
+            embeddings = {layer.name: layer.weights[0]
+                          for layer in self.model.layers
+                          if layer.name in embeddings_layer_names}
+
+            embeddings_metadata = {}
+
+            if not isinstance(self.embeddings_metadata, str):
+                embeddings_metadata = self.embeddings_metadata
+            else:
+                embeddings_metadata = {layer_name: self.embeddings_metadata
+                                       for layer_name in embeddings.keys()}
+
+            config = projector.ProjectorConfig()
+            self.embeddings_logs = []
+
+            for layer_name, tensor in embeddings.items():
+                embedding = config.embeddings.add()
+                embedding.tensor_name = tensor.name
+
+                self.embeddings_logs.append(os.path.join(self.log_dir,
+                                                         layer_name + '.ckpt'))
+
+                if layer_name in embeddings_metadata:
+                    embedding.metadata_path = embeddings_metadata[layer_name]
+
+            projector.visualize_embeddings(self.writer, config)
+
     def on_epoch_end(self, epoch, logs=None):
         logs = logs or {}
 
@@ -654,6 +706,11 @@ class TensorBoard(Callback):
                 summary_str = result[0]
                 self.writer.add_summary(summary_str, epoch)
 
+        if self.embeddings_freq and self.embeddings_logs:
+            if epoch % self.embeddings_freq == 0:
+                for log in self.embeddings_logs:
+                    self.saver.save(self.sess, log, epoch)
+
         for name, value in logs.items():
             if name in ['batch', 'size']:
                 continue
@@ -678,9 +735,9 @@ class ReduceLROnPlateau(Callback):
 
     # Example
         ```python
-            reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
-                                          patience=5, min_lr=0.001)
-            model.fit(X_train, Y_train, callbacks=[reduce_lr])
+        reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
+                                      patience=5, min_lr=0.001)
+        model.fit(X_train, Y_train, callbacks=[reduce_lr])
         ```
 
     # Arguments
@@ -787,8 +844,8 @@ class CSVLogger(Callback):
 
     # Example
         ```python
-            csv_logger = CSVLogger('training.log')
-            model.fit(X_train, Y_train, callbacks=[csv_logger])
+        csv_logger = CSVLogger('training.log')
+        model.fit(X_train, Y_train, callbacks=[csv_logger])
         ```
 
     # Arguments
@@ -805,16 +862,17 @@ class CSVLogger(Callback):
         self.writer = None
         self.keys = None
         self.append_header = True
+        self.file_flags = 'b' if six.PY2 and os.name == 'nt' else ''
         super(CSVLogger, self).__init__()
 
     def on_train_begin(self, logs=None):
         if self.append:
             if os.path.exists(self.filename):
-                with open(self.filename) as f:
+                with open(self.filename, 'r' + self.file_flags) as f:
                     self.append_header = not bool(len(f.readline()))
-            self.csv_file = open(self.filename, 'a')
+            self.csv_file = open(self.filename, 'a' + self.file_flags)
         else:
-            self.csv_file = open(self.filename, 'w')
+            self.csv_file = open(self.filename, 'w' + self.file_flags)
 
     def on_epoch_end(self, epoch, logs=None):
         logs = logs or {}
@@ -853,6 +911,7 @@ class LambdaCallback(Callback):
     This callback is constructed with anonymous functions that will be called
     at the appropriate time. Note that the callbacks expects positional
     arguments, as:
+
      - `on_epoch_begin` and `on_epoch_end` expect two positional arguments:
         `epoch`, `logs`
      - `on_batch_begin` and `on_batch_end` expect two positional arguments:

keras/datasets/imdb.py

@@ -83,10 +83,10 @@ def load_data(path='imdb.npz', num_words=None, skip_top=0,
                 new_labels.append(y)
         xs = new_xs
         labels = new_labels
-    if not xs:
-        raise ValueError('After filtering for sequences shorter than maxlen=' +
-                         str(maxlen) + ', no sequence was kept. '
-                         'Increase maxlen.')
+        if not xs:
+            raise ValueError('After filtering for sequences shorter than maxlen=' +
+                             str(maxlen) + ', no sequence was kept. '
+                             'Increase maxlen.')
     if not num_words:
         num_words = max([max(x) for x in xs])
 
@@ -100,7 +100,7 @@ def load_data(path='imdb.npz', num_words=None, skip_top=0,
         for x in xs:
             nx = []
             for w in x:
-                if w >= num_words or w < skip_top:
+                if skip_top <= w < num_words:
                     nx.append(w)
             new_xs.append(nx)
         xs = new_xs

keras/datasets/reuters.py

@@ -84,7 +84,7 @@ def load_data(path='reuters.npz', num_words=None, skip_top=0,
         for x in xs:
             nx = []
             for w in x:
-                if w >= num_words or w < skip_top:
+                if skip_top <= w < num_words:
                     nx.append(w)
             new_xs.append(nx)
         xs = new_xs

keras/engine/topology.py

@@ -252,7 +252,11 @@ class Layer(object):
         self._trainable_weights = []
         self._non_trainable_weights = []
         self._constraints = {}  # dict {tensor: constraint instance}
-        self.built = False
+        self._losses = []
+        self._updates = []
+        self._per_input_losses = {}
+        self._per_input_updates = {}
+        self._built = False
 
         # These lists will be filled via successive calls
         # to self._add_inbound_node().
@@ -308,6 +312,22 @@ class Layer(object):
         else:
             self._initial_weights = None
 
+    @property
+    def losses(self):
+        return self._losses
+
+    @property
+    def updates(self):
+        return self._updates
+
+    @property
+    def built(self):
+        return self._built
+
+    @built.setter
+    def built(self, value):
+        self._built = value
+
     @property
     def constraints(self):
         return self._constraints
@@ -340,28 +360,35 @@ class Layer(object):
     def non_trainable_weights(self, weights):
         self._non_trainable_weights = weights
 
-    def add_weight(self, shape, initializer,
-                   name=None,
-                   trainable=True,
+    @interfaces.legacy_add_weight_support
+    def add_weight(self,
+                   name,
+                   shape,
+                   dtype=None,
+                   initializer=None,
                    regularizer=None,
+                   trainable=True,
                    constraint=None):
         """Adds a weight variable to the layer.
 
         # Arguments
-            shape: The shape tuple of the weight.
-            initializer: An Initializer instance (callable).
             name: String, the name for the weight variable.
+            shape: The shape tuple of the weight.
+            dtype: The dtype of the weight.
+            initializer: An Initializer instance (callable).
+            regularizer: An optional Regularizer instance.
             trainable: A boolean, whether the weight should
                 be trained via backprop or not (assuming
                 that the layer itself is also trainable).
-            regularizer: An optional Regularizer instance.
             constraint: An optional Constraint instance.
 
         # Returns
             The created weight variable.
         """
         initializer = initializers.get(initializer)
-        weight = K.variable(initializer(shape), dtype=K.floatx(), name=name)
+        if dtype is None:
+            dtype = K.floatx()
+        weight = K.variable(initializer(shape), dtype=dtype, name=name)
         if regularizer is not None:
             self.add_loss(regularizer(weight))
         if constraint is not None:
@@ -398,7 +425,7 @@ class Layer(object):
                              str(len(input_spec)) + ' inputs, '
                              'but it received ' + str(len(inputs)) +
                              ' input tensors. Input received: ' +
-                             str(input))
+                             str(inputs))
         for input_index, (x, spec) in enumerate(zip(inputs, input_spec)):
             if spec is None:
                 continue
@@ -467,11 +494,12 @@ class Layer(object):
                                     str(spec.shape) + ', found shape=' +
                                     str(x_shape))
 
-    def call(self, inputs):
+    def call(self, inputs, **kwargs):
         """This is where the layer's logic lives.
 
         # Arguments
-            inputs: input tensor, or list/tuple of input tensors.
+            inputs: Input tensor, or list/tuple of input tensors.
+            **kwargs: Additional keyword arguments.
 
         # Returns
             A tensor or list/tuple of tensors.
@@ -486,7 +514,7 @@ class Layer(object):
             - If necessary, we `build` the layer to match
                 the _keras_shape of the input(s).
             - We update the _keras_shape of every input tensor with
-                its new shape (obtained via self.get_output_shape_for).
+                its new shape (obtained via self.compute_output_shape).
                 This is done as part of _add_inbound_node().
             - We update the _keras_history of the output tensor(s)
                 with the current layer.
@@ -503,6 +531,8 @@ class Layer(object):
             ValueError: in case the layer is missing shape information
                 for its `build` call.
         """
+        if isinstance(inputs, list):
+            inputs = inputs[:]
         with K.name_scope(self.name):
             # Handle laying building (weight creating, input spec locking).
             if not self.built:
@@ -540,6 +570,7 @@ class Layer(object):
 
             # Handle mask propagation.
             previous_mask = _collect_previous_mask(inputs)
+            user_kwargs = copy.copy(kwargs)
             if not _is_all_none(previous_mask):
                 # The previous layer generated a mask.
                 if 'mask' in inspect.getargspec(self.call).args:
@@ -554,6 +585,20 @@ class Layer(object):
             output = self.call(inputs, **kwargs)
             output_mask = self.compute_mask(inputs, previous_mask)
 
+            # If the layer returns tensors from its inputs, unmodified,
+            # we copy them to avoid loss of tensor metadata.
+            output_ls = _to_list(output)
+            inputs_ls = _to_list(inputs)
+            output_ls_copy = []
+            for x in output_ls:
+                if x in inputs_ls:
+                    x = K.identity(x)
+                output_ls_copy.append(x)
+            if len(output_ls_copy) == 1:
+                output = output_ls_copy[0]
+            else:
+                output = output_ls_copy
+
             # Infering the output shape is only relevant for Theano.
             if all([s is not None for s in _to_list(input_shape)]):
                 output_shape = self.compute_output_shape(input_shape)
@@ -571,7 +616,7 @@ class Layer(object):
             self._add_inbound_node(input_tensors=inputs, output_tensors=output,
                                    input_masks=previous_mask, output_masks=output_mask,
                                    input_shapes=input_shape, output_shapes=output_shape,
-                                   arguments=kwargs)
+                                   arguments=user_kwargs)
 
             # Apply activity regularizer if any:
             if hasattr(self, 'activity_regularizer') and self.activity_regularizer is not None:
@@ -656,6 +701,10 @@ class Layer(object):
         # Returns
             An input shape tuple.
         """
+        if hasattr(self, 'get_output_shape_for'):
+            msg = "Class `{}.{}` defines `get_output_shape_for` but does not override `compute_output_shape`. " + \
+                  "If this is a Keras 1 layer, please implement `compute_output_shape` to support Keras 2."
+            warnings.warn(msg.format(type(self).__module__, type(self).__name__), stacklevel=2)
         return input_shape
 
     def compute_mask(self, inputs, mask=None):
@@ -1021,23 +1070,15 @@ class Layer(object):
                 (e.g. L2 weight regularization, which only depends
                 on the layer's weights variables, not on any inputs tensors).
         """
-        if losses is None:
+        if losses is None or losses == []:
             return
         # Update self.losses
         losses = _to_list(losses)
-        if not hasattr(self, 'losses'):
-            self.losses = []
-        try:
-            self.losses += losses
-        except AttributeError:
-            # In case self.losses isn't settable
-            # (i.e. it's a getter method).
-            # In that case the `losses` property is
-            # auto-computed and shouldn't be set.
-            pass
+        if hasattr(self, '_losses'):
+            self._losses += losses
         # Update self._per_input_updates
-        if not hasattr(self, '_per_input_losses'):
-            self._per_input_losses = {}
+        if inputs == []:
+            inputs = None
         if inputs is not None:
             inputs_hash = _object_list_uid(inputs)
         else:
@@ -1061,23 +1102,15 @@ class Layer(object):
                 the updates as conditional on these inputs.
                 If None is passed, the updates are assumed unconditional.
         """
-        if updates is None:
+        if updates is None or updates == []:
             return
         # Update self.updates
         updates = _to_list(updates)
-        if not hasattr(self, 'updates'):
-            self.updates = []
-        try:
-            self.updates += updates
-        except AttributeError:
-            # In case self.updates isn't settable
-            # (i.e. it's a getter method).
-            # In that case the `updates` property is
-            # auto-computed and shouldn't be set.
-            pass
+        if hasattr(self, '_updates'):
+            self._updates += updates
         # Update self._per_input_updates
-        if not hasattr(self, '_per_input_updates'):
-            self._per_input_updates = {}
+        if inputs == []:
+            inputs = None
         if inputs is not None:
             inputs_hash = _object_list_uid(inputs)
         else:
@@ -1089,8 +1122,6 @@ class Layer(object):
         self._per_input_updates[inputs_hash] += updates
 
     def get_updates_for(self, inputs):
-        if not hasattr(self, '_per_input_updates'):
-            return []
         if inputs is not None:
             inputs_hash = _object_list_uid(inputs)
         else:
@@ -1100,8 +1131,6 @@ class Layer(object):
         return []
 
     def get_losses_for(self, inputs):
-        if not hasattr(self, '_per_input_losses'):
-            return []
         if inputs is not None:
             inputs_hash = _object_list_uid(inputs)
         else:
@@ -1241,6 +1270,7 @@ class InputLayer(Layer):
         name: Name of the layer (string).
     """
 
+    @interfaces.legacy_input_support
     def __init__(self, input_shape=None, batch_size=None,
                  batch_input_shape=None,
                  dtype=None, input_tensor=None, sparse=False, name=None):
@@ -1422,7 +1452,7 @@ class Container(Layer):
         get_weights
         set_weights
         get_config
-        get_output_shape_for
+        compute_output_shape
 
     # Class Methods
         from_config
@@ -1438,6 +1468,8 @@ class Container(Layer):
 
         self.supports_masking = False
         self.trainable = True
+        self._per_input_losses = {}
+        self._per_input_updates = {}
 
         # Container-specific properties.
         if isinstance(inputs, (list, tuple)):
@@ -1576,56 +1608,53 @@ class Container(Layer):
         nodes_depths = {}  # dict {node: depth value}
         layers_depths = {}  # dict {layer: depth value}
         layer_indices = {}  # dict {layer: index in traversal}
+        nodes_in_decreasing_depth = []
 
-        def make_node_marker(node, depth):
-            return str(id(node)) + '-' + str(depth)
-
-        def build_map_of_graph(tensor, seen_nodes=None, depth=0,
+        def build_map_of_graph(tensor, finished_nodes, nodes_in_progress,
                                layer=None, node_index=None, tensor_index=None):
             """Builds a map of the graph of layers.
 
-            This recursively updates the maps `nodes_depths`,
-            `layers_depths` and the set `container_nodes`.
-
-            Does not try to detect cycles in the graph.
+            This recursively updates the map `layer_indices`,
+            the list `nodes_in_decreasing_depth` and the set `container_nodes`.
 
             # Arguments
                 tensor: Some tensor in a graph.
-                seen_nodes: Set of node ids ("{layer.name}_ib-{node_index}")
-                    of nodes seen so far. Useful to prevent infinite loops.
-                depth: Current depth in the graph (0 = last output).
+                finished_nodes: Set of nodes whose subgraphs have been traversed
+                    completely. Useful to prevent duplicated work.
+                nodes_in_progress: Set of nodes that are currently active on the
+                    recursion stack. Useful to detect cycles.
                 layer: Layer from which `tensor` comes from. If not provided,
                     will be obtained from `tensor._keras_history`.
                 node_index: Node index from which `tensor` comes from.
                 tensor_index: Tensor_index from which `tensor` comes from.
+
+            # Raises
+                RuntimeError: if a cycle is detected.
             """
-            seen_nodes = seen_nodes or set()
             if not layer or node_index is None or tensor_index is None:
                 layer, node_index, tensor_index = tensor._keras_history
             node = layer.inbound_nodes[node_index]
 
             # Prevent cycles.
-            seen_nodes.add(make_node_marker(node, depth))
+            if node in nodes_in_progress:
+                raise RuntimeError(
+                    'The tensor ' + str(tensor) + ' at layer "' +
+                    layer.name + '" is part of a cycle.')
+
+            # Don't repeat work for shared subgraphs
+            if node in finished_nodes:
+                return
 
             node_key = layer.name + '_ib-' + str(node_index)
             # Update container_nodes.
             container_nodes.add(node_key)
-            # Update nodes_depths.
-            node_depth = nodes_depths.get(node)
-            if node_depth is None:
-                nodes_depths[node] = depth
-            else:
-                nodes_depths[node] = max(depth, node_depth)
-            # Update layers_depths.
-            previously_seen_depth = layers_depths.get(layer)
-            if previously_seen_depth is None:
-                current_depth = depth
-            else:
-                current_depth = max(depth, previously_seen_depth)
-            layers_depths[layer] = current_depth
+
+            # Store the traversal order for layer sorting.
             if layer not in layer_indices:
                 layer_indices[layer] = len(layer_indices)
 
+            nodes_in_progress.add(node)
+
             # Propagate to all previous tensors connected to this node.
             for i in range(len(node.inbound_layers)):
                 x = node.input_tensors[i]
@@ -1633,15 +1662,34 @@ class Container(Layer):
                 node_index = node.node_indices[i]
                 tensor_index = node.tensor_indices[i]
                 next_node = layer.inbound_nodes[node_index]
-                # use node_marker to prevent cycles
-                node_marker = make_node_marker(next_node, current_depth + 1)
-                if node_marker not in seen_nodes:
-                    build_map_of_graph(x, seen_nodes, current_depth + 1,
-                                       layer, node_index, tensor_index)
+                build_map_of_graph(x, finished_nodes, nodes_in_progress,
+                                   layer, node_index, tensor_index)
 
+            finished_nodes.add(node)
+            nodes_in_progress.remove(node)
+
+            nodes_in_decreasing_depth.append(node)
+
+        finished_nodes = set()
+        nodes_in_progress = set()
         for x in self.outputs:
-            seen_nodes = set()
-            build_map_of_graph(x, seen_nodes, depth=0)
+            build_map_of_graph(x, finished_nodes, nodes_in_progress)
+
+        for node in reversed(nodes_in_decreasing_depth):
+            # If the depth is not set, the node has no outbound nodes (depth 0).
+            depth = nodes_depths.setdefault(node, 0)
+
+            # Update the depth of inbound nodes.
+            for i in range(len(node.inbound_layers)):
+                inbound_layer = node.inbound_layers[i]
+                node_index = node.node_indices[i]
+                inbound_node = inbound_layer.inbound_nodes[node_index]
+                previous_depth = nodes_depths.get(inbound_node, 0)
+                nodes_depths[inbound_node] = max(depth + 1, previous_depth)
+
+            # Update the depth of the corresponding layer
+            previous_depth = layers_depths.get(node.outbound_layer, 0)
+            layers_depths[node.outbound_layer] = max(depth, previous_depth)
 
         # Build a dict {depth: list of nodes with this depth}
         nodes_by_depth = {}
@@ -1792,19 +1840,16 @@ class Container(Layer):
         updates = []
         for layer in self.layers:
             if hasattr(layer, 'updates'):
-                if len(layer.inbound_nodes) == 1:
-                    updates += layer.updates
-                else:
-                    # Collect updates that are dependent on inputs
-                    # that are part of the model.
-                    for node_index, node in enumerate(layer.inbound_nodes):
-                        node_key = layer.name + '_ib-' + str(node_index)
-                        if node_key in self.container_nodes:
-                            # The model owns this layer node.
-                            inputs = node.input_tensors
-                            updates += layer.get_updates_for(inputs)
-                    # Collect unconditional updates.
-                    updates += layer.get_updates_for(None)
+                # Collect updates that are dependent on inputs
+                # that are part of the model.
+                for node_index, node in enumerate(layer.inbound_nodes):
+                    node_key = layer.name + '_ib-' + str(node_index)
+                    if node_key in self.container_nodes:
+                        # The model owns this layer node.
+                        inputs = node.input_tensors
+                        updates += layer.get_updates_for(inputs)
+                # Collect unconditional updates.
+                updates += layer.get_updates_for(None)
         return updates
 
     @property
@@ -1823,22 +1868,18 @@ class Container(Layer):
         # Retrieve losses for all internal layers.
         for layer in self.layers:
             if hasattr(layer, 'losses'):
-                if len(layer.inbound_nodes) == 1:
-                    losses += layer.losses
-                else:
-                    # Collect losses that are dependent on inputs
-                    # that are part of the model.
-                    for node_index, node in enumerate(layer.inbound_nodes):
-                        node_key = layer.name + '_ib-' + str(node_index)
-                        if node_key in self.container_nodes:
-                            # The model owns this layer node.
-                            inputs = node.input_tensors
-                            losses += layer.get_losses_for(inputs)
-                    # Collect unconditional losses.
-                    losses += layer.get_losses_for(None)
+                # Collect losses that are dependent on inputs
+                # that are part of the model.
+                for node_index, node in enumerate(layer.inbound_nodes):
+                    node_key = layer.name + '_ib-' + str(node_index)
+                    if node_key in self.container_nodes:
+                        # The model owns this layer node.
+                        inputs = node.input_tensors
+                        losses += layer.get_losses_for(inputs)
+                # Collect unconditional losses.
+                losses += layer.get_losses_for(None)
         # Add any potential unconditional model-level loss.
-        if hasattr(self, '_per_input_losses'):
-            losses += self._per_input_losses.get(None, [])
+        losses += self.get_losses_for(None)
         return losses
 
     @property
@@ -2019,7 +2060,7 @@ class Container(Layer):
             for i in range(len(input_shapes)):
                 layer = self.input_layers[i]
                 input_shape = input_shapes[i]
-                # It's an input layer: get_output_shape_for is identity,
+                # It's an input layer: compute_output_shape is identity,
                 # and there is only one node and one tensor output.
                 shape_key = layer.name + '_0_0'
                 layers_to_output_shapes[shape_key] = input_shape
@@ -2121,6 +2162,7 @@ class Container(Layer):
                 for x in reference_input_tensors:
                     if str(id(x)) in tensor_map:
                         computed_data.append(tensor_map[str(id(x))])
+
                 if len(computed_data) == len(reference_input_tensors):
                     # call layer
                     with K.name_scope(layer.name):
@@ -2148,16 +2190,20 @@ class Container(Layer):
                             output_masks = _to_list(layer.compute_mask(computed_tensors,
                                                                        computed_masks))
 
+                        # Apply activity regularizer if any:
+                        if hasattr(layer, 'activity_regularizer') and layer.activity_regularizer is not None:
+                            regularization_losses = [layer.activity_regularizer(x) for x in computed_tensors]
+                            layer.add_loss(regularization_losses, computed_tensors)
+
                     # Update model updates and losses:
-                    layer_inputs = [x[0] for x in computed_data]
                     # Keep track of updates that depend on the inputs
                     # (e.g. BN updates).
-                    self.add_update(layer.get_updates_for(layer_inputs), inputs)
+                    self.add_update(layer.get_updates_for(computed_tensors), inputs)
                     # Keep track of unconditional updates (e.g. a counter).
                     self.add_update(layer.get_updates_for(None), None)
                     # Keep track of losses that depend on the inputs
                     # (e.g. activity regularizers).
-                    self.add_loss(layer.get_losses_for(layer_inputs), inputs)
+                    self.add_loss(layer.get_losses_for(computed_tensors), inputs)
                     # Keep track of unconditional losses
                     # (e.g. weight regularizers).
                     self.add_loss(layer.get_losses_for(None), None)
@@ -2386,7 +2432,7 @@ class Container(Layer):
             output_tensors.append(layer_output_tensors[tensor_index])
         return cls(inputs=input_tensors, outputs=output_tensors, name=name)
 
-    def save(self, filepath, overwrite=True):
+    def save(self, filepath, overwrite=True, include_optimizer=True):
         """Save the model to a single HDF5 file.
 
         The savefile includes:
@@ -2407,6 +2453,7 @@ class Container(Layer):
             filepath: String, path to the file to save the weights to.
             overwrite: Whether to silently overwrite any existing file at the
                 target location, or provide the user with a manual prompt.
+            include_optimizer: If True, save optimizer's state together.
 
         # Example
 
@@ -2422,7 +2469,7 @@ class Container(Layer):
         ```
         """
         from ..models import save_model
-        save_model(self, filepath, overwrite)
+        save_model(self, filepath, overwrite, include_optimizer)
 
     def save_weights(self, filepath, overwrite=True):
         """Dumps all layer weights to a HDF5 file.
@@ -2738,6 +2785,25 @@ def preprocess_weights_for_loading(layer, weights,
         A list of weights values (Numpy arrays).
     """
     if original_keras_version == '1':
+        if layer.__class__.__name__ == 'Bidirectional':
+            num_weights_per_layer = len(weights) // 2
+
+            forward_weights = preprocess_weights_for_loading(layer.forward_layer,
+                                                             weights[:num_weights_per_layer],
+                                                             original_keras_version,
+                                                             original_backend)
+            backward_weights = preprocess_weights_for_loading(layer.backward_layer,
+                                                              weights[num_weights_per_layer:],
+                                                              original_keras_version,
+                                                              original_backend)
+            weights = forward_weights + backward_weights
+
+        if layer.__class__.__name__ == 'TimeDistributed':
+            weights = preprocess_weights_for_loading(layer.layer,
+                                                     weights,
+                                                     original_keras_version,
+                                                     original_backend)
+
         if layer.__class__.__name__ == 'Conv1D':
             shape = weights[0].shape
             # Handle Keras 1.1 format

keras/engine/training.py

@@ -50,6 +50,8 @@ def _standardize_input_data(data, names, shapes=None,
     # Raises
         ValueError: in case of improperly formatted user-provided data.
     """
+    if not names:
+        return []
     if data is None:
         return [None for _ in range(len(names))]
     if isinstance(data, dict):
@@ -63,7 +65,8 @@ def _standardize_input_data(data, names, shapes=None,
     elif isinstance(data, list):
         if len(data) != len(names):
             if data and hasattr(data[0], 'shape'):
-                raise ValueError('Error when checking ' + exception_prefix +
+                raise ValueError('Error when checking model ' +
+                                 exception_prefix +
                                  ': the list of Numpy arrays '
                                  'that you are passing to your model '
                                  'is not the size the model expected. '
@@ -77,7 +80,8 @@ def _standardize_input_data(data, names, shapes=None,
                     data = [np.asarray(data)]
                 else:
                     raise ValueError(
-                        'Error when checking ' + exception_prefix +
+                        'Error when checking model ' +
+                        exception_prefix +
                         ': you are passing a list as '
                         'input to your model, '
                         'but the model expects '
@@ -88,15 +92,17 @@ def _standardize_input_data(data, names, shapes=None,
         arrays = data
     else:
         if not hasattr(data, 'shape'):
-            raise TypeError('Error when checking ' + exception_prefix +
+            raise TypeError('Error when checking model ' +
+                            exception_prefix +
                             ': data should be a Numpy array, '
                             'or list/dict of Numpy arrays. '
                             'Found: ' + str(data)[:200] + '...')
-        if len(names) != 1:
+        if len(names) > 1:
             # Case: model expects multiple inputs but only received
             # a single Numpy array.
             raise ValueError('The model expects ' + str(len(names)) +
-                             ' input arrays, but only received one array. '
+                             exception_prefix +
+                             ' arrays, but only received one array. '
                              'Found: array with shape ' + str(data.shape))
         arrays = [data]
 
@@ -679,6 +685,8 @@ class Model(Container):
                 See [losses](/losses).
                 If the model has multiple outputs, you can use a different loss
                 on each output by passing a dictionary or a list of losses.
+                The loss value that will be minimized by the model
+                will then be the sum of all individual losses.
             metrics: list of metrics to be evaluated by the model
                 during training and testing.
                 Typically you will use `metrics=['accuracy']`.
@@ -688,6 +696,9 @@ class Model(Container):
             loss_weights: Optional list or dictionary specifying scalar
                 coefficients (Python floats) to weight the loss contributions
                 of different model outputs.
+                The loss value that will be minimized by the model
+                will then be the *weighted sum* of all individual losses,
+                weighted by the `loss_weights` coefficients.
                 If a list, it is expected to have a 1:1 mapping
                 to the model's outputs. If a tensor, it is expected to map
                 output names (strings) to scalar coefficients.
@@ -726,7 +737,7 @@ class Model(Container):
                                   'We assume this was done on purpose, '
                                   'and we will not be expecting '
                                   'any data to be passed to "' + name +
-                                  '" during training.')
+                                  '" during training.', stacklevel=2)
                 loss_functions.append(losses.get(loss.get(name)))
         elif isinstance(loss, list):
             if len(loss) != len(self.outputs):
@@ -939,7 +950,8 @@ class Model(Container):
                     # (because of class mode duality)
                     output_shape = self.internal_output_shapes[i]
                     acc_fn = None
-                    if output_shape[-1] == 1 or self.loss_functions[i] == losses.binary_crossentropy:
+                    if (output_shape[-1] == 1 or
+                       self.loss_functions[i] == losses.binary_crossentropy):
                         # case: binary accuracy
                         acc_fn = metrics_module.binary_accuracy
                     elif self.loss_functions[i] == losses.sparse_categorical_crossentropy:
@@ -1125,7 +1137,7 @@ class Model(Container):
                 batch_ids = index_array[batch_start:batch_end]
                 try:
                     if isinstance(ins[-1], float):
-                        # do not slice the training phase flag
+                        # Do not slice the training phase flag.
                         ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
                     else:
                         ins_batch = _slice_arrays(ins, batch_ids)
@@ -1145,16 +1157,14 @@ class Model(Container):
 
                 callbacks.on_batch_end(batch_index, batch_logs)
 
-                if batch_index == len(batches) - 1:  # last batch
-                    # validation
+                if batch_index == len(batches) - 1:  # Last batch.
                     if do_validation:
-                        # replace with self._evaluate
                         val_outs = self._test_loop(val_f, val_ins,
                                                    batch_size=batch_size,
                                                    verbose=0)
                         if not isinstance(val_outs, list):
                             val_outs = [val_outs]
-                        # same labels assumed
+                        # Same labels assumed.
                         for l, o in zip(out_labels, val_outs):
                             epoch_logs['val_' + l] = o
             callbacks.on_epoch_end(epoch, epoch_logs)
@@ -1194,7 +1204,7 @@ class Model(Container):
         for batch_index, (batch_start, batch_end) in enumerate(batches):
             batch_ids = index_array[batch_start:batch_end]
             if ins and isinstance(ins[-1], float):
-                # do not slice the training phase flag
+                # Do not slice the training phase flag.
                 ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
             else:
                 ins_batch = _slice_arrays(ins, batch_ids)
@@ -1205,7 +1215,7 @@ class Model(Container):
             if batch_index == 0:
                 for batch_out in batch_outs:
                     shape = (samples,) + batch_out.shape[1:]
-                    outs.append(np.zeros(shape, dtype=K.floatx()))
+                    outs.append(np.zeros(shape, dtype=batch_out.dtype))
 
             for i, batch_out in enumerate(batch_outs):
                 outs[i][batch_start:batch_end] = batch_out
@@ -1248,7 +1258,7 @@ class Model(Container):
         for batch_index, (batch_start, batch_end) in enumerate(batches):
             batch_ids = index_array[batch_start:batch_end]
             if isinstance(ins[-1], float):
-                # do not slice the training phase flag
+                # Do not slice the training phase flag.
                 ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
             else:
                 ins_batch = _slice_arrays(ins, batch_ids)
@@ -1292,11 +1302,11 @@ class Model(Container):
         x = _standardize_input_data(x, self._feed_input_names,
                                     self._feed_input_shapes,
                                     check_batch_axis=False,
-                                    exception_prefix='model input')
+                                    exception_prefix='input')
         y = _standardize_input_data(y, self._feed_output_names,
                                     output_shapes,
                                     check_batch_axis=False,
-                                    exception_prefix='model target')
+                                    exception_prefix='target')
         sample_weights = _standardize_sample_weights(sample_weight,
                                                      self._feed_output_names)
         class_weights = _standardize_class_weights(class_weight,
@@ -1317,6 +1327,20 @@ class Model(Container):
                                  str(x[0].shape[0]) + ' samples')
         return x, y, sample_weights
 
+    def _get_deduped_metrics_names(self):
+        out_labels = self.metrics_names
+
+        # Rename duplicated metrics name
+        # (can happen with an output layer shared among multiple dataflows).
+        deduped_out_labels = []
+        for i, label in enumerate(out_labels):
+            new_label = label
+            if out_labels.count(label) > 1:
+                dup_idx = out_labels[:i].count(label)
+                new_label += '_' + str(dup_idx + 1)
+            deduped_out_labels.append(new_label)
+        return deduped_out_labels
+
     def fit(self, x=None,
             y=None,
             batch_size=32,
@@ -1346,7 +1370,7 @@ class Model(Container):
             batch_size: integer. Number of samples per gradient update.
             epochs: integer, the number of times to iterate
                 over the training data arrays.
-                verbose: 0, 1, or 2. Verbosity mode.
+            verbose: 0, 1, or 2. Verbosity mode.
                 0 = silent, 1 = verbose, 2 = one log line per epoch.
             callbacks: list of callbacks to be called during training.
                 See [callbacks](/callbacks).
@@ -1391,19 +1415,19 @@ class Model(Container):
         # Legacy support
         if 'nb_epoch' in kwargs:
             warnings.warn('The `nb_epoch` argument in `fit` '
-                          'has been renamed `epochs`.')
+                          'has been renamed `epochs`.', stacklevel=2)
             epochs = kwargs.pop('nb_epoch')
         if kwargs:
             raise TypeError('Unrecognized keyword arguments: ' + str(kwargs))
 
-        # validate user data
+        # Validate user data.
         x, y, sample_weights = self._standardize_user_data(
             x, y,
             sample_weight=sample_weight,
             class_weight=class_weight,
             check_batch_axis=False,
             batch_size=batch_size)
-        # prepare validation data
+        # Prepare validation data.
         if validation_data:
             do_validation = True
             if len(validation_data) == 2:
@@ -1449,7 +1473,7 @@ class Model(Container):
             val_f = None
             val_ins = None
 
-        # prepare input arrays and training function
+        # Prepare input arrays and training function.
         if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
             ins = x + y + sample_weights + [1.]
         else:
@@ -1457,26 +1481,15 @@ class Model(Container):
         self._make_train_function()
         f = self.train_function
 
-        # prepare display labels
-        out_labels = self.metrics_names
-
-        # rename duplicated metrics name
-        # (can happen with an output layer shared among multiple dataflows)
-        deduped_out_labels = []
-        for i, label in enumerate(out_labels):
-            new_label = label
-            if out_labels.count(label) > 1:
-                dup_idx = out_labels[:i].count(label)
-                new_label += '_' + str(dup_idx + 1)
-            deduped_out_labels.append(new_label)
-        out_labels = deduped_out_labels
+        # Prepare display labels.
+        out_labels = self._get_deduped_metrics_names()
 
         if do_validation:
             callback_metrics = copy.copy(out_labels) + ['val_' + n for n in out_labels]
         else:
             callback_metrics = copy.copy(out_labels)
 
-        # delegate logic to _fit_loop
+        # Delegate logic to `_fit_loop`.
         return self._fit_loop(f, ins, out_labels=out_labels,
                               batch_size=batch_size, epochs=epochs,
                               verbose=verbose, callbacks=callbacks,
@@ -1511,13 +1524,13 @@ class Model(Container):
             and/or metrics). The attribute `model.metrics_names` will give you
             the display labels for the scalar outputs.
         """
-        # validate user data
+        # Validate user data.
         x, y, sample_weights = self._standardize_user_data(
             x, y,
             sample_weight=sample_weight,
             check_batch_axis=False,
             batch_size=batch_size)
-        # prepare inputs, delegate logic to _test_loop
+        # Prepare inputs, delegate logic to `_test_loop`.
         if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
             ins = x + y + sample_weights + [0.]
         else:
@@ -1548,7 +1561,7 @@ class Model(Container):
                 or in case a stateful model receives a number of samples
                 that is not a multiple of the batch size.
         """
-        # validate user data
+        # Validate user data.
         x = _standardize_input_data(x, self._feed_input_names,
                                     self._feed_input_shapes,
                                     check_batch_axis=False)
@@ -1561,7 +1574,7 @@ class Model(Container):
                                  str(x[0].shape[0]) + ' samples. '
                                  'Batch size: ' + str(batch_size) + '.')
 
-        # prepare inputs, delegate logic to _predict_loop
+        # Prepare inputs, delegate logic to `_predict_loop`.
         if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
             ins = x + [0.]
         else:
@@ -1711,8 +1724,8 @@ class Model(Container):
                 - a tuple (inputs, targets, sample_weights).
                 All arrays should contain the same number of samples.
                 The generator is expected to loop over its data
-                indefinitely. An epoch finishes when `samples_per_epoch`
-                samples have been seen by the model.
+                indefinitely. An epoch finishes when `steps_per_epoch`
+                batches have been seen by the model.
             steps_per_epoch: Total number of steps (batches of samples)
                 to yield from `generator` before declaring one epoch
                 finished and starting the next epoch. It should typically
@@ -1760,7 +1773,7 @@ class Model(Container):
                     f.close()
 
             model.fit_generator(generate_arrays_from_file('/my_file.txt'),
-                                samples_per_epoch=10000, epochs=10)
+                                steps_per_epoch=10000, epochs=10)
         ```
 
         # Raises
@@ -1784,7 +1797,8 @@ class Model(Container):
                              'you must specify a value for '
                              '`validation_steps`.')
 
-        out_labels = self.metrics_names
+        # Prepare display labels.
+        out_labels = self._get_deduped_metrics_names()
         callback_metrics = out_labels + ['val_' + n for n in out_labels]
 
         # prepare callbacks
@@ -1930,7 +1944,7 @@ class Model(Container):
         The generator should return the same kind of data
         as accepted by `test_on_batch`.
 
-        Arguments:
+        # Arguments
             generator: Generator yielding tuples (inputs, targets)
                 or (inputs, targets, sample_weights)
             steps: Total number of steps (batches of samples)
@@ -2021,7 +2035,8 @@ class Model(Container):
 
     @interfaces.legacy_generator_methods_support
     def predict_generator(self, generator, steps,
-                          max_q_size=10, workers=1, pickle_safe=False):
+                          max_q_size=10, workers=1,
+                          pickle_safe=False, verbose=0):
         """Generates predictions for the input samples from a data generator.
 
         The generator should return the same kind of data as accepted by
@@ -2041,6 +2056,7 @@ class Model(Container):
                 non picklable arguments to the generator
                 as they can't be passed
                 easily to children processes.
+            verbose: verbosity mode, 0 or 1.
 
         # Returns
             Numpy array(s) of predictions.
@@ -2060,6 +2076,9 @@ class Model(Container):
             enqueuer = GeneratorEnqueuer(generator, pickle_safe=pickle_safe)
             enqueuer.start(workers=workers, max_q_size=max_q_size)
 
+            if verbose == 1:
+                progbar = Progbar(target=steps)
+
             while steps_done < steps:
                 generator_output = None
                 while enqueuer.is_running():
@@ -2097,6 +2116,8 @@ class Model(Container):
                 for i, out in enumerate(outs):
                     all_outs[i].append(out)
                 steps_done += 1
+                if verbose == 1:
+                    progbar.update(steps_done)
 
         finally:
             if enqueuer is not None:

keras/initializers.py

@@ -22,14 +22,16 @@ class Initializer(object):
 
 
 class Zeros(Initializer):
-    """Initializer that generates tensors initialized to 0."""
+    """Initializer that generates tensors initialized to 0.
+    """
 
     def __call__(self, shape, dtype=None):
         return K.constant(0, shape=shape, dtype=dtype)
 
 
 class Ones(Initializer):
-    """Initializer that generates tensors initialized to 1."""
+    """Initializer that generates tensors initialized to 1.
+    """
 
     def __call__(self, shape, dtype=None):
         return K.constant(1, shape=shape, dtype=dtype)
@@ -111,7 +113,7 @@ class RandomUniform(Initializer):
 class TruncatedNormal(Initializer):
     """Initializer that generates a truncated normal distribution.
 
-    These values are similar to values from a `random_normal_initializer`
+    These values are similar to values from a `RandomNormal`
     except that values more than two standard deviations from the mean
     are discarded and re-drawn. This is the recommended initializer for
     neural network weights and filters.
@@ -146,6 +148,7 @@ class VarianceScaling(Initializer):
 
     With `distribution="normal"`, samples are drawn from a truncated normal
     distribution centered on zero, with `stddev = sqrt(scale / n)` where n is:
+
         - number of input units in the weight tensor, if mode = "fan_in"
         - number of output units, if mode = "fan_out"
         - average of the numbers of input and output units, if mode = "fan_avg"

keras/layers/__init__.py

@@ -21,6 +21,14 @@ from ..legacy.layers import *
 
 
 def serialize(layer):
+    """Serialize a layer.
+
+    # Arguments
+        layer: a Layer object.
+
+    # Returns
+        dictionary with config.
+    """
     return {'class_name': layer.__class__.__name__,
             'config': layer.get_config()}
 

keras/layers/advanced_activations.py

@@ -104,7 +104,7 @@ class PReLU(Layer):
             for i in self.shared_axes:
                 param_shape[i - 1] = 1
                 self.param_broadcast[i - 1] = True
-        self.alpha = self.add_weight(param_shape,
+        self.alpha = self.add_weight(shape=param_shape,
                                      name='alpha',
                                      initializer=self.alpha_initializer,
                                      regularizer=self.alpha_regularizer,

keras/layers/convolutional.py

@@ -127,13 +127,13 @@ class _Conv(Layer):
         input_dim = input_shape[channel_axis]
         kernel_shape = self.kernel_size + (input_dim, self.filters)
 
-        self.kernel = self.add_weight(kernel_shape,
+        self.kernel = self.add_weight(shape=kernel_shape,
                                       initializer=self.kernel_initializer,
                                       name='kernel',
                                       regularizer=self.kernel_regularizer,
                                       constraint=self.kernel_constraint)
         if self.use_bias:
-            self.bias = self.add_weight((self.filters,),
+            self.bias = self.add_weight(shape=(self.filters,),
                                         initializer=self.bias_initializer,
                                         name='bias',
                                         regularizer=self.bias_regularizer,
@@ -257,7 +257,7 @@ class Conv1D(_Conv):
             any `dilation_rate` value != 1.
         padding: One of `"valid"`, `"causal"` or `"same"` (case-insensitive).
             `"causal"` results in causal (dilated) convolutions, e.g. output[t]
-            depends solely on input[:t-1]. Useful when modeling temporal data
+            does not depend on input[t+1:]. Useful when modeling temporal data
             where the model should not violate the temporal order.
             See [WaveNet: A Generative Model for Raw Audio, section 2.1](https://arxiv.org/abs/1609.03499).
         dilation_rate: an integer or tuple/list of a single integer, specifying
@@ -370,9 +370,9 @@ class Conv2D(_Conv):
             one of `channels_last` (default) or `channels_first`.
             The ordering of the dimensions in the inputs.
             `channels_last` corresponds to inputs with shape
-            `(batch, width, height, channels)` while `channels_first`
+            `(batch, height, width, channels)` while `channels_first`
             corresponds to inputs with shape
-            `(batch, channels, width, height)`.
+            `(batch, channels, height, width)`.
             It defaults to the `image_data_format` value found in your
             Keras config file at `~/.keras/keras.json`.
             If you never set it, then it will be "channels_last".
@@ -604,7 +604,7 @@ class Conv2DTranspose(Conv2D):
 
     # Arguments
         filters: Integer, the dimensionality of the output space
-            (i.e. the number output of filters in the convolution).
+            (i.e. the number of output filters in the convolution).
         kernel_size: An integer or tuple/list of 2 integers, specifying the
             width and height of the 2D convolution window.
             Can be a single integer to specify the same value for
@@ -620,9 +620,9 @@ class Conv2DTranspose(Conv2D):
             one of `channels_last` (default) or `channels_first`.
             The ordering of the dimensions in the inputs.
             `channels_last` corresponds to inputs with shape
-            `(batch, width, height, channels)` while `channels_first`
+            `(batch, height, width, channels)` while `channels_first`
             corresponds to inputs with shape
-            `(batch, channels, width, height)`.
+            `(batch, channels, height, width)`.
             It defaults to the `image_data_format` value found in your
             Keras config file at `~/.keras/keras.json`.
             If you never set it, then it will be "channels_last".
@@ -677,7 +677,7 @@ class Conv2DTranspose(Conv2D):
                  kernel_size,
                  strides=(1, 1),
                  padding='valid',
-                 data_format='channels_last',
+                 data_format=None,
                  activation=None,
                  use_bias=True,
                  kernel_initializer='glorot_uniform',
@@ -721,13 +721,13 @@ class Conv2DTranspose(Conv2D):
         input_dim = input_shape[channel_axis]
         kernel_shape = self.kernel_size + (self.filters, input_dim)
 
-        self.kernel = self.add_weight(kernel_shape,
+        self.kernel = self.add_weight(shape=kernel_shape,
                                       initializer=self.kernel_initializer,
                                       name='kernel',
                                       regularizer=self.kernel_regularizer,
                                       constraint=self.kernel_constraint)
         if self.use_bias:
-            self.bias = self.add_weight((self.filters,),
+            self.bias = self.add_weight(shape=(self.filters,),
                                         initializer=self.bias_initializer,
                                         name='bias',
                                         regularizer=self.bias_regularizer,
@@ -835,9 +835,9 @@ class SeparableConv2D(Conv2D):
             one of `channels_last` (default) or `channels_first`.
             The ordering of the dimensions in the inputs.
             `channels_last` corresponds to inputs with shape
-            `(batch, width, height, channels)` while `channels_first`
+            `(batch, height, width, channels)` while `channels_first`
             corresponds to inputs with shape
-            `(batch, channels, width, height)`.
+            `(batch, channels, height, width)`.
             It defaults to the `image_data_format` value found in your
             Keras config file at `~/.keras/keras.json`.
             If you never set it, then it will be "channels_last".
@@ -952,20 +952,20 @@ class SeparableConv2D(Conv2D):
                                   self.filters)
 
         self.depthwise_kernel = self.add_weight(
-            depthwise_kernel_shape,
+            shape=depthwise_kernel_shape,
             initializer=self.depthwise_initializer,
             name='depthwise_kernel',
             regularizer=self.depthwise_regularizer,
             constraint=self.depthwise_constraint)
         self.pointwise_kernel = self.add_weight(
-            pointwise_kernel_shape,
+            shape=pointwise_kernel_shape,
             initializer=self.pointwise_initializer,
             name='pointwise_kernel',
             regularizer=self.pointwise_regularizer,
             constraint=self.pointwise_constraint)
 
         if self.use_bias:
-            self.bias = self.add_weight((self.filters,),
+            self.bias = self.add_weight(shape=(self.filters,),
                                         initializer=self.bias_initializer,
                                         name='bias',
                                         regularizer=self.bias_regularizer,
@@ -1077,9 +1077,9 @@ class UpSampling2D(Layer):
             one of `channels_last` (default) or `channels_first`.
             The ordering of the dimensions in the inputs.
             `channels_last` corresponds to inputs with shape
-            `(batch, width, height, channels)` while `channels_first`
+            `(batch, height, width, channels)` while `channels_first`
             corresponds to inputs with shape
-            `(batch, channels, width, height)`.
+            `(batch, channels, height, width)`.
             It defaults to the `image_data_format` value found in your
             Keras config file at `~/.keras/keras.json`.
             If you never set it, then it will be "channels_last".
@@ -1232,7 +1232,10 @@ class ZeroPadding1D(Layer):
         self.input_spec = InputSpec(ndim=3)
 
     def compute_output_shape(self, input_shape):
-        length = input_shape[1] + self.padding[0] + self.padding[1] if input_shape[1] is not None else None
+        if input_shape[1] is not None:
+            length = input_shape[1] + self.padding[0] + self.padding[1]
+        else:
+            length = None
         return (input_shape[0],
                 length,
                 input_shape[2])
@@ -1259,7 +1262,7 @@ class ZeroPadding2D(Layer):
             - If tuple of 2 ints:
                 interpreted as two different
                 symmetric padding values for height and width:
-                `(symmetric_height_pad, symmetrc_width_pad)`.
+                `(symmetric_height_pad, symmetric_width_pad)`.
             - If tuple of 2 tuples of 2 ints:
                 interpreted as
                 `((top_pad, bottom_pad), (left_pad, right_pad))`
@@ -1267,9 +1270,9 @@ class ZeroPadding2D(Layer):
             one of `channels_last` (default) or `channels_first`.
             The ordering of the dimensions in the inputs.
             `channels_last` corresponds to inputs with shape
-            `(batch, width, height, channels)` while `channels_first`
+            `(batch, height, width, channels)` while `channels_first`
             corresponds to inputs with shape
-            `(batch, channels, width, height)`.
+            `(batch, channels, height, width)`.
             It defaults to the `image_data_format` value found in your
             Keras config file at `~/.keras/keras.json`.
             If you never set it, then it will be "channels_last".
@@ -1318,15 +1321,27 @@ class ZeroPadding2D(Layer):
 
     def compute_output_shape(self, input_shape):
         if self.data_format == 'channels_first':
-            rows = input_shape[2] + self.padding[0][0] + self.padding[0][1] if input_shape[2] is not None else None
-            cols = input_shape[3] + self.padding[1][0] + self.padding[1][1] if input_shape[3] is not None else None
+            if input_shape[2] is not None:
+                rows = input_shape[2] + self.padding[0][0] + self.padding[0][1]
+            else:
+                rows = None
+            if input_shape[3] is not None:
+                cols = input_shape[3] + self.padding[1][0] + self.padding[1][1]
+            else:
+                cols = None
             return (input_shape[0],
                     input_shape[1],
                     rows,
                     cols)
         elif self.data_format == 'channels_last':
-            rows = input_shape[1] + self.padding[0][0] + self.padding[0][1] if input_shape[1] is not None else None
-            cols = input_shape[2] + self.padding[1][0] + self.padding[1][1] if input_shape[2] is not None else None
+            if input_shape[1] is not None:
+                rows = input_shape[1] + self.padding[0][0] + self.padding[0][1]
+            else:
+                rows = None
+            if input_shape[2] is not None:
+                cols = input_shape[2] + self.padding[1][0] + self.padding[1][1]
+            else:
+                cols = None
             return (input_shape[0],
                     rows,
                     cols,
@@ -1414,18 +1429,36 @@ class ZeroPadding3D(Layer):
 
     def compute_output_shape(self, input_shape):
         if self.data_format == 'channels_first':
-            dim1 = input_shape[2] + 2 * self.padding[0][0] if input_shape[2] is not None else None
-            dim2 = input_shape[3] + 2 * self.padding[1][0] if input_shape[3] is not None else None
-            dim3 = input_shape[4] + 2 * self.padding[2][0] if input_shape[4] is not None else None
+            if input_shape[2] is not None:
+                dim1 = input_shape[2] + 2 * self.padding[0][0]
+            else:
+                dim1 = None
+            if input_shape[3] is not None:
+                dim2 = input_shape[3] + 2 * self.padding[1][0]
+            else:
+                dim2 = None
+            if input_shape[4] is not None:
+                dim3 = input_shape[4] + 2 * self.padding[2][0]
+            else:
+                dim3 = None
             return (input_shape[0],
                     input_shape[1],
                     dim1,
                     dim2,
                     dim3)
         elif self.data_format == 'channels_last':
-            dim1 = input_shape[1] + 2 * self.padding[0][1] if input_shape[1] is not None else None
-            dim2 = input_shape[2] + 2 * self.padding[1][1] if input_shape[2] is not None else None
-            dim3 = input_shape[3] + 2 * self.padding[2][1] if input_shape[3] is not None else None
+            if input_shape[1] is not None:
+                dim1 = input_shape[1] + 2 * self.padding[0][1]
+            else:
+                dim1 = None
+            if input_shape[2] is not None:
+                dim2 = input_shape[2] + 2 * self.padding[1][1]
+            else:
+                dim2 = None
+            if input_shape[3] is not None:
+                dim3 = input_shape[3] + 2 * self.padding[2][1]
+            else:
+                dim3 = None
             return (input_shape[0],
                     dim1,
                     dim2,
@@ -1501,7 +1534,7 @@ class Cropping2D(Layer):
             - If tuple of 2 ints:
                 interpreted as two different
                 symmetric cropping values for height and width:
-                `(symmetric_height_crop, symmetrc_width_crop)`.
+                `(symmetric_height_crop, symmetric_width_crop)`.
             - If tuple of 2 tuples of 2 ints:
                 interpreted as
                 `((top_crop, bottom_crop), (left_crop, right_crop))`
@@ -1509,9 +1542,9 @@ class Cropping2D(Layer):
             one of `channels_last` (default) or `channels_first`.
             The ordering of the dimensions in the inputs.
             `channels_last` corresponds to inputs with shape
-            `(batch, width, height, channels)` while `channels_first`
+            `(batch, height, width, channels)` while `channels_first`
             corresponds to inputs with shape
-            `(batch, channels, width, height)`.
+            `(batch, channels, height, width)`.
             It defaults to the `image_data_format` value found in your
             Keras config file at `~/.keras/keras.json`.
             If you never set it, then it will be "channels_last".
@@ -1538,7 +1571,7 @@ class Cropping2D(Layer):
         model.add(Cropping2D(cropping=((2, 2), (4, 4)),
                              input_shape=(28, 28, 3)))
         # now model.output_shape == (None, 24, 20, 3)
-        model.add(Conv2D(64, (3, 3), padding='same))
+        model.add(Conv2D(64, (3, 3), padding='same'))
         model.add(Cropping2D(cropping=((2, 2), (2, 2))))
         # now model.output_shape == (None, 20, 16. 64)
     ```
@@ -1705,18 +1738,36 @@ class Cropping3D(Layer):
 
     def compute_output_shape(self, input_shape):
         if self.data_format == 'channels_first':
-            dim1 = input_shape[2] - self.cropping[0][0] - self.cropping[0][1] if input_shape[2] is not None else None
-            dim2 = input_shape[3] - self.cropping[1][0] - self.cropping[1][1] if input_shape[3] is not None else None
-            dim3 = input_shape[4] - self.cropping[2][0] - self.cropping[2][1] if input_shape[4] is not None else None
+            if input_shape[2] is not None:
+                dim1 = input_shape[2] - self.cropping[0][0] - self.cropping[0][1]
+            else:
+                dim1 = None
+            if input_shape[3] is not None:
+                dim2 = input_shape[3] - self.cropping[1][0] - self.cropping[1][1]
+            else:
+                dim2 = None
+            if input_shape[4] is not None:
+                dim3 = input_shape[4] - self.cropping[2][0] - self.cropping[2][1]
+            else:
+                dim3 = None
             return (input_shape[0],
                     input_shape[1],
                     dim1,
                     dim2,
                     dim3)
         elif self.data_format == 'channels_last':
-            dim1 = input_shape[1] - self.cropping[0][0] - self.cropping[0][1] if input_shape[1] is not None else None
-            dim2 = input_shape[2] - self.cropping[1][0] - self.cropping[1][1] if input_shape[2] is not None else None
-            dim3 = input_shape[3] - self.cropping[2][0] - self.cropping[2][1] if input_shape[3] is not None else None
+            if input_shape[1] is not None:
+                dim1 = input_shape[1] - self.cropping[0][0] - self.cropping[0][1]
+            else:
+                dim1 = None
+            if input_shape[2] is not None:
+                dim2 = input_shape[2] - self.cropping[1][0] - self.cropping[1][1]
+            else:
+                dim2 = None
+            if input_shape[3] is not None:
+                dim3 = input_shape[3] - self.cropping[2][0] - self.cropping[2][1]
+            else:
+                dim3 = None
             return (input_shape[0],
                     dim1,
                     dim2,

keras/layers/convolutional_recurrent.py

@@ -340,7 +340,7 @@ class ConvLSTM2D(ConvRecurrent2D):
             self.states = [None, None]
 
         if self.data_format == 'channels_first':
-            channel_axis = 1
+            channel_axis = 2
         else:
             channel_axis = -1
         if input_shape[channel_axis] is None:
@@ -351,19 +351,19 @@ class ConvLSTM2D(ConvRecurrent2D):
         self.kernel_shape = kernel_shape
         recurrent_kernel_shape = self.kernel_size + (self.filters, self.filters * 4)
 
-        self.kernel = self.add_weight(kernel_shape,
+        self.kernel = self.add_weight(shape=kernel_shape,
                                       initializer=self.kernel_initializer,
                                       name='kernel',
                                       regularizer=self.kernel_regularizer,
                                       constraint=self.kernel_constraint)
         self.recurrent_kernel = self.add_weight(
-            recurrent_kernel_shape,
+            shape=recurrent_kernel_shape,
             initializer=self.recurrent_initializer,
             name='recurrent_kernel',
             regularizer=self.recurrent_regularizer,
             constraint=self.recurrent_constraint)
         if self.use_bias:
-            self.bias = self.add_weight((self.filters * 4,),
+            self.bias = self.add_weight(shape=(self.filters * 4,),
                                         initializer=self.bias_initializer,
                                         name='bias',
                                         regularizer=self.bias_regularizer,
@@ -396,7 +396,7 @@ class ConvLSTM2D(ConvRecurrent2D):
             self.bias_o = None
         self.built = True
 
-    def get_initial_states(self, inputs):
+    def get_initial_state(self, inputs):
         # (samples, timesteps, rows, cols, filters)
         initial_state = K.zeros_like(inputs)
         # (samples, rows, cols, filters)

keras/layers/core.py

@@ -73,7 +73,7 @@ class Dropout(Layer):
     """Applies Dropout to the input.
 
     Dropout consists in randomly setting
-    a fraction `p` of input units to 0 at each update during training time,
+    a fraction `rate` of input units to 0 at each update during training time,
     which helps prevent overfitting.
 
     # Arguments
@@ -129,7 +129,7 @@ class SpatialDropout1D(Dropout):
     between feature maps and should be used instead.
 
     # Arguments
-        p: float between 0 and 1. Fraction of the input units to drop.
+        rate: float between 0 and 1. Fraction of the input units to drop.
 
     # Input shape
         3D tensor with shape:
@@ -820,13 +820,13 @@ class Dense(Layer):
         assert len(input_shape) >= 2
         input_dim = input_shape[-1]
 
-        self.kernel = self.add_weight((input_dim, self.units),
+        self.kernel = self.add_weight(shape=(input_dim, self.units),
                                       initializer=self.kernel_initializer,
                                       name='kernel',
                                       regularizer=self.kernel_regularizer,
                                       constraint=self.kernel_constraint)
         if self.use_bias:
-            self.bias = self.add_weight((self.units,),
+            self.bias = self.add_weight(shape=(self.units,),
                                         initializer=self.bias_initializer,
                                         name='bias',
                                         regularizer=self.bias_regularizer,

keras/layers/embeddings.py

@@ -31,17 +31,17 @@ class Embedding(Layer):
     ```
 
     # Arguments
-      input_dim: int > 0. Size of the vocabulary, ie.
-          1 + maximum integer index occurring in the input data.
+      input_dim: int > 0. Size of the vocabulary,
+          i.e. maximum integer index + 1.
       output_dim: int >= 0. Dimension of the dense embedding.
       embeddings_initializer: Initializer for the `embeddings` matrix
-            (see [initializers](../initializers.md)).
+          (see [initializers](../initializers.md)).
       embeddings_regularizer: Regularizer function applied to
-            the `embeddings` matrix
-            (see [regularizer](../regularizers.md)).
+          the `embeddings` matrix
+          (see [regularizer](../regularizers.md)).
       embeddings_constraint: Constraint function applied to
-            the `embeddings` matrix
-            (see [constraints](../constraints.md)).
+          the `embeddings` matrix
+          (see [constraints](../constraints.md)).
       mask_zero: Whether or not the input value 0 is a special "padding"
           value that should be masked out.
           This is useful when using [recurrent layers](recurrent.md)
@@ -49,7 +49,8 @@ class Embedding(Layer):
           If this is `True` then all subsequent layers
           in the model need to support masking or an exception will be raised.
           If mask_zero is set to True, as a consequence, index 0 cannot be
-          used in the vocabulary (input_dim should equal `|vocabulary| + 2`).
+          used in the vocabulary (input_dim should equal size of
+          vocabulary + 1).
       input_length: Length of input sequences, when it is constant.
           This argument is required if you are going to connect
           `Flatten` then `Dense` layers upstream
@@ -93,7 +94,7 @@ class Embedding(Layer):
 
     def build(self, input_shape):
         self.embeddings = self.add_weight(
-            (self.input_dim, self.output_dim),
+            shape=(self.input_dim, self.output_dim),
             initializer=self.embeddings_initializer,
             name='embeddings',
             regularizer=self.embeddings_regularizer,

keras/layers/local.py

@@ -41,7 +41,8 @@ class LocallyConnected1D(Layer):
             specifying the stride length of the convolution.
             Specifying any stride value != 1 is incompatible with specifying
             any `dilation_rate` value != 1.
-        padding: One of `"valid"` or `"same"` (case-insensitive).
+        padding: Currently only supports `"valid"` (case-insensitive).
+            `"same"` may be supported in the future.
         activation: Activation function to use
             (see [activations](../activations.md)).
             If you don't specify anything, no activation is applied
@@ -121,14 +122,14 @@ class LocallyConnected1D(Layer):
                              self.kernel_size[0] * input_dim,
                              self.filters)
         self.kernel = self.add_weight(
-            self.kernel_shape,
+            shape=self.kernel_shape,
             initializer=self.kernel_initializer,
             name='kernel',
             regularizer=self.kernel_regularizer,
             constraint=self.kernel_constraint)
         if self.use_bias:
             self.bias = self.add_weight(
-                (output_length, self.filters),
+                shape=(output_length, self.filters),
                 initializer=self.bias_initializer,
                 name='bias',
                 regularizer=self.bias_regularizer,
@@ -219,16 +220,15 @@ class LocallyConnected2D(Layer):
             specifying the strides of the convolution along the width and height.
             Can be a single integer to specify the same value for
             all spatial dimensions.
-            Specifying any stride value != 1 is incompatible with specifying
-            any `dilation_rate` value != 1.
-        padding: one of `"valid"` or `"same"` (case-insensitive).
+        padding: Currently only support `"valid"` (case-insensitive).
+            `"same"` will be supported in future.
         data_format: A string,
             one of `channels_last` (default) or `channels_first`.
             The ordering of the dimensions in the inputs.
             `channels_last` corresponds to inputs with shape
-            `(batch, width, height, channels)` while `channels_first`
+            `(batch, height, width, channels)` while `channels_first`
             corresponds to inputs with shape
-            `(batch, channels, width, height)`.
+            `(batch, channels, height, width)`.
             It defaults to the `image_data_format` value found in your
             Keras config file at `~/.keras/keras.json`.
             If you never set it, then it will be "channels_last".
@@ -325,13 +325,13 @@ class LocallyConnected2D(Layer):
         self.kernel_shape = (output_row * output_col,
                              self.kernel_size[0] * self.kernel_size[1] * input_filter,
                              self.filters)
-        self.kernel = self.add_weight(self.kernel_shape,
+        self.kernel = self.add_weight(shape=self.kernel_shape,
                                       initializer=self.kernel_initializer,
                                       name='kernel',
                                       regularizer=self.kernel_regularizer,
                                       constraint=self.kernel_constraint)
         if self.use_bias:
-            self.bias = self.add_weight((output_row, output_col, self.filters),
+            self.bias = self.add_weight(shape=(output_row, output_col, self.filters),
                                         initializer=self.bias_initializer,
                                         name='bias',
                                         regularizer=self.bias_regularizer,

keras/layers/merge.py

@@ -18,6 +18,44 @@ class _Merge(Layer):
     def _merge_function(self, inputs):
         raise NotImplementedError
 
+    def _compute_elemwise_op_output_shape(self, shape1, shape2):
+        """Computes the shape of the resultant of an elementwise operation.
+
+        # Arguments
+            shape1: tuple or None. Shape of the first tensor
+            shape2: tuple or None. Shape of the second tensor
+
+        # Returns
+            expected output shape when an element-wise operation is
+            carried out on 2 tensors with shapes shape1 and shape2.
+            tuple or None.
+
+        # Raises
+            ValueError: if shape1 and shape2 are not compatible for
+                element-wise operations.
+        """
+        if None in [shape1, shape2]:
+            return None
+        elif len(shape1) < len(shape2):
+            return self._compute_elemwise_op_output_shape(shape2, shape1)
+        elif len(shape2) == 0:
+            return shape1
+        output_shape = list(shape1[:-len(shape2)])
+        for i, j in zip(shape1[-len(shape2):], shape2):
+            if i is None or j is None:
+                output_shape.append(None)
+            elif i == 1:
+                output_shape.append(j)
+            elif j == 1:
+                output_shape.append(i)
+            else:
+                if i != j:
+                    raise ValueError('Operands could not be broadcast '
+                                     'together with shapes ' +
+                                     str(shape1) + ' ' + str(shape2))
+                output_shape.append(i)
+        return tuple(output_shape)
+
     def build(self, input_shape):
         # Used purely for shape validation.
         if not isinstance(input_shape, list):
@@ -27,19 +65,105 @@ class _Merge(Layer):
             raise ValueError('A merge layer should be called '
                              'on a list of at least 2 inputs. '
                              'Got ' + str(len(input_shape)) + ' inputs.')
-        if all([shape is None for shape in input_shape]):
-            return
-        # TODO: handle shapes with None entries.
-        input_shapes_set = set(input_shape)
-        if None in input_shapes_set:
-            input_shapes_set.remove(None)
-        if len(input_shapes_set) > 1:
-            raise ValueError('Only tensors of same shape can '
-                             'be merged by layer' + self.name +
-                             ' Got input shapes: %s' % input_shape)
+        batch_sizes = [s[0] for s in input_shape if s is not None]
+        batch_sizes = set(batch_sizes)
+        batch_sizes -= set([None])
+        if len(batch_sizes) > 1:
+            raise ValueError('Can not merge tensors with different '
+                             'batch sizes. Got tensors with shapes : ' +
+                             str(input_shape))
+        if input_shape[0] is None:
+            output_shape = None
+        else:
+            output_shape = input_shape[0][1:]
+        for i in range(1, len(input_shape)):
+            if input_shape[i] is None:
+                shape = None
+            else:
+                shape = input_shape[i][1:]
+            output_shape = self._compute_elemwise_op_output_shape(output_shape, shape)
+        # If the inputs have different ranks, we have to reshape them
+        # to make them broadcastable.
+        if None not in input_shape and len(set(map(len, input_shape))) == 1:
+            self._reshape_required = False
+        else:
+            self._reshape_required = True
 
     def call(self, inputs):
-        return self._merge_function(inputs)
+        if self._reshape_required:
+            reshaped_inputs = []
+            input_ndims = list(map(K.ndim, inputs))
+            if None not in input_ndims:
+                # If ranks of all inputs are available,
+                # we simply expand each of them at axis=1
+                # until all of them have the same rank.
+                max_ndim = max(input_ndims)
+                for x in inputs:
+                    x_ndim = K.ndim(x)
+                    for _ in range(max_ndim - x_ndim):
+                        x = K.expand_dims(x, 1)
+                    reshaped_inputs.append(x)
+                return self._merge_function(reshaped_inputs)
+            else:
+                # Transpose all inputs so that batch size is the last dimension.
+                # (batch_size, dim1, dim2, ... ) -> (dim1, dim2, ... , batch_size)
+                transposed = False
+                for x in inputs:
+                    x_ndim = K.ndim(x)
+                    if x_ndim is None:
+                        x_shape = K.shape(x)
+                        batch_size = x_shape[0]
+                        new_shape = K.concatenate([x_shape[1:], K.expand_dims(batch_size)])
+                        x_transposed = K.reshape(x, K.stack([batch_size, K.prod(x_shape[1:])]))
+                        x_transposed = K.permute_dimensions(x_transposed, (1, 0))
+                        x_transposed = K.reshape(x_transposed, new_shape)
+                        reshaped_inputs.append(x_transposed)
+                        transposed = True
+                    elif x_ndim > 1:
+                        dims = list(range(1, x_ndim)) + [0]
+                        reshaped_inputs.append(K.permute_dimensions(x, dims))
+                        transposed = True
+                    else:
+                        # We don't transpose inputs if they are 1D vectors or scalars.
+                        reshaped_inputs.append(x)
+                y = self._merge_function(reshaped_inputs)
+                y_ndim = K.ndim(y)
+                if transposed:
+                    # If inputs have been transposed, we have to transpose the output too.
+                    if y_ndim is None:
+                        y_shape = K.shape(y)
+                        y_ndim = K.shape(y_shape)[0]
+                        batch_size = y_shape[y_ndim - 1]
+                        new_shape = K.concatenate([K.expand_dims(batch_size), y_shape[:y_ndim - 1]])
+                        y = K.reshape(y, (-1, batch_size))
+                        y = K.permute_dimensions(y, (1, 0))
+                        y = K.reshape(y, new_shape)
+                    elif y_ndim > 1:
+                        dims = [y_ndim - 1] + list(range(y_ndim - 1))
+                        y = K.permute_dimensions(y, dims)
+                return y
+        else:
+            return self._merge_function(inputs)
+
+    def compute_output_shape(self, input_shape):
+        if input_shape[0] is None:
+            output_shape = None
+        else:
+            output_shape = input_shape[0][1:]
+        for i in range(1, len(input_shape)):
+            if input_shape[i] is None:
+                shape = None
+            else:
+                shape = input_shape[i][1:]
+            output_shape = self._compute_elemwise_op_output_shape(output_shape, shape)
+        batch_sizes = [s[0] for s in input_shape if s is not None]
+        batch_sizes = set(batch_sizes)
+        batch_sizes -= set([None])
+        if len(batch_sizes) == 1:
+            output_shape = (list(batch_sizes)[0],) + output_shape
+        else:
+            output_shape = (None,) + output_shape
+        return output_shape
 
     def compute_mask(self, inputs, mask=None):
         if mask is None:
@@ -190,8 +314,8 @@ class Concatenate(_Merge):
         for input_i, mask_i in zip(inputs, mask):
             if mask_i is None:
                 # Input is unmasked. Append all 1s to masks,
-                # but cast it to uint8 first
-                masks.append(K.cast(K.ones_like(input_i), 'uint8'))
+                # but cast it to bool first
+                masks.append(K.cast(K.ones_like(input_i), 'bool'))
             elif K.ndim(mask_i) < K.ndim(input_i):
                 # Mask is smaller than the input, expand it
                 masks.append(K.expand_dims(mask_i))

keras/layers/normalization.py

@@ -96,7 +96,7 @@ class BatchNormalization(Layer):
         shape = (dim,)
 
         if self.scale:
-            self.gamma = self.add_weight(shape,
+            self.gamma = self.add_weight(shape=shape,
                                          name='gamma',
                                          initializer=self.gamma_initializer,
                                          regularizer=self.gamma_regularizer,
@@ -104,7 +104,7 @@ class BatchNormalization(Layer):
         else:
             self.gamma = None
         if self.center:
-            self.beta = self.add_weight(shape,
+            self.beta = self.add_weight(shape=shape,
                                         name='beta',
                                         initializer=self.beta_initializer,
                                         regularizer=self.beta_regularizer,
@@ -112,12 +112,12 @@ class BatchNormalization(Layer):
         else:
             self.beta = None
         self.moving_mean = self.add_weight(
-            shape,
+            shape=shape,
             name='moving_mean',
             initializer=self.moving_mean_initializer,
             trainable=False)
         self.moving_variance = self.add_weight(
-            shape,
+            shape=shape,
             name='moving_variance',
             initializer=self.moving_variance_initializer,
             trainable=False)
@@ -133,57 +133,59 @@ class BatchNormalization(Layer):
         broadcast_shape[self.axis] = input_shape[self.axis]
 
         # Determines whether broadcasting is needed.
-        needs_broadcasting = (sorted(reduction_axes) != range(ndim)[:-1])
+        needs_broadcasting = (sorted(reduction_axes) != list(range(ndim))[:-1])
 
-        normed, mean, variance = K.normalize_batch_in_training(
+        def normalize_inference():
+            if needs_broadcasting:
+                # In this case we must explictly broadcast all parameters.
+                broadcast_moving_mean = K.reshape(self.moving_mean,
+                                                  broadcast_shape)
+                broadcast_moving_variance = K.reshape(self.moving_variance,
+                                                      broadcast_shape)
+                if self.center:
+                    broadcast_beta = K.reshape(self.beta, broadcast_shape)
+                else:
+                    broadcast_beta = None
+                if self.scale:
+                    broadcast_gamma = K.reshape(self.gamma,
+                                                broadcast_shape)
+                else:
+                    broadcast_gamma = None
+                return K.batch_normalization(
+                    inputs,
+                    broadcast_moving_mean,
+                    broadcast_moving_variance,
+                    broadcast_beta,
+                    broadcast_gamma,
+                    epsilon=self.epsilon)
+            else:
+                return K.batch_normalization(
+                    inputs,
+                    self.moving_mean,
+                    self.moving_variance,
+                    self.beta,
+                    self.gamma,
+                    epsilon=self.epsilon)
+
+        # If the learning phase is *static* and set to inference:
+        if training in {0, False}:
+            return normalize_inference()
+
+        # If the learning is either dynamic, or set to training:
+        normed_training, mean, variance = K.normalize_batch_in_training(
             inputs, self.gamma, self.beta, reduction_axes,
             epsilon=self.epsilon)
 
-        if training in {0, False}:
-            return normed
-        else:
-            self.add_update([K.moving_average_update(self.moving_mean,
-                                                     mean,
-                                                     self.momentum),
-                             K.moving_average_update(self.moving_variance,
-                                                     variance,
-                                                     self.momentum)],
-                            inputs)
-
-            def normalize_inference():
-                if needs_broadcasting:
-                    # In this case we must explictly broadcast all parameters.
-                    broadcast_moving_mean = K.reshape(self.moving_mean,
-                                                      broadcast_shape)
-                    broadcast_moving_variance = K.reshape(self.moving_variance,
-                                                          broadcast_shape)
-                    if self.center:
-                        broadcast_beta = K.reshape(self.beta, broadcast_shape)
-                    else:
-                        broadcast_beta = None
-                    if self.scale:
-                        broadcast_gamma = K.reshape(self.gamma,
-                                                    broadcast_shape)
-                    else:
-                        broadcast_gamma = None
-                    return K.batch_normalization(
-                        inputs,
-                        broadcast_moving_mean,
-                        broadcast_moving_variance,
-                        broadcast_beta,
-                        broadcast_gamma,
-                        epsilon=self.epsilon)
-                else:
-                    return K.batch_normalization(
-                        inputs,
-                        self.moving_mean,
-                        self.moving_variance,
-                        self.beta,
-                        self.gamma,
-                        epsilon=self.epsilon)
+        self.add_update([K.moving_average_update(self.moving_mean,
+                                                 mean,
+                                                 self.momentum),
+                         K.moving_average_update(self.moving_variance,
+                                                 variance,
+                                                 self.momentum)],
+                        inputs)
 
         # Pick the normalized form corresponding to the training phase.
-        return K.in_train_phase(normed,
+        return K.in_train_phase(normed_training,
                                 normalize_inference,
                                 training=training)
 

keras/layers/pooling.py

@@ -180,9 +180,9 @@ class MaxPooling2D(_Pooling2D):
             one of `channels_last` (default) or `channels_first`.
             The ordering of the dimensions in the inputs.
             `channels_last` corresponds to inputs with shape
-            `(batch, width, height, channels)` while `channels_first`
+            `(batch, height, width, channels)` while `channels_first`
             corresponds to inputs with shape
-            `(batch, channels, width, height)`.
+            `(batch, channels, height, width)`.
             It defaults to the `image_data_format` value found in your
             Keras config file at `~/.keras/keras.json`.
             If you never set it, then it will be "channels_last".
@@ -235,9 +235,9 @@ class AveragePooling2D(_Pooling2D):
             one of `channels_last` (default) or `channels_first`.
             The ordering of the dimensions in the inputs.
             `channels_last` corresponds to inputs with shape
-            `(batch, width, height, channels)` while `channels_first`
+            `(batch, height, width, channels)` while `channels_first`
             corresponds to inputs with shape
-            `(batch, channels, width, height)`.
+            `(batch, channels, height, width)`.
             It defaults to the `image_data_format` value found in your
             Keras config file at `~/.keras/keras.json`.
             If you never set it, then it will be "channels_last".
@@ -511,9 +511,9 @@ class GlobalAveragePooling2D(_GlobalPooling2D):
             one of `channels_last` (default) or `channels_first`.
             The ordering of the dimensions in the inputs.
             `channels_last` corresponds to inputs with shape
-            `(batch, width, height, channels)` while `channels_first`
+            `(batch, height, width, channels)` while `channels_first`
             corresponds to inputs with shape
-            `(batch, channels, width, height)`.
+            `(batch, channels, height, width)`.
             It defaults to the `image_data_format` value found in your
             Keras config file at `~/.keras/keras.json`.
             If you never set it, then it will be "channels_last".
@@ -546,9 +546,9 @@ class GlobalMaxPooling2D(_GlobalPooling2D):
             one of `channels_last` (default) or `channels_first`.
             The ordering of the dimensions in the inputs.
             `channels_last` corresponds to inputs with shape
-            `(batch, width, height, channels)` while `channels_first`
+            `(batch, height, width, channels)` while `channels_first`
             corresponds to inputs with shape
-            `(batch, channels, width, height)`.
+            `(batch, channels, height, width)`.
             It defaults to the `image_data_format` value found in your
             Keras config file at `~/.keras/keras.json`.
             If you never set it, then it will be "channels_last".

keras/layers/recurrent.py

@@ -78,12 +78,16 @@ class Recurrent(Layer):
         # now model.output_shape == (None, 32)
         # note: `None` is the batch dimension.
 
-        # the following is identical:
-        model = Sequential()
-        model.add(LSTM(32, input_dim=64, input_length=10))
-
-        # for subsequent layers, not need to specify the input size:
+        # for subsequent layers, no need to specify the input size:
         model.add(LSTM(16))
+
+        # to stack recurrent layers, you must use return_sequences=True
+        # on any recurrent layer that feeds into another recurrent layer.
+        # note that you only need to specify the input size on the first layer.
+        model = Sequential()
+        model.add(LSTM(64, input_dim=64, input_length=10, return_sequences=True))
+        model.add(LSTM(32, return_sequences=True))
+        model.add(LSTM(10))
     ```
 
     # Arguments
@@ -93,7 +97,8 @@ class Recurrent(Layer):
         return_sequences: Boolean. Whether to return the last output
             in the output sequence, or the full sequence.
         go_backwards: Boolean (default False).
-            If True, process the input sequence backwards.
+            If True, process the input sequence backwards and return the
+            reversed sequence.
         stateful: Boolean (default False). If True, the last state
             for each sample at index i in a batch will be used as initial
             state for the sample of index i in the following batch.
@@ -165,11 +170,16 @@ class Recurrent(Layer):
         To reset the states of your model, call `.reset_states()` on either
         a specific layer, or on your entire model.
 
-    # Note on specifying initial states in RNNs
-        You can specify the initial state of RNN layers by calling them with
-        the keyword argument `initial_state`. The value of `initial_state`
-        should be a tensor or list of tensors representing the initial state
-        of the RNN layer.
+    # Note on specifying the initial state of RNNs
+        You can specify the initial state of RNN layers symbolically by
+        calling them with the keyword argument `initial_state`. The value of
+        `initial_state` should be a tensor or list of tensors representing
+        the initial state of the RNN layer.
+
+        You can specify the initial state of RNN layers numerically by
+        calling `reset_states` with the keyword argument `states`. The value of
+        `states` should be a numpy array or list of numpy arrays representing
+        the initial state of the RNN layer.
     """
 
     def __init__(self, return_sequences=False,
@@ -185,7 +195,7 @@ class Recurrent(Layer):
         self.unroll = unroll
         self.implementation = implementation
         self.supports_masking = True
-        self.input_spec = InputSpec(ndim=3)
+        self.input_spec = [InputSpec(ndim=3)]
         self.state_spec = None
         self.dropout = 0
         self.recurrent_dropout = 0
@@ -200,6 +210,8 @@ class Recurrent(Layer):
 
     def compute_mask(self, inputs, mask):
         if self.return_sequences:
+            if isinstance(mask, list):
+                return mask[0]
             return mask
         else:
             return None
@@ -210,14 +222,14 @@ class Recurrent(Layer):
     def get_constants(self, inputs, training=None):
         return []
 
-    def get_initial_states(self, inputs):
+    def get_initial_state(self, inputs):
         # build an all-zero tensor of shape (samples, output_dim)
         initial_state = K.zeros_like(inputs)  # (samples, timesteps, input_dim)
         initial_state = K.sum(initial_state, axis=(1, 2))  # (samples,)
         initial_state = K.expand_dims(initial_state)  # (samples, 1)
         initial_state = K.tile(initial_state, [1, self.units])  # (samples, output_dim)
-        initial_states = [initial_state for _ in range(len(self.states))]
-        return initial_states
+        initial_state = [initial_state for _ in range(len(self.states))]
+        return initial_state
 
     def preprocess_input(self, inputs, training=None):
         return inputs
@@ -227,51 +239,61 @@ class Recurrent(Layer):
         # and if it a Keras tensor,
         # then add it to the inputs and temporarily
         # modify the input spec to include the state.
-        if initial_state is not None:
-            if hasattr(initial_state, '_keras_history'):
-                # Compute the full input spec, including state
-                input_spec = self.input_spec
-                state_spec = self.state_spec
-                if not isinstance(state_spec, list):
-                    state_spec = [state_spec]
-                self.input_spec = [input_spec] + state_spec
+        if initial_state is None:
+            return super(Recurrent, self).__call__(inputs, **kwargs)
 
-                # Compute the full inputs, including state
-                if not isinstance(initial_state, (list, tuple)):
-                    initial_state = [initial_state]
-                inputs = [inputs] + list(initial_state)
+        if not isinstance(initial_state, (list, tuple)):
+            initial_state = [initial_state]
 
-                # Perform the call
-                output = super(Recurrent, self).__call__(inputs, **kwargs)
+        is_keras_tensor = hasattr(initial_state[0], '_keras_history')
+        for tensor in initial_state:
+            if hasattr(tensor, '_keras_history') != is_keras_tensor:
+                raise ValueError('The initial state of an RNN layer cannot be'
+                                 ' specified with a mix of Keras tensors and'
+                                 ' non-Keras tensors')
 
-                # Restore original input spec
-                self.input_spec = input_spec
-                return output
-            else:
-                kwargs['initial_state'] = initial_state
-        return super(Recurrent, self).__call__(inputs, **kwargs)
+        if is_keras_tensor:
+            # Compute the full input spec, including state
+            input_spec = self.input_spec
+            state_spec = self.state_spec
+            if not isinstance(state_spec, list):
+                state_spec = [state_spec]
+            self.input_spec = input_spec + state_spec
 
-    def call(self, inputs, mask=None, initial_state=None, training=None):
+            # Compute the full inputs, including state
+            inputs = [inputs] + list(initial_state)
+
+            # Perform the call
+            output = super(Recurrent, self).__call__(inputs, **kwargs)
+
+            # Restore original input spec
+            self.input_spec = input_spec
+            return output
+        else:
+            kwargs['initial_state'] = initial_state
+            return super(Recurrent, self).__call__(inputs, **kwargs)
+
+    def call(self, inputs, mask=None, training=None, initial_state=None):
         # input shape: `(samples, time (padded with zeros), input_dim)`
         # note that the .build() method of subclasses MUST define
         # self.input_spec and self.state_spec with complete input shapes.
-        if initial_state is not None:
-            if not isinstance(initial_state, (list, tuple)):
-                initial_states = [initial_state]
-            else:
-                initial_states = list(initial_state)
         if isinstance(inputs, list):
-            initial_states = inputs[1:]
+            initial_state = inputs[1:]
             inputs = inputs[0]
+        elif initial_state is not None:
+            pass
         elif self.stateful:
-            initial_states = self.states
+            initial_state = self.states
         else:
-            initial_states = self.get_initial_states(inputs)
+            initial_state = self.get_initial_state(inputs)
 
-        if len(initial_states) != len(self.states):
+        if isinstance(mask, list):
+            mask = mask[0]
+
+        if len(initial_state) != len(self.states):
             raise ValueError('Layer has ' + str(len(self.states)) +
                              ' states but was passed ' +
-                             str(len(initial_states)) +
+                             str(len(initial_state)) +
                              ' initial states.')
         input_shape = K.int_shape(inputs)
         if self.unroll and input_shape[1] is None:
@@ -290,7 +312,7 @@ class Recurrent(Layer):
         preprocessed_input = self.preprocess_input(inputs, training=None)
         last_output, outputs, states = K.rnn(self.step,
                                              preprocessed_input,
-                                             initial_states,
+                                             initial_state,
                                              go_backwards=self.go_backwards,
                                              mask=mask,
                                              constants=constants,
@@ -312,13 +334,10 @@ class Recurrent(Layer):
         else:
             return last_output
 
-    def reset_states(self, states_value=None):
+    def reset_states(self, states=None):
         if not self.stateful:
             raise AttributeError('Layer must be stateful.')
-        if not self.input_spec:
-            raise RuntimeError('Layer has never been called '
-                               'and thus has no states.')
-        batch_size = self.input_spec.shape[0]
+        batch_size = self.input_spec[0].shape[0]
         if not batch_size:
             raise ValueError('If a RNN is stateful, it needs to know '
                              'its batch size. Specify the batch size '
@@ -330,31 +349,30 @@ class Recurrent(Layer):
                              '- If using the functional API, specify '
                              'the time dimension by passing a '
                              '`batch_shape` argument to your Input layer.')
-        if states_value is not None:
-            if not isinstance(states_value, (list, tuple)):
-                states_value = [states_value]
-            if len(states_value) != len(self.states):
-                raise ValueError('The layer has ' + str(len(self.states)) +
-                                 ' states, but the `states_value` '
-                                 'argument passed '
-                                 'only has ' + str(len(states_value)) +
-                                 ' entries')
+        # initialize state if None
         if self.states[0] is None:
             self.states = [K.zeros((batch_size, self.units))
                            for _ in self.states]
-            if not states_value:
-                return
-        for i, state in enumerate(self.states):
-            if states_value:
-                value = states_value[i]
+        elif states is None:
+            for state in self.states:
+                K.set_value(state, np.zeros((batch_size, self.units)))
+        else:
+            if not isinstance(states, (list, tuple)):
+                states = [states]
+            if len(states) != len(self.states):
+                raise ValueError('Layer ' + self.name + ' expects ' +
+                                 str(len(self.states)) + ' states, '
+                                 'but it received ' + str(len(states)) +
+                                 ' state values. Input received: ' +
+                                 str(states))
+            for index, (value, state) in enumerate(zip(states, self.states)):
                 if value.shape != (batch_size, self.units):
-                    raise ValueError(
-                        'Expected state #' + str(i) +
-                        ' to have shape ' + str((batch_size, self.units)) +
-                        ' but got array with shape ' + str(value.shape))
-            else:
-                value = np.zeros((batch_size, self.units))
-            K.set_value(state, value)
+                    raise ValueError('State ' + str(index) +
+                                     ' is incompatible with layer ' +
+                                     self.name + ': expected shape=' +
+                                     str((batch_size, self.units)) +
+                                     ', found shape=' + str(value.shape))
+                K.set_value(state, value)
 
     def get_config(self):
         config = {'return_sequences': self.return_sequences,
@@ -373,7 +391,7 @@ class SimpleRNN(Recurrent):
         units: Positive integer, dimensionality of the output space.
         activation: Activation function to use
             (see [activations](../activations.md)).
-            If you don't specify anything, no activation is applied
+            If you pass None, no activation is applied
             (ie. "linear" activation: `a(x) = x`).
         use_bias: Boolean, whether the layer uses a bias vector.
         kernel_initializer: Initializer for the `kernel` weights matrix,
@@ -452,6 +470,7 @@ class SimpleRNN(Recurrent):
 
         self.dropout = min(1., max(0., dropout))
         self.recurrent_dropout = min(1., max(0., recurrent_dropout))
+        self.state_spec = InputSpec(shape=(None, self.units))
 
     def build(self, input_shape):
         if isinstance(input_shape, list):
@@ -459,26 +478,25 @@ class SimpleRNN(Recurrent):
 
         batch_size = input_shape[0] if self.stateful else None
         self.input_dim = input_shape[2]
-        self.input_spec = InputSpec(shape=(batch_size, None, self.input_dim))
-        self.state_spec = InputSpec(shape=(batch_size, self.units))
+        self.input_spec[0] = InputSpec(shape=(batch_size, None, self.input_dim))
 
         self.states = [None]
         if self.stateful:
             self.reset_states()
 
-        self.kernel = self.add_weight((self.input_dim, self.units),
+        self.kernel = self.add_weight(shape=(self.input_dim, self.units),
                                       name='kernel',
                                       initializer=self.kernel_initializer,
                                       regularizer=self.kernel_regularizer,
                                       constraint=self.kernel_constraint)
         self.recurrent_kernel = self.add_weight(
-            (self.units, self.units),
+            shape=(self.units, self.units),
             name='recurrent_kernel',
             initializer=self.recurrent_initializer,
             regularizer=self.recurrent_regularizer,
             constraint=self.recurrent_constraint)
         if self.use_bias:
-            self.bias = self.add_weight((self.units,),
+            self.bias = self.add_weight(shape=(self.units,),
                                         name='bias',
                                         initializer=self.bias_initializer,
                                         regularizer=self.bias_regularizer,
@@ -528,7 +546,7 @@ class SimpleRNN(Recurrent):
 
     def get_constants(self, inputs, training=None):
         constants = []
-        if self.implementation == 0 and 0 < self.dropout < 1:
+        if self.implementation != 0 and 0 < self.dropout < 1:
             input_shape = K.int_shape(inputs)
             input_dim = input_shape[-1]
             ones = K.ones_like(K.reshape(inputs[:, 0, 0], (-1, 1)))
@@ -585,7 +603,7 @@ class GRU(Recurrent):
         units: Positive integer, dimensionality of the output space.
         activation: Activation function to use
             (see [activations](../activations.md)).
-            If you don't specify anything, no activation is applied
+            If you pass None, no activation is applied
             (ie. "linear" activation: `a(x) = x`).
         recurrent_activation: Activation function to use
             for the recurrent step
@@ -671,6 +689,7 @@ class GRU(Recurrent):
 
         self.dropout = min(1., max(0., dropout))
         self.recurrent_dropout = min(1., max(0., recurrent_dropout))
+        self.state_spec = InputSpec(shape=(None, self.units))
 
     def build(self, input_shape):
         if isinstance(input_shape, list):
@@ -678,29 +697,28 @@ class GRU(Recurrent):
 
         batch_size = input_shape[0] if self.stateful else None
         self.input_dim = input_shape[2]
-        self.input_spec = InputSpec(shape=(batch_size, None, self.input_dim))
-        self.state_spec = InputSpec(shape=(batch_size, self.units))
+        self.input_spec[0] = InputSpec(shape=(batch_size, None, self.input_dim))
 
         self.states = [None]
         if self.stateful:
             self.reset_states()
 
-        self.kernel = self.add_weight((self.input_dim, self.units * 3),
+        self.kernel = self.add_weight(shape=(self.input_dim, self.units * 3),
                                       name='kernel',
                                       initializer=self.kernel_initializer,
                                       regularizer=self.kernel_regularizer,
                                       constraint=self.kernel_constraint)
         self.recurrent_kernel = self.add_weight(
-            (self.units, self.units * 3),
+            shape=(self.units, self.units * 3),
             name='recurrent_kernel',
             initializer=self.recurrent_initializer,
             regularizer=self.recurrent_regularizer,
             constraint=self.recurrent_constraint)
 
         if self.use_bias:
-            self.bias = self.add_weight((self.units * 3,),
+            self.bias = self.add_weight(shape=(self.units * 3,),
                                         name='bias',
-                                        initializer='zero',
+                                        initializer=self.bias_initializer,
                                         regularizer=self.bias_regularizer,
                                         constraint=self.bias_constraint)
         else:
@@ -746,7 +764,7 @@ class GRU(Recurrent):
 
     def get_constants(self, inputs, training=None):
         constants = []
-        if self.implementation == 0 and 0 < self.dropout < 1:
+        if self.implementation != 0 and 0 < self.dropout < 1:
             input_shape = K.int_shape(inputs)
             input_dim = input_shape[-1]
             ones = K.ones_like(K.reshape(inputs[:, 0, 0], (-1, 1)))
@@ -812,7 +830,7 @@ class GRU(Recurrent):
                 if self.use_bias:
                     x_z = K.bias_add(x_z, self.bias_z)
                     x_r = K.bias_add(x_r, self.bias_r)
-                    x_h = K.bias_add(x_r, self.bias_h)
+                    x_h = K.bias_add(x_h, self.bias_h)
             else:
                 raise ValueError('Unknown `implementation` mode.')
             z = self.recurrent_activation(x_z + K.dot(h_tm1 * rec_dp_mask[0],
@@ -858,7 +876,7 @@ class LSTM(Recurrent):
         units: Positive integer, dimensionality of the output space.
         activation: Activation function to use
             (see [activations](../activations.md)).
-            If you don't specify anything, no activation is applied
+            If you pass None, no activation is applied
             (ie. "linear" activation: `a(x) = x`).
         recurrent_activation: Activation function to use
             for the recurrent step
@@ -950,6 +968,8 @@ class LSTM(Recurrent):
 
         self.dropout = min(1., max(0., dropout))
         self.recurrent_dropout = min(1., max(0., recurrent_dropout))
+        self.state_spec = [InputSpec(shape=(None, self.units)),
+                           InputSpec(shape=(None, self.units))]
 
     def build(self, input_shape):
         if isinstance(input_shape, list):
@@ -957,36 +977,39 @@ class LSTM(Recurrent):
 
         batch_size = input_shape[0] if self.stateful else None
         self.input_dim = input_shape[2]
-        self.input_spec = InputSpec(shape=(batch_size, None, self.input_dim))
-        self.state_spec = [InputSpec(shape=(batch_size, self.units)),
-                           InputSpec(shape=(batch_size, self.units))]
+        self.input_spec[0] = InputSpec(shape=(batch_size, None, self.input_dim))
 
         self.states = [None, None]
         if self.stateful:
             self.reset_states()
 
-        self.kernel = self.add_weight((self.input_dim, self.units * 4),
+        self.kernel = self.add_weight(shape=(self.input_dim, self.units * 4),
                                       name='kernel',
                                       initializer=self.kernel_initializer,
                                       regularizer=self.kernel_regularizer,
                                       constraint=self.kernel_constraint)
         self.recurrent_kernel = self.add_weight(
-            (self.units, self.units * 4),
+            shape=(self.units, self.units * 4),
             name='recurrent_kernel',
             initializer=self.recurrent_initializer,
             regularizer=self.recurrent_regularizer,
             constraint=self.recurrent_constraint)
 
         if self.use_bias:
-            self.bias = self.add_weight((self.units * 4,),
+            if self.unit_forget_bias:
+                def bias_initializer(shape, *args, **kwargs):
+                    return K.concatenate([
+                        self.bias_initializer((self.units,), *args, **kwargs),
+                        initializers.Ones()((self.units,), *args, **kwargs),
+                        self.bias_initializer((self.units * 2,), *args, **kwargs),
+                    ])
+            else:
+                bias_initializer = self.bias_initializer
+            self.bias = self.add_weight(shape=(self.units * 4,),
                                         name='bias',
-                                        initializer=self.bias_initializer,
+                                        initializer=bias_initializer,
                                         regularizer=self.bias_regularizer,
                                         constraint=self.bias_constraint)
-            if self.unit_forget_bias:
-                bias_value = np.zeros((self.units * 4,))
-                bias_value[self.units: self.units * 2] = 1.
-                K.set_value(self.bias, bias_value)
         else:
             self.bias = None
 
@@ -1036,7 +1059,7 @@ class LSTM(Recurrent):
 
     def get_constants(self, inputs, training=None):
         constants = []
-        if self.implementation == 0 and 0 < self.dropout < 1:
+        if self.implementation != 0 and 0 < self.dropout < 1:
             input_shape = K.int_shape(inputs)
             input_dim = input_shape[-1]
             ones = K.ones_like(K.reshape(inputs[:, 0, 0], (-1, 1)))

keras/layers/wrappers.py

@@ -2,6 +2,7 @@
 from __future__ import absolute_import
 
 import copy
+import inspect
 from ..engine import Layer
 from ..engine import InputSpec
 from .. import backend as K
@@ -23,18 +24,53 @@ class Wrapper(Layer):
         super(Wrapper, self).__init__(**kwargs)
 
     def build(self, input_shape=None):
-        # Assumes that self.layer is already set.
-        # Should be called at the end of .build() in the children classes.
-        self.trainable_weights = getattr(self.layer, 'trainable_weights', [])
-        self.non_trainable_weights = getattr(self.layer, 'non_trainable_weights', [])
-        self.updates = getattr(self.layer, 'updates', [])
-        self.losses = getattr(self.layer, 'losses', [])
-        self.constraints = getattr(self.layer, 'constraints', {})
         self.built = True
 
+    @property
+    def activity_regularizer(self):
+        if hasattr(self.layer, 'activity_regularizer'):
+            return self.layer.activity_regularizer
+        else:
+            return None
+
+    @property
+    def trainable_weights(self):
+        return self.layer.trainable_weights
+
+    @property
+    def non_trainable_weights(self):
+        return self.layer.non_trainable_weights
+
+    @property
+    def updates(self):
+        if hasattr(self.layer, 'updates'):
+            return self.layer.updates
+        return []
+
+    def get_updates_for(self, inputs=None):
+        if inputs is None:
+            updates = self.layer.get_updates_for(None)
+            return updates + super(Wrapper, self).get_updates_for(None)
+        return super(Wrapper, self).get_updates_for(inputs)
+
+    @property
+    def losses(self):
+        if hasattr(self.layer, 'losses'):
+            return self.layer.losses
+        return []
+
+    def get_losses_for(self, inputs=None):
+        if inputs is None:
+            losses = self.layer.get_losses_for(None)
+            return losses + super(Wrapper, self).get_losses_for(None)
+        return super(Wrapper, self).get_losses_for(inputs)
+
+    @property
+    def constraints(self):
+        return self.layer.constraints
+
     def get_weights(self):
-        weights = self.layer.get_weights()
-        return weights
+        return self.layer.get_weights()
 
     def set_weights(self, weights):
         self.layer.set_weights(weights)
@@ -46,9 +82,9 @@ class Wrapper(Layer):
         return dict(list(base_config.items()) + list(config.items()))
 
     @classmethod
-    def from_config(cls, config):
+    def from_config(cls, config, custom_objects=None):
         from . import deserialize as deserialize_layer
-        layer = deserialize_layer(config.pop('layer'))
+        layer = deserialize_layer(config.pop('layer'), custom_objects=custom_objects)
         return cls(layer, **config)
 
 
@@ -71,13 +107,18 @@ class TimeDistributed(Wrapper):
         model = Sequential()
         model.add(TimeDistributed(Dense(8), input_shape=(10, 16)))
         # now model.output_shape == (None, 10, 8)
+    ```
 
-        # subsequent layers: no need for input_shape
+    The output will then have shape `(32, 10, 8)`.
+
+    In subsequent layers, there is no need for the `input_shape`:
+
+    ```python
         model.add(TimeDistributed(Dense(32)))
         # now model.output_shape == (None, 10, 32)
     ```
 
-    The output will then have shape `(32, 10, 8)`.
+    The output will then have shape `(32, 10, 32)`.
 
     `TimeDistributed` can be used with arbitrary layers, not just `Dense`,
     for instance with a `Conv2D` layer:
@@ -157,6 +198,9 @@ class Bidirectional(Wrapper):
             If None, the outputs will not be combined,
             they will be returned as a list.
 
+    # Raises
+        ValueError: In case of invalid `merge_mode` argument.
+
     # Examples
 
     ```python
@@ -208,29 +252,47 @@ class Bidirectional(Wrapper):
         elif self.merge_mode is None:
             return [self.forward_layer.compute_output_shape(input_shape)] * 2
 
-    def call(self, inputs, mask=None):
-        y = self.forward_layer.call(inputs, mask)
-        y_rev = self.backward_layer.call(inputs, mask)
+    def call(self, inputs, training=None, mask=None):
+        kwargs = {}
+        func_args = inspect.getargspec(self.layer.call).args
+        if 'training' in func_args:
+            kwargs['training'] = training
+        if 'mask' in func_args:
+            kwargs['mask'] = mask
+
+        y = self.forward_layer.call(inputs, **kwargs)
+        y_rev = self.backward_layer.call(inputs, **kwargs)
         if self.return_sequences:
             y_rev = K.reverse(y_rev, 1)
         if self.merge_mode == 'concat':
-            return K.concatenate([y, y_rev])
+            output = K.concatenate([y, y_rev])
         elif self.merge_mode == 'sum':
-            return y + y_rev
+            output = y + y_rev
         elif self.merge_mode == 'ave':
-            return (y + y_rev) / 2
+            output = (y + y_rev) / 2
         elif self.merge_mode == 'mul':
-            return y * y_rev
+            output = y * y_rev
         elif self.merge_mode is None:
-            return [y, y_rev]
+            output = [y, y_rev]
+
+        # Properly set learning phase
+        if 0 < self.layer.dropout + self.layer.recurrent_dropout:
+            if self.merge_mode is None:
+                for out in output:
+                    out._uses_learning_phase = True
+            else:
+                output._uses_learning_phase = True
+        return output
 
     def reset_states(self):
         self.forward_layer.reset_states()
         self.backward_layer.reset_states()
 
     def build(self, input_shape):
-        self.forward_layer.build(input_shape)
-        self.backward_layer.build(input_shape)
+        with K.name_scope(self.forward_layer.name):
+            self.forward_layer.build(input_shape)
+        with K.name_scope(self.backward_layer.name):
+            self.backward_layer.build(input_shape)
         self.built = True
 
     def compute_mask(self, inputs, mask):

keras/legacy/interfaces.py

@@ -3,6 +3,7 @@
 import six
 import warnings
 import functools
+import inspect
 import numpy as np
 
 
@@ -83,8 +84,9 @@ def generate_legacy_interface(allowed_positional_args=None,
                         signature += ', '
                 signature += ')`'
                 warnings.warn('Update your `' + object_name +
-                              '` call to the Keras 2 API: ' + signature)
+                              '` call to the Keras 2 API: ' + signature, stacklevel=2)
             return func(*args, **kwargs)
+        wrapper._legacy_support_signature = inspect.getargspec(func)
         return wrapper
     return legacy_support
 
@@ -122,7 +124,7 @@ def embedding_kwargs_preprocessor(args, kwargs):
         kwargs.pop('dropout')
         warnings.warn('The `dropout` argument is no longer support in `Embedding`. '
                       'You can apply a `keras.layers.SpatialDropout1D` layer '
-                      'right after the `Embedding` layer to get the same behavior.')
+                      'right after the `Embedding` layer to get the same behavior.', stacklevel=3)
     return args, kwargs, converted
 
 legacy_embedding_support = generate_legacy_interface(
@@ -159,7 +161,7 @@ def recurrent_args_preprocessor(args, kwargs):
             kwargs.pop('forget_bias_init')
             warnings.warn('The `forget_bias_init` argument '
                           'has been ignored. Use `unit_forget_bias=True` '
-                          'instead to intialize with ones.')
+                          'instead to intialize with ones.', stacklevel=3)
     if 'input_dim' in kwargs:
         input_length = kwargs.pop('input_length', None)
         input_dim = kwargs.pop('input_dim')
@@ -168,7 +170,7 @@ def recurrent_args_preprocessor(args, kwargs):
         converted.append(('input_dim', 'input_shape'))
         warnings.warn('The `input_dim` and `input_length` arguments '
                       'in recurrent layers are deprecated. '
-                      'Use `input_shape` instead.')
+                      'Use `input_shape` instead.', stacklevel=3)
     return args, kwargs, converted
 
 legacy_recurrent_support = generate_legacy_interface(
@@ -459,7 +461,7 @@ def convlstm2d_args_preprocessor(args, kwargs):
         else:
             warnings.warn('The `forget_bias_init` argument '
                           'has been ignored. Use `unit_forget_bias=True` '
-                          'instead to intialize with ones.')
+                          'instead to intialize with ones.', stacklevel=3)
     args, kwargs, _converted = conv2d_args_preprocessor(args, kwargs)
     return args, kwargs, converted + _converted
 
@@ -502,7 +504,7 @@ def zeropadding2d_args_preprocessor(args, kwargs):
             kwargs['padding'] = ((top_pad, bottom_pad), (left_pad, right_pad))
             warnings.warn('The `padding` argument in the Keras 2 API no longer'
                           'accepts dict types. You can now input argument as: '
-                          '`padding=(top_pad, bottom_pad, left_pad, right_pad)`.')
+                          '`padding=(top_pad, bottom_pad, left_pad, right_pad)`.', stacklevel=3)
     elif len(args) == 2 and isinstance(args[1], dict):
         if set(args[1].keys()) <= {'top_pad', 'bottom_pad',
                                    'left_pad', 'right_pad'}:
@@ -513,7 +515,7 @@ def zeropadding2d_args_preprocessor(args, kwargs):
             args = (args[0], ((top_pad, bottom_pad), (left_pad, right_pad)))
             warnings.warn('The `padding` argument in the Keras 2 API no longer'
                           'accepts dict types. You can now input argument as: '
-                          '`padding=((top_pad, bottom_pad), (left_pad, right_pad))`')
+                          '`padding=((top_pad, bottom_pad), (left_pad, right_pad))`', stacklevel=3)
     return args, kwargs, converted
 
 legacy_zeropadding2d_support = generate_legacy_interface(
@@ -580,7 +582,7 @@ def generator_methods_args_preprocessor(args, kwargs):
                               'Keras 1 argument `samples_per_epoch`. '
                               '`steps_per_epoch` is the number of batches '
                               'to draw from the generator at each epoch. '
-                              'Update your method calls accordingly.')
+                              'Update your method calls accordingly.', stacklevel=3)
                 kwargs['steps_per_epoch'] = samples_per_epoch
             converted.append(('samples_per_epoch', 'steps_per_epoch'))
     return args, kwargs, converted
@@ -600,3 +602,24 @@ legacy_model_constructor_support = generate_legacy_interface(
     allowed_positional_args=None,
     conversions=[('input', 'inputs'),
                  ('output', 'outputs')])
+
+legacy_input_support = generate_legacy_interface(
+    allowed_positional_args=None,
+    conversions=[('input_dtype', 'dtype')])
+
+
+def add_weight_args_preprocessing(args, kwargs):
+    if len(args) > 1:
+        if isinstance(args[1], (tuple, list)):
+            kwargs['shape'] = args[1]
+            args = (args[0],) + args[2:]
+            if len(args) > 1:
+                if isinstance(args[1], six.string_types):
+                    kwargs['name'] = args[1]
+                    args = (args[0],) + args[2:]
+    return args, kwargs, []
+
+
+legacy_add_weight_support = generate_legacy_interface(
+    allowed_positional_args=['name', 'shape'],
+    preprocessor=add_weight_args_preprocessing)

keras/legacy/layers.py

@@ -46,7 +46,7 @@ class Merge(Layer):
             (1:1 mapping to input tensors)
             and return a single shape tuple, including the
             batch size (same convention as the
-            `get_output_shape_for` method of layers).
+            `compute_output_shape` method of layers).
         node_indices: Optional list of integers containing
             the output node index for each input layer
             (in case some input layers have multiple output nodes).
@@ -76,6 +76,10 @@ class Merge(Layer):
         self._output_mask = output_mask
         self.arguments = arguments if arguments else {}
         self._initial_weights = None
+        self._updates = []
+        self._losses = []
+        self._per_input_updates = {}
+        self._per_input_losses = {}
 
         # Layer parameters.
         self.inbound_nodes = []
@@ -297,8 +301,8 @@ class Merge(Layer):
             for input_i, mask_i in zip(inputs, mask):
                 if mask_i is None:
                     # Input is unmasked. Append all 1s to masks,
-                    # but cast it to uint8 first
-                    masks.append(K.cast(K.ones_like(input_i), 'uint8'))
+                    # but cast it to bool first
+                    masks.append(K.cast(K.ones_like(input_i), 'bool'))
                 elif K.ndim(mask_i) < K.ndim(input_i):
                     # Mask is smaller than the input, expand it
                     masks.append(K.expand_dims(mask_i))
@@ -418,7 +422,7 @@ def merge(inputs, mode='sum', concat_axis=-1,
             If the latter case, it should take as input a list of shape tuples
             (1:1 mapping to input tensors) and return a single shape tuple,
             including the batch size
-            (same convention as the `get_output_shape_for` method of layers).
+            (same convention as the `compute_output_shape` method of layers).
         node_indices: Optional list of integers containing
             the output node index for each input layer
             (in case some input layers have multiple output nodes).
@@ -430,7 +434,7 @@ def merge(inputs, mode='sum', concat_axis=-1,
     warnings.warn('The `merge` function is deprecated '
                   'and will be removed after 08/2017. '
                   'Use instead layers from `keras.layers.merge`, '
-                  'e.g. `sum`, `concatenate`, etc.', stacklevel=2)
+                  'e.g. `add`, `concatenate`, etc.', stacklevel=2)
     all_keras_tensors = True
     for x in inputs:
         if not hasattr(x, '_keras_history'):

keras/losses.py

@@ -33,6 +33,12 @@ def hinge(y_true, y_pred):
     return K.mean(K.maximum(1. - y_true * y_pred, 0.), axis=-1)
 
 
+def logcosh(y_true, y_pred):
+    def cosh(x):
+        return (K.exp(x) + K.exp(-x)) / 2
+    return K.mean(K.log(cosh(y_pred - y_true)), axis=-1)
+
+
 def categorical_crossentropy(y_true, y_pred):
     return K.categorical_crossentropy(y_pred, y_true)
 

keras/metrics.py

@@ -6,6 +6,7 @@ from .losses import mean_absolute_error
 from .losses import mean_absolute_percentage_error
 from .losses import mean_squared_logarithmic_error
 from .losses import hinge
+from .losses import logcosh
 from .losses import squared_hinge
 from .losses import categorical_crossentropy
 from .losses import sparse_categorical_crossentropy
@@ -21,13 +22,15 @@ def binary_accuracy(y_true, y_pred):
 
 
 def categorical_accuracy(y_true, y_pred):
-    return K.equal(K.argmax(y_true, axis=-1),
-                   K.argmax(y_pred, axis=-1))
+    return K.cast(K.equal(K.argmax(y_true, axis=-1),
+                          K.argmax(y_pred, axis=-1)),
+                  K.floatx())
 
 
 def sparse_categorical_accuracy(y_true, y_pred):
-    return K.equal(K.max(y_true, axis=-1),
-                   K.cast(K.argmax(y_pred, axis=-1), K.floatx()))
+    return K.cast(K.equal(K.max(y_true, axis=-1),
+                          K.cast(K.argmax(y_pred, axis=-1), K.floatx())),
+                  K.floatx())
 
 
 def top_k_categorical_accuracy(y_true, y_pred, k=5):

keras/models.py

@@ -27,7 +27,7 @@ except ImportError:
     h5py = None
 
 
-def save_model(model, filepath, overwrite=True):
+def save_model(model, filepath, overwrite=True, include_optimizer=True):
     """Save a model to a HDF5 file.
 
     The saved model contains:
@@ -45,6 +45,7 @@ def save_model(model, filepath, overwrite=True):
         overwrite: Whether we should overwrite any existing
             model at the target location, or instead
             ask the user with a manual prompt.
+        include_optimizer: If True, save optimizer's state together.
 
     # Raises
         ImportError: if h5py is not available.
@@ -108,7 +109,7 @@ def save_model(model, filepath, overwrite=True):
         model_layers = model.layers
     topology.save_weights_to_hdf5_group(model_weights_group, model_layers)
 
-    if hasattr(model, 'optimizer'):
+    if include_optimizer and hasattr(model, 'optimizer'):
         if isinstance(model.optimizer, optimizers.TFOptimizer):
             warnings.warn(
                 'TensorFlow optimizers do not '
@@ -186,7 +187,7 @@ def load_model(filepath, custom_objects=None):
         ValueError: In case of an invalid savefile.
     """
     if h5py is None:
-        raise ImportError('`save_model` requires h5py.')
+        raise ImportError('`load_model` requires h5py.')
 
     if not custom_objects:
         custom_objects = {}
@@ -213,7 +214,14 @@ def load_model(filepath, custom_objects=None):
         if isinstance(obj, dict):
             deserialized = {}
             for key, value in obj.items():
-                if value in custom_objects:
+                deserialized[key] = []
+                if isinstance(value, list):
+                    for element in value:
+                        if element in custom_objects:
+                            deserialized[key].append(custom_objects[element])
+                        else:
+                            deserialized[key].append(element)
+                elif value in custom_objects:
                     deserialized[key] = custom_objects[value]
                 else:
                     deserialized[key] = value
@@ -285,9 +293,12 @@ def model_from_config(config, custom_objects=None):
 
     # Returns
         A Keras model instance (uncompiled).
+
+    # Raises
+        TypeError if `config` is not a dictionary
     """
     if isinstance(config, list):
-        raise TypeError('`model_fom_config` expects a dictionary, not a list. '
+        raise TypeError('`model_from_config` expects a dictionary, not a list. '
                         'Maybe you meant to use '
                         '`Sequential.from_config(config)`?')
     return layer_module.deserialize(config, custom_objects=custom_objects)
@@ -736,7 +747,7 @@ class Sequential(Model):
             optimizer: str (name of optimizer) or optimizer object.
                 See [optimizers](/optimizers).
             loss: str (name of objective function) or objective function.
-                See [objectives](/objectives).
+                See [losses](/losses).
             metrics: list of metrics to be evaluated by the model
                 during training and testing.
                 Typically you will use `metrics=['accuracy']`.
@@ -1025,12 +1036,12 @@ class Sequential(Model):
                 - a tuple (inputs, targets, sample_weights).
                 All arrays should contain the same number of samples.
                 The generator is expected to loop over its data
-                indefinitely. An epoch finishes when `samples_per_epoch`
-                samples have been seen by the model.
+                indefinitely. An epoch finishes when `steps_per_epoch`
+                batches have been seen by the model.
             steps_per_epoch: Total number of steps (batches of samples)
                 to yield from `generator` before declaring one epoch
                 finished and starting the next epoch. It should typically
-                be equal to the number of unique samples if your dataset
+                be equal to the number of unique samples of your dataset
                 divided by the batch size.
             epochs: Integer, total number of iterations on the data.
             verbose: Verbosity mode, 0, 1, or 2.
@@ -1041,8 +1052,10 @@ class Sequential(Model):
                 - A tuple (inputs, targets, sample_weights).
             validation_steps: Only relevant if `validation_data`
                 is a generator.
-                Number of samples to use from validation generator
-                at the end of every epoch.
+                Number of steps to yield from validation generator
+                at the end of every epoch. It should typically
+                be equal to the number of unique samples of your
+                validation dataset divided by the batch size.
             class_weight: Dictionary mapping class indices to a weight
                 for the class.
             max_q_size: Maximum size for the generator queue
@@ -1074,10 +1087,10 @@ class Sequential(Model):
                         # and labels, from each line in the file
                         x, y = process_line(line)
                         yield (x, y)
-                    f.close()
+                        f.close()
 
             model.fit_generator(generate_arrays_from_file('/my_file.txt'),
-                                samples_per_epoch=10000, epochs=10)
+                                steps_per_epoch=1000, epochs=10)
         ```
         """
         if self.model is None:
@@ -1138,7 +1151,8 @@ class Sequential(Model):
 
     @interfaces.legacy_generator_methods_support
     def predict_generator(self, generator, steps,
-                          max_q_size=10, workers=1, pickle_safe=False):
+                          max_q_size=10, workers=1,
+                          pickle_safe=False, verbose=0):
         """Generates predictions for the input samples from a data generator.
 
         The generator should return the same kind of data as accepted by
@@ -1155,6 +1169,7 @@ class Sequential(Model):
                 relies on multiprocessing, you should not pass
                 non picklable arguments to the generator
                 as they can't be passed easily to children processes.
+            verbose: verbosity mode, 0 or 1.
 
         # Returns
             A Numpy array of predictions.
@@ -1164,7 +1179,8 @@ class Sequential(Model):
         return self.model.predict_generator(generator, steps,
                                             max_q_size=max_q_size,
                                             workers=workers,
-                                            pickle_safe=pickle_safe)
+                                            pickle_safe=pickle_safe,
+                                            verbose=verbose)
 
     def get_config(self):
         if isinstance(self.layers[0], legacy_layers.Merge):
@@ -1177,13 +1193,13 @@ class Sequential(Model):
         return copy.deepcopy(config)
 
     @classmethod
-    def from_config(cls, config):
+    def from_config(cls, config, custom_objects=None):
         if 'class_name' not in config[0] or config[0]['class_name'] == 'Merge':
             return cls.legacy_from_config(config)
 
         model = cls()
         for conf in config:
-            layer = layer_module.deserialize(conf)
+            layer = layer_module.deserialize(conf, custom_objects=custom_objects)
             model.add(layer)
         return model
 
@@ -1214,6 +1230,15 @@ class Sequential(Model):
 
     @classmethod
     def legacy_from_config(cls, config, layer_cache=None):
+        """Load a model from a legacy configuration.
+
+        # Arguments
+            config: dictionary with configuration.
+            layer_cache: cache to draw pre-existing layer.
+
+        # Returns
+            The loaded Model.
+        """
         if not layer_cache:
             layer_cache = {}
 

keras/preprocessing/image.py

@@ -325,9 +325,9 @@ def load_img(path, grayscale=False, target_size=None):
         if img.mode != 'RGB':
             img = img.convert('RGB')
     if target_size:
-        wh_tuple = (target_size[1], target_size[0])
-        if img.size != wh_tuple:
-            img = img.resize(wh_tuple)
+        hw_tuple = (target_size[1], target_size[0])
+        if img.size != hw_tuple:
+            img = img.resize(hw_tuple)
     return img
 
 
@@ -708,7 +708,7 @@ class Iterator(object):
                     index_array = np.random.permutation(n)
 
             current_index = (self.batch_index * batch_size) % n
-            if n >= current_index + batch_size:
+            if n > current_index + batch_size:
                 current_batch_size = batch_size
                 self.batch_index += 1
             else:

keras/utils/conv_utils.py

@@ -78,6 +78,7 @@ def convert_kernel(kernel):
     # Raises
         ValueError: in case of invalid kernel shape or invalid data_format.
     """
+    kernel = np.asarray(kernel)
     if not 4 <= kernel.ndim <= 5:
         raise ValueError('Invalid kernel shape:', kernel.shape)
     slices = [slice(None, None, -1) for _ in range(kernel.ndim)]

keras/utils/data_utils.py

@@ -4,10 +4,12 @@ from __future__ import print_function
 
 import functools
 import tarfile
+import zipfile
 import os
 import sys
 import shutil
 import hashlib
+import six
 from six.moves.urllib.request import urlopen
 from six.moves.urllib.error import URLError
 from six.moves.urllib.error import HTTPError
@@ -55,24 +57,108 @@ else:
     from six.moves.urllib.request import urlretrieve
 
 
-def get_file(fname, origin, untar=False,
-             md5_hash=None, cache_subdir='datasets'):
-    """Downloads a file from a URL if it not already in the cache.
-
-    Passing the MD5 hash will verify the file after download
-    as well as if it is already present in the cache.
+def _extract_archive(file_path, path='.', archive_format='auto'):
+    """Extracts an archive if it matches tar, tar.gz, tar.bz, or zip formats.
 
     # Arguments
-        fname: name of the file
-        origin: original URL of the file
-        untar: boolean, whether the file should be decompressed
-        md5_hash: MD5 hash of the file for verification
-        cache_subdir: directory being used as the cache
+        file_path: path to the archive file
+        path: path to extract the archive file
+        archive_format: Archive format to try for extracting the file.
+            Options are 'auto', 'tar', 'zip', and None.
+            'tar' includes tar, tar.gz, and tar.bz files.
+            The default 'auto' is ['tar', 'zip'].
+            None or an empty list will return no matches found.
+
+    # Returns
+        True if a match was found and an archive extraction was completed,
+        False otherwise.
+    """
+    if archive_format is None:
+        return False
+    if archive_format is 'auto':
+        archive_format = ['tar', 'zip']
+    if isinstance(archive_format, six.string_types):
+        archive_format = [archive_format]
+
+    for archive_type in archive_format:
+        if archive_type is 'tar':
+            open_fn = tarfile.open
+            is_match_fn = tarfile.is_tarfile
+        if archive_type is 'zip':
+            open_fn = zipfile.ZipFile
+            is_match_fn = zipfile.is_zipfile
+
+        if is_match_fn(file_path):
+            with open_fn(file_path) as archive:
+                try:
+                    archive.extractall(path)
+                except (tarfile.TarError, RuntimeError,
+                        KeyboardInterrupt):
+                    if os.path.exists(path):
+                        if os.path.isfile(path):
+                            os.remove(path)
+                        else:
+                            shutil.rmtree(path)
+                    raise
+            return True
+    return False
+
+
+def get_file(fname,
+             origin,
+             untar=False,
+             md5_hash=None,
+             file_hash=None,
+             cache_subdir='datasets',
+             hash_algorithm='auto',
+             extract=False,
+             archive_format='auto',
+             cache_dir=None):
+    """Downloads a file from a URL if it not already in the cache.
+
+    By default the file at the url `origin` is downloaded to the
+    cache_dir `~/.keras`, placed in the cache_subdir `datasets`,
+    and given the filename `fname`. The final location of a file
+    `example.txt` would therefore be `~/.keras/datasets/example.txt`.
+
+    Files in tar, tar.gz, tar.bz, and zip formats can also be extracted.
+    Passing a hash will verify the file after download. The command line
+    programs `shasum` and `sha256sum` can compute the hash.
+
+    # Arguments
+        fname: Name of the file. If an absolute path `/path/to/file.txt` is
+            specified the file will be saved at that location.
+        origin: Original URL of the file.
+        untar: Deprecated in favor of 'extract'.
+            boolean, whether the file should be decompressed
+        md5_hash: Deprecated in favor of 'file_hash'.
+            md5 hash of the file for verification
+        file_hash: The expected hash string of the file after download.
+            The sha256 and md5 hash algorithms are both supported.
+        cache_subdir: Subdirectory under the Keras cache dir where the file is
+            saved. If an absolute path `/path/to/folder` is
+            specified the file will be saved at that location.
+        hash_algorithm: Select the hash algorithm to verify the file.
+            options are 'md5', 'sha256', and 'auto'.
+            The default 'auto' detects the hash algorithm in use.
+        extract: True tries extracting the file as an Archive, like tar or zip.
+        archive_format: Archive format to try for extracting the file.
+            Options are 'auto', 'tar', 'zip', and None.
+            'tar' includes tar, tar.gz, and tar.bz files.
+            The default 'auto' is ['tar', 'zip'].
+            None or an empty list will return no matches found.
+        cache_dir: Location to store cached files, when None it
+            defaults to the [Keras Directory](/faq/#where-is-the-keras-configuration-filed-stored).
 
     # Returns
         Path to the downloaded file
     """
-    datadir_base = os.path.expanduser(os.path.join('~', '.keras'))
+    if cache_dir is None:
+        cache_dir = os.path.expanduser(os.path.join('~', '.keras'))
+    if md5_hash is not None and file_hash is None:
+        file_hash = md5_hash
+        hash_algorithm = 'md5'
+    datadir_base = os.path.expanduser(cache_dir)
     if not os.access(datadir_base, os.W_OK):
         datadir_base = os.path.join('/tmp', '.keras')
     datadir = os.path.join(datadir_base, cache_subdir)
@@ -88,10 +174,12 @@ def get_file(fname, origin, untar=False,
     download = False
     if os.path.exists(fpath):
         # File found; verify integrity if a hash was provided.
-        if md5_hash is not None:
-            if not validate_file(fpath, md5_hash):
+        if file_hash is not None:
+            if not validate_file(fpath, file_hash, algorithm=hash_algorithm):
                 print('A local file was found, but it seems to be '
-                      'incomplete or outdated.')
+                      'incomplete or outdated because the ' + hash_algorithm +
+                      ' file hash does not match the original value of ' +
+                      file_hash + ' so we will re-download the data.')
                 download = True
     else:
         download = True
@@ -123,38 +211,68 @@ def get_file(fname, origin, untar=False,
 
     if untar:
         if not os.path.exists(untar_fpath):
-            print('Untaring file...')
-            tfile = tarfile.open(fpath, 'r:gz')
-            try:
-                tfile.extractall(path=datadir)
-            except (Exception, KeyboardInterrupt) as e:
-                if os.path.exists(untar_fpath):
-                    if os.path.isfile(untar_fpath):
-                        os.remove(untar_fpath)
-                    else:
-                        shutil.rmtree(untar_fpath)
-                raise
-            tfile.close()
+            _extract_archive(fpath, datadir, archive_format='tar')
         return untar_fpath
 
+    if extract:
+        _extract_archive(fpath, datadir, archive_format)
+
     return fpath
 
 
-def validate_file(fpath, md5_hash):
-    """Validates a file against a MD5 hash.
+def _hash_file(fpath, algorithm='sha256', chunk_size=65535):
+    """Calculates a file sha256 or md5 hash.
+
+    # Example
+
+    ```python
+       >>> from keras.data_utils import _hash_file
+       >>> _hash_file('/path/to/file.zip')
+       'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'
+    ```
 
     # Arguments
         fpath: path to the file being validated
-        md5_hash: the MD5 hash being validated against
+        algorithm: hash algorithm, one of 'auto', 'sha256', or 'md5'.
+            The default 'auto' detects the hash algorithm in use.
+        chunk_size: Bytes to read at a time, important for large files.
+
+    # Returns
+        The file hash
+    """
+    if (algorithm is 'sha256') or (algorithm is 'auto' and len(hash) is 64):
+        hasher = hashlib.sha256()
+    else:
+        hasher = hashlib.md5()
+
+    with open(fpath, 'rb') as fpath_file:
+        for chunk in iter(lambda: fpath_file.read(chunk_size), b''):
+            hasher.update(chunk)
+
+    return hasher.hexdigest()
+
+
+def validate_file(fpath, file_hash, algorithm='auto', chunk_size=65535):
+    """Validates a file against a sha256 or md5 hash.
+
+    # Arguments
+        fpath: path to the file being validated
+        file_hash:  The expected hash string of the file.
+            The sha256 and md5 hash algorithms are both supported.
+        algorithm: Hash algorithm, one of 'auto', 'sha256', or 'md5'.
+            The default 'auto' detects the hash algorithm in use.
+        chunk_size: Bytes to read at a time, important for large files.
 
     # Returns
         Whether the file is valid
     """
-    hasher = hashlib.md5()
-    with open(fpath, 'rb') as f:
-        buf = f.read()
-        hasher.update(buf)
-    if str(hasher.hexdigest()) == str(md5_hash):
+    if ((algorithm is 'sha256') or
+            (algorithm is 'auto' and len(file_hash) is 64)):
+        hasher = 'sha256'
+    else:
+        hasher = 'md5'
+
+    if str(_hash_file(fpath, hasher, chunk_size)) == str(file_hash):
         return True
     else:
         return False

keras/utils/generic_utils.py

@@ -27,8 +27,8 @@ class CustomObjectScope(object):
     Consider a custom object `MyObject`
 
     ```python
-        with CustomObjectScope({"MyObject":MyObject}):
-            layer = Dense(..., W_regularizer="MyObject")
+        with CustomObjectScope({'MyObject':MyObject}):
+            layer = Dense(..., kernel_regularizer='MyObject')
             # save, load, etc. will recognize custom object by name
     ```
     """
@@ -63,8 +63,8 @@ def custom_object_scope(*args):
     Consider a custom object `MyObject`
 
     ```python
-        with custom_object_scope({"MyObject":MyObject}):
-            layer = Dense(..., W_regularizer="MyObject")
+        with custom_object_scope({'MyObject':MyObject}):
+            layer = Dense(..., kernel_regularizer='MyObject')
             # save, load, etc. will recognize custom object by name
     ```
 
@@ -89,7 +89,7 @@ def get_custom_objects():
 
     ```python
         get_custom_objects().clear()
-        get_custom_objects()["MyObject"] = MyObject
+        get_custom_objects()['MyObject'] = MyObject
     ```
 
     # Returns
@@ -154,17 +154,13 @@ def deserialize_keras_object(identifier, module_objects=None,
             fn = module_objects.get(function_name)
             if fn is None:
                 raise ValueError('Unknown ' + printable_module_name,
-                                 ':' + class_name)
+                                 ':' + function_name)
         return fn
     else:
         raise ValueError('Could not interpret serialized ' +
                          printable_module_name + ': ' + identifier)
 
 
-def make_tuple(*args):
-    return args
-
-
 def func_dump(func):
     """Serializes a user defined function.
 

keras/utils/io_utils.py

@@ -62,9 +62,14 @@ class HDF5Matrix(object):
         return self.end - self.start
 
     def __getitem__(self, key):
+        start, stop = key.start, key.stop
         if isinstance(key, slice):
-            if key.stop + self.start <= self.end:
-                idx = slice(key.start + self.start, key.stop + self.start)
+            if start is None:
+                start = 0
+            if stop is None:
+                stop = self.data.shape[0]
+            if stop + self.start <= self.end:
+                idx = slice(start + self.start, stop + self.start)
             else:
                 raise IndexError
         elif isinstance(key, int):

keras/utils/layer_utils.py

@@ -19,8 +19,11 @@ def print_summary(model, line_length=None, positions=None):
     else:
         sequential_like = True
         for v in model.nodes_by_depth.values():
-            if len(v) > 1:
+            if (len(v) > 1) or (len(v) == 1 and len(v[0].inbound_layers) > 1):
+                # if the model has multiple nodes or if the nodes have multiple inbound_layers
+                # the model is no longer sequential
                 sequential_like = False
+                break
 
     if sequential_like:
         line_length = line_length or 65
@@ -75,12 +78,10 @@ def print_summary(model, line_length=None, positions=None):
         except AttributeError:
             output_shape = 'multiple'
         connections = []
-        for node_index, node in enumerate(layer.inbound_nodes):
-            if relevant_nodes:
-                node_key = layer.name + '_ib-' + str(node_index)
-                if node_key not in relevant_nodes:
-                    # node is node part of the current network
-                    continue
+        for node in layer.inbound_nodes:
+            if relevant_nodes and node not in relevant_nodes:
+                # node is not part of the current network
+                continue
             for i in range(len(node.inbound_layers)):
                 inbound_layer = node.inbound_layers[i].name
                 inbound_node_index = node.node_indices[i]
@@ -111,7 +112,10 @@ def print_summary(model, line_length=None, positions=None):
         else:
             print('_' * line_length)
 
-    trainable_count, non_trainable_count = count_total_params(layers, layer_set=None)
+    trainable_count = int(
+        np.sum([K.count_params(p) for p in set(model.trainable_weights)]))
+    non_trainable_count = int(
+        np.sum([K.count_params(p) for p in set(model.non_trainable_weights)]))
 
     print('Total params: {:,}'.format(trainable_count + non_trainable_count))
     print('Trainable params: {:,}'.format(trainable_count))
@@ -119,35 +123,6 @@ def print_summary(model, line_length=None, positions=None):
     print('_' * line_length)
 
 
-def count_total_params(layers, layer_set=None):
-    """Counts the number of parameters in a list of layers.
-
-    # Arguments
-        layers: list of layers.
-        layer_set: set of layers already seen
-            (so that we don't count their weights twice).
-
-    # Returns
-        A tuple (count of trainable weights, count of non-trainable weights.)
-    """
-    if layer_set is None:
-        layer_set = set()
-    trainable_count = 0
-    non_trainable_count = 0
-    for layer in layers:
-        if layer in layer_set:
-            continue
-        layer_set.add(layer)
-        if hasattr(layer, 'layers'):
-            t, nt = count_total_params(layer.layers, layer_set)
-            trainable_count += t
-            non_trainable_count += nt
-        else:
-            trainable_count += np.sum([K.count_params(p) for p in layer.trainable_weights])
-            non_trainable_count += np.sum([K.count_params(p) for p in layer.non_trainable_weights])
-    return trainable_count, non_trainable_count
-
-
 def convert_all_kernels_in_model(model):
     """Converts all convolution kernels in a model from Theano to TensorFlow.
 

keras/utils/vis_utils.py

@@ -13,7 +13,10 @@ except ImportError:
 
 
 def _check_pydot():
-    if not (pydot and pydot.find_graphviz()):
+    try:
+        # Attempt to create an image of a blank graph to check the pydot/graphviz installation.
+        pydot.Dot.create(pydot.Dot())
+    except Exception:  # pydot raises a generic Exception here, so no specific class can be caught.
         raise ImportError('Failed to import pydot. You must install pydot'
                           ' and graphviz for `pydotprint` to work.')
 

keras/wrappers/scikit_learn.py

@@ -175,6 +175,36 @@ class KerasClassifier(BaseWrapper):
     """Implementation of the scikit-learn classifier API for Keras.
     """
 
+    def fit(self, x, y, **kwargs):
+        """Constructs a new model with `build_fn` & fit the model to `(x, y)`.
+
+        # Arguments
+            x : array-like, shape `(n_samples, n_features)`
+                Training samples where n_samples in the number of samples
+                and n_features is the number of features.
+            y : array-like, shape `(n_samples,)` or `(n_samples, n_outputs)`
+                True labels for X.
+            **kwargs: dictionary arguments
+                Legal arguments are the arguments of `Sequential.fit`
+
+        # Returns
+            history : object
+                details about the training history at each epoch.
+
+        # Raises
+            ValueError: In case of invalid shape for `y` argument.
+        """
+        y = np.array(y)
+        if len(y.shape) == 2 and y.shape[1] > 1:
+            self.classes_ = np.arange(y.shape[1])
+        elif (len(y.shape) == 2 and y.shape[1] == 1) or len(y.shape) == 1:
+            self.classes_ = np.unique(y)
+            y = np.searchsorted(self.classes_, y)
+        else:
+            raise ValueError('Invalid shape for y: ' + str(y.shape))
+        self.n_classes_ = len(self.classes_)
+        return super(KerasClassifier, self).fit(x, y, **kwargs)
+
     def predict(self, x, **kwargs):
         """Returns the class predictions for the given test data.
 
@@ -191,7 +221,8 @@ class KerasClassifier(BaseWrapper):
                 Class predictions.
         """
         kwargs = self.filter_sk_params(Sequential.predict_classes, kwargs)
-        return self.model.predict_classes(x, **kwargs)
+        classes = self.model.predict_classes(x, **kwargs)
+        return self.classes_[classes]
 
     def predict_proba(self, x, **kwargs):
         """Returns class probability estimates for the given test data.
@@ -242,6 +273,7 @@ class KerasClassifier(BaseWrapper):
                 compute accuracy. You should pass `metrics=["accuracy"]` to
                 the `.compile()` method of the model.
         """
+        y = np.searchsorted(self.classes_, y)
         kwargs = self.filter_sk_params(Sequential.evaluate, kwargs)
 
         loss_name = self.model.loss

setup.py

@@ -3,12 +3,12 @@ from setuptools import find_packages
 
 
 setup(name='Keras',
-      version='2.0.1',
+      version='2.0.4',
       description='Deep Learning for Python',
       author='Francois Chollet',
       author_email='francois.chollet@gmail.com',
       url='https://github.com/fchollet/keras',
-      download_url='https://github.com/fchollet/keras/tarball/2.0.1',
+      download_url='https://github.com/fchollet/keras/tarball/2.0.4',
       license='MIT',
       install_requires=['theano', 'pyyaml', 'six'],
       extras_require={
@@ -16,6 +16,7 @@ setup(name='Keras',
           'visualize': ['pydot-ng'],
           'tests': ['pytest',
                     'pytest-pep8',
-                    'pytest-xdist'],
+                    'pytest-xdist',
+                    'pytest-cov'],
       },
       packages=find_packages())

tests/keras/backend/backend_test.py

@@ -83,6 +83,10 @@ class TestBackend(object):
 
         check_two_tensor_operation('batch_dot', (4, 2, 3), (4, 5, 3),
                                    axes=(2, 2))
+        check_two_tensor_operation('batch_dot', (4, 2, 3), (4, 3),
+                                   axes=(2, 1))
+        check_two_tensor_operation('batch_dot', (4, 2), (4, 2, 3),
+                                   axes=(1, 1))
         check_two_tensor_operation('batch_dot', (32, 20), (32, 20), axes=1)
         check_two_tensor_operation('batch_dot', (32, 20), (32, 20), axes=(1, 1))
         check_single_tensor_operation('transpose', (4, 2))
@@ -131,6 +135,20 @@ class TestBackend(object):
                                          'squeeze', {'axis': 2},
                                          (4, 3, 1, 1))
 
+    def test_none_shape_operations(self):
+        # Test shape inference when input
+        # shape has `None` entries
+        if K.backend() == 'theano':
+            x = KTH.placeholder((3, None, 4))
+
+            y = KTH.batch_flatten(x)
+            if hasattr(y, '_keras_shape'):
+                assert y._keras_shape == (3, None)
+
+            y = KTH.flatten(x)
+            if hasattr(y, '_keras_shape'):
+                assert y._keras_shape == (None, )
+
     def test_repeat_elements(self):
         reps = 3
         for ndims in [1, 2, 3]:
@@ -153,6 +171,15 @@ class TestBackend(object):
                 if hasattr(th_z, '_keras_shape'):
                     assert th_z._keras_shape == th_rep.shape
 
+                # test theano shape inference when
+                # input shape has None entries
+                if K.backend() == 'theano':
+                    shape = list(shape)
+                    shape[rep_axis] = None
+                    x = K.placeholder(shape=shape)
+                    y = K.repeat_elements(x, reps, axis=rep_axis)
+                    assert y._keras_shape == tuple(shape)
+
     def test_tile(self):
         shape = (3, 4)
         arr = np.arange(np.prod(shape)).reshape(shape)
@@ -167,6 +194,17 @@ class TestBackend(object):
         if hasattr(th_z, '_keras_shape'):
             assert th_z._keras_shape == th_rep.shape
 
+        # test theano shape inference when
+        # input shape has None entries
+        if K.backend() == 'theano':
+            x = K.placeholder(shape=(None, 4))
+            n = 2
+            y = KTH.tile(x, n)
+            assert y._keras_shape == (None, 8)
+            n = (4, 3)
+            y = K.tile(x, n)
+            assert y._keras_shape == (None, 12)
+
     def test_gather(self):
         shape = (10, 2, 3)
         ref = np.arange(np.prod(shape)).reshape(shape)
@@ -185,6 +223,14 @@ class TestBackend(object):
         if hasattr(th_z, '_keras_shape'):
             assert th_z._keras_shape == th_result.shape
 
+        # test theano shape inference when
+        # input shape has None entries
+        if K.backend() == 'theano':
+            x = K.placeholder(shape=(None, 3, 4))
+            indices = K.placeholder(shape=(5, 6), dtype='int32')
+            y = K.gather(x, indices)
+            assert y._keras_shape == (5, 6, 3, 4)
+
     def test_value_manipulation(self):
         val = np.random.random((4, 2))
         xth = KTH.variable(val)
@@ -241,6 +287,12 @@ class TestBackend(object):
         check_single_tensor_operation('prod', (4, 2), axis=1, keepdims=True)
         check_single_tensor_operation('prod', (4, 2, 3), axis=[1, -1])
 
+        check_single_tensor_operation('cumsum', (4, 2))
+        check_single_tensor_operation('cumsum', (4, 2), axis=1)
+
+        check_single_tensor_operation('cumprod', (4, 2))
+        check_single_tensor_operation('cumprod', (4, 2), axis=1)
+
         # does not work yet, wait for bool <-> int casting in TF (coming soon)
         # check_single_tensor_operation('any', (4, 2))
         # check_single_tensor_operation('any', (4, 2), axis=1, keepdims=True)
@@ -528,6 +580,41 @@ class TestBackend(object):
         assert_allclose(tf_last_output, th_last_output, atol=1e-04)
         assert_allclose(tf_outputs, th_outputs, atol=1e-04)
 
+    @pytest.mark.parametrize('x_np,axis,keepdims', [
+        (np.array([1.1, 0.8, 0.9]), 0, False),
+        (np.array([[1.1, 0.8, 0.9]]), 0, False),
+        (np.array([[1.1, 0.8, 0.9]]), 1, False),
+        (np.array([[1.1, 0.8, 0.9]]), -1, False),
+        (np.array([[1.1, 0.8, 0.9]]), 1, True),
+        (np.array([[1.1], [1.2]]), 0, False),
+        (np.array([[1.1], [1.2]]), 1, False),
+        (np.array([[1.1], [1.2]]), -1, False),
+        (np.array([[1.1], [1.2]]), -1, True),
+        (np.array([[1.1, 1.2, 1.3], [0.9, 0.7, 1.4]]), None, False),
+        (np.array([[1.1, 1.2, 1.3], [0.9, 0.7, 1.4]]), 0, False),
+        (np.array([[1.1, 1.2, 1.3], [0.9, 0.7, 1.4]]), 1, False),
+        (np.array([[1.1, 1.2, 1.3], [0.9, 0.7, 1.4]]), -1, False),
+    ])
+    @pytest.mark.parametrize('K', [KTH, KTF], ids=["KTH", "KTF"])
+    def test_logsumexp(self, x_np, axis, keepdims, K):
+        '''
+        Check if K.logsumexp works properly for values close to one.
+        '''
+        x = K.variable(x_np)
+        assert_allclose(K.eval(K.logsumexp(x, axis=axis, keepdims=keepdims)),
+                        np.log(np.sum(np.exp(x_np), axis=axis, keepdims=keepdims)),
+                        rtol=1e-5)
+
+    @pytest.mark.parametrize('K', [KTH, KTF], ids=["KTH", "KTF"])
+    def test_logsumexp_optim(self, K):
+        '''
+        Check if optimization works.
+        '''
+        x_np = np.array([1e+4, 1e-4])
+        assert_allclose(K.eval(K.logsumexp(K.variable(x_np), axis=0)),
+                        1e4,
+                        rtol=1e-5)
+
     def test_switch(self):
         val = np.random.random()
         xth = KTH.variable(val)
@@ -570,6 +657,46 @@ class TestBackend(object):
         check_single_tensor_operation('l2_normalize', (4, 3), axis=-1)
         check_single_tensor_operation('l2_normalize', (4, 3), axis=1)
 
+    def test_in_top_k(self):
+        batch_size = 20
+        num_classes = 10
+
+        # Random prediction test case
+        predictions = np.random.random((batch_size, num_classes)).astype('float32')
+        targets = np.random.randint(num_classes, size=batch_size, dtype='int32')
+
+        predictions_th = KTH.variable(predictions, dtype='float32')
+        targets_th = KTH.variable(targets, dtype='int32')
+        predictions_tf = KTF.variable(predictions, dtype='float32')
+        targets_tf = KTF.variable(targets, dtype='int32')
+
+        for k in range(1, num_classes + 1):
+            res_th = KTH.eval(KTH.in_top_k(predictions_th, targets_th, k))
+            res_tf = KTF.eval(KTF.in_top_k(predictions_tf, targets_tf, k))
+
+            assert res_th.shape == res_tf.shape
+            assert_allclose(res_th, res_tf, atol=1e-05)
+
+        # Identical prediction test case:
+        # randomly set half of the predictions to an identical value
+        num_identical = num_classes // 2
+        for i in range(batch_size):
+            idx_identical = np.random.choice(num_classes, size=num_identical, replace=False)
+            predictions[i, idx_identical] = predictions[i, 0]
+        targets = np.zeros(batch_size, dtype='int32')
+
+        predictions_th = KTH.variable(predictions, dtype='float32')
+        targets_th = KTH.variable(targets, dtype='int32')
+        predictions_tf = KTF.variable(predictions, dtype='float32')
+        targets_tf = KTF.variable(targets, dtype='int32')
+
+        for k in range(1, num_classes + 1):
+            res_th = KTH.eval(KTH.in_top_k(predictions_th, targets_th, k))
+            res_tf = KTF.eval(KTF.in_top_k(predictions_tf, targets_tf, k))
+
+            assert res_th.shape == res_tf.shape
+            assert_allclose(res_th, res_tf, atol=1e-05)
+
     def test_conv2d(self):
         # TF kernel shape: (rows, cols, input_depth, depth)
 
@@ -939,15 +1066,25 @@ class TestBackend(object):
     def test_map(self):
         x = np.random.rand(10, 3).astype(np.float32)
         for K in [KTF, KTH]:
-            kx = K.eval(K.map_fn(K.sum, x))
+            vx = K.variable(x)
+            kx = K.eval(K.map_fn(K.sum, vx))
+            # make sure we can also walk the indexes in tensorflow which we
+            # can't without specifying dtype
+            kx2 = K.eval(K.map_fn(
+                lambda i: K.sum(vx[i]),
+                K.arange(10),
+                dtype=K.floatx()
+            ))
 
             assert (10,) == kx.shape
+            assert (10,) == kx2.shape
             assert_allclose(x.sum(axis=1), kx, atol=1e-05)
+            assert_allclose(kx, kx2, atol=1e-05)
 
     def test_foldl(self):
         x = np.random.rand(10, 3).astype(np.float32)
         for K in [KTF, KTH]:
-            kx = K.eval(K.foldl(lambda a, b: a + b, x))
+            kx = K.eval(K.foldl(lambda a, b: a + b, K.variable(x)))
 
             assert (3,) == kx.shape
             assert_allclose(x.sum(axis=0), kx, atol=1e-05)
@@ -959,8 +1096,9 @@ class TestBackend(object):
         # right to left we have no such problem and the result is larger
         x = np.array([1e-20, 1e-20, 10, 10, 10], dtype=np.float32)
         for K in [KTF, KTH]:
-            p1 = K.eval(K.foldl(lambda a, b: a * b, x))
-            p2 = K.eval(K.foldr(lambda a, b: a * b, x))
+            vx = K.variable(x)
+            p1 = K.eval(K.foldl(lambda a, b: a * b, vx))
+            p2 = K.eval(K.foldr(lambda a, b: a * b, vx))
 
             assert p1 < p2
             assert 9e-38 < p2 <= 1e-37

tests/keras/engine/test_topology.py

@@ -489,5 +489,93 @@ def test_recursion():
         Dense(2)(x)
 
 
+@keras_test
+def test_load_layers():
+    from keras.layers import ConvLSTM2D, TimeDistributed, Bidirectional, Conv2D, Input
+    from keras.models import Model
+    from keras.engine.topology import preprocess_weights_for_loading
+
+    if K.backend() == 'tensorflow':
+        inputs = Input(shape=(10, 20, 20, 1))
+    else:
+        inputs = Input(shape=(10, 1, 20, 20))
+    td_conv = TimeDistributed(Conv2D(15, (5, 5)))(inputs)
+    bi_convlstm2d = Bidirectional(ConvLSTM2D(10, (3, 3)), merge_mode='concat')(td_conv)
+    model = Model(inputs=inputs, outputs=bi_convlstm2d)
+
+    weight_value_tuples = []
+
+    # TimeDistributed Conv2D layer
+    # use 'channels_first' data format to check that the function is being called correctly for Conv2D
+    # old: (filters, stack_size, kernel_rows, kernel_cols)
+    # new: (kernel_rows, kernel_cols, stack_size, filters)
+    weight_tensor_td_conv_old = list()
+    weight_tensor_td_conv_old.append(np.zeros((15, 1, 5, 5)))
+    weight_tensor_td_conv_old.append(np.zeros((15,)))
+    td_conv_layer = model.layers[1]
+    td_conv_layer.layer.data_format = 'channels_first'
+    weight_tensor_td_conv_new = preprocess_weights_for_loading(td_conv_layer,
+                                                               weight_tensor_td_conv_old,
+                                                               original_keras_version='1')
+    symbolic_weights = td_conv_layer.weights
+    assert (len(symbolic_weights) == len(weight_tensor_td_conv_new))
+    weight_value_tuples += zip(symbolic_weights, weight_tensor_td_conv_new)
+
+    # Bidirectional ConvLSTM2D layer
+    # old ConvLSTM2D took a list of 12 weight tensors, returns a list of 3 concatenated larger tensors.
+    weight_tensor_bi_convlstm_old = []
+    for j in range(2):  # bidirectional
+        for i in range(4):
+            weight_tensor_bi_convlstm_old.append(np.zeros((3, 3, 15, 10)))  # kernel
+            weight_tensor_bi_convlstm_old.append(np.zeros((3, 3, 10, 10)))  # recurrent kernel
+            weight_tensor_bi_convlstm_old.append(np.zeros((10,)))  # bias
+
+    bi_convlstm_layer = model.layers[2]
+    weight_tensor_bi_convlstm_new = preprocess_weights_for_loading(bi_convlstm_layer,
+                                                                   weight_tensor_bi_convlstm_old,
+                                                                   original_keras_version='1')
+
+    symbolic_weights = bi_convlstm_layer.weights
+    assert (len(symbolic_weights) == len(weight_tensor_bi_convlstm_new))
+    weight_value_tuples += zip(symbolic_weights, weight_tensor_bi_convlstm_new)
+
+    K.batch_set_value(weight_value_tuples)
+
+    assert np.all(K.eval(model.layers[1].weights[0]) == weight_tensor_td_conv_new[0])
+    assert np.all(K.eval(model.layers[1].weights[1]) == weight_tensor_td_conv_new[1])
+    assert np.all(K.eval(model.layers[2].weights[0]) == weight_tensor_bi_convlstm_new[0])
+    assert np.all(K.eval(model.layers[2].weights[1]) == weight_tensor_bi_convlstm_new[1])
+    assert np.all(K.eval(model.layers[2].weights[2]) == weight_tensor_bi_convlstm_new[2])
+    assert np.all(K.eval(model.layers[2].weights[3]) == weight_tensor_bi_convlstm_new[3])
+    assert np.all(K.eval(model.layers[2].weights[4]) == weight_tensor_bi_convlstm_new[4])
+    assert np.all(K.eval(model.layers[2].weights[5]) == weight_tensor_bi_convlstm_new[5])
+
+
+def test_recursion_with_bn_and_loss():
+    model1 = Sequential([
+        layers.Dense(5, input_dim=5, activity_regularizer='l1'),
+        layers.BatchNormalization(),
+        layers.Dense(5),
+    ])
+
+    print('NEW MODEL')
+    inputs = layers.Input(shape=(5,))
+    outputs = model1(inputs)
+    model2 = Model(inputs=inputs, outputs=outputs)
+
+    assert len(model1.updates) == 2
+    assert len(model2.updates) == 2
+    assert len(model1.losses) == 1
+    assert len(model2.losses) == 1, model2.layers[1]._per_input_losses
+
+    model1.compile(optimizer='sgd', loss='categorical_crossentropy')
+    model2.compile(optimizer='sgd', loss='categorical_crossentropy')
+
+    x = np.ones((3, 5))
+    y = np.ones((3, 5))
+    model1.fit(x, y, verbose=0, epochs=1)
+    model2.fit(x, y, verbose=0, epochs=1)
+
+
 if __name__ == '__main__':
     pytest.main([__file__])

tests/keras/layers/local_test.py

@@ -12,63 +12,51 @@ def test_locallyconnected_1d():
     input_dim = 5
     filter_length = 3
     filters = 4
+    padding = 'valid'
+    strides = 1
 
-    for padding in ['valid']:
-        for strides in [1]:
-            if padding == 'same' and strides != 1:
-                continue
-            layer_test(local.LocallyConnected1D,
-                       kwargs={'filters': filters,
-                               'kernel_size': filter_length,
-                               'padding': padding,
-                               'strides': strides},
-                       input_shape=(num_samples, num_steps, input_dim))
-
-            layer_test(local.LocallyConnected1D,
-                       kwargs={'filters': filters,
-                               'kernel_size': filter_length,
-                               'padding': padding,
-                               'kernel_regularizer': 'l2',
-                               'bias_regularizer': 'l2',
-                               'activity_regularizer': 'l2',
-                               'strides': strides},
-                       input_shape=(num_samples, num_steps, input_dim))
+    layer_test(local.LocallyConnected1D,
+               kwargs={'filters': filters,
+                       'kernel_size': filter_length,
+                       'padding': padding,
+                       'kernel_regularizer': 'l2',
+                       'bias_regularizer': 'l2',
+                       'activity_regularizer': 'l2',
+                       'strides': strides},
+               input_shape=(num_samples, num_steps, input_dim))
 
 
 @keras_test
 def test_locallyconnected_2d():
-    num_samples = 8
+    num_samples = 5
     filters = 3
     stack_size = 4
     num_row = 6
-    num_col = 10
+    num_col = 8
+    padding = 'valid'
 
-    for padding in ['valid']:
-        for strides in [(1, 1), (2, 2)]:
-            if padding == 'same' and strides != (1, 1):
-                continue
+    for strides in [(1, 1), (2, 2)]:
+        layer_test(local.LocallyConnected2D,
+                   kwargs={'filters': filters,
+                           'kernel_size': 3,
+                           'padding': padding,
+                           'kernel_regularizer': 'l2',
+                           'bias_regularizer': 'l2',
+                           'activity_regularizer': 'l2',
+                           'strides': strides,
+                           'data_format': 'channels_last'},
+                   input_shape=(num_samples, num_row, num_col, stack_size))
 
-            layer_test(local.LocallyConnected2D,
-                       kwargs={'filters': filters,
-                               'kernel_size': 3,
-                               'padding': padding,
-                               'kernel_regularizer': 'l2',
-                               'bias_regularizer': 'l2',
-                               'activity_regularizer': 'l2',
-                               'strides': strides,
-                               'data_format': 'channels_last'},
-                       input_shape=(num_samples, num_row, num_col, stack_size))
-
-            layer_test(local.LocallyConnected2D,
-                       kwargs={'filters': filters,
-                               'kernel_size': (3, 3),
-                               'padding': padding,
-                               'kernel_regularizer': 'l2',
-                               'bias_regularizer': 'l2',
-                               'activity_regularizer': 'l2',
-                               'strides': strides,
-                               'data_format': 'channels_first'},
-                       input_shape=(num_samples, stack_size, num_row, num_col))
+        layer_test(local.LocallyConnected2D,
+                   kwargs={'filters': filters,
+                           'kernel_size': (3, 3),
+                           'padding': padding,
+                           'kernel_regularizer': 'l2',
+                           'bias_regularizer': 'l2',
+                           'activity_regularizer': 'l2',
+                           'strides': strides,
+                           'data_format': 'channels_first'},
+                   input_shape=(num_samples, stack_size, num_row, num_col))
 
 
 if __name__ == '__main__':

tests/keras/layers/merge_test.py

@@ -3,6 +3,7 @@ import numpy as np
 from numpy.testing import assert_allclose
 from keras import layers
 from keras import models
+from keras import backend as K
 from keras.utils.test_utils import layer_test
 from keras.utils.test_utils import keras_test
 from keras.layers import merge
@@ -17,6 +18,10 @@ def test_merge_add():
     assert o._keras_shape == (None, 4, 5)
     model = models.Model([i1, i2, i3], o)
 
+    add_layer = layers.Add()
+    o2 = add_layer([i1, i2, i3])
+    assert add_layer.output_shape == (None, 4, 5)
+
     x1 = np.random.random((2, 4, 5))
     x2 = np.random.random((2, 4, 5))
     x3 = np.random.random((2, 4, 5))
@@ -34,6 +39,10 @@ def test_merge_multiply():
     assert o._keras_shape == (None, 4, 5)
     model = models.Model([i1, i2, i3], o)
 
+    mul_layer = layers.Multiply()
+    o2 = mul_layer([i1, i2, i3])
+    assert mul_layer.output_shape == (None, 4, 5)
+
     x1 = np.random.random((2, 4, 5))
     x2 = np.random.random((2, 4, 5))
     x3 = np.random.random((2, 4, 5))
@@ -50,6 +59,10 @@ def test_merge_average():
     assert o._keras_shape == (None, 4, 5)
     model = models.Model([i1, i2], o)
 
+    avg_layer = layers.Average()
+    o2 = avg_layer([i1, i2])
+    assert avg_layer.output_shape == (None, 4, 5)
+
     x1 = np.random.random((2, 4, 5))
     x2 = np.random.random((2, 4, 5))
     out = model.predict([x1, x2])
@@ -65,6 +78,10 @@ def test_merge_maximum():
     assert o._keras_shape == (None, 4, 5)
     model = models.Model([i1, i2], o)
 
+    max_layer = layers.Maximum()
+    o2 = max_layer([i1, i2])
+    assert max_layer.output_shape == (None, 4, 5)
+
     x1 = np.random.random((2, 4, 5))
     x2 = np.random.random((2, 4, 5))
     out = model.predict([x1, x2])
@@ -80,12 +97,30 @@ def test_merge_concatenate():
     assert o._keras_shape == (None, 8, 5)
     model = models.Model([i1, i2], o)
 
+    concat_layer = layers.Concatenate(axis=1)
+    o2 = concat_layer([i1, i2])
+    assert concat_layer.output_shape == (None, 8, 5)
+
     x1 = np.random.random((2, 4, 5))
     x2 = np.random.random((2, 4, 5))
     out = model.predict([x1, x2])
     assert out.shape == (2, 8, 5)
     assert_allclose(out, np.concatenate([x1, x2], axis=1), atol=1e-4)
 
+    x3 = np.random.random((1, 1, 1))
+    nb_layers = 4
+    x_i = layers.Input(shape=(None, None))
+    x_list = [x_i]
+    x = x_i
+    for i in range(nb_layers):
+        x_list.append(x)
+        x = layers.concatenate(x_list, axis=1)
+    concat_model = models.Model(x_i, x)
+    concat_out = concat_model.predict([x3])
+    x3 = np.repeat(x3, 16, axis=1)
+    assert concat_out.shape == (1, 16, 1)
+    assert_allclose(concat_out, x3)
+
 
 @keras_test
 def test_merge_dot():
@@ -95,6 +130,10 @@ def test_merge_dot():
     assert o._keras_shape == (None, 1)
     model = models.Model([i1, i2], o)
 
+    dot_layer = layers.Dot(axes=1)
+    o2 = dot_layer([i1, i2])
+    assert dot_layer.output_shape == (None, 1)
+
     x1 = np.random.random((2, 4))
     x2 = np.random.random((2, 4))
     out = model.predict([x1, x2])
@@ -113,5 +152,55 @@ def test_merge_dot():
     assert_allclose(out, expected, atol=1e-4)
 
 
+@keras_test
+def test_merge_broadcast():
+    # shapes provided
+    i1 = layers.Input(shape=(4, 5))
+    i2 = layers.Input(shape=(5,))
+    ops = [layers.add, layers.maximum]
+    for op in ops:
+        o = op([i1, i2])
+        assert o._keras_shape == (None, 4, 5)
+        model = models.Model([i1, i2], o)
+
+        x1 = np.random.random((2, 4, 5))
+        x2 = np.random.random((2, 5))
+        out = model.predict([x1, x2])
+        assert out.shape == (2, 4, 5)
+
+    # shapes not provided
+    i1 = layers.Input(shape=(None, None))
+    i2 = layers.Input(shape=(None,))
+    ops = [layers.add, layers.maximum]
+    for op in ops:
+        o = op([i1, i2])
+        assert o._keras_shape == (None, None, None)
+        model = models.Model([i1, i2], o)
+
+        x1 = np.random.random((2, 4, 5))
+        x2 = np.random.random((2, 5))
+        out = model.predict([x1, x2])
+        assert out.shape == (2, 4, 5)
+
+    # ndim not provided
+    if K.backend() == 'tensorflow':
+        k_ndim = K.ndim
+        K.ndim = lambda _: None
+
+        i1 = layers.Input(shape=(None, None))
+        i2 = layers.Input(shape=(None,))
+        ops = [layers.add, layers.maximum]
+        for op in ops:
+            o = op([i1, i2])
+            assert o._keras_shape == (None, None, None)
+            model = models.Model([i1, i2], o)
+
+            x1 = np.random.random((2, 4, 5))
+            x2 = np.random.random((2, 5))
+            out = model.predict([x1, x2])
+            assert out.shape == (2, 4, 5)
+        K.ndim = k_ndim
+
+
 if __name__ == '__main__':
     pytest.main([__file__])

tests/keras/layers/normalization_test.py

@@ -52,6 +52,31 @@ def test_batchnorm_correctness():
     assert_allclose(out.std(), 1.0, atol=1e-1)
 
 
+@keras_test
+def test_batchnorm_training_argument():
+    bn1 = normalization.BatchNormalization(input_shape=(10,))
+    x1 = Input(shape=(10,))
+    y1 = bn1(x1, training=True)
+    assert bn1.updates
+
+    model1 = Model(x1, y1)
+    np.random.seed(123)
+    x = np.random.normal(loc=5.0, scale=10.0, size=(20, 10))
+    output_a = model1.predict(x)
+
+    model1.compile(loss='mse', optimizer='rmsprop')
+    model1.fit(x, x, epochs=1, verbose=0)
+    output_b = model1.predict(x)
+    assert np.abs(np.sum(output_a - output_b)) > 0.1
+    assert_allclose(output_b.mean(), 0.0, atol=1e-1)
+    assert_allclose(output_b.std(), 1.0, atol=1e-1)
+
+    bn2 = normalization.BatchNormalization(input_shape=(10,))
+    x2 = Input(shape=(10,))
+    bn2(x2, training=False)
+    assert not bn2.updates
+
+
 @keras_test
 def test_batchnorm_mode_twice():
     # This is a regression test for issue #4881 with the old

tests/keras/layers/recurrent_test.py

@@ -57,6 +57,14 @@ def test_dropout(layer_class):
                        'dropout': 0.1,
                        'recurrent_dropout': 0.1},
                input_shape=(num_samples, timesteps, embedding_dim))
+    # Test that dropout is not applied during testing
+    x = np.random.random((num_samples, timesteps, embedding_dim))
+    layer = layer_class(units, dropout=0.5, recurrent_dropout=0.5,
+                        input_shape=(timesteps, embedding_dim))
+    model = Sequential([layer])
+    y1 = model.predict(x)
+    y2 = model.predict(x)
+    assert_allclose(y1, y2)
 
 
 @rnn_test
@@ -169,29 +177,41 @@ def test_from_config(layer_class):
 
 
 @rnn_test
-def test_specify_initial_state(layer_class):
+def test_specify_initial_state_keras_tensor(layer_class):
     num_states = 2 if layer_class is recurrent.LSTM else 1
 
     # Test with Keras tensor
     inputs = Input((timesteps, embedding_dim))
     initial_state = [Input((units,)) for _ in range(num_states)]
     layer = layer_class(units)
-    output = layer(inputs, initial_state=initial_state)
+    if len(initial_state) == 1:
+        output = layer(inputs, initial_state=initial_state[0])
+    else:
+        output = layer(inputs, initial_state=initial_state)
+    assert initial_state[0] in layer.inbound_nodes[0].input_tensors
+
     model = Model([inputs] + initial_state, output)
     model.compile(loss='categorical_crossentropy', optimizer='adam')
 
     inputs = np.random.random((num_samples, timesteps, embedding_dim))
-    initial_states = [np.random.random((num_samples, units))
-                      for _ in range(num_states)]
+    initial_state = [np.random.random((num_samples, units))
+                     for _ in range(num_states)]
     targets = np.random.random((num_samples, units))
-    model.fit([inputs] + initial_states, targets)
+    model.fit([inputs] + initial_state, targets)
+
+
+@rnn_test
+def test_specify_initial_state_non_keras_tensor(layer_class):
+    num_states = 2 if layer_class is recurrent.LSTM else 1
 
     # Test with non-Keras tensor
     inputs = Input((timesteps, embedding_dim))
-    initial_state = [K.random_normal_variable((units,), 0, 1) for _ in range(num_states)]
+    initial_state = [K.random_normal_variable((num_samples, units), 0, 1)
+                     for _ in range(num_states)]
     layer = layer_class(units)
     output = layer(inputs, initial_state=initial_state)
-    model = Model([inputs], output)
+
+    model = Model(inputs, output)
     model.compile(loss='categorical_crossentropy', optimizer='adam')
 
     inputs = np.random.random((num_samples, timesteps, embedding_dim))
@@ -213,10 +233,38 @@ def test_reset_states_with_values(layer_class):
                                atol=1e-4)
     state_shapes = [K.int_shape(state) for state in layer.states]
     values = [np.ones(shape) for shape in state_shapes]
+    if len(values) == 1:
+        values = values[0]
     layer.reset_states(values)
     np.testing.assert_allclose(K.eval(layer.states[0]),
                                np.ones(K.int_shape(layer.states[0])),
                                atol=1e-4)
 
+    # Test fit with invalid data
+    with pytest.raises(ValueError):
+        layer.reset_states([1] * (len(layer.states) + 1))
+
+
+@rnn_test
+def test_specify_state_with_masking(layer_class):
+    ''' This test based on a previously failing issue here:
+    https://github.com/fchollet/keras/issues/1567
+    '''
+    num_states = 2 if layer_class is recurrent.LSTM else 1
+
+    inputs = Input((timesteps, embedding_dim))
+    masked_inputs = Masking()(inputs)
+    initial_state = [Input((units,)) for _ in range(num_states)]
+    output = layer_class(units)(inputs, initial_state=initial_state)
+
+    model = Model([inputs] + initial_state, output)
+    model.compile(loss='categorical_crossentropy', optimizer='adam')
+
+    inputs = np.random.random((num_samples, timesteps, embedding_dim))
+    initial_state = [np.random.random((num_samples, units))
+                     for _ in range(num_states)]
+    targets = np.random.random((num_samples, units))
+    model.fit([inputs] + initial_state, targets)
+
 if __name__ == '__main__':
     pytest.main([__file__])

tests/keras/layers/wrappers_test.py

@@ -90,7 +90,18 @@ def test_TimeDistributed():
 @keras_test
 def test_regularizers():
     model = Sequential()
-    model.add(wrappers.TimeDistributed(core.Dense(2, kernel_regularizer='l1'), input_shape=(3, 4)))
+    model.add(wrappers.TimeDistributed(
+        core.Dense(2, kernel_regularizer='l1'), input_shape=(3, 4)))
+    model.add(core.Activation('relu'))
+    model.compile(optimizer='rmsprop', loss='mse')
+    assert len(model.layers[0].layer.losses) == 1
+    assert len(model.layers[0].losses) == 1
+    assert len(model.layers[0].get_losses_for(None)) == 1
+    assert len(model.losses) == 1
+
+    model = Sequential()
+    model.add(wrappers.TimeDistributed(
+        core.Dense(2, activity_regularizer='l1'), input_shape=(3, 4)))
     model.add(core.Activation('relu'))
     model.compile(optimizer='rmsprop', loss='mse')
     assert len(model.losses) == 1
@@ -103,6 +114,7 @@ def test_Bidirectional():
     dim = 2
     timesteps = 2
     output_dim = 2
+    dropout_rate = 0.2
     for mode in ['sum', 'concat']:
         x = np.random.random((samples, timesteps, dim))
         target_dim = 2 * output_dim if mode == 'concat' else output_dim
@@ -110,7 +122,8 @@ def test_Bidirectional():
 
         # test with Sequential model
         model = Sequential()
-        model.add(wrappers.Bidirectional(rnn(output_dim),
+        model.add(wrappers.Bidirectional(rnn(output_dim, dropout=dropout_rate,
+                                             recurrent_dropout=dropout_rate),
                                          merge_mode=mode, input_shape=(timesteps, dim)))
         model.compile(loss='mse', optimizer='sgd')
         model.fit(x, y, epochs=1, batch_size=1)
@@ -130,7 +143,9 @@ def test_Bidirectional():
 
         # test with functional API
         input = Input((timesteps, dim))
-        output = wrappers.Bidirectional(rnn(output_dim), merge_mode=mode)(input)
+        output = wrappers.Bidirectional(rnn(output_dim, dropout=dropout_rate,
+                                            recurrent_dropout=dropout_rate),
+                                        merge_mode=mode)(input)
         model = Model(input, output)
         model.compile(loss='mse', optimizer='sgd')
         model.fit(x, y, epochs=1, batch_size=1)

tests/keras/losses_test.py

@@ -14,7 +14,8 @@ allobj = [losses.mean_squared_error,
           losses.binary_crossentropy,
           losses.kullback_leibler_divergence,
           losses.poisson,
-          losses.cosine_proximity]
+          losses.cosine_proximity,
+          losses.logcosh]
 
 
 def test_objective_shapes_3d():

tests/keras/metrics_test.py

@@ -17,6 +17,7 @@ all_metrics = [
     metrics.binary_crossentropy,
     metrics.poisson,
     metrics.cosine_proximity,
+    metrics.logcosh,
 ]
 
 all_sparse_metrics = [

tests/keras/test_sequential_model.py

@@ -122,7 +122,7 @@ def test_sequential():
 
     loss = model.evaluate(x_test, y_test)
 
-    prediction = model.predict_generator(data_generator(x_test, y_test), 1, max_q_size=2)
+    prediction = model.predict_generator(data_generator(x_test, y_test), 1, max_q_size=2, verbose=1)
     gen_loss = model.evaluate_generator(data_generator(x_test, y_test, 50), 1, max_q_size=2)
     pred_loss = K.eval(K.mean(losses.get(model.loss)(K.variable(y_test), K.variable(prediction))))
 

tests/keras/utils/data_utils_test.py

@@ -0,0 +1,59 @@
+"""Tests for functions in data_utils.py.
+"""
+import os
+import pytest
+import tarfile
+import zipfile
+from six.moves.urllib.request import pathname2url
+from six.moves.urllib.parse import urljoin
+from keras.utils.data_utils import get_file
+from keras.utils.data_utils import validate_file
+from keras.utils.data_utils import _hash_file
+from keras import activations
+from keras import regularizers
+
+
+def test_data_utils():
+    """Tests get_file from a url, plus extraction and validation.
+    """
+    dirname = 'data_utils'
+
+    with open('test.txt', 'w') as text_file:
+        text_file.write('Float like a butterfly, sting like a bee.')
+
+    with tarfile.open('test.tar.gz', 'w:gz') as tar_file:
+        tar_file.add('test.txt')
+
+    with zipfile.ZipFile('test.zip', 'w') as zip_file:
+        zip_file.write('test.txt')
+
+    origin = urljoin('file://', pathname2url(os.path.abspath('test.tar.gz')))
+
+    path = get_file(dirname, origin, untar=True)
+    filepath = path + '.tar.gz'
+    hashval_sha256 = _hash_file(filepath)
+    hashval_md5 = _hash_file(filepath, algorithm='md5')
+    path = get_file(dirname, origin, md5_hash=hashval_md5, untar=True)
+    path = get_file(filepath, origin, file_hash=hashval_sha256, extract=True)
+    assert os.path.exists(filepath)
+    assert validate_file(filepath, hashval_sha256)
+    assert validate_file(filepath, hashval_md5)
+    os.remove(filepath)
+    os.remove('test.tar.gz')
+
+    origin = urljoin('file://', pathname2url(os.path.abspath('test.zip')))
+
+    hashval_sha256 = _hash_file('test.zip')
+    hashval_md5 = _hash_file('test.zip', algorithm='md5')
+    path = get_file(dirname, origin, md5_hash=hashval_md5, extract=True)
+    path = get_file(dirname, origin, file_hash=hashval_sha256, extract=True)
+    assert os.path.exists(path)
+    assert validate_file(path, hashval_sha256)
+    assert validate_file(path, hashval_md5)
+
+    os.remove(path)
+    os.remove('test.txt')
+    os.remove('test.zip')
+
+if __name__ == '__main__':
+    pytest.main([__file__])

tests/keras/wrappers/scikit_learn_test.py

@@ -39,15 +39,16 @@ def build_fn_clf(hidden_dims):
     return model
 
 
-def test_clasify_build_fn():
+def test_classify_build_fn():
     clf = KerasClassifier(
         build_fn=build_fn_clf, hidden_dims=hidden_dims,
         batch_size=batch_size, epochs=epochs)
 
     assert_classification_works(clf)
+    assert_string_classification_works(clf)
 
 
-def test_clasify_class_build_fn():
+def test_classify_class_build_fn():
     class ClassBuildFnClf(object):
 
         def __call__(self, hidden_dims):
@@ -58,9 +59,10 @@ def test_clasify_class_build_fn():
         batch_size=batch_size, epochs=epochs)
 
     assert_classification_works(clf)
+    assert_string_classification_works(clf)
 
 
-def test_clasify_inherit_class_build_fn():
+def test_classify_inherit_class_build_fn():
     class InheritClassBuildFnClf(KerasClassifier):
 
         def __call__(self, hidden_dims):
@@ -71,6 +73,7 @@ def test_clasify_inherit_class_build_fn():
         batch_size=batch_size, epochs=epochs)
 
     assert_classification_works(clf)
+    assert_string_classification_works(clf)
 
 
 def assert_classification_works(clf):
@@ -89,6 +92,25 @@ def assert_classification_works(clf):
     assert np.allclose(np.sum(proba, axis=1), np.ones(num_test))
 
 
+def assert_string_classification_works(clf):
+    string_classes = ['cls{}'.format(x) for x in range(num_class)]
+    str_y_train = np.array(string_classes)[y_train]
+
+    clf.fit(X_train, str_y_train, batch_size=batch_size, epochs=epochs)
+
+    score = clf.score(X_train, str_y_train, batch_size=batch_size)
+    assert np.isscalar(score) and np.isfinite(score)
+
+    preds = clf.predict(X_test, batch_size=batch_size)
+    assert preds.shape == (num_test, )
+    for prediction in np.unique(preds):
+        assert prediction in string_classes
+
+    proba = clf.predict_proba(X_test, batch_size=batch_size)
+    assert proba.shape == (num_test, num_class)
+    assert np.allclose(np.sum(proba, axis=1), np.ones(num_test))
+
+
 def build_fn_reg(hidden_dims=50):
     model = Sequential()
     model.add(Dense(input_dim, input_shape=(input_dim,)))

tests/test_documentation.py

@@ -0,0 +1,151 @@
+import importlib
+import inspect
+import re
+import sys
+from itertools import compress
+
+import pytest
+
+modules = ['keras.layers', 'keras.models', 'keras', 'keras.backend.tensorflow_backend']
+accepted_name = ['from_config']
+accepted_module = ['keras.legacy.layers', 'keras.utils.generic_utils']
+
+# Functions or classes with less than 'MIN_CODE_SIZE' lines can be ignored
+MIN_CODE_SIZE = 10
+
+
+def handle_class(name, member):
+    if is_accepted(name, member):
+        return
+
+    if member.__doc__ is None and not member_too_small(member):
+        raise ValueError("{} class doesn't have any documentation".format(name),
+                         member.__module__, inspect.getmodule(member).__file__)
+    for n, met in inspect.getmembers(member):
+        if inspect.ismethod(met):
+            handle_method(n, met)
+
+
+def handle_function(name, member):
+    if is_accepted(name, member):
+        return
+    doc = member.__doc__
+    if doc is None and not member_too_small(member):
+        raise ValueError("{} function doesn't have any documentation".format(name),
+                         member.__module__, inspect.getmodule(member).__file__)
+    args = list(inspect.signature(member).parameters.keys())
+    assert_args_presence(args, doc, member, name)
+    assert_function_style(name, member, doc, args)
+    assert_doc_style(name, member, doc)
+
+
+def assert_doc_style(name, member, doc):
+    lines = doc.split("\n")
+    first_line = lines[0]
+    if len(first_line.strip()) == 0:
+        raise ValueError("{} the documentation should be on the first line.".format(name),
+                         member.__module__)
+    if first_line.strip()[-1] != '.':
+        raise ValueError("{} first line should end with a '.'".format(name),
+                         member.__module__)
+
+
+def assert_function_style(name, member, doc, args):
+    code = inspect.getsource(member)
+    has_return = re.findall(r"\s*return \S+", code, re.MULTILINE)
+    if has_return and "# Returns" not in doc:
+        innerfunction = [inspect.getsource(x) for x in member.__code__.co_consts if
+                         inspect.iscode(x)]
+        return_in_sub = [ret for code_inner in innerfunction for ret in
+                         re.findall(r"\s*return \S+", code_inner, re.MULTILINE)]
+        if len(return_in_sub) < len(has_return):
+            raise ValueError("{} needs a '# Returns' section".format(name),
+                             member.__module__)
+
+    has_raise = re.findall(r"^\s*raise \S+", code, re.MULTILINE)
+    if has_raise and "# Raises" not in doc:
+        innerfunction = [inspect.getsource(x) for x in member.__code__.co_consts if
+                         inspect.iscode(x)]
+        raise_in_sub = [ret for code_inner in innerfunction for ret in
+                        re.findall(r"\s*raise \S+", code_inner, re.MULTILINE)]
+        if len(raise_in_sub) < len(has_raise):
+            raise ValueError("{} needs a '# Raises' section".format(name),
+                             member.__module__)
+
+    if len(args) > 0 and "# Arguments" not in doc:
+        raise ValueError("{} needs a '# Arguments' section".format(name),
+                         member.__module__)
+
+    assert_blank_before(name, member, doc, ['# Arguments', '# Raises', '# Returns'])
+
+
+def assert_blank_before(name, member, doc, keywords):
+    doc_lines = [x.strip() for x in doc.split('\n')]
+    for keyword in keywords:
+        if keyword in doc_lines:
+            index = doc_lines.index(keyword)
+            if doc_lines[index - 1] != '':
+                raise ValueError(
+                    "{} '{}' should have a blank line above.".format(name, keyword),
+                    member.__module__)
+
+
+def is_accepted(name, member):
+    if 'keras' not in str(member.__module__):
+        return True
+    return name in accepted_name or member.__module__ in accepted_module
+
+
+def member_too_small(member):
+    code = inspect.getsource(member).split('\n')
+    return len(code) < MIN_CODE_SIZE
+
+
+def assert_args_presence(args, doc, member, name):
+    args_not_in_doc = [arg not in doc for arg in args]
+    if any(args_not_in_doc):
+        raise ValueError(
+            "{} {} arguments are not present in documentation ".format(name, list(
+                compress(args, args_not_in_doc))), member.__module__)
+    words = doc.replace('*', '').split()
+    # Check arguments styling
+    styles = [arg + ":" not in words for arg in args]
+    if any(styles):
+        raise ValueError(
+            "{} {} are not style properly 'argument': documentation".format(name, list(
+                compress(args, styles))), member.__module__)
+
+    # Check arguments order
+    indexes = [words.index(arg + ":") for arg in args]
+    if indexes != sorted(indexes):
+        raise ValueError(
+            "{} arguments order is different from the documentation".format(name),
+            member.__module__)
+
+
+def handle_method(name, member):
+    if name in accepted_name or member.__module__ in accepted_module:
+        return
+    handle_function(name, member)
+
+
+def handle_module(mod):
+    for name, mem in inspect.getmembers(mod):
+        if inspect.isclass(mem):
+            handle_class(name, mem)
+        elif inspect.isfunction(mem):
+            handle_function(name, mem)
+        elif 'keras' in name and inspect.ismodule(mem):
+            # Only test keras' modules
+            handle_module(mem)
+
+
+@pytest.mark.skipif(sys.version_info < (3, 3), reason="requires python3.3")
+def test_doc():
+    for module in modules:
+        mod = importlib.import_module(module)
+        handle_module(mod)
+
+
+if __name__ == '__main__':
+    pytest.main([__file__])

tests/test_model_saving.py

@@ -100,6 +100,35 @@ def test_fuctional_model_saving():
     assert_allclose(out, out2, atol=1e-05)
 
 
+@keras_test
+def test_saving_multiple_metrics_outputs():
+    input = Input(shape=(5,))
+    x = Dense(5)(input)
+    output1 = Dense(1, name='output1')(x)
+    output2 = Dense(1, name='output2')(x)
+
+    model = Model(inputs=input, outputs=[output1, output2])
+
+    metrics = {'output1': ['mse', 'binary_accuracy'],
+               'output2': ['mse', 'binary_accuracy']
+               }
+    loss = {'output1': 'mse', 'output2': 'mse'}
+
+    model.compile(loss=loss, optimizer='sgd', metrics=metrics)
+
+    # assure that model is working
+    x = np.array([[1, 1, 1, 1, 1]])
+    out = model.predict(x)
+    _, fname = tempfile.mkstemp('.h5')
+    save_model(model, fname)
+
+    model = load_model(fname)
+    os.remove(fname)
+
+    out2 = model.predict(x)
+    assert_allclose(out, out2, atol=1e-05)
+
+
 @keras_test
 def test_saving_without_compilation():
     model = Sequential()

tests/test_multiprocessing.py

@@ -1,4 +1,5 @@
 from __future__ import print_function
+import os
 import pytest
 import numpy as np
 from keras.models import Sequential
@@ -85,6 +86,8 @@ def test_multiprocessing_training_fromfile():
                         max_q_size=10,
                         pickle_safe=False)
 
+    os.remove('data.npz')
+
 
 @keras_test
 def test_multiprocessing_predicting():