1# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""Tests for training routines."""
16
17from __future__ import absolute_import
18from __future__ import division
19from __future__ import print_function
20
21import io
22import logging
23import sys
24
25from absl.testing import parameterized
26import numpy as np
27import six
28
29from tensorflow.python import keras
30from tensorflow.python import tf2
31from tensorflow.python.data.ops import dataset_ops
32from tensorflow.python.eager import context
33from tensorflow.python.eager import function
34from tensorflow.python.framework import ops
35from tensorflow.python.framework import tensor_shape
36from tensorflow.python.framework import test_util as tf_test_util
37from tensorflow.python.keras import keras_parameterized
38from tensorflow.python.keras import losses
39from tensorflow.python.keras import metrics as metrics_module
40from tensorflow.python.keras import testing_utils
41from tensorflow.python.keras.callbacks import Callback
42from tensorflow.python.ops import array_ops
43from tensorflow.python.ops import math_ops
44from tensorflow.python.ops import sparse_ops
45from tensorflow.python.ops import state_ops
46from tensorflow.python.ops import variables as variables_lib
47from tensorflow.python.platform import test
48from tensorflow.python.platform import tf_logging as logging
49from tensorflow.python.training.rmsprop import RMSPropOptimizer
50
51try:
52  import scipy.sparse as scipy_sparse  # pylint: disable=g-import-not-at-top
53except ImportError:
54  scipy_sparse = None
55
56
57class CompileTest(keras_parameterized.TestCase):
58
59  def _get_multi_output_model(self):
60    input_a = keras.layers.Input(shape=(3,), name='input_a')
61    output_a = keras.layers.Dense(1, name='dense_1')(input_a)
62    output_b = keras.layers.Dense(1, name='dense_2')(input_a)
63    return keras.models.Model(input_a, [output_a, output_b])
64
65  def _do_test_compile_with_model_and_single_loss(self, model, loss):
66    model.compile(optimizer='adam', loss=loss)
67    self.assertEqual(model.loss, loss)
68
69    loss = losses.get(loss)
70    if not isinstance(loss, list):
71      loss_list = [loss] * len(model.outputs)
72
73    self.assertEqual(len(model.loss_functions), len(loss_list))
74    for i in range(len(loss_list)):
75      self.assertIsInstance(model.loss_functions[i], losses.LossFunctionWrapper)
76      if not isinstance(loss_list[i], losses.LossFunctionWrapper):
77        self.assertEqual(model.loss_functions[i].fn, loss_list[i])
78    self.assertAllEqual(model.loss_weights_list, [1.] * len(loss_list))
79
80  @keras_parameterized.run_all_keras_modes
81  @parameterized.named_parameters(('loss_string', 'mse'),
82                                  ('loss_function', losses.mean_squared_error),
83                                  ('loss_instance', losses.MeanSquaredError()))
84  def test_compile_with_single_output(self, loss):
85    model = testing_utils.get_small_sequential_mlp(
86        num_hidden=10, num_classes=2, input_dim=3)
87    self._do_test_compile_with_model_and_single_loss(model, loss)
88
89  @keras_parameterized.run_all_keras_modes
90  @parameterized.named_parameters(('loss_string', 'mse'),
91                                  ('loss_function', losses.mean_squared_error),
92                                  ('loss_instance', losses.MeanSquaredError()))
93  def test_compile_with_multi_output(self, loss):
94    model = self._get_multi_output_model()
95    self._do_test_compile_with_model_and_single_loss(model, loss)
96
97  @keras_parameterized.run_all_keras_modes
98  def test_compile_with_multi_output_and_multi_loss(self):
99    model = self._get_multi_output_model()
100    # Test loss is a list.
101    loss = ['mse', 'mae']
102    model.compile(optimizer='adam', loss=loss)
103    self.assertEqual(model.loss_functions[0].fn, losses.mean_squared_error)
104    self.assertEqual(model.loss_functions[1].fn, losses.mean_absolute_error)
105    self.assertAllEqual(model.loss_weights_list, [1., 1.])
106
107    # Test loss is a dict.
108    loss = {'dense_1': 'mae', 'dense_2': 'mse'}
109    model.compile(optimizer='adam', loss=loss)
110    self.assertEqual(model.loss_functions[0].fn, losses.mean_absolute_error)
111    self.assertEqual(model.loss_functions[1].fn, losses.mean_squared_error)
112    self.assertAllEqual(model.loss_weights_list, [1., 1.])
113
114  @keras_parameterized.run_all_keras_modes
115  def test_compile_with_multi_output_and_loss_weights_list(self):
116    model = self._get_multi_output_model()
117    loss_weights = [1., 2.]
118    model.compile(optimizer='adam', loss='mse', loss_weights=loss_weights)
119    self.assertAllEqual(model.loss_weights_list, [1., 2.])
120
121  def test_compile_with_multi_output_and_loss_weights_dict(self):
122    with context.graph_mode():
123      model = self._get_multi_output_model()
124      loss_weights = {'dense_1': 1., 'dense_2': 2.}
125      model.compile(optimizer='adam', loss='mse', loss_weights=loss_weights)
126      self.assertAllEqual(model.loss_weights_list, [1., 2.])
127
128      input_np = np.random.random((10, 3))
129      output_a_np = np.random.random((10, 1))
130      output_b_np = np.random.random((10, 1))
131
132      with self.cached_session() as sess:
133        sess.run(variables_lib.global_variables_initializer())
134        total_loss, y_preds = sess.run(
135            [model.total_loss, model.outputs],
136            feed_dict={
137                'input_a:0': input_np,
138                'dense_1_target:0': output_a_np,
139                'dense_2_target:0': output_b_np
140            })
141        self.assertAllClose(
142            total_loss,
143            np.mean(
144                np.add((output_a_np - y_preds[0])**2,
145                       2 * (output_b_np - y_preds[1])**2)))
146
147  @keras_parameterized.run_all_keras_modes
148  def test_compile_with_incorrect_loss_size(self):
149    model = testing_utils.get_small_sequential_mlp(
150        num_hidden=10, num_classes=2, input_dim=3)
151    with self.assertRaisesRegexp(ValueError, 'The model has 1 outputs'):
152      model.compile(optimizer='adam', loss=['mse', 'mae'])
153
154  @keras_parameterized.run_all_keras_modes
155  def test_compile_with_incorrect_loss_key(self):
156    model = testing_utils.get_small_sequential_mlp(
157        num_hidden=10, num_classes=2, input_dim=3)
158    with self.assertRaisesRegexp(
159        ValueError, 'Unknown entry in loss dictionary: unknown_output'):
160      model.compile(optimizer='adam', loss={'unknown_output': 'mse'})
161
162  @keras_parameterized.run_all_keras_modes
163  def test_compile_with_incorrect_loss_weights_size(self):
164    model = testing_utils.get_small_sequential_mlp(
165        num_hidden=10, num_classes=2, input_dim=3)
166    with self.assertRaisesRegexp(ValueError,
167                                 'it should have one entry per model output'):
168      model.compile(optimizer='adam', loss='mse', loss_weights=[1., 2.])
169
170  @keras_parameterized.run_all_keras_modes
171  def test_compile_with_incorrect_loss_weights_key(self):
172    model = testing_utils.get_small_sequential_mlp(
173        num_hidden=10, num_classes=2, input_dim=3)
174    with self.assertRaisesRegexp(
175        ValueError, 'Unknown entry in loss_weights dictionary: unknown_output'):
176      model.compile(
177          optimizer='adam', loss='mse', loss_weights={'unknown_output': 1.})
178
179
180class TrainingTest(keras_parameterized.TestCase):
181
182  @keras_parameterized.run_with_all_model_types(exclude_models='sequential')
183  @keras_parameterized.run_all_keras_modes
184  def test_fit_on_arrays(self):
185    input_a = keras.layers.Input(shape=(3,), name='input_a')
186    input_b = keras.layers.Input(shape=(3,), name='input_b')
187
188    dense = keras.layers.Dense(4, name='dense')
189    dropout = keras.layers.Dropout(0.5, name='dropout')
190    branch_a = [input_a, dense]
191    branch_b = [input_b, dense, dropout]
192
193    model = testing_utils.get_multi_io_model(branch_a, branch_b)
194
195    optimizer = RMSPropOptimizer(learning_rate=0.001)
196    loss = 'mse'
197    loss_weights = [1., 0.5]
198    model.compile(
199        optimizer,
200        loss,
201        metrics=[metrics_module.CategoricalAccuracy(), 'mae'],
202        loss_weights=loss_weights,
203        run_eagerly=testing_utils.should_run_eagerly())
204
205    input_a_np = np.random.random((10, 3))
206    input_b_np = np.random.random((10, 3))
207
208    output_d_np = np.random.random((10, 4))
209    output_e_np = np.random.random((10, 4))
210
211    # Test fit at different verbosity
212    model.fit(
213        [input_a_np, input_b_np], [output_d_np, output_e_np],
214        epochs=1,
215        batch_size=5,
216        verbose=0)
217    model.fit(
218        [input_a_np, input_b_np], [output_d_np, output_e_np],
219        epochs=1,
220        batch_size=5,
221        verbose=1)
222    model.fit(
223        [input_a_np, input_b_np], [output_d_np, output_e_np],
224        epochs=2,
225        batch_size=5,
226        verbose=2)
227    model.train_on_batch([input_a_np, input_b_np], [output_d_np, output_e_np])
228
229    # Test model with input data as a list of lists
230    model.fit(
231        [np.ndarray.tolist(input_a_np), np.ndarray.tolist(input_b_np)],
232        [output_d_np, output_e_np],
233        epochs=2,
234        batch_size=5,
235        verbose=2)
236
237    # Test with validation data
238    model.fit(
239        [input_a_np, input_b_np], [output_d_np, output_e_np],
240        validation_data=([input_a_np, input_b_np], [output_d_np,
241                                                    output_e_np]),
242        epochs=1,
243        batch_size=5,
244        verbose=0)
245    model.fit(
246        [input_a_np, input_b_np], [output_d_np, output_e_np],
247        validation_data=([input_a_np, input_b_np], [output_d_np,
248                                                    output_e_np]),
249        epochs=2,
250        batch_size=5,
251        verbose=1)
252    model.fit(
253        [input_a_np, input_b_np], [output_d_np, output_e_np],
254        validation_data=([input_a_np, input_b_np], [output_d_np,
255                                                    output_e_np]),
256        epochs=2,
257        batch_size=5,
258        verbose=2)
259    # Test with validation split
260    model.fit(
261        [input_a_np, input_b_np], [output_d_np, output_e_np],
262        epochs=2,
263        batch_size=5,
264        verbose=0,
265        validation_split=0.2)
266
267    if testing_utils.get_model_type() == 'functional':
268      # Test with dictionary inputs
269      model.fit(
270          {
271              'input_a': input_a_np,
272              'input_b': input_b_np
273          }, {
274              'dense': output_d_np,
275              'dropout': output_e_np
276          },
277          epochs=1,
278          batch_size=5,
279          verbose=0)
280      model.fit(
281          {
282              'input_a': input_a_np,
283              'input_b': input_b_np
284          }, {
285              'dense': output_d_np,
286              'dropout': output_e_np
287          },
288          epochs=1,
289          batch_size=5,
290          verbose=1)
291      model.fit(
292          {
293              'input_a': input_a_np,
294              'input_b': input_b_np
295          }, {
296              'dense': output_d_np,
297              'dropout': output_e_np
298          },
299          validation_data=({
300              'input_a': input_a_np,
301              'input_b': input_b_np
302          }, {
303              'dense': output_d_np,
304              'dropout': output_e_np
305          }),
306          epochs=1,
307          batch_size=5,
308          verbose=0)
309      model.train_on_batch({
310          'input_a': input_a_np,
311          'input_b': input_b_np
312      }, {
313          'dense': output_d_np,
314          'dropout': output_e_np
315      })
316
317    # Test with lists for loss, metrics
318    loss = ['mae', 'mse']
319    model.compile(
320        optimizer,
321        loss,
322        metrics=[metrics_module.CategoricalAccuracy(), 'mae'],
323        run_eagerly=testing_utils.should_run_eagerly())
324    model.fit(
325        [input_a_np, input_b_np], [output_d_np, output_e_np],
326        epochs=1,
327        batch_size=5,
328        verbose=0)
329
330    # Test with dictionaries for loss, metrics, loss weights
331    if testing_utils.get_model_type() == 'functional':
332      loss = {'dense': 'mse', 'dropout': 'mae'}
333      loss_weights = {'dense': 1., 'dropout': 0.5}
334      metrics = {
335          'dense': 'mse',
336          'dropout': metrics_module.CategoricalAccuracy()
337      }
338      model.compile(optimizer, loss, metrics=metrics, loss_weights=loss_weights,
339                    run_eagerly=testing_utils.should_run_eagerly())
340    model.fit(
341        [input_a_np, input_b_np], [output_d_np, output_e_np],
342        epochs=1,
343        batch_size=5,
344        verbose=0)
345
346    # Invalid use cases
347    with self.assertRaises(ValueError):
348      model.train_on_batch({'input_a': input_a_np},
349                           [output_d_np, output_e_np])
350    with self.assertRaises(AttributeError):
351      model.fit(
352          [input_a_np, input_b_np], [output_d_np, output_e_np],
353          epochs=1,
354          validation_data=([input_a_np, input_b_np], 0, 0),
355          verbose=0)
356    with self.assertRaises(ValueError):
357      model.train_on_batch([input_a_np], [output_d_np, output_e_np])
358    with self.assertRaises(AttributeError):
359      model.train_on_batch(1, [output_d_np, output_e_np])
360    with self.assertRaises(ValueError):
361      model.train_on_batch(input_a_np, [output_d_np, output_e_np])
362    with self.assertRaises(ValueError):
363      bad_input = np.random.random((11, 3))
364      model.train_on_batch([bad_input, input_b_np],
365                           [output_d_np, output_e_np])
366    with self.assertRaises(ValueError):
367      bad_target = np.random.random((11, 4))
368      model.train_on_batch([input_a_np, input_b_np],
369                           [bad_target, output_e_np])
370
371    # Build single-input model
372    x = keras.layers.Input(shape=(3,), name='input_a')
373    y = keras.layers.Dense(4)(x)
374    model = keras.models.Model(x, y)
375    model.compile(optimizer, loss='mse',
376                  run_eagerly=testing_utils.should_run_eagerly())
377    # This will work
378    model.fit([input_a_np], output_d_np, epochs=1)
379    # TODO(gsundeep) Test only works in eager, file ticket
380    if testing_utils.should_run_eagerly() and context.executing_eagerly():
381      with self.assertRaises(ValueError):
382        model.fit([input_a_np, input_a_np], output_d_np, epochs=1)
383
384    # Test model on a list of floats
385    input_a_np = np.random.random((10, 3))
386    input_b_np = np.random.random((10, 4))
387
388    model.fit([np.ndarray.tolist(input_a_np)],
389              [np.ndarray.tolist(input_b_np)],
390              epochs=2,
391              batch_size=5,
392              verbose=2)
393
394  @keras_parameterized.run_all_keras_modes
395  def test_evaluate_predict_on_arrays(self):
396    a = keras.layers.Input(shape=(3,), name='input_a')
397    b = keras.layers.Input(shape=(3,), name='input_b')
398
399    dense = keras.layers.Dense(4, name='dense')
400    c = dense(a)
401    d = dense(b)
402    e = keras.layers.Dropout(0.5, name='dropout')(c)
403
404    model = keras.models.Model([a, b], [d, e])
405
406    optimizer = RMSPropOptimizer(learning_rate=0.001)
407    loss = 'mse'
408    loss_weights = [1., 0.5]
409    model.compile(
410        optimizer,
411        loss,
412        metrics=['mae', metrics_module.CategoricalAccuracy()],
413        loss_weights=loss_weights,
414        sample_weight_mode=None,
415        run_eagerly=testing_utils.should_run_eagerly())
416
417    input_a_np = np.random.random((10, 3))
418    input_b_np = np.random.random((10, 3))
419
420    output_d_np = np.random.random((10, 4))
421    output_e_np = np.random.random((10, 4))
422
423    # Test evaluate at different verbosity
424    out = model.evaluate(
425        [input_a_np, input_b_np], [output_d_np, output_e_np],
426        batch_size=5,
427        verbose=0)
428    self.assertEqual(len(out), 7)
429    out = model.evaluate(
430        [input_a_np, input_b_np], [output_d_np, output_e_np],
431        batch_size=5,
432        verbose=1)
433    self.assertEqual(len(out), 7)
434    out = model.evaluate(
435        [input_a_np, input_b_np], [output_d_np, output_e_np],
436        batch_size=5,
437        verbose=2)
438    self.assertEqual(len(out), 7)
439    out = model.test_on_batch([input_a_np, input_b_np],
440                              [output_d_np, output_e_np])
441    self.assertEqual(len(out), 7)
442
443    # Test evaluate with dictionary inputs
444    model.evaluate(
445        {
446            'input_a': input_a_np,
447            'input_b': input_b_np
448        }, {
449            'dense': output_d_np,
450            'dropout': output_e_np
451        },
452        batch_size=5,
453        verbose=0)
454    model.evaluate(
455        {
456            'input_a': input_a_np,
457            'input_b': input_b_np
458        }, {
459            'dense': output_d_np,
460            'dropout': output_e_np
461        },
462        batch_size=5,
463        verbose=1)
464
465    # Test predict
466    out = model.predict([input_a_np, input_b_np], batch_size=5)
467    self.assertEqual(len(out), 2)
468    out = model.predict({'input_a': input_a_np, 'input_b': input_b_np})
469    self.assertEqual(len(out), 2)
470    out = model.predict_on_batch({
471        'input_a': input_a_np,
472        'input_b': input_b_np
473    })
474    self.assertEqual(len(out), 2)
475
476  @keras_parameterized.run_all_keras_modes
477  @keras_parameterized.run_with_all_model_types
478  def test_activity_regularizer_fit(self):
479    loss = {}
480    for reg in [None, 'l2']:
481      layers = [
482          keras.layers.Dense(
483              10, activation='relu', activity_regularizer=reg,
484              kernel_initializer='ones', use_bias=False),
485          keras.layers.Dense(
486              1, activation='sigmoid', kernel_initializer='ones',
487              use_bias=False),
488      ]
489
490      model = testing_utils.get_model_from_layers(
491          layers, input_shape=(10,))
492
493      x = np.ones((10, 10), 'float32')
494      y = np.ones((10, 1), 'float32')
495
496      optimizer = RMSPropOptimizer(learning_rate=0.001)
497      model.compile(optimizer, 'binary_crossentropy',
498                    run_eagerly=testing_utils.should_run_eagerly())
499      model.fit(x, y, batch_size=2, epochs=5)
500      loss[reg] = model.evaluate(x, y)
501    self.assertLess(loss[None], loss['l2'])
502
503  @keras_parameterized.run_all_keras_modes
504  @keras_parameterized.run_with_all_model_types
505  def test_activity_regularizer_loss_value(self):
506    layer = keras.layers.Dense(
507        1, kernel_initializer=keras.initializers.zeros(),
508        bias_initializer=keras.initializers.ones(), activity_regularizer='l2')
509
510    model = testing_utils.get_model_from_layers([layer], input_shape=(10,))
511
512    x = np.ones((10, 10), 'float32')
513    y = np.ones((10, 1), 'float32')
514    optimizer = RMSPropOptimizer(learning_rate=0.001)
515    model.compile(optimizer, 'binary_crossentropy',
516                  run_eagerly=testing_utils.should_run_eagerly())
517    loss = model.test_on_batch(x, y)
518    self.assertAlmostEqual(0.01, loss, places=4)
519
520  @keras_parameterized.run_all_keras_modes
521  def test_activity_regularizer_batch_independent(self):
522    inputs = keras.layers.Input(shape=(10,))
523    x = keras.layers.Dense(
524        10, activation='relu', activity_regularizer='l2')(
525            inputs)
526    outputs = keras.layers.Dense(1, activation='sigmoid')(x)
527    model = keras.Model(inputs, outputs)
528
529    optimizer = RMSPropOptimizer(learning_rate=0.001)
530    model.compile(optimizer, 'binary_crossentropy',
531                  run_eagerly=testing_utils.should_run_eagerly())
532
533    x = np.ones((10, 10), 'float32')
534    y = np.ones((10, 1), 'float32')
535    loss_small_batch = model.test_on_batch(x, y)
536
537    x2 = np.ones((20, 10), 'float32')
538    y2 = np.ones((20, 1), 'float32')
539    loss_big_batch = model.test_on_batch(x2, y2)
540
541    self.assertAlmostEqual(loss_small_batch, loss_big_batch, places=4)
542
543  @keras_parameterized.run_all_keras_modes
544  def test_activity_regularizer_in_model_call(self):
545
546    class MyModel(keras.Model):
547
548      def call(self, inputs):
549        self.add_loss(inputs)
550        return inputs
551
552    x = ops.convert_to_tensor(1.)
553    model = MyModel()
554    _ = model(x)
555    self.assertEqual(1, len(model.losses))
556
557  @keras_parameterized.run_all_keras_modes
558  def test_custom_mapping_in_config(self):
559
560    class MyModel(keras.Model):
561
562      def call(self, inputs):
563        return inputs
564
565      def get_config(self):
566        self.a = {}
567        return {'a': self.a}
568
569    model = MyModel()
570    self.assertIn('{"a": {}}', model.to_json())
571
572  @keras_parameterized.run_all_keras_modes
573  def test_training_on_sparse_data_with_dense_placeholders(self):
574    # TODO(kaftan) Test seems to not work, file ticket
575    if testing_utils.should_run_eagerly() and context.executing_eagerly():
576      self.skipTest('Skipping running model eagerly.')
577
578    if scipy_sparse is None:
579      return
580
581    test_inputs = [
582        scipy_sparse.random(6, 3, density=0.25).tocsr() for _ in range(2)
583    ]
584    test_outputs = [
585        scipy_sparse.random(6, i, density=0.25).tocsr() for i in range(3, 5)
586    ]
587    in1 = keras.layers.Input(shape=(3,))
588    in2 = keras.layers.Input(shape=(3,))
589    out1 = keras.layers.Dropout(0.5, name='dropout')(in1)
590    out2 = keras.layers.Dense(4, name='dense_1')(in2)
591    model = keras.Model([in1, in2], [out1, out2])
592    model.predict(test_inputs, batch_size=2)
593    optimizer = RMSPropOptimizer(learning_rate=0.001)
594    model.compile(
595        optimizer,
596        'mse',
597        metrics=['mae', metrics_module.CategoricalAccuracy()],
598        run_eagerly=testing_utils.should_run_eagerly())
599    model.fit(test_inputs, test_outputs,
600              epochs=1, batch_size=2, validation_split=0.5)
601    model.evaluate(test_inputs, test_outputs, batch_size=2)
602
603  @keras_parameterized.run_all_keras_modes
604  def test_compile_with_sparse_placeholders(self):
605    # TODO(kaftan) Test seems to not work, file ticket
606    if testing_utils.should_run_eagerly() and context.executing_eagerly():
607      self.skipTest('Skipping running model eagerly.')
608
609    input_layer = keras.layers.Input(shape=(10,), sparse=True)
610    weights = variables_lib.Variable(
611        np.ones((10, 1)).astype(np.float32), name='weights')
612    weights_mult = lambda x: sparse_ops.sparse_tensor_dense_matmul(x, weights)
613    output_layer = keras.layers.Lambda(weights_mult)(input_layer)
614    model = keras.Model([input_layer], output_layer)
615    model.compile(
616        loss='binary_crossentropy',
617        optimizer=keras.optimizers.Adam(lr=0.0001),
618        metrics=['accuracy'],
619        run_eagerly=testing_utils.should_run_eagerly())
620
621  def test_that_trainable_disables_updates(self):
622    val_a = np.random.random((10, 4))
623    val_out = np.random.random((10, 4))
624
625    with self.cached_session():
626      a = keras.layers.Input(shape=(4,))
627      layer = keras.layers.BatchNormalization(input_shape=(4,))
628      b = layer(a)
629      model = keras.Model(a, b)
630
631      model.trainable = False
632      assert not model.updates
633
634      model.compile('sgd', 'mse')
635      assert not model.updates
636
637      x1 = model.predict(val_a)
638      model.train_on_batch(val_a, val_out)
639      x2 = model.predict(val_a)
640      self.assertAllClose(x1, x2, atol=1e-7)
641
642      model.trainable = True
643      model.compile('sgd', 'mse')
644      assert model.updates
645
646      model.train_on_batch(val_a, val_out)
647      x2 = model.predict(val_a)
648      assert np.abs(np.sum(x1 - x2)) > 1e-5
649
650      layer.trainable = False
651      model.compile('sgd', 'mse')
652      assert not model.updates
653
654      x1 = model.predict(val_a)
655      model.train_on_batch(val_a, val_out)
656      x2 = model.predict(val_a)
657      self.assertAllClose(x1, x2, atol=1e-7)
658
659  def test_logs_passed_to_callbacks(self):
660    with self.cached_session():
661      input_dim = 5
662      num_classes = 1
663
664      class TestCallback(Callback):
665
666        def __init__(self):
667          super(TestCallback, self).__init__()
668          self.epoch_end_logs = None
669          self.batch_end_logs = None
670          self.epoch_end_call_count = 0
671          self.batch_end_call_count = 0
672
673        def on_epoch_end(self, epoch, logs=None):
674          self.epoch_end_logs = logs
675          self.epoch_end_call_count += 1
676
677        def on_batch_end(self, batch, logs=None):
678          self.batch_end_logs = logs
679          self.batch_end_call_count += 1
680
681      model = testing_utils.get_small_sequential_mlp(
682          num_hidden=10, num_classes=num_classes, input_dim=input_dim)
683      model.compile(
684          loss='binary_crossentropy',
685          metrics=['acc'],
686          weighted_metrics=['mae'],
687          optimizer=RMSPropOptimizer(learning_rate=0.01))
688
689      np.random.seed(1337)
690      (x_train, y_train), (_, _) = testing_utils.get_test_data(
691          train_samples=10,
692          test_samples=10,
693          input_shape=(input_dim,),
694          num_classes=num_classes)
695
696      test_callback = TestCallback()
697      model.fit(
698          x_train,
699          y_train,
700          batch_size=2,
701          epochs=2,
702          verbose=0,
703          callbacks=[test_callback],
704          validation_data=(x_train, y_train))
705      self.assertEqual(test_callback.batch_end_call_count, 10)
706      self.assertEqual(test_callback.epoch_end_call_count, 2)
707
708      weighted_metric = ('mae'
709                         if tf2.enabled() else 'weighted_mean_absolute_error')
710      self.assertSetEqual(
711          set(test_callback.batch_end_logs.keys()),
712          set(['batch', 'size', 'acc', 'loss', weighted_metric]))
713      self.assertSetEqual(
714          set(test_callback.epoch_end_logs.keys()),
715          set([
716              'acc', 'loss', weighted_metric, 'val_acc', 'val_loss',
717              'val_' + weighted_metric
718          ]))
719
720  @keras_parameterized.run_all_keras_modes
721  def test_mismatched_output_shape_and_target_shape(self):
722    model = keras.Sequential([
723        keras.layers.Dense(2, input_shape=(3, 4)),
724        keras.layers.Dense(5),
725    ])
726    model.compile(RMSPropOptimizer(learning_rate=0.001),
727                  loss='sparse_categorical_crossentropy',
728                  run_eagerly=testing_utils.should_run_eagerly())
729    # Test with Numpy data
730    x_train = np.random.random((10, 3, 4))
731    y_train = np.random.randint(0, 5, size=(10, 3))
732    model.fit(x_train, y_train, batch_size=5, epochs=1)
733
734    # Test with iterator
735    dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train))
736    dataset = dataset.repeat(10)
737    dataset = dataset.batch(10)
738    iterator = dataset_ops.make_one_shot_iterator(dataset)
739    model.fit(iterator, epochs=1, steps_per_epoch=2)
740
741    if context.executing_eagerly():
742      # Test with eager execution
743      model.compile(RMSPropOptimizer(learning_rate=0.001),
744                    loss='sparse_categorical_crossentropy',
745                    run_eagerly=True)
746      model.fit(x_train, y_train, batch_size=5, epochs=1)
747
748      # Test with eager execution and iterator
749      model.fit(iterator, epochs=1, steps_per_epoch=2)
750
751  def test_losses_in_defun(self):
752    with context.eager_mode():
753      layer = keras.layers.Dense(1, kernel_regularizer='l1')
754      layer(array_ops.ones([1, 10]))
755
756      @function.defun
757      def get_losses():
758        return layer.losses
759
760      self.assertAllEqual(
761          self.evaluate(layer.losses), self.evaluate(get_losses()))
762
763  @keras_parameterized.run_all_keras_modes
764  def test_logging(self):
765    mock_stdout = io.BytesIO() if six.PY2 else io.StringIO()
766    model = keras.models.Sequential()
767    model.add(keras.layers.Dense(10, activation='relu'))
768    model.add(keras.layers.Dense(1, activation='sigmoid'))
769    model.compile(
770        RMSPropOptimizer(learning_rate=0.001), loss='binary_crossentropy',
771        run_eagerly=testing_utils.should_run_eagerly())
772    with test.mock.patch.object(sys, 'stdout', mock_stdout):
773      model.fit(
774          np.ones((10, 10), 'float32'), np.ones((10, 1), 'float32'), epochs=10)
775    self.assertTrue('Epoch 5/10' in mock_stdout.getvalue())
776
777  @tf_test_util.run_in_graph_and_eager_modes
778  def test_training_with_loss_instance(self):
779    a = keras.layers.Input(shape=(3,), name='input_a')
780    b = keras.layers.Input(shape=(3,), name='input_b')
781
782    dense = keras.layers.Dense(4, name='dense')
783    c = dense(a)
784    d = dense(b)
785    e = keras.layers.Dropout(0.5, name='dropout')(c)
786
787    model = keras.models.Model([a, b], [d, e])
788    loss_weights = [1., 0.5]
789    model.compile(
790        RMSPropOptimizer(learning_rate=0.001),
791        loss=keras.losses.MeanSquaredError(),
792        metrics=[metrics_module.CategoricalAccuracy(), 'mae'],
793        loss_weights=loss_weights)
794
795    input_a_np = np.random.random((10, 3))
796    input_b_np = np.random.random((10, 3))
797
798    output_d_np = np.random.random((10, 4))
799    output_e_np = np.random.random((10, 4))
800
801    model.fit([input_a_np, input_b_np], [output_d_np, output_e_np],
802              epochs=1,
803              batch_size=5)
804
805  @tf_test_util.run_in_graph_and_eager_modes
806  def test_static_batch_in_input_layer(self):
807
808    class Counter(keras.callbacks.Callback):
809
810      def __init__(self):
811        self.batches = 0
812
813      def on_batch_end(self, batch, logs=None):
814        self.batches += 1
815
816    x, y = np.ones((64, 10), 'float32'), np.ones((64, 1), 'float32')
817
818    for batch_size, expected_batches in [(None, 2), (4, 16)]:
819      inputs = keras.Input(batch_size=batch_size, shape=(10,))
820      outputs = keras.layers.Dense(1, activation='sigmoid')(inputs)
821      model = keras.Model(inputs, outputs)
822
823      model.compile(keras.optimizer_v2.adam.Adam(0.001), 'binary_crossentropy')
824      counter = Counter()
825      model.fit(x, y, callbacks=[counter])
826      self.assertEqual(counter.batches, expected_batches)
827
828      model = keras.Sequential(
829          [keras.layers.Dense(1, batch_input_shape=(batch_size, 10))])
830      model.compile(keras.optimizer_v2.adam.Adam(0.001), 'binary_crossentropy')
831      counter = Counter()
832      model.fit(x, y, callbacks=[counter])
833      self.assertEqual(counter.batches, expected_batches)
834
835  @tf_test_util.run_in_graph_and_eager_modes
836  def test_static_batch_in_input_layer_consistency_checks(self):
837    x, y = np.ones((64, 10), 'float32'), np.ones((64, 1), 'float32')
838
839    inputs = keras.Input(batch_size=2, shape=(10,))
840    outputs = keras.layers.Dense(1, activation='sigmoid')(inputs)
841    model = keras.Model(inputs, outputs)
842    model.compile(keras.optimizer_v2.adam.Adam(0.001), 'binary_crossentropy')
843    with self.assertRaisesRegexp(ValueError,
844                                 'incompatible with the specified batch size'):
845      model.fit(x, y, batch_size=4)
846
847    data = dataset_ops.DatasetV2.from_tensor_slices((x, y))
848    data = data.batch(4, drop_remainder=True)
849    with self.assertRaisesRegexp(ValueError,
850                                 'incompatible with the specified batch size'):
851      model.fit(data, steps_per_epoch=16)
852
853  @tf_test_util.run_in_graph_and_eager_modes
854  def test_compatible_batch_size_functional_model(self):
855
856    class MyLayer(keras.layers.Layer):
857
858      def call(self, inputs):
859        return array_ops.concat(inputs, axis=0)
860
861    input1 = keras.Input(batch_size=2, shape=(10,))
862    input2 = keras.Input(batch_size=3, shape=(10,))
863    outputs = MyLayer()([input1, input2])
864    with self.assertRaisesRegexp(ValueError,
865                                 'specified batch sizes of the Input Layers'):
866      keras.Model([input1, input2], outputs)
867
868  @tf_test_util.run_in_graph_and_eager_modes
869  def test_calling_subclass_model_on_different_datasets(self):
870
871    class SubclassedModel(keras.models.Model):
872
873      def call(self, inputs):
874        return inputs * 2
875
876    model = SubclassedModel()
877    dataset_one = dataset_ops.Dataset.range(2).batch(2)
878    dataset_two = dataset_ops.Dataset.range(3, 10).batch(2)
879    self.assertAllEqual([[0], [2]], model.predict(dataset_one, steps=1))
880    self.assertAllEqual([[6], [8], [10], [12]],
881                        model.predict(dataset_two, steps=2))
882
883  def test_training_on_sparse_categorical_crossentropy_loss_with_softmax(self):
884    with context.eager_mode():
885      np.random.seed(1337)
886      train_x = np.ones((100, 4))
887      train_y = np.random.randint(0, 1, size=(100, 1))
888
889      reference_model = testing_utils.get_small_sequential_mlp(16, 2,
890                                                               input_dim=4)
891      reference_model.compile(loss='sparse_categorical_crossentropy',
892                              optimizer=RMSPropOptimizer(learning_rate=0.001),
893                              run_eagerly=True)
894      fixed_weights = reference_model.get_weights()
895      reference_model_loss = reference_model.train_on_batch(train_x, train_y)
896
897      test_model = testing_utils.get_small_sequential_mlp(16, 2, input_dim=4)
898      test_model.compile(loss='sparse_categorical_crossentropy',
899                         optimizer=RMSPropOptimizer(learning_rate=0.001),
900                         run_eagerly=False)
901      test_model.set_weights(fixed_weights)
902      test_model_loss = test_model.train_on_batch(train_x, train_y)
903      self.assertAlmostEqual(test_model_loss, reference_model_loss, places=4)
904
905  def test_training_on_categorical_crossentropy_loss_with_softmax(self):
906    with context.eager_mode():
907      np.random.seed(1337)
908      train_x = np.ones((100, 4))
909      train_y = keras.utils.to_categorical(np.random.randint(0, 1,
910                                                             size=(100, 1)), 2)
911
912      reference_model = testing_utils.get_small_sequential_mlp(16, 2,
913                                                               input_dim=4)
914      reference_model.compile(loss='categorical_crossentropy',
915                              optimizer=RMSPropOptimizer(learning_rate=0.001),
916                              run_eagerly=True)
917      fixed_weights = reference_model.get_weights()
918      reference_model_loss = reference_model.train_on_batch(train_x, train_y)
919
920      test_model = testing_utils.get_small_sequential_mlp(16, 2, input_dim=4)
921      test_model.compile(loss='categorical_crossentropy',
922                         optimizer=RMSPropOptimizer(learning_rate=0.001),
923                         run_eagerly=False)
924      test_model.set_weights(fixed_weights)
925      test_model_loss = test_model.train_on_batch(train_x, train_y)
926      self.assertAlmostEqual(test_model_loss, reference_model_loss, places=4)
927
928  def test_training_on_binary_crossentropy_loss(self):
929    with context.eager_mode():
930      train_x = np.ones((100, 4), dtype=np.float32)
931      train_y = np.ones((100, 1), dtype=np.float32)
932      reference_model = testing_utils.get_small_sequential_mlp(16, 1,
933                                                               input_dim=4)
934      reference_model.compile(loss='binary_crossentropy',
935                              optimizer=RMSPropOptimizer(learning_rate=0.001),
936                              run_eagerly=True)
937      fixed_weights = reference_model.get_weights()
938      reference_model_loss = reference_model.train_on_batch(train_x, train_y)
939
940      test_model = testing_utils.get_small_sequential_mlp(16, 1, input_dim=4)
941      test_model.compile(loss='binary_crossentropy',
942                         optimizer=RMSPropOptimizer(learning_rate=0.001),
943                         run_eagerly=False)
944      test_model.set_weights(fixed_weights)
945      test_model_loss = test_model.train_on_batch(train_x, train_y)
946      self.assertAlmostEqual(test_model_loss, reference_model_loss, places=4)
947
948  @keras_parameterized.run_with_all_model_types
949  @keras_parameterized.run_all_keras_modes
950  @parameterized.named_parameters(
951      ('default', 1, 4), ('integer_two', 2, 2), ('integer_four', 4, 1),
952      ('simple_list', [1, 3, 4], 3), ('duplicated_list', [4, 2, 2], 2))
953  def test_validation_freq(self, validation_freq, expected_runs):
954    x, y = np.ones((10, 10)), np.ones((10, 1))
955    model = testing_utils.get_small_mlp(2, 1, 10)
956    model.compile('sgd', 'mse')
957
958    class ValCounter(keras.callbacks.Callback):
959
960      def __init__(self):
961        self.val_runs = 0
962
963      def on_test_begin(self, logs=None):
964        self.val_runs += 1
965
966    val_counter = ValCounter()
967    model.fit(
968        x,
969        y,
970        epochs=4,
971        validation_data=(x, y),
972        validation_freq=validation_freq,
973        callbacks=[val_counter])
974    self.assertEqual(val_counter.val_runs, expected_runs)
975
976  @keras_parameterized.run_all_keras_modes
977  def test_add_loss_correctness(self):
978    class Bias(keras.layers.Layer):
979
980      def build(self, input_shape):
981        self.bias = self.add_variable('bias', (1,), initializer='zeros')
982
983      def call(self, inputs):
984        return inputs + self.bias
985
986    inputs = keras.Input(shape=(1,))
987    outputs = Bias()(inputs)
988    model = keras.Model(inputs, outputs)
989    targets = keras.Input(shape=(1,))
990
991    model.add_loss(
992        math_ops.reduce_mean(
993            keras.losses.mean_absolute_error(targets, outputs)))
994
995    # If we want to use the loss class instance as shown below, we will need to
996    # add graph scope as the reduction logic involves some eager mode checks.
997    with keras.backend.get_graph().as_default():
998      model.add_loss(keras.losses.MeanAbsoluteError()(targets, outputs))
999
1000    if testing_utils.should_run_eagerly():
1001      with self.assertRaisesRegex(
1002          ValueError,
1003          'We currently do not support enabling `run_eagerly` on compile if '
1004          r'`model.add_loss\(tensor\)` or `model.add_metric\(tensor\)` '
1005          'has been called.'):
1006        model.compile('sgd', run_eagerly=True)
1007      return
1008    else:
1009      model.compile(
1010          keras.optimizer_v2.gradient_descent.SGD(0.033333),
1011          loss=keras.losses.MeanAbsoluteError(),
1012          target_tensors=[targets],
1013          run_eagerly=False)
1014
1015      x = np.array([[0.], [1.], [2.]])
1016      y = np.array([[0.5], [2.], [3.5]])
1017      history = model.fit(x, y, batch_size=3, epochs=5)
1018      self.assertAllClose(history.history['loss'], [3., 2.7, 2.4, 2.1, 1.8],
1019                          1e-3)
1020
1021  @keras_parameterized.run_all_keras_modes
1022  def test_clear_losses(self):
1023
1024    class LayerWithSharedNestedLossLayer(keras.layers.Layer):
1025
1026      def __init__(self):
1027        super(LayerWithSharedNestedLossLayer, self).__init__()
1028        self.loss_layer = keras.layers.ActivityRegularization()
1029        self.add_weight(shape=(1,), regularizer='l2')
1030
1031      def call(self, x):
1032        x = self.loss_layer(x)
1033        return self.loss_layer(x)
1034
1035    inputs = keras.Input(shape=(1,))
1036    outputs = LayerWithSharedNestedLossLayer()(inputs)
1037    model = keras.Model(inputs, outputs)
1038
1039    model(array_ops.ones((1, 1)))
1040    self.assertEqual(len(model.losses), 3)  # Weight loss + 2 activity losses.
1041
1042    model(array_ops.ones((1, 1)))
1043    self.assertEqual(len(model.losses), 3)  # Losses are reset upon __call__.
1044
1045  @keras_parameterized.run_with_all_model_types
1046  @keras_parameterized.run_all_keras_modes
1047  def test_layer_with_variable_output(self):
1048
1049    class VariableOutputLayer(keras.layers.Layer):
1050
1051      def build(self, input_shape):
1052        self.v = self.add_weight('output_var', shape=(2, 5), initializer='ones')
1053
1054      def call(self, inputs):
1055        return self.v
1056
1057    model = testing_utils.get_model_from_layers(
1058        [VariableOutputLayer(), keras.layers.Dense(1)], input_shape=(10,))
1059    # TODO(omalleyt): Make this work with `run_eagerly=True`.
1060    model.compile('sgd', 'mse', run_eagerly=False)
1061    model.fit(np.ones((10, 10)), np.ones((10, 1)), batch_size=2, epochs=5)
1062
1063    self.assertLen(model.trainable_variables, 3)
1064
1065
1066class TestExceptionsAndWarnings(keras_parameterized.TestCase):
1067
1068  @keras_parameterized.run_all_keras_modes
1069  def test_invalid_loss(self):
1070    num_classes = 5
1071    train_samples = 1000
1072    test_samples = 1000
1073    input_dim = 5
1074
1075    model = testing_utils.get_small_sequential_mlp(
1076        num_hidden=10, num_classes=num_classes, input_dim=input_dim)
1077    optimizer = RMSPropOptimizer(learning_rate=0.001)
1078    model.compile(optimizer, loss='categorical_crossentropy')
1079    np.random.seed(1337)
1080    (x_train, y_train), (_, _) = testing_utils.get_test_data(
1081        train_samples=train_samples,
1082        test_samples=test_samples,
1083        input_shape=(input_dim,),
1084        num_classes=num_classes)
1085
1086    with self.assertRaises(ValueError):
1087      model.fit(x_train, np.concatenate([y_train, y_train], axis=-1))
1088
1089    if not context.executing_eagerly():
1090      # TODO(psv): Investigate these use cases in eager mode.
1091      with self.assertRaises(ValueError):
1092        model.fit(x_train, y_train)
1093
1094      with self.assertRaises(ValueError):
1095        model.compile(optimizer, loss=None,
1096                      run_eagerly=testing_utils.should_run_eagerly())
1097
1098  @keras_parameterized.run_all_keras_modes
1099  def test_compile_warning_for_loss_missing_output(self):
1100    with self.cached_session():
1101      inp = keras.layers.Input(shape=(16,), name='input_a')
1102      out_1 = keras.layers.Dense(8, name='dense_1')(inp)
1103      out_2 = keras.layers.Dense(3, activation='softmax', name='dense_2')(out_1)
1104      model = keras.models.Model(inputs=[inp], outputs=[out_1, out_2])
1105      optimizer = RMSPropOptimizer(learning_rate=0.001)
1106
1107      with test.mock.patch.object(logging, 'warning') as mock_log:
1108        model.compile(
1109            optimizer,
1110            loss={
1111                'dense_2': 'categorical_crossentropy',
1112            },
1113            metrics={
1114                'dense_2': 'categorical_accuracy',
1115                'dense_1': metrics_module.CategoricalAccuracy(),
1116            },
1117            run_eagerly=testing_utils.should_run_eagerly())
1118        msg = ('Output dense_1 missing from loss dictionary. We assume this '
1119               'was done on purpose. The fit and evaluate APIs will not be '
1120               'expecting any data to be passed to dense_1.')
1121        self.assertRegexpMatches(str(mock_log.call_args), msg)
1122
1123
1124class LossWeightingTest(keras_parameterized.TestCase):
1125
1126  @keras_parameterized.run_all_keras_modes
1127  def test_class_weights(self):
1128    num_classes = 5
1129    batch_size = 5
1130    epochs = 10
1131    weighted_class = 3
1132    weight = 10.
1133    train_samples = 1000
1134    test_samples = 1000
1135    input_dim = 5
1136    learning_rate = 0.001
1137
1138    model = testing_utils.get_small_sequential_mlp(
1139        num_hidden=10, num_classes=num_classes, input_dim=input_dim)
1140    model.compile(
1141        loss='categorical_crossentropy',
1142        metrics=['acc', metrics_module.CategoricalAccuracy()],
1143        weighted_metrics=['mae', metrics_module.CategoricalAccuracy()],
1144        optimizer=RMSPropOptimizer(learning_rate=learning_rate),
1145        run_eagerly=testing_utils.should_run_eagerly())
1146
1147    np.random.seed(1337)
1148    (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
1149        train_samples=train_samples,
1150        test_samples=test_samples,
1151        input_shape=(input_dim,),
1152        num_classes=num_classes)
1153    int_y_test = y_test.copy()
1154    int_y_train = y_train.copy()
1155    # convert class vectors to binary class matrices
1156    y_train = keras.utils.to_categorical(y_train, num_classes)
1157    y_test = keras.utils.to_categorical(y_test, num_classes)
1158    test_ids = np.where(int_y_test == np.array(weighted_class))[0]
1159
1160    class_weight = dict([(i, 1.) for i in range(num_classes)])
1161    class_weight[weighted_class] = weight
1162
1163    model.fit(
1164        x_train,
1165        y_train,
1166        batch_size=batch_size,
1167        epochs=epochs // 3,
1168        verbose=0,
1169        class_weight=class_weight,
1170        validation_data=(x_train, y_train))
1171    model.fit(
1172        x_train,
1173        y_train,
1174        batch_size=batch_size,
1175        epochs=epochs // 2,
1176        verbose=0,
1177        class_weight=class_weight)
1178    model.fit(
1179        x_train,
1180        y_train,
1181        batch_size=batch_size,
1182        epochs=epochs // 2,
1183        verbose=0,
1184        class_weight=class_weight,
1185        validation_split=0.1)
1186
1187    model.train_on_batch(
1188        x_train[:batch_size], y_train[:batch_size], class_weight=class_weight)
1189    ref_score = model.evaluate(x_test, y_test, verbose=0)
1190    score = model.evaluate(
1191        x_test[test_ids, :], y_test[test_ids, :], verbose=0)
1192    self.assertLess(score[0], ref_score[0])
1193
1194  @keras_parameterized.run_all_keras_modes
1195  def test_sample_weights(self):
1196    num_classes = 5
1197    batch_size = 5
1198    epochs = 10
1199    weighted_class = 3
1200    weight = 10.
1201    train_samples = 1000
1202    test_samples = 1000
1203    input_dim = 5
1204    learning_rate = 0.001
1205
1206    model = testing_utils.get_small_sequential_mlp(
1207        num_hidden=10, num_classes=num_classes, input_dim=input_dim)
1208    model.compile(
1209        RMSPropOptimizer(learning_rate=learning_rate),
1210        metrics=['acc', metrics_module.CategoricalAccuracy()],
1211        weighted_metrics=['mae', metrics_module.CategoricalAccuracy()],
1212        loss='categorical_crossentropy',
1213        run_eagerly=testing_utils.should_run_eagerly())
1214
1215    np.random.seed(43)
1216    (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
1217        train_samples=train_samples,
1218        test_samples=test_samples,
1219        input_shape=(input_dim,),
1220        num_classes=num_classes)
1221    int_y_test = y_test.copy()
1222    int_y_train = y_train.copy()
1223    # convert class vectors to binary class matrices
1224    y_train = keras.utils.to_categorical(y_train, num_classes)
1225    y_test = keras.utils.to_categorical(y_test, num_classes)
1226    test_ids = np.where(int_y_test == np.array(weighted_class))[0]
1227
1228    sample_weight = np.ones((y_train.shape[0]))
1229    sample_weight[int_y_train == weighted_class] = weight
1230
1231    model.fit(
1232        x_train,
1233        y_train,
1234        batch_size=batch_size,
1235        epochs=epochs // 3,
1236        verbose=0,
1237        sample_weight=sample_weight)
1238    model.fit(
1239        x_train,
1240        y_train,
1241        batch_size=batch_size,
1242        epochs=epochs // 3,
1243        verbose=0,
1244        sample_weight=sample_weight,
1245        validation_split=0.1)
1246
1247    model.train_on_batch(
1248        x_train[:batch_size],
1249        y_train[:batch_size],
1250        sample_weight=sample_weight[:batch_size])
1251    model.test_on_batch(
1252        x_train[:batch_size],
1253        y_train[:batch_size],
1254        sample_weight=sample_weight[:batch_size])
1255    ref_score = model.evaluate(x_test, y_test, verbose=0)
1256    if not context.executing_eagerly():
1257      score = model.evaluate(
1258          x_test[test_ids, :], y_test[test_ids, :], verbose=0)
1259      self.assertLess(score[0], ref_score[0])
1260
1261  @keras_parameterized.run_all_keras_modes
1262  def test_temporal_sample_weights(self):
1263    num_classes = 5
1264    batch_size = 5
1265    epochs = 10
1266    weighted_class = 3
1267    weight = 10.
1268    train_samples = 1000
1269    test_samples = 1000
1270    input_dim = 5
1271    timesteps = 3
1272    learning_rate = 0.001
1273
1274    with self.cached_session():
1275      model = keras.models.Sequential()
1276      model.add(
1277          keras.layers.TimeDistributed(
1278              keras.layers.Dense(num_classes),
1279              input_shape=(timesteps, input_dim)))
1280      model.add(keras.layers.Activation('softmax'))
1281
1282      np.random.seed(1337)
1283      (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
1284          train_samples=train_samples,
1285          test_samples=test_samples,
1286          input_shape=(input_dim,),
1287          num_classes=num_classes)
1288      int_y_test = y_test.copy()
1289      int_y_train = y_train.copy()
1290      # convert class vectors to binary class matrices
1291      y_train = keras.utils.to_categorical(y_train, num_classes)
1292      y_test = keras.utils.to_categorical(y_test, num_classes)
1293      test_ids = np.where(int_y_test == np.array(weighted_class))[0]
1294
1295      sample_weight = np.ones((y_train.shape[0]))
1296      sample_weight[int_y_train == weighted_class] = weight
1297
1298      temporal_x_train = np.reshape(x_train, (len(x_train), 1,
1299                                              x_train.shape[1]))
1300      temporal_x_train = np.repeat(temporal_x_train, timesteps, axis=1)
1301      temporal_x_test = np.reshape(x_test, (len(x_test), 1, x_test.shape[1]))
1302      temporal_x_test = np.repeat(temporal_x_test, timesteps, axis=1)
1303
1304      temporal_y_train = np.reshape(y_train, (len(y_train), 1,
1305                                              y_train.shape[1]))
1306      temporal_y_train = np.repeat(temporal_y_train, timesteps, axis=1)
1307      temporal_y_test = np.reshape(y_test, (len(y_test), 1, y_test.shape[1]))
1308      temporal_y_test = np.repeat(temporal_y_test, timesteps, axis=1)
1309
1310      temporal_sample_weight = np.reshape(sample_weight, (len(sample_weight),
1311                                                          1))
1312      temporal_sample_weight = np.repeat(
1313          temporal_sample_weight, timesteps, axis=1)
1314
1315      model.compile(
1316          RMSPropOptimizer(learning_rate=learning_rate),
1317          loss='categorical_crossentropy',
1318          metrics=['acc', metrics_module.CategoricalAccuracy()],
1319          weighted_metrics=['mae', metrics_module.CategoricalAccuracy()],
1320          sample_weight_mode='temporal',
1321          run_eagerly=testing_utils.should_run_eagerly())
1322
1323      model.fit(
1324          temporal_x_train,
1325          temporal_y_train,
1326          batch_size=batch_size,
1327          epochs=epochs // 3,
1328          verbose=0,
1329          sample_weight=temporal_sample_weight)
1330      model.fit(
1331          temporal_x_train,
1332          temporal_y_train,
1333          batch_size=batch_size,
1334          epochs=epochs // 3,
1335          verbose=0,
1336          sample_weight=temporal_sample_weight,
1337          validation_split=0.1)
1338
1339      model.train_on_batch(
1340          temporal_x_train[:batch_size],
1341          temporal_y_train[:batch_size],
1342          sample_weight=temporal_sample_weight[:batch_size])
1343      model.test_on_batch(
1344          temporal_x_train[:batch_size],
1345          temporal_y_train[:batch_size],
1346          sample_weight=temporal_sample_weight[:batch_size])
1347      ref_score = model.evaluate(temporal_x_test, temporal_y_test, verbose=0)
1348      if not context.executing_eagerly():
1349        score = model.evaluate(
1350            temporal_x_test[test_ids], temporal_y_test[test_ids], verbose=0)
1351        self.assertLess(score[0], ref_score[0])
1352
1353  @keras_parameterized.run_all_keras_modes
1354  def test_class_weight_invalid_use_case(self):
1355    num_classes = 5
1356    train_samples = 1000
1357    test_samples = 1000
1358    input_dim = 5
1359    timesteps = 3
1360    learning_rate = 0.001
1361
1362    with self.cached_session():
1363      model = keras.models.Sequential()
1364      model.add(
1365          keras.layers.TimeDistributed(
1366              keras.layers.Dense(num_classes),
1367              input_shape=(timesteps, input_dim)))
1368      model.add(keras.layers.Activation('softmax'))
1369      optimizer = RMSPropOptimizer(learning_rate=learning_rate)
1370      model.compile(optimizer, loss='binary_crossentropy',
1371                    run_eagerly=testing_utils.should_run_eagerly())
1372
1373      (x_train, y_train), _ = testing_utils.get_test_data(
1374          train_samples=train_samples,
1375          test_samples=test_samples,
1376          input_shape=(input_dim,),
1377          num_classes=num_classes)
1378      # convert class vectors to binary class matrices
1379      y_train = keras.utils.to_categorical(y_train, num_classes)
1380      class_weight = dict([(i, 1.) for i in range(num_classes)])
1381
1382      del class_weight[1]
1383      with self.assertRaises(ValueError):
1384        model.fit(x_train, y_train,
1385                  epochs=0, verbose=0, class_weight=class_weight)
1386
1387      with self.assertRaises(ValueError):
1388        model.compile(
1389            optimizer, loss='binary_crossentropy', sample_weight_mode=[],
1390            run_eagerly=testing_utils.should_run_eagerly())
1391
1392      # Build multi-output model
1393      x = keras.Input((3,))
1394      y1 = keras.layers.Dense(4, name='1')(x)
1395      y2 = keras.layers.Dense(4, name='2')(x)
1396      model = keras.models.Model(x, [y1, y2])
1397      model.compile(optimizer, loss='mse',
1398                    run_eagerly=testing_utils.should_run_eagerly())
1399      x_np = np.random.random((10, 3))
1400      y_np = np.random.random((10, 4))
1401      w_np = np.random.random((10,))
1402      # This will work
1403      model.fit(x_np, [y_np, y_np], epochs=1,
1404                sample_weight={'1': w_np})
1405      # These will not
1406      with self.assertRaises(ValueError):
1407        model.fit(x_np, [y_np, y_np], epochs=1,
1408                  sample_weight=[w_np])
1409      with self.assertRaises(TypeError):
1410        model.fit(x_np, [y_np, y_np], epochs=1,
1411                  sample_weight=w_np)
1412      with self.assertRaises(ValueError):
1413        bad_w_np = np.random.random((11,))
1414        model.fit(x_np, [y_np, y_np], epochs=1,
1415                  sample_weight={'1': bad_w_np})
1416      with self.assertRaises(ValueError):
1417        bad_w_np = np.random.random((10, 2))
1418        model.fit(x_np, [y_np, y_np], epochs=1,
1419                  sample_weight={'1': bad_w_np})
1420      with self.assertRaises(ValueError):
1421        bad_w_np = np.random.random((10, 2, 2))
1422        model.fit(x_np, [y_np, y_np], epochs=1,
1423                  sample_weight={'1': bad_w_np})
1424
1425  @keras_parameterized.run_all_keras_modes
1426  def test_default_sample_weight(self):
1427    """Verifies that fit works without having to set sample_weight."""
1428
1429    num_classes = 5
1430    input_dim = 5
1431    timesteps = 3
1432    learning_rate = 0.001
1433
1434    with self.cached_session():
1435      model = keras.models.Sequential()
1436      model.add(
1437          keras.layers.TimeDistributed(
1438              keras.layers.Dense(num_classes),
1439              input_shape=(timesteps, input_dim)))
1440
1441      x = np.random.random((10, timesteps, input_dim))
1442      y = np.random.random((10, timesteps, num_classes))
1443      optimizer = RMSPropOptimizer(learning_rate=learning_rate)
1444
1445      # sample_weight_mode is a list and mode value is None
1446      model.compile(optimizer, loss='mse', sample_weight_mode=[None],
1447                    run_eagerly=testing_utils.should_run_eagerly())
1448      model.fit(x, y, epochs=1, batch_size=10)
1449
1450      # sample_weight_mode is a list and mode value is `temporal`
1451      model.compile(optimizer, loss='mse', sample_weight_mode=['temporal'],
1452                    run_eagerly=testing_utils.should_run_eagerly())
1453      model.fit(x, y, epochs=1, batch_size=10)
1454
1455      # sample_weight_mode is a dict and mode value is None
1456      model.compile(
1457          optimizer, loss='mse', sample_weight_mode={'time_distributed': None},
1458          run_eagerly=testing_utils.should_run_eagerly())
1459      model.fit(x, y, epochs=1, batch_size=10)
1460
1461      # sample_weight_mode is a dict and mode value is `temporal`
1462      model.compile(
1463          optimizer,
1464          loss='mse',
1465          sample_weight_mode={'time_distributed': 'temporal'},
1466          run_eagerly=testing_utils.should_run_eagerly())
1467      model.fit(x, y, epochs=1, batch_size=10)
1468
1469      # sample_weight_mode is a not a list/dict and mode value is None
1470      model.compile(optimizer, loss='mse', sample_weight_mode=None,
1471                    run_eagerly=testing_utils.should_run_eagerly())
1472      model.fit(x, y, epochs=1, batch_size=10)
1473
1474      # sample_weight_mode is a not a list/dict and mode value is `temporal`
1475      model.compile(optimizer, loss='mse', sample_weight_mode='temporal',
1476                    run_eagerly=testing_utils.should_run_eagerly())
1477      model.fit(x, y, epochs=1, batch_size=10)
1478
1479
1480@keras_parameterized.run_all_keras_modes
1481class MaskingTest(keras_parameterized.TestCase):
1482
1483  def _get_model(self, input_shape=None):
1484    layers = [
1485        keras.layers.Masking(mask_value=0),
1486        keras.layers.TimeDistributed(
1487            keras.layers.Dense(1, kernel_initializer='one'))
1488    ]
1489    model = testing_utils.get_model_from_layers(layers, input_shape)
1490    model.compile(
1491        loss='mse',
1492        optimizer=RMSPropOptimizer(learning_rate=0.001),
1493        run_eagerly=testing_utils.should_run_eagerly())
1494    return model
1495
1496  @keras_parameterized.run_with_all_model_types
1497  def test_masking(self):
1498    model = self._get_model(input_shape=(2, 1))
1499    x = np.array([[[1], [1]], [[0], [0]]])
1500    y = np.array([[[1], [1]], [[1], [1]]])
1501    loss = model.train_on_batch(x, y)
1502    self.assertEqual(loss, 0)
1503
1504  @keras_parameterized.run_with_all_model_types(exclude_models='functional')
1505  def test_masking_deferred(self):
1506    model = self._get_model()
1507    x = np.array([[[1], [1]], [[0], [0]]])
1508    y = np.array([[[1], [1]], [[1], [1]]])
1509    loss = model.train_on_batch(x, y)
1510    self.assertEqual(loss, 0)
1511
1512  def test_mask_argument_in_layer(self):
1513    # Test that the mask argument gets correctly passed to a layer in the
1514    # functional API.
1515
1516    class CustomMaskedLayer(keras.layers.Layer):
1517
1518      def __init__(self):
1519        super(CustomMaskedLayer, self).__init__()
1520        self.supports_masking = True
1521
1522      def call(self, inputs, mask=None):
1523        assert mask is not None
1524        return inputs
1525
1526      def compute_output_shape(self, input_shape):
1527        return input_shape
1528
1529    x = np.random.random((5, 3))
1530    inputs = keras.layers.Input((3,))
1531    masked = keras.layers.Masking(mask_value=0)(inputs)
1532    outputs = CustomMaskedLayer()(masked)
1533
1534    model = keras.Model(inputs, outputs)
1535    model.compile(
1536        loss='mse',
1537        optimizer=RMSPropOptimizer(learning_rate=0.001),
1538        run_eagerly=testing_utils.should_run_eagerly())
1539    y = np.random.random((5, 3))
1540    model.train_on_batch(x, y)
1541
1542
1543class TestDynamicTrainability(keras_parameterized.TestCase):
1544
1545  def test_trainable_warning(self):
1546    with self.cached_session():
1547      x = np.random.random((5, 3))
1548      y = np.random.random((5, 2))
1549
1550      model = keras.models.Sequential()
1551      model.add(keras.layers.Dense(2, input_dim=3))
1552      model.trainable = False
1553      model.compile('rmsprop', 'mse')
1554      model.trainable = True
1555      model.train_on_batch(x, y)
1556      self.assertRaises(Warning)
1557
1558  def test_trainable_argument(self):
1559    with self.cached_session():
1560      x = np.random.random((5, 3))
1561      y = np.random.random((5, 2))
1562
1563      model = keras.models.Sequential()
1564      model.add(keras.layers.Dense(2, input_dim=3, trainable=False))
1565      model.compile('rmsprop', 'mse')
1566      out = model.predict(x)
1567      model.train_on_batch(x, y)
1568      out_2 = model.predict(x)
1569      self.assertAllClose(out, out_2)
1570
1571      # test with nesting
1572      inputs = keras.layers.Input(shape=(3,))
1573      output = model(inputs)
1574      model = keras.models.Model(inputs, output)
1575      model.compile('rmsprop', 'mse')
1576      out = model.predict(x)
1577      model.train_on_batch(x, y)
1578      out_2 = model.predict(x)
1579      self.assertAllClose(out, out_2)
1580
1581  def test_layer_trainability_switch(self):
1582    with self.cached_session():
1583      # with constructor argument, in Sequential
1584      model = keras.models.Sequential()
1585      model.add(keras.layers.Dense(2, trainable=False, input_dim=1))
1586      self.assertListEqual(model.trainable_weights, [])
1587
1588      # by setting the `trainable` argument, in Sequential
1589      model = keras.models.Sequential()
1590      layer = keras.layers.Dense(2, input_dim=1)
1591      model.add(layer)
1592      self.assertListEqual(model.trainable_weights, layer.trainable_weights)
1593      layer.trainable = False
1594      self.assertListEqual(model.trainable_weights, [])
1595
1596      # with constructor argument, in Model
1597      x = keras.layers.Input(shape=(1,))
1598      y = keras.layers.Dense(2, trainable=False)(x)
1599      model = keras.models.Model(x, y)
1600      self.assertListEqual(model.trainable_weights, [])
1601
1602      # by setting the `trainable` argument, in Model
1603      x = keras.layers.Input(shape=(1,))
1604      layer = keras.layers.Dense(2)
1605      y = layer(x)
1606      model = keras.models.Model(x, y)
1607      self.assertListEqual(model.trainable_weights, layer.trainable_weights)
1608      layer.trainable = False
1609      self.assertListEqual(model.trainable_weights, [])
1610
1611  def test_model_trainability_switch(self):
1612    with self.cached_session():
1613      # a non-trainable model has no trainable weights
1614      x = keras.layers.Input(shape=(1,))
1615      y = keras.layers.Dense(2)(x)
1616      model = keras.models.Model(x, y)
1617      model.trainable = False
1618      self.assertListEqual(model.trainable_weights, [])
1619
1620      # same for Sequential
1621      model = keras.models.Sequential()
1622      model.add(keras.layers.Dense(2, input_dim=1))
1623      model.trainable = False
1624      self.assertListEqual(model.trainable_weights, [])
1625
1626  def test_nested_model_trainability(self):
1627    with self.cached_session():
1628      # a Sequential inside a Model
1629      inner_model = keras.models.Sequential()
1630      inner_model.add(keras.layers.Dense(2, input_dim=1))
1631
1632      x = keras.layers.Input(shape=(1,))
1633      y = inner_model(x)
1634      outer_model = keras.models.Model(x, y)
1635      self.assertListEqual(outer_model.trainable_weights,
1636                           inner_model.trainable_weights)
1637      inner_model.trainable = False
1638      self.assertListEqual(outer_model.trainable_weights, [])
1639      inner_model.trainable = True
1640      inner_model.layers[-1].trainable = False
1641      self.assertListEqual(outer_model.trainable_weights, [])
1642
1643      # a Sequential inside a Sequential
1644      inner_model = keras.models.Sequential()
1645      inner_model.add(keras.layers.Dense(2, input_dim=1))
1646      outer_model = keras.models.Sequential()
1647      outer_model.add(inner_model)
1648      self.assertListEqual(outer_model.trainable_weights,
1649                           inner_model.trainable_weights)
1650      inner_model.trainable = False
1651      self.assertListEqual(outer_model.trainable_weights, [])
1652      inner_model.trainable = True
1653      inner_model.layers[-1].trainable = False
1654      self.assertListEqual(outer_model.trainable_weights, [])
1655
1656      # a Model inside a Model
1657      x = keras.layers.Input(shape=(1,))
1658      y = keras.layers.Dense(2)(x)
1659      inner_model = keras.models.Model(x, y)
1660      x = keras.layers.Input(shape=(1,))
1661      y = inner_model(x)
1662      outer_model = keras.models.Model(x, y)
1663      self.assertListEqual(outer_model.trainable_weights,
1664                           inner_model.trainable_weights)
1665      inner_model.trainable = False
1666      self.assertListEqual(outer_model.trainable_weights, [])
1667      inner_model.trainable = True
1668      inner_model.layers[-1].trainable = False
1669      self.assertListEqual(outer_model.trainable_weights, [])
1670
1671      # a Model inside a Sequential
1672      x = keras.layers.Input(shape=(1,))
1673      y = keras.layers.Dense(2)(x)
1674      inner_model = keras.models.Model(x, y)
1675      outer_model = keras.models.Sequential()
1676      outer_model.add(inner_model)
1677      self.assertListEqual(outer_model.trainable_weights,
1678                           inner_model.trainable_weights)
1679      inner_model.trainable = False
1680      self.assertListEqual(outer_model.trainable_weights, [])
1681      inner_model.trainable = True
1682      inner_model.layers[-1].trainable = False
1683      self.assertListEqual(outer_model.trainable_weights, [])
1684
1685
1686class TestTrainingWithDataTensors(keras_parameterized.TestCase):
1687
1688  @keras_parameterized.run_all_keras_modes
1689  def test_training_and_eval_methods_on_symbolic_tensors_single_io(self):
1690    # TODO(kaftan) Test seems to not work, file ticket
1691    if  context.executing_eagerly():
1692      self.skipTest('Skipping eager execution.')
1693
1694    x = keras.layers.Input(shape=(3,), name='input')
1695    y = keras.layers.Dense(4, name='dense')(x)
1696    model = keras.Model(x, y)
1697
1698    optimizer = RMSPropOptimizer(learning_rate=0.001)
1699    loss = 'mse'
1700    model.compile(
1701        optimizer,
1702        loss,
1703        metrics=['mae', metrics_module.CategoricalAccuracy()],
1704        run_eagerly=testing_utils.should_run_eagerly())
1705
1706    inputs = keras.backend.zeros(shape=(10, 3))
1707    targets = keras.backend.zeros(shape=(10, 4))
1708
1709    model.fit(inputs, targets, epochs=1, steps_per_epoch=2, verbose=0)
1710    model.evaluate(inputs, targets, steps=2, verbose=0)
1711    model.predict(inputs, steps=2)
1712    model.train_on_batch(inputs, targets)
1713    model.test_on_batch(inputs, targets)
1714    model.fit(inputs, targets,
1715              epochs=1, steps_per_epoch=2, verbose=0,
1716              validation_data=(inputs, targets), validation_steps=2)
1717
1718    # Test with dynamic shape
1719    inputs = array_ops.placeholder_with_default(
1720        np.zeros((2, 3)), shape=tensor_shape.TensorShape([None, 3]))
1721    targets = array_ops.placeholder_with_default(
1722        np.zeros((2, 4)), shape=tensor_shape.TensorShape([None, 4]))
1723    self.assertEqual(inputs.shape.dims[0].value, None)
1724    model.fit(inputs, targets, epochs=1, steps_per_epoch=2, verbose=0)
1725    model.evaluate(inputs, targets, steps=2, verbose=0)
1726    model.predict(inputs, steps=2)
1727    model.train_on_batch(inputs, targets)
1728    model.test_on_batch(inputs, targets)
1729    model.fit(inputs, targets,
1730              epochs=1, steps_per_epoch=2, verbose=0,
1731              validation_data=(inputs, targets), validation_steps=2)
1732
1733  @keras_parameterized.run_all_keras_modes
1734  def test_training_and_eval_methods_on_symbolic_tensors_multi_io(self):
1735    # TODO(kaftan) Test seems to not work, file ticket
1736    if context.executing_eagerly():
1737      self.skipTest('Skipping eager execution.')
1738
1739    a = keras.layers.Input(shape=(3,), name='input_a')
1740    b = keras.layers.Input(shape=(3,), name='input_b')
1741
1742    dense = keras.layers.Dense(4, name='dense')
1743    c = dense(a)
1744    d = dense(b)
1745    e = keras.layers.Dropout(0.5, name='dropout')(c)
1746
1747    model = keras.models.Model([a, b], [d, e])
1748
1749    optimizer = 'rmsprop'
1750    loss = 'mse'
1751    loss_weights = [1., 0.5]
1752    model.compile(
1753        optimizer,
1754        loss,
1755        metrics=['mae', metrics_module.CategoricalAccuracy()],
1756        loss_weights=loss_weights,
1757        run_eagerly=testing_utils.should_run_eagerly())
1758
1759    input_a_tf = keras.backend.zeros(shape=(10, 3))
1760    input_b_tf = keras.backend.zeros(shape=(10, 3))
1761
1762    output_d_tf = keras.backend.zeros(shape=(10, 4))
1763    output_e_tf = keras.backend.zeros(shape=(10, 4))
1764
1765    model.fit(
1766        [input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
1767        epochs=1,
1768        steps_per_epoch=2,
1769        verbose=0)
1770    with self.assertRaisesRegexp(ValueError,
1771                                 'should specify the `steps_per_epoch`'):
1772      model.fit(
1773          [input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
1774          epochs=1,
1775          batch_size=5,
1776          verbose=0)
1777    model.train_on_batch([input_a_tf, input_b_tf], [output_d_tf, output_e_tf])
1778
1779    # Test with dictionary inputs
1780    model.fit(
1781        {'input_a': input_a_tf,
1782         'input_b': input_b_tf},
1783        {'dense': output_d_tf,
1784         'dropout': output_e_tf},
1785        epochs=1,
1786        steps_per_epoch=2,
1787        verbose=0)
1788    model.fit(
1789        {'input_a': input_a_tf,
1790         'input_b': input_b_tf},
1791        {'dense': output_d_tf,
1792         'dropout': output_e_tf},
1793        validation_data=({'input_a': input_a_tf,
1794                          'input_b': input_b_tf},
1795                         {'dense': output_d_tf,
1796                          'dropout': output_e_tf}),
1797        epochs=1,
1798        steps_per_epoch=2,
1799        validation_steps=2,
1800        verbose=0)
1801    model.train_on_batch(
1802        {'input_a': input_a_tf,
1803         'input_b': input_b_tf},
1804        {'dense': output_d_tf,
1805         'dropout': output_e_tf})
1806
1807    # Test with validation data
1808    model.fit(
1809        [input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
1810        validation_data=([input_a_tf, input_b_tf],
1811                         [output_d_tf, output_e_tf]),
1812        epochs=1,
1813        steps_per_epoch=2,
1814        validation_steps=2,
1815        verbose=0)
1816    # Test with validation split
1817    with self.assertRaisesRegexp(ValueError,
1818                                 'you cannot use `validation_split`'):
1819      model.fit(
1820          [input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
1821          epochs=2,
1822          steps_per_epoch=2,
1823          verbose=0,
1824          validation_split=0.2,
1825          validation_steps=2)
1826
1827    # Test evaluation / prediction methods
1828    model.evaluate([input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
1829                   steps=2, verbose=0)
1830    model.predict([input_a_tf, input_b_tf], steps=2)
1831    model.test_on_batch([input_a_tf, input_b_tf], [output_d_tf, output_e_tf])
1832
1833  @tf_test_util.run_deprecated_v1
1834  def test_model_with_input_feed_tensor(self):
1835    """We test building a model with a TF variable as input.
1836
1837    We should be able to call fit, evaluate, predict,
1838    by only passing them data for the placeholder inputs
1839    in the model.
1840    """
1841    with self.cached_session():
1842      input_a_np = np.random.random((10, 3))
1843      input_b_np = np.random.random((10, 3))
1844
1845      output_a_np = np.random.random((10, 4))
1846      output_b_np = np.random.random((10, 3))
1847
1848      input_v = keras.backend.variables_module.Variable(
1849          input_a_np, dtype='float32')
1850      self.evaluate(variables_lib.variables_initializer([input_v]))
1851      a = keras.Input(tensor=input_v)
1852      b = keras.Input(shape=(3,), name='input_b')
1853
1854      a_2 = keras.layers.Dense(4, name='dense_1')(a)
1855      dp = keras.layers.Dropout(0.5, name='dropout')
1856      b_2 = dp(b)
1857
1858      model = keras.models.Model([a, b], [a_2, b_2])
1859      model.summary()
1860
1861      optimizer = 'rmsprop'
1862      loss = 'mse'
1863      loss_weights = [1., 0.5]
1864      model.compile(optimizer, loss, metrics=['mean_squared_error'],
1865                    loss_weights=loss_weights,
1866                    sample_weight_mode=None)
1867
1868      # test train_on_batch
1869      out = model.train_on_batch(input_b_np,
1870                                 [output_a_np, output_b_np])
1871      out = model.train_on_batch({'input_b': input_b_np},
1872                                 [output_a_np, output_b_np])
1873      out = model.test_on_batch({'input_b': input_b_np},
1874                                [output_a_np, output_b_np])
1875      out = model.predict_on_batch({'input_b': input_b_np})
1876
1877      # test fit
1878      out = model.fit({'input_b': input_b_np},
1879                      [output_a_np, output_b_np], epochs=1, batch_size=10)
1880      out = model.fit(input_b_np,
1881                      [output_a_np, output_b_np], epochs=1, batch_size=10)
1882
1883      # test evaluate
1884      out = model.evaluate({'input_b': input_b_np},
1885                           [output_a_np, output_b_np], batch_size=10)
1886      out = model.evaluate(input_b_np,
1887                           [output_a_np, output_b_np], batch_size=10)
1888
1889      # test predict
1890      out = model.predict({'input_b': input_b_np}, batch_size=10)
1891      out = model.predict(input_b_np, batch_size=10)
1892      self.assertEqual(len(out), 2)
1893
1894      # Now test a model with a single input
1895      # i.e. we don't pass any data to fit the model.
1896      self.evaluate(variables_lib.variables_initializer([input_v]))
1897      a = keras.Input(tensor=input_v)
1898      a_2 = keras.layers.Dense(4, name='dense_1')(a)
1899      a_2 = keras.layers.Dropout(0.5, name='dropout')(a_2)
1900      model = keras.models.Model(a, a_2)
1901      model.summary()
1902
1903      optimizer = 'rmsprop'
1904      loss = 'mse'
1905      model.compile(optimizer, loss, metrics=['mean_squared_error'])
1906
1907      # test train_on_batch
1908      out = model.train_on_batch(None,
1909                                 output_a_np)
1910      out = model.train_on_batch(None,
1911                                 output_a_np)
1912      out = model.test_on_batch(None,
1913                                output_a_np)
1914      out = model.predict_on_batch(None)
1915      out = model.train_on_batch([],
1916                                 output_a_np)
1917      out = model.train_on_batch({},
1918                                 output_a_np)
1919
1920      # test fit
1921      _ = model.fit(None, output_a_np, epochs=1, steps_per_epoch=3)
1922      _ = model.fit(None, output_a_np, epochs=1, steps_per_epoch=3)
1923
1924      # test evaluate
1925      _ = model.evaluate(None, output_a_np, steps=3)
1926      _ = model.evaluate(None, output_a_np, steps=3)
1927
1928      # test predict
1929      out = model.predict(None, steps=3)
1930      out = model.predict(None, steps=3)
1931      self.assertEqual(out.shape, (10 * 3, 4))
1932
1933      # Same, without learning phase
1934      # i.e. we don't pass any data to fit the model.
1935      self.evaluate(variables_lib.variables_initializer([input_v]))
1936      a = keras.Input(tensor=input_v)
1937      a_2 = keras.layers.Dense(4, name='dense_1')(a)
1938      model = keras.models.Model(a, a_2)
1939      model.summary()
1940
1941      optimizer = 'rmsprop'
1942      loss = 'mse'
1943      model.compile(optimizer, loss, metrics=['mean_squared_error'])
1944
1945      # test train_on_batch
1946      out = model.train_on_batch(None,
1947                                 output_a_np)
1948      out = model.train_on_batch(None,
1949                                 output_a_np)
1950      out = model.test_on_batch(None,
1951                                output_a_np)
1952      out = model.predict_on_batch(None)
1953      out = model.train_on_batch([],
1954                                 output_a_np)
1955      out = model.train_on_batch({},
1956                                 output_a_np)
1957
1958      # test fit
1959      _ = model.fit(None, output_a_np, epochs=1, steps_per_epoch=10)
1960      _ = model.fit(None, output_a_np, epochs=1, steps_per_epoch=10)
1961
1962      # test evaluate
1963      _ = model.evaluate(None, output_a_np, steps=10)
1964      _ = model.evaluate(None, output_a_np, steps=10)
1965
1966      # test predict
1967      out = model.predict(None, steps=3)
1968      out = model.predict(None, steps=3)
1969      self.assertEqual(out.shape, (10 * 3, 4))
1970
1971  def test_model_with_partial_loss(self):
1972    with self.cached_session():
1973      a = keras.Input(shape=(3,), name='input_a')
1974      a_2 = keras.layers.Dense(4, name='dense_1')(a)
1975      dp = keras.layers.Dropout(0.5, name='dropout')
1976      a_3 = dp(a_2)
1977      model = keras.models.Model(a, [a_2, a_3])
1978
1979      optimizer = 'rmsprop'
1980      loss = {'dropout': 'mse'}
1981      model.compile(optimizer, loss, metrics=['mae'])
1982
1983      input_a_np = np.random.random((10, 3))
1984      output_a_np = np.random.random((10, 4))
1985
1986      # test train_on_batch
1987      _ = model.train_on_batch(input_a_np, output_a_np)
1988      _ = model.test_on_batch(input_a_np, output_a_np)
1989      # fit
1990      _ = model.fit(input_a_np, [output_a_np])
1991      # evaluate
1992      _ = model.evaluate(input_a_np, [output_a_np])
1993
1994      # Same without dropout.
1995      a = keras.Input(shape=(3,), name='input_a')
1996      a_2 = keras.layers.Dense(4, name='dense_1')(a)
1997      a_3 = keras.layers.Dense(4, name='dense_2')(a_2)
1998      model = keras.models.Model(a, [a_2, a_3])
1999
2000      optimizer = 'rmsprop'
2001      loss = {'dense_2': 'mse'}
2002      model.compile(optimizer, loss, metrics={'dense_1': 'mae'})
2003
2004      # test train_on_batch
2005      _ = model.train_on_batch(input_a_np, output_a_np)
2006      _ = model.test_on_batch(input_a_np, output_a_np)
2007      # fit
2008      _ = model.fit(input_a_np, [output_a_np])
2009      # evaluate
2010      _ = model.evaluate(input_a_np, [output_a_np])
2011
2012  @tf_test_util.run_deprecated_v1
2013  def test_model_with_external_loss(self):
2014    with self.cached_session():
2015      # None loss, only regularization loss.
2016      a = keras.Input(shape=(3,), name='input_a')
2017      a_2 = keras.layers.Dense(4, name='dense_1',
2018                               kernel_regularizer='l1',
2019                               bias_regularizer='l2')(a)
2020      dp = keras.layers.Dropout(0.5, name='dropout')
2021      a_3 = dp(a_2)
2022
2023      model = keras.models.Model(a, [a_2, a_3])
2024
2025      optimizer = 'rmsprop'
2026      loss = None
2027      model.compile(optimizer, loss, metrics=['mae'])
2028
2029      input_a_np = np.random.random((10, 3))
2030
2031      # test train_on_batch
2032      out = model.train_on_batch(input_a_np, None)
2033      out = model.test_on_batch(input_a_np, None)
2034      # fit
2035      out = model.fit(input_a_np, None)
2036      # evaluate
2037      out = model.evaluate(input_a_np, None)
2038
2039      # No dropout, external loss.
2040      a = keras.Input(shape=(3,), name='input_a')
2041      a_2 = keras.layers.Dense(4, name='dense_1')(a)
2042      a_3 = keras.layers.Dense(4, name='dense_2')(a)
2043
2044      model = keras.models.Model(a, [a_2, a_3])
2045      model.add_loss(keras.backend.mean(a_3 + a_2))
2046
2047      optimizer = 'rmsprop'
2048      loss = None
2049      model.compile(optimizer, loss, metrics=['mae'])
2050
2051      # test train_on_batch
2052      out = model.train_on_batch(input_a_np, None)
2053      out = model.test_on_batch(input_a_np, None)
2054      # fit
2055      out = model.fit(input_a_np, None)
2056      # evaluate
2057      out = model.evaluate(input_a_np, None)
2058
2059      # Test model with no external data at all.
2060      input_v = keras.backend.variables_module.Variable(
2061          input_a_np, dtype='float32')
2062      self.evaluate(variables_lib.variables_initializer([input_v]))
2063      a = keras.Input(tensor=input_v)
2064      a_2 = keras.layers.Dense(4, name='dense_1')(a)
2065      a_2 = keras.layers.Dropout(0.5, name='dropout')(a_2)
2066      model = keras.models.Model(a, a_2)
2067      model.add_loss(keras.backend.mean(a_2))
2068
2069      model.compile(optimizer='rmsprop',
2070                    loss=None,
2071                    metrics=['mean_squared_error'])
2072
2073      # test train_on_batch
2074      out = model.train_on_batch(None, None)
2075      out = model.test_on_batch(None, None)
2076      out = model.predict_on_batch(None)
2077
2078      # test fit
2079      with self.assertRaises(ValueError):
2080        out = model.fit(None, None, epochs=1, batch_size=10)
2081      out = model.fit(None, None, epochs=1, steps_per_epoch=1)
2082
2083      # test fit with validation data
2084      with self.assertRaises(ValueError):
2085        out = model.fit(None, None, epochs=1,
2086                        steps_per_epoch=None,
2087                        validation_steps=2)
2088      out = model.fit(None, None, epochs=1,
2089                      steps_per_epoch=2,
2090                      validation_steps=2)
2091
2092      # test evaluate
2093      with self.assertRaises(ValueError):
2094        out = model.evaluate(None, None, batch_size=10)
2095      out = model.evaluate(None, None, steps=3)
2096
2097      # test predict
2098      with self.assertRaises(ValueError):
2099        out = model.predict(None, batch_size=10)
2100      out = model.predict(None, steps=3)
2101      self.assertEqual(out.shape, (10 * 3, 4))
2102
2103      # Test multi-output model with no external data at all.
2104      self.evaluate(variables_lib.variables_initializer([input_v]))
2105      a = keras.Input(tensor=input_v)
2106      a_1 = keras.layers.Dense(4, name='dense_1')(a)
2107      a_2 = keras.layers.Dropout(0.5, name='dropout')(a_1)
2108      model = keras.models.Model(a, [a_1, a_2])
2109      model.add_loss(keras.backend.mean(a_2))
2110
2111      model.compile(optimizer='rmsprop',
2112                    loss=None,
2113                    metrics=['mean_squared_error'])
2114
2115      # test train_on_batch
2116      out = model.train_on_batch(None, None)
2117      out = model.test_on_batch(None, None)
2118      out = model.predict_on_batch(None)
2119
2120      # test fit
2121      with self.assertRaises(ValueError):
2122        out = model.fit(None, None, epochs=1, batch_size=10)
2123      out = model.fit(None, None, epochs=1, steps_per_epoch=1)
2124
2125      # test fit with validation data
2126      out = model.fit(None, None, epochs=1,
2127                      steps_per_epoch=2,
2128                      validation_steps=2)
2129
2130      # test evaluate
2131      with self.assertRaises(ValueError):
2132        out = model.evaluate(None, None, batch_size=10)
2133      out = model.evaluate(None, None, steps=3)
2134
2135      # test predict
2136      with self.assertRaises(ValueError):
2137        out = model.predict(None, batch_size=10, verbose=1)
2138      out = model.predict(None, steps=3)
2139      self.assertEqual(len(out), 2)
2140      self.assertEqual(out[0].shape, (10 * 3, 4))
2141      self.assertEqual(out[1].shape, (10 * 3, 4))
2142
2143  def test_target_tensors(self):
2144    with self.cached_session():
2145      # single-output, as list
2146      model = keras.models.Sequential()
2147      model.add(keras.layers.Dense(4, input_shape=(4,), name='dense'))
2148      input_val = np.random.random((10, 4))
2149      target_val = np.random.random((10, 4))
2150      target = keras.backend.variable(target_val)
2151      model.compile(optimizer='rmsprop', loss='mse', target_tensors=[target])
2152      model.train_on_batch(input_val, None)
2153
2154      # single-output, as single tensor
2155      model.compile(optimizer='rmsprop', loss='mse', target_tensors=target)
2156      model.train_on_batch(input_val, None)
2157
2158      # single-output, as dict
2159      model.compile(optimizer='rmsprop', loss='mse',
2160                    target_tensors={'dense': target})
2161      model.train_on_batch(input_val, None)
2162
2163      # test invalid arguments
2164      with self.assertRaises(TypeError):
2165        model.compile(optimizer='rmsprop', loss='mse',
2166                      target_tensors=set())
2167      with self.assertRaises(ValueError):
2168        model.compile(optimizer='rmsprop', loss='mse',
2169                      target_tensors=[target, target])
2170      with self.assertRaises(ValueError):
2171        model.compile(optimizer='rmsprop', loss='mse',
2172                      target_tensors={'dense2': None})
2173      with self.assertRaises(ValueError):
2174        model.compile(optimizer='rmsprop', loss='mse',
2175                      target_tensors=[target])
2176        model.train_on_batch(input_val, target_val)
2177
2178      # multi-output, as list
2179      input_val = np.random.random((10, 4))
2180      target_val_a = np.random.random((10, 4))
2181      target_val_b = np.random.random((10, 4))
2182      target_a = keras.backend.variable(target_val_a)
2183      target_b = keras.backend.variable(target_val_b)
2184
2185      inputs = keras.layers.Input(shape=(4,))
2186      output_a = keras.layers.Dense(4, name='dense_a')(inputs)
2187      output_b = keras.layers.Dense(4, name='dense_b')(inputs)
2188      model = keras.models.Model(inputs, [output_a, output_b])
2189      model.compile(optimizer='rmsprop', loss='mse',
2190                    target_tensors=[target_a, target_b])
2191      model.train_on_batch(input_val, None)
2192
2193      # multi-output, as dict
2194      model.compile(optimizer='rmsprop', loss='mse',
2195                    target_tensors={'dense_a': target_a,
2196                                    'dense_b': target_b})
2197      model.train_on_batch(input_val, None)
2198
2199      # test with sample weights
2200      model.compile(
2201          optimizer='rmsprop',
2202          loss='mse',
2203          metrics=['mae', metrics_module.CategoricalAccuracy()],
2204          target_tensors=[target_a, target_b])
2205      model.train_on_batch(input_val, None,
2206                           sample_weight={'dense_a': np.random.random((10,))})
2207
2208  @tf_test_util.run_deprecated_v1
2209  def test_model_custom_target_tensors(self):
2210    with self.cached_session():
2211      a = keras.Input(shape=(3,), name='input_a')
2212      b = keras.Input(shape=(3,), name='input_b')
2213
2214      a_2 = keras.layers.Dense(4, name='dense_1')(a)
2215      dp = keras.layers.Dropout(0.5, name='dropout')
2216      b_2 = dp(b)
2217
2218      y = keras.backend.placeholder([10, 4], name='y')
2219      y1 = keras.backend.placeholder([10, 3], name='y1')
2220      y2 = keras.backend.placeholder([7, 5], name='y2')
2221      model = keras.models.Model([a, b], [a_2, b_2])
2222
2223      optimizer = 'rmsprop'
2224      loss = 'mse'
2225      loss_weights = [1., 0.5]
2226
2227      # test list of target tensors
2228      with self.assertRaises(ValueError):
2229        model.compile(optimizer, loss, metrics=[], loss_weights=loss_weights,
2230                      sample_weight_mode=None, target_tensors=[y, y1, y2])
2231      model.compile(optimizer, loss, metrics=[], loss_weights=loss_weights,
2232                    sample_weight_mode=None, target_tensors=[y, y1])
2233      input_a_np = np.random.random((10, 3))
2234      input_b_np = np.random.random((10, 3))
2235
2236      output_a_np = np.random.random((10, 4))
2237      output_b_np = np.random.random((10, 3))
2238
2239      _ = model.train_on_batch([input_a_np, input_b_np],
2240                               [output_a_np, output_b_np],
2241                               {y: np.random.random((10, 4)),
2242                                y1: np.random.random((10, 3))})
2243      # test dictionary of target_tensors
2244      with self.assertRaises(ValueError):
2245        model.compile(optimizer, loss,
2246                      metrics=[],
2247                      loss_weights=loss_weights,
2248                      sample_weight_mode=None,
2249                      target_tensors={'does_not_exist': y2})
2250      # test dictionary of target_tensors
2251      model.compile(optimizer, loss,
2252                    metrics=[],
2253                    loss_weights=loss_weights,
2254                    sample_weight_mode=None,
2255                    target_tensors={'dense_1': y, 'dropout': y1})
2256      _ = model.train_on_batch([input_a_np, input_b_np],
2257                               [output_a_np, output_b_np],
2258                               {y: np.random.random((10, 4)),
2259                                y1: np.random.random((10, 3))})
2260
2261      # test with custom TF placeholder as target
2262      pl_target_a = keras.backend.array_ops.placeholder('float32',
2263                                                        shape=(None, 4))
2264      model.compile(optimizer='rmsprop', loss='mse',
2265                    target_tensors={'dense_1': pl_target_a})
2266      model.train_on_batch([input_a_np, input_b_np],
2267                           [output_a_np, output_b_np])
2268
2269
2270class TestTrainingWithMetrics(keras_parameterized.TestCase):
2271  """Training tests related to metrics."""
2272
2273  @keras_parameterized.run_all_keras_modes
2274  def test_metrics_names(self):
2275    a = keras.layers.Input(shape=(3,), name='input_a')
2276    b = keras.layers.Input(shape=(3,), name='input_b')
2277
2278    dense = keras.layers.Dense(4, name='dense')
2279    c = dense(a)
2280    d = dense(b)
2281    e = keras.layers.Dropout(0.5, name='dropout')(c)
2282
2283    model = keras.models.Model([a, b], [d, e])
2284
2285    optimizer = RMSPropOptimizer(learning_rate=0.001)
2286    metrics = ['mse', metrics_module.BinaryAccuracy()]
2287    model.compile(optimizer, loss='mae', metrics=metrics,
2288                  run_eagerly=testing_utils.should_run_eagerly())
2289
2290    mse_metric = 'mse' if tf2.enabled() else 'mean_squared_error'
2291    reference_metric_names = [
2292        'loss', 'dense_loss', 'dropout_loss', 'dense_' + mse_metric,
2293        'dense_binary_accuracy', 'dropout_' + mse_metric,
2294        'dropout_binary_accuracy'
2295    ]
2296    self.assertEqual(reference_metric_names, model.metrics_names)
2297
2298    # Verify that model metric names are not altered during training.
2299    input_a_np = np.random.random((10, 3))
2300    input_b_np = np.random.random((10, 3))
2301
2302    output_d_np = np.random.random((10, 4))
2303    output_e_np = np.random.random((10, 4))
2304
2305    model.fit([input_a_np, input_b_np], [output_d_np, output_e_np],
2306              epochs=1,
2307              batch_size=5)
2308    self.assertEqual(reference_metric_names, model.metrics_names)
2309
2310  @keras_parameterized.run_all_keras_modes
2311  def test_metric_state_reset_between_fit_and_evaluate(self):
2312    model = keras.Sequential()
2313    model.add(keras.layers.Dense(3, activation='relu', input_dim=4))
2314    model.add(keras.layers.Dense(1, activation='sigmoid'))
2315    acc_obj = metrics_module.BinaryAccuracy()
2316    model.compile(
2317        loss='mae',
2318        metrics=[acc_obj],
2319        optimizer=RMSPropOptimizer(learning_rate=0.001),
2320        run_eagerly=testing_utils.should_run_eagerly())
2321
2322    x_train = np.random.random((100, 4))
2323    y_train = np.random.random((100, 1))
2324    model.fit(x_train, y_train, batch_size=5, epochs=2)
2325    self.assertEqual(self.evaluate(acc_obj.count), 100)
2326
2327    x_test = np.random.random((10, 4))
2328    y_test = np.random.random((10, 1))
2329    model.evaluate(x_test, y_test, batch_size=5)
2330    self.assertEqual(self.evaluate(acc_obj.count), 10)
2331
2332  @keras_parameterized.run_with_all_model_types(exclude_models=['sequential'])
2333  @keras_parameterized.run_all_keras_modes
2334  def test_metrics_valid_compile_input_formats(self):
2335    inp_1 = keras.layers.Input(shape=(1,), name='input_1')
2336    inp_2 = keras.layers.Input(shape=(1,), name='input_2')
2337    x = keras.layers.Dense(3, kernel_initializer='ones', trainable=False)
2338    out_1 = keras.layers.Dense(
2339        1, kernel_initializer='ones', name='output_1', trainable=False)
2340    out_2 = keras.layers.Dense(
2341        1, kernel_initializer='ones', name='output_2', trainable=False)
2342
2343    branch_a = [inp_1, x, out_1]
2344    branch_b = [inp_2, x, out_2]
2345    model = testing_utils.get_multi_io_model(branch_a, branch_b)
2346
2347    # list of metrics.
2348    model.compile(
2349        optimizer='rmsprop',
2350        loss='mse',
2351        metrics=[keras.metrics.MeanSquaredError()],
2352        weighted_metrics=[keras.metrics.MeanSquaredError()],
2353        run_eagerly=testing_utils.should_run_eagerly())
2354
2355    # list of list of metrics.
2356    model.compile(
2357        optimizer='rmsprop',
2358        loss='mse',
2359        metrics=[
2360            keras.metrics.MeanSquaredError(),
2361            [keras.metrics.MeanSquaredError(),
2362             keras.metrics.Accuracy()]
2363        ],
2364        weighted_metrics=[
2365            keras.metrics.MeanSquaredError(),
2366            [keras.metrics.MeanSquaredError(),
2367             keras.metrics.Accuracy()]
2368        ],
2369        run_eagerly=testing_utils.should_run_eagerly())
2370
2371    # dict of metrics.
2372    model.compile(
2373        optimizer='rmsprop',
2374        loss='mse',
2375        metrics={
2376            'output_1':
2377                keras.metrics.MeanSquaredError(),
2378            'output_2': [
2379                keras.metrics.MeanSquaredError(),
2380                keras.metrics.Accuracy()
2381            ],
2382        },
2383        weighted_metrics={
2384            'output_1':
2385                keras.metrics.MeanSquaredError(),
2386            'output_2': [
2387                keras.metrics.MeanSquaredError(),
2388                keras.metrics.Accuracy()
2389            ],
2390        },
2391        run_eagerly=testing_utils.should_run_eagerly())
2392
2393  @keras_parameterized.run_all_keras_modes
2394  def test_invalid_metrics(self):
2395    num_classes = 5
2396    input_dim = 5
2397
2398    model = testing_utils.get_small_sequential_mlp(
2399        num_hidden=10, num_classes=num_classes, input_dim=input_dim)
2400
2401    with self.assertRaisesRegexp(
2402        TypeError, 'Type of `metrics` argument not understood. '
2403        'Expected a list or dictionary, found: '):
2404      model.compile(
2405          RMSPropOptimizer(learning_rate=0.001),
2406          loss='categorical_crossentropy',
2407          metrics=metrics_module.CategoricalAccuracy(),
2408          run_eagerly=testing_utils.should_run_eagerly())
2409
2410    inp = keras.layers.Input(shape=(1,))
2411    x = keras.layers.Dense(3, activation='relu')(inp)
2412    out_1 = keras.layers.Dense(1, activation='sigmoid', name='output_1')(x)
2413    out_2 = keras.layers.Dense(1, activation='sigmoid', name='output_2')(x)
2414    model = keras.models.Model(inp, [out_1, out_2])
2415    with self.assertRaisesRegex(
2416        ValueError, 'When passing a list of lists as `metrics`, '
2417        'it should have one entry per model output. '
2418        'The model has 2 outputs, but you passed metrics='):
2419      model.compile('rmsprop', loss='mse', metrics=[['mse']])
2420
2421  @keras_parameterized.run_all_keras_modes
2422  def test_metrics_masking(self):
2423    if testing_utils.should_run_eagerly():
2424      self.skipTest('b/120495761')
2425    with self.cached_session():
2426      np.random.seed(1337)
2427      model = keras.models.Sequential()
2428      model.add(keras.layers.Masking(mask_value=0, input_shape=(2, 1)))
2429      model.add(
2430          keras.layers.TimeDistributed(
2431              keras.layers.Dense(1, kernel_initializer='ones')))
2432      model.compile(
2433          RMSPropOptimizer(learning_rate=0.001),
2434          loss='mse',
2435          weighted_metrics=['accuracy'],
2436          run_eagerly=testing_utils.should_run_eagerly())
2437
2438      # verify that masking is applied.
2439      x = np.array([[[1], [1]], [[1], [1]], [[0], [0]]])
2440      y = np.array([[[1], [1]], [[0], [1]], [[1], [1]]])
2441      scores = model.train_on_batch(x, y)
2442      self.assertArrayNear(scores, [0.25, 0.75], 0.1)
2443
2444      # verify that masking is combined with sample weights.
2445      w = np.array([3, 2, 4])
2446      scores = model.train_on_batch(x, y, sample_weight=w)
2447      self.assertArrayNear(scores, [0.3328, 0.8], 0.001)
2448
2449  @keras_parameterized.run_all_keras_modes
2450  def test_add_metric_with_tensor_on_model(self):
2451    x = keras.layers.Input(shape=(1,))
2452    y = keras.layers.Dense(1, kernel_initializer='ones')(x)
2453    model = keras.models.Model(x, y)
2454    model.add_metric(
2455        math_ops.reduce_sum(y), name='metric_1', aggregation='mean')
2456
2457    # test with a metric which does not have the standard signature:
2458    # (y_true, y_pred, sample_Weight)
2459    with keras.backend.get_graph().as_default():
2460      model.add_metric(metrics_module.Mean(name='metric_2')(y))
2461
2462    if testing_utils.should_run_eagerly():
2463      with self.assertRaisesRegex(
2464          ValueError,
2465          'We currently do not support enabling `run_eagerly` on compile if '
2466          r'`model.add_loss\(tensor\)` or `model.add_metric\(tensor\)` '
2467          'has been called.'):
2468        model.compile('sgd', run_eagerly=True)
2469      return
2470    else:
2471      model.compile('sgd', loss='mse', run_eagerly=False)
2472
2473    inputs = np.ones(shape=(10, 1))
2474    targets = np.ones(shape=(10, 1))
2475    history = model.fit(
2476        inputs,
2477        targets,
2478        epochs=2,
2479        batch_size=5,
2480        validation_data=(inputs, targets))
2481    self.assertEqual(history.history['metric_1'][-1], 5)
2482    self.assertEqual(history.history['metric_2'][-1], 1)
2483    self.assertEqual(history.history['val_metric_1'][-1], 5)
2484    self.assertEqual(history.history['val_metric_2'][-1], 1)
2485
2486    eval_results = model.evaluate(inputs, targets, batch_size=5)
2487    self.assertEqual(eval_results[-1], 1)
2488    self.assertEqual(eval_results[-2], 5)
2489
2490    model.predict(inputs, batch_size=5)
2491    model.train_on_batch(inputs, targets)
2492    model.test_on_batch(inputs, targets)
2493
2494  @keras_parameterized.run_all_keras_modes
2495  def test_add_metric_in_model_call(self):
2496
2497    class TestModel(keras.Model):
2498
2499      def __init__(self):
2500        super(TestModel, self).__init__(name='test_model')
2501        self.dense1 = keras.layers.Dense(2, kernel_initializer='ones')
2502        self.mean = metrics_module.Mean(name='metric_1')
2503
2504      def call(self, x):
2505        self.add_metric(
2506            math_ops.reduce_sum(x), name='metric_2', aggregation='mean')
2507        # Provide same name as in the instance created in __init__
2508        # for eager mode
2509        self.add_metric(self.mean(x), name='metric_1')
2510        return self.dense1(x)
2511
2512    model = TestModel()
2513    model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01),
2514                  run_eagerly=testing_utils.should_run_eagerly())
2515
2516    x = np.ones(shape=(10, 1))
2517    y = np.ones(shape=(10, 2))
2518    history = model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
2519    self.assertAlmostEqual(history.history['metric_1'][-1], 1, 0)
2520    self.assertAlmostEqual(history.history['val_metric_1'][-1], 1, 0)
2521    self.assertAlmostEqual(history.history['metric_2'][-1], 5, 0)
2522    self.assertAlmostEqual(history.history['val_metric_2'][-1], 5, 0)
2523
2524    eval_results = model.evaluate(x, y, batch_size=5)
2525    self.assertAlmostEqual(eval_results[1], 1, 0)
2526    self.assertAlmostEqual(eval_results[2], 5, 0)
2527
2528    model.predict(x, batch_size=5)
2529    model.train_on_batch(x, y)
2530    model.test_on_batch(x, y)
2531
2532  @keras_parameterized.run_with_all_model_types
2533  @keras_parameterized.run_all_keras_modes
2534  def test_add_metric_in_layer_call(self):
2535
2536    class TestLayer(keras.layers.Layer):
2537
2538      def build(self, input_shape):
2539        self.a = self.add_variable(
2540            'a', (1, 1), initializer='ones', trainable=False)
2541        self.built = True
2542
2543      def call(self, inputs):
2544        self.add_metric(
2545            math_ops.reduce_sum(inputs), name='metric_1', aggregation='mean')
2546        return inputs + 1
2547
2548    layers = [
2549        TestLayer(input_shape=(1,)),
2550        keras.layers.Dense(2, kernel_initializer='ones')
2551    ]
2552    model = testing_utils.get_model_from_layers(layers, input_shape=(1,))
2553    model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01),
2554                  run_eagerly=testing_utils.should_run_eagerly())
2555
2556    x = np.ones(shape=(10, 1))
2557    y = np.ones(shape=(10, 2))
2558    history = model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
2559    self.assertEqual(history.history['metric_1'][-1], 5)
2560    self.assertAlmostEqual(history.history['val_metric_1'][-1], 5, 0)
2561
2562  @keras_parameterized.run_all_keras_modes
2563  def test_model_metrics_list(self):
2564    x = keras.layers.Input(shape=(1,))
2565    y = keras.layers.Dense(1, kernel_initializer='ones')(x)
2566    model = keras.models.Model(x, y)
2567    model.add_metric(
2568        math_ops.reduce_sum(y), name='metric_1', aggregation='mean')
2569    with keras.backend.get_graph().as_default():
2570      model.add_metric(metrics_module.Mean(name='metric_2')(y))
2571
2572    if testing_utils.should_run_eagerly():
2573      with self.assertRaisesRegex(
2574          ValueError,
2575          'We currently do not support enabling `run_eagerly` on compile if '
2576          r'`model.add_loss\(tensor\)` or `model.add_metric\(tensor\)` '
2577          'has been called.'):
2578        model.compile('sgd', run_eagerly=True)
2579      return
2580    else:
2581      model.compile(
2582          'sgd',
2583          loss='mse',
2584          metrics=[metrics_module.Accuracy('acc')],
2585          run_eagerly=False)
2586
2587    # Verify that the metrics added using `compile` and `add_metric` API are
2588    # included
2589    self.assertEqual([m.name for m in model._compile_metrics], ['acc'])
2590    self.assertEqual([m.name for m in model.metrics],
2591                     ['acc', 'metric_1', 'metric_2'])
2592
2593  @keras_parameterized.run_all_keras_modes
2594  def test_model_metrics_list_in_call(self):
2595
2596    class TestModel(keras.Model):
2597
2598      def __init__(self):
2599        super(TestModel, self).__init__(name='test_model')
2600        self.dense1 = keras.layers.Dense(2, kernel_initializer='ones')
2601
2602      def call(self, x):
2603        self.add_metric(
2604            math_ops.reduce_sum(x), name='metric_1', aggregation='mean')
2605        return self.dense1(x)
2606
2607    model = TestModel()
2608    model.compile(
2609        loss='mse',
2610        optimizer=RMSPropOptimizer(0.01),
2611        metrics=[metrics_module.Accuracy('acc')],
2612        run_eagerly=testing_utils.should_run_eagerly())
2613    x = np.ones(shape=(10, 1))
2614    y = np.ones(shape=(10, 2))
2615    model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
2616
2617    self.assertEqual([m.name for m in model._compile_metrics], ['acc'])
2618    self.assertEqual([m.name for m in model.metrics], ['acc', 'metric_1'])
2619
2620  @keras_parameterized.run_all_keras_modes
2621  def test_multiple_add_metric_calls(self):
2622
2623    class TestModel(keras.Model):
2624
2625      def __init__(self):
2626        super(TestModel, self).__init__(name='test_model')
2627        self.dense1 = keras.layers.Dense(2, kernel_initializer='ones')
2628        self.mean1 = metrics_module.Mean(name='metric_1')
2629        self.mean2 = metrics_module.Mean(name='metric_2')
2630
2631      def call(self, x):
2632        self.add_metric(self.mean2(x), name='metric_2')
2633        self.add_metric(self.mean1(x), name='metric_1')
2634        self.add_metric(
2635            math_ops.reduce_sum(x), name='metric_3', aggregation='mean')
2636        return self.dense1(x)
2637
2638    model = TestModel()
2639    model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01),
2640                  run_eagerly=testing_utils.should_run_eagerly())
2641
2642    x = np.ones(shape=(10, 1))
2643    y = np.ones(shape=(10, 2))
2644    history = model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
2645    self.assertAlmostEqual(history.history['metric_1'][-1], 1, 0)
2646    self.assertAlmostEqual(history.history['metric_2'][-1], 1, 0)
2647    self.assertAlmostEqual(history.history['metric_3'][-1], 5, 0)
2648
2649    eval_results = model.evaluate(x, y, batch_size=5)
2650    self.assertArrayNear(eval_results[1:4], [1, 1, 5], 0.1)
2651
2652    model.predict(x, batch_size=5)
2653    model.train_on_batch(x, y)
2654    model.test_on_batch(x, y)
2655
2656  @keras_parameterized.run_with_all_model_types
2657  @keras_parameterized.run_all_keras_modes
2658  def test_invalid_metric_tensor(self):
2659
2660    class TestLayer(keras.layers.Layer):
2661
2662      def build(self, input_shape):
2663        self.built = True
2664
2665      def call(self, inputs):
2666        self.add_metric(math_ops.reduce_mean(inputs), name='metric_1')
2667        return inputs + 1
2668
2669    layers = [TestLayer(input_shape=(1,))]
2670    layers.append(keras.layers.Dense(2, kernel_initializer='ones'))
2671    x = np.ones(shape=(10, 1))
2672    y = np.ones(shape=(10, 2))
2673
2674    with self.assertRaisesRegexp(
2675        ValueError,
2676        'We do not support adding an aggregated metric result tensor that is '
2677        'not the output of a `tf.keras.metrics.Metric` metric instance.'):
2678      model = testing_utils.get_model_from_layers(layers, input_shape=(1,))
2679      model.compile(
2680          loss='mse',
2681          optimizer=RMSPropOptimizer(0.01),
2682          run_eagerly=testing_utils.should_run_eagerly())
2683      model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
2684
2685  @keras_parameterized.run_all_keras_modes
2686  def test_duplicate_metric_name_in_add_metric(self):
2687
2688    class TestModel(keras.Model):
2689
2690      def __init__(self):
2691        super(TestModel, self).__init__(name='test_model')
2692        self.dense1 = keras.layers.Dense(2, kernel_initializer='ones')
2693        self.mean = metrics_module.Mean(name='metric_1')
2694        self.mean2 = metrics_module.Mean(name='metric_1')
2695
2696      def call(self, x):
2697        self.add_metric(self.mean(x), name='metric_1')
2698        return self.dense1(x)
2699
2700    model = TestModel()
2701    model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01),
2702                  run_eagerly=testing_utils.should_run_eagerly())
2703
2704    x = np.ones(shape=(10, 1))
2705    y = np.ones(shape=(10, 2))
2706    with self.assertRaisesRegexp(
2707        ValueError,
2708        'Please provide different names for the metrics you have added. '
2709        'We found 2 metrics with the name: "metric_1"'):
2710      model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
2711
2712  @keras_parameterized.run_all_keras_modes
2713  def test_add_metric_without_name(self):
2714
2715    class TestModel(keras.Model):
2716
2717      def __init__(self):
2718        super(TestModel, self).__init__(name='test_model')
2719        self.dense1 = keras.layers.Dense(2, kernel_initializer='ones')
2720
2721      def call(self, x):
2722        self.add_metric(math_ops.reduce_sum(x), aggregation='mean')
2723        return self.dense1(x)
2724
2725    model = TestModel()
2726    model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01),
2727                  run_eagerly=testing_utils.should_run_eagerly())
2728    x = np.ones(shape=(10, 1))
2729    y = np.ones(shape=(10, 2))
2730
2731    with self.assertRaisesRegex(ValueError,
2732                                'Please provide a name for your metric like'):
2733      model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
2734
2735  @keras_parameterized.run_all_keras_modes
2736  def test_add_metric_correctness(self):
2737    inputs = keras.Input(shape=(1,))
2738    targets = keras.Input(shape=(1,))
2739
2740    class Bias(keras.layers.Layer):
2741
2742      def build(self, input_shape):
2743        self.bias = self.add_variable('bias', (1,), initializer='zeros')
2744        self.mae = metrics_module.MeanAbsoluteError(name='mae_1')
2745
2746      def call(self, inputs):
2747        outputs = inputs + self.bias
2748        self.add_metric(self.mae(targets, outputs), name='mae_1')
2749        return outputs
2750
2751    outputs = Bias()(inputs)
2752    model = keras.Model(inputs, outputs)
2753
2754    model.add_metric(
2755        metrics_module.mean_absolute_error(targets, outputs),
2756        name='mae_2',
2757        aggregation='mean')
2758
2759    # If we want to use the metric class instance as shown below, we will need
2760    # to add graph scope as the reduction logic involves some eager mode checks.
2761    with keras.backend.get_graph().as_default():
2762      model.add_metric(
2763          metrics_module.MeanAbsoluteError(name='mae_3')(targets, outputs))
2764
2765    if testing_utils.should_run_eagerly():
2766      with self.assertRaisesRegex(
2767          ValueError,
2768          'We currently do not support enabling `run_eagerly` on compile if '
2769          r'`model.add_loss\(tensor\)` or `model.add_metric\(tensor\)` '
2770          'has been called.'):
2771        model.compile('sgd', run_eagerly=True)
2772      return
2773    else:
2774      model.compile(
2775          loss='mae',
2776          optimizer=keras.optimizer_v2.gradient_descent.SGD(0.1),
2777          metrics=[metrics_module.MeanAbsoluteError(name='mae_4')],
2778          target_tensors=[targets],
2779          run_eagerly=False)
2780
2781    x = np.array([[0.], [1.], [2.]])
2782    y = np.array([[0.5], [2.], [3.5]])
2783    history = model.fit(x, y, batch_size=3, epochs=5)
2784
2785    expected_val = [1., 0.9, 0.8, 0.7, 0.6]
2786    for key in ['loss', 'mae_1', 'mae_2', 'mae_3', 'mae_4']:
2787      self.assertAllClose(history.history[key], expected_val, 1e-3)
2788
2789
2790class BareUpdateLayer(keras.layers.Layer):
2791
2792  def build(self, input_shape):
2793    self.counter = self.add_weight(
2794        'counter',
2795        dtype='int32',
2796        shape=(),
2797        initializer='zeros',
2798        trainable=False)
2799
2800  def call(self, inputs):
2801    state_ops.assign_add(self.counter, 1)
2802    return math_ops.cast(self.counter, inputs.dtype) * inputs
2803
2804
2805class AddUpdateLayer(keras.layers.Layer):
2806
2807  def build(self, input_shape):
2808    self.counter = self.add_weight(
2809        'counter',
2810        dtype='int32',
2811        shape=(),
2812        initializer='zeros',
2813        trainable=False)
2814
2815  def call(self, inputs):
2816    # Make sure update isn't run twice.
2817    self.add_update(state_ops.assign_add(self.counter, 1))
2818    return math_ops.cast(self.counter, inputs.dtype) * inputs
2819
2820
2821class NestedUpdateLayer(keras.layers.Layer):
2822
2823  def build(self, input_shape):
2824    self.layer = BareUpdateLayer()
2825    self.layer.build(input_shape)
2826
2827  @property
2828  def counter(self):
2829    return self.layer.counter
2830
2831  def call(self, inputs):
2832    return self.layer(inputs)
2833
2834
2835@keras_parameterized.run_all_keras_modes(always_skip_v1=True)
2836class TestAutoUpdates(keras_parameterized.TestCase):
2837
2838  @keras_parameterized.run_with_all_model_types
2839  @parameterized.named_parameters(('bare_update', BareUpdateLayer()),
2840                                  ('add_update', AddUpdateLayer()),
2841                                  ('nested_update', NestedUpdateLayer()))
2842  def test_updates_in_model(self, layer):
2843    x, y = np.ones((10, 10)), np.ones((10, 1))
2844    model = testing_utils.get_model_from_layers(
2845        [layer, keras.layers.Dense(1)], input_shape=(10,))
2846    model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly())
2847    model.fit(x, y, batch_size=2, epochs=1)
2848    if not testing_utils.should_run_eagerly():
2849      # Check that `trainable=False` disables updates.
2850      layer.trainable = False
2851      model.compile(
2852          'sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly())
2853      model.fit(x, y, batch_size=2, epochs=1)
2854    self.assertEqual(self.evaluate(layer.counter), 5)
2855
2856  @parameterized.named_parameters(('bare_update', BareUpdateLayer()),
2857                                  ('add_update', AddUpdateLayer()),
2858                                  ('nested_update', NestedUpdateLayer()))
2859  def test_updates_standalone_layer(self, layer):
2860    y = layer(np.ones((10, 10)))
2861    self.evaluate(layer.counter.initializer)
2862    self.evaluate(y)
2863    self.assertEqual(self.evaluate(layer.counter), 1)
2864
2865  def test_trainable_false(self):
2866    x = keras.backend.placeholder(shape=(10, 10), dtype='float32')
2867    layer = NestedUpdateLayer()
2868    layer.trainable = False
2869    y = layer(x)
2870    func = keras.backend.function([x], [y])
2871    x_val = np.ones((10, 10))
2872    func(x_val)
2873    counter = keras.backend.get_value(layer.counter)
2874    self.assertEqual(counter, 0)
2875
2876  @keras_parameterized.run_with_all_model_types
2877  def test_batchnorm_trainable_false(self):
2878    bn = keras.layers.BatchNormalization()
2879    bn.trainable = False
2880    model = testing_utils.get_model_from_layers([bn, keras.layers.Dense(1)],
2881                                                input_shape=(10,))
2882    model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly())
2883    x, y = np.ones((10, 10)), np.ones((10, 1))
2884    model.fit(x, y, batch_size=2, epochs=1)
2885    self.assertAllEqual(self.evaluate(bn.moving_mean), np.zeros((10,)))
2886    self.assertAllEqual(self.evaluate(bn.moving_variance), np.ones((10,)))
2887
2888
2889if __name__ == '__main__':
2890  test.main()
2891