• Home
  • History
  • Annotate
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 # Copyright 2016 The TensorFlow Authors. All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 # ==============================================================================
15 """Tests for training routines."""
16 
17 from __future__ import absolute_import
18 from __future__ import division
19 from __future__ import print_function
20 
21 import io
22 import logging
23 import sys
24 
25 from absl.testing import parameterized
26 import numpy as np
27 import six
28 
29 from tensorflow.python import keras
30 from tensorflow.python import tf2
31 from tensorflow.python.data.ops import dataset_ops
32 from tensorflow.python.eager import context
33 from tensorflow.python.eager import function
34 from tensorflow.python.framework import ops
35 from tensorflow.python.framework import tensor_shape
36 from tensorflow.python.framework import test_util as tf_test_util
37 from tensorflow.python.keras import keras_parameterized
38 from tensorflow.python.keras import losses
39 from tensorflow.python.keras import metrics as metrics_module
40 from tensorflow.python.keras import testing_utils
41 from tensorflow.python.keras.callbacks import Callback
42 from tensorflow.python.ops import array_ops
43 from tensorflow.python.ops import math_ops
44 from tensorflow.python.ops import sparse_ops
45 from tensorflow.python.ops import state_ops
46 from tensorflow.python.ops import variables as variables_lib
47 from tensorflow.python.platform import test
48 from tensorflow.python.platform import tf_logging as logging
49 from tensorflow.python.training.rmsprop import RMSPropOptimizer
50 
51 try:
52   import scipy.sparse as scipy_sparse  # pylint: disable=g-import-not-at-top
53 except ImportError:
54   scipy_sparse = None
55 
56 
57 class CompileTest(keras_parameterized.TestCase):
58 
59   def _get_multi_output_model(self):
60     input_a = keras.layers.Input(shape=(3,), name='input_a')
61     output_a = keras.layers.Dense(1, name='dense_1')(input_a)
62     output_b = keras.layers.Dense(1, name='dense_2')(input_a)
63     return keras.models.Model(input_a, [output_a, output_b])
64 
65   def _do_test_compile_with_model_and_single_loss(self, model, loss):
66     model.compile(optimizer='adam', loss=loss)
67     self.assertEqual(model.loss, loss)
68 
69     loss = losses.get(loss)
70     if not isinstance(loss, list):
71       loss_list = [loss] * len(model.outputs)
72 
73     self.assertEqual(len(model.loss_functions), len(loss_list))
74     for i in range(len(loss_list)):
75       self.assertIsInstance(model.loss_functions[i], losses.LossFunctionWrapper)
76       if not isinstance(loss_list[i], losses.LossFunctionWrapper):
77         self.assertEqual(model.loss_functions[i].fn, loss_list[i])
78     self.assertAllEqual(model.loss_weights_list, [1.] * len(loss_list))
79 
80   @keras_parameterized.run_all_keras_modes
81   @parameterized.named_parameters(('loss_string', 'mse'),
82                                   ('loss_function', losses.mean_squared_error),
83                                   ('loss_instance', losses.MeanSquaredError()))
84   def test_compile_with_single_output(self, loss):
85     model = testing_utils.get_small_sequential_mlp(
86         num_hidden=10, num_classes=2, input_dim=3)
87     self._do_test_compile_with_model_and_single_loss(model, loss)
88 
89   @keras_parameterized.run_all_keras_modes
90   @parameterized.named_parameters(('loss_string', 'mse'),
91                                   ('loss_function', losses.mean_squared_error),
92                                   ('loss_instance', losses.MeanSquaredError()))
93   def test_compile_with_multi_output(self, loss):
94     model = self._get_multi_output_model()
95     self._do_test_compile_with_model_and_single_loss(model, loss)
96 
97   @keras_parameterized.run_all_keras_modes
98   def test_compile_with_multi_output_and_multi_loss(self):
99     model = self._get_multi_output_model()
100     # Test loss is a list.
101     loss = ['mse', 'mae']
102     model.compile(optimizer='adam', loss=loss)
103     self.assertEqual(model.loss_functions[0].fn, losses.mean_squared_error)
104     self.assertEqual(model.loss_functions[1].fn, losses.mean_absolute_error)
105     self.assertAllEqual(model.loss_weights_list, [1., 1.])
106 
107     # Test loss is a dict.
108     loss = {'dense_1': 'mae', 'dense_2': 'mse'}
109     model.compile(optimizer='adam', loss=loss)
110     self.assertEqual(model.loss_functions[0].fn, losses.mean_absolute_error)
111     self.assertEqual(model.loss_functions[1].fn, losses.mean_squared_error)
112     self.assertAllEqual(model.loss_weights_list, [1., 1.])
113 
114   @keras_parameterized.run_all_keras_modes
115   def test_compile_with_multi_output_and_loss_weights_list(self):
116     model = self._get_multi_output_model()
117     loss_weights = [1., 2.]
118     model.compile(optimizer='adam', loss='mse', loss_weights=loss_weights)
119     self.assertAllEqual(model.loss_weights_list, [1., 2.])
120 
121   def test_compile_with_multi_output_and_loss_weights_dict(self):
122     with context.graph_mode():
123       model = self._get_multi_output_model()
124       loss_weights = {'dense_1': 1., 'dense_2': 2.}
125       model.compile(optimizer='adam', loss='mse', loss_weights=loss_weights)
126       self.assertAllEqual(model.loss_weights_list, [1., 2.])
127 
128       input_np = np.random.random((10, 3))
129       output_a_np = np.random.random((10, 1))
130       output_b_np = np.random.random((10, 1))
131 
132       with self.cached_session() as sess:
133         sess.run(variables_lib.global_variables_initializer())
134         total_loss, y_preds = sess.run(
135             [model.total_loss, model.outputs],
136             feed_dict={
137                 'input_a:0': input_np,
138                 'dense_1_target:0': output_a_np,
139                 'dense_2_target:0': output_b_np
140             })
141         self.assertAllClose(
142             total_loss,
143             np.mean(
144                 np.add((output_a_np - y_preds[0])**2,
145                        2 * (output_b_np - y_preds[1])**2)))
146 
147   @keras_parameterized.run_all_keras_modes
148   def test_compile_with_incorrect_loss_size(self):
149     model = testing_utils.get_small_sequential_mlp(
150         num_hidden=10, num_classes=2, input_dim=3)
151     with self.assertRaisesRegexp(ValueError, 'The model has 1 outputs'):
152       model.compile(optimizer='adam', loss=['mse', 'mae'])
153 
154   @keras_parameterized.run_all_keras_modes
155   def test_compile_with_incorrect_loss_key(self):
156     model = testing_utils.get_small_sequential_mlp(
157         num_hidden=10, num_classes=2, input_dim=3)
158     with self.assertRaisesRegexp(
159         ValueError, 'Unknown entry in loss dictionary: unknown_output'):
160       model.compile(optimizer='adam', loss={'unknown_output': 'mse'})
161 
162   @keras_parameterized.run_all_keras_modes
163   def test_compile_with_incorrect_loss_weights_size(self):
164     model = testing_utils.get_small_sequential_mlp(
165         num_hidden=10, num_classes=2, input_dim=3)
166     with self.assertRaisesRegexp(ValueError,
167                                  'it should have one entry per model output'):
168       model.compile(optimizer='adam', loss='mse', loss_weights=[1., 2.])
169 
170   @keras_parameterized.run_all_keras_modes
171   def test_compile_with_incorrect_loss_weights_key(self):
172     model = testing_utils.get_small_sequential_mlp(
173         num_hidden=10, num_classes=2, input_dim=3)
174     with self.assertRaisesRegexp(
175         ValueError, 'Unknown entry in loss_weights dictionary: unknown_output'):
176       model.compile(
177           optimizer='adam', loss='mse', loss_weights={'unknown_output': 1.})
178 
179 
180 class TrainingTest(keras_parameterized.TestCase):
181 
182   @keras_parameterized.run_with_all_model_types(exclude_models='sequential')
183   @keras_parameterized.run_all_keras_modes
184   def test_fit_on_arrays(self):
185     input_a = keras.layers.Input(shape=(3,), name='input_a')
186     input_b = keras.layers.Input(shape=(3,), name='input_b')
187 
188     dense = keras.layers.Dense(4, name='dense')
189     dropout = keras.layers.Dropout(0.5, name='dropout')
190     branch_a = [input_a, dense]
191     branch_b = [input_b, dense, dropout]
192 
193     model = testing_utils.get_multi_io_model(branch_a, branch_b)
194 
195     optimizer = RMSPropOptimizer(learning_rate=0.001)
196     loss = 'mse'
197     loss_weights = [1., 0.5]
198     model.compile(
199         optimizer,
200         loss,
201         metrics=[metrics_module.CategoricalAccuracy(), 'mae'],
202         loss_weights=loss_weights,
203         run_eagerly=testing_utils.should_run_eagerly())
204 
205     input_a_np = np.random.random((10, 3))
206     input_b_np = np.random.random((10, 3))
207 
208     output_d_np = np.random.random((10, 4))
209     output_e_np = np.random.random((10, 4))
210 
211     # Test fit at different verbosity
212     model.fit(
213         [input_a_np, input_b_np], [output_d_np, output_e_np],
214         epochs=1,
215         batch_size=5,
216         verbose=0)
217     model.fit(
218         [input_a_np, input_b_np], [output_d_np, output_e_np],
219         epochs=1,
220         batch_size=5,
221         verbose=1)
222     model.fit(
223         [input_a_np, input_b_np], [output_d_np, output_e_np],
224         epochs=2,
225         batch_size=5,
226         verbose=2)
227     model.train_on_batch([input_a_np, input_b_np], [output_d_np, output_e_np])
228 
229     # Test model with input data as a list of lists
230     model.fit(
231         [np.ndarray.tolist(input_a_np), np.ndarray.tolist(input_b_np)],
232         [output_d_np, output_e_np],
233         epochs=2,
234         batch_size=5,
235         verbose=2)
236 
237     # Test with validation data
238     model.fit(
239         [input_a_np, input_b_np], [output_d_np, output_e_np],
240         validation_data=([input_a_np, input_b_np], [output_d_np,
241                                                     output_e_np]),
242         epochs=1,
243         batch_size=5,
244         verbose=0)
245     model.fit(
246         [input_a_np, input_b_np], [output_d_np, output_e_np],
247         validation_data=([input_a_np, input_b_np], [output_d_np,
248                                                     output_e_np]),
249         epochs=2,
250         batch_size=5,
251         verbose=1)
252     model.fit(
253         [input_a_np, input_b_np], [output_d_np, output_e_np],
254         validation_data=([input_a_np, input_b_np], [output_d_np,
255                                                     output_e_np]),
256         epochs=2,
257         batch_size=5,
258         verbose=2)
259     # Test with validation split
260     model.fit(
261         [input_a_np, input_b_np], [output_d_np, output_e_np],
262         epochs=2,
263         batch_size=5,
264         verbose=0,
265         validation_split=0.2)
266 
267     if testing_utils.get_model_type() == 'functional':
268       # Test with dictionary inputs
269       model.fit(
270           {
271               'input_a': input_a_np,
272               'input_b': input_b_np
273           }, {
274               'dense': output_d_np,
275               'dropout': output_e_np
276           },
277           epochs=1,
278           batch_size=5,
279           verbose=0)
280       model.fit(
281           {
282               'input_a': input_a_np,
283               'input_b': input_b_np
284           }, {
285               'dense': output_d_np,
286               'dropout': output_e_np
287           },
288           epochs=1,
289           batch_size=5,
290           verbose=1)
291       model.fit(
292           {
293               'input_a': input_a_np,
294               'input_b': input_b_np
295           }, {
296               'dense': output_d_np,
297               'dropout': output_e_np
298           },
299           validation_data=({
300               'input_a': input_a_np,
301               'input_b': input_b_np
302           }, {
303               'dense': output_d_np,
304               'dropout': output_e_np
305           }),
306           epochs=1,
307           batch_size=5,
308           verbose=0)
309       model.train_on_batch({
310           'input_a': input_a_np,
311           'input_b': input_b_np
312       }, {
313           'dense': output_d_np,
314           'dropout': output_e_np
315       })
316 
317     # Test with lists for loss, metrics
318     loss = ['mae', 'mse']
319     model.compile(
320         optimizer,
321         loss,
322         metrics=[metrics_module.CategoricalAccuracy(), 'mae'],
323         run_eagerly=testing_utils.should_run_eagerly())
324     model.fit(
325         [input_a_np, input_b_np], [output_d_np, output_e_np],
326         epochs=1,
327         batch_size=5,
328         verbose=0)
329 
330     # Test with dictionaries for loss, metrics, loss weights
331     if testing_utils.get_model_type() == 'functional':
332       loss = {'dense': 'mse', 'dropout': 'mae'}
333       loss_weights = {'dense': 1., 'dropout': 0.5}
334       metrics = {
335           'dense': 'mse',
336           'dropout': metrics_module.CategoricalAccuracy()
337       }
338       model.compile(optimizer, loss, metrics=metrics, loss_weights=loss_weights,
339                     run_eagerly=testing_utils.should_run_eagerly())
340     model.fit(
341         [input_a_np, input_b_np], [output_d_np, output_e_np],
342         epochs=1,
343         batch_size=5,
344         verbose=0)
345 
346     # Invalid use cases
347     with self.assertRaises(ValueError):
348       model.train_on_batch({'input_a': input_a_np},
349                            [output_d_np, output_e_np])
350     with self.assertRaises(AttributeError):
351       model.fit(
352           [input_a_np, input_b_np], [output_d_np, output_e_np],
353           epochs=1,
354           validation_data=([input_a_np, input_b_np], 0, 0),
355           verbose=0)
356     with self.assertRaises(ValueError):
357       model.train_on_batch([input_a_np], [output_d_np, output_e_np])
358     with self.assertRaises(AttributeError):
359       model.train_on_batch(1, [output_d_np, output_e_np])
360     with self.assertRaises(ValueError):
361       model.train_on_batch(input_a_np, [output_d_np, output_e_np])
362     with self.assertRaises(ValueError):
363       bad_input = np.random.random((11, 3))
364       model.train_on_batch([bad_input, input_b_np],
365                            [output_d_np, output_e_np])
366     with self.assertRaises(ValueError):
367       bad_target = np.random.random((11, 4))
368       model.train_on_batch([input_a_np, input_b_np],
369                            [bad_target, output_e_np])
370 
371     # Build single-input model
372     x = keras.layers.Input(shape=(3,), name='input_a')
373     y = keras.layers.Dense(4)(x)
374     model = keras.models.Model(x, y)
375     model.compile(optimizer, loss='mse',
376                   run_eagerly=testing_utils.should_run_eagerly())
377     # This will work
378     model.fit([input_a_np], output_d_np, epochs=1)
379     # TODO(gsundeep) Test only works in eager, file ticket
380     if testing_utils.should_run_eagerly() and context.executing_eagerly():
381       with self.assertRaises(ValueError):
382         model.fit([input_a_np, input_a_np], output_d_np, epochs=1)
383 
384     # Test model on a list of floats
385     input_a_np = np.random.random((10, 3))
386     input_b_np = np.random.random((10, 4))
387 
388     model.fit([np.ndarray.tolist(input_a_np)],
389               [np.ndarray.tolist(input_b_np)],
390               epochs=2,
391               batch_size=5,
392               verbose=2)
393 
394   @keras_parameterized.run_all_keras_modes
395   def test_evaluate_predict_on_arrays(self):
396     a = keras.layers.Input(shape=(3,), name='input_a')
397     b = keras.layers.Input(shape=(3,), name='input_b')
398 
399     dense = keras.layers.Dense(4, name='dense')
400     c = dense(a)
401     d = dense(b)
402     e = keras.layers.Dropout(0.5, name='dropout')(c)
403 
404     model = keras.models.Model([a, b], [d, e])
405 
406     optimizer = RMSPropOptimizer(learning_rate=0.001)
407     loss = 'mse'
408     loss_weights = [1., 0.5]
409     model.compile(
410         optimizer,
411         loss,
412         metrics=['mae', metrics_module.CategoricalAccuracy()],
413         loss_weights=loss_weights,
414         sample_weight_mode=None,
415         run_eagerly=testing_utils.should_run_eagerly())
416 
417     input_a_np = np.random.random((10, 3))
418     input_b_np = np.random.random((10, 3))
419 
420     output_d_np = np.random.random((10, 4))
421     output_e_np = np.random.random((10, 4))
422 
423     # Test evaluate at different verbosity
424     out = model.evaluate(
425         [input_a_np, input_b_np], [output_d_np, output_e_np],
426         batch_size=5,
427         verbose=0)
428     self.assertEqual(len(out), 7)
429     out = model.evaluate(
430         [input_a_np, input_b_np], [output_d_np, output_e_np],
431         batch_size=5,
432         verbose=1)
433     self.assertEqual(len(out), 7)
434     out = model.evaluate(
435         [input_a_np, input_b_np], [output_d_np, output_e_np],
436         batch_size=5,
437         verbose=2)
438     self.assertEqual(len(out), 7)
439     out = model.test_on_batch([input_a_np, input_b_np],
440                               [output_d_np, output_e_np])
441     self.assertEqual(len(out), 7)
442 
443     # Test evaluate with dictionary inputs
444     model.evaluate(
445         {
446             'input_a': input_a_np,
447             'input_b': input_b_np
448         }, {
449             'dense': output_d_np,
450             'dropout': output_e_np
451         },
452         batch_size=5,
453         verbose=0)
454     model.evaluate(
455         {
456             'input_a': input_a_np,
457             'input_b': input_b_np
458         }, {
459             'dense': output_d_np,
460             'dropout': output_e_np
461         },
462         batch_size=5,
463         verbose=1)
464 
465     # Test predict
466     out = model.predict([input_a_np, input_b_np], batch_size=5)
467     self.assertEqual(len(out), 2)
468     out = model.predict({'input_a': input_a_np, 'input_b': input_b_np})
469     self.assertEqual(len(out), 2)
470     out = model.predict_on_batch({
471         'input_a': input_a_np,
472         'input_b': input_b_np
473     })
474     self.assertEqual(len(out), 2)
475 
476   @keras_parameterized.run_all_keras_modes
477   @keras_parameterized.run_with_all_model_types
478   def test_activity_regularizer_fit(self):
479     loss = {}
480     for reg in [None, 'l2']:
481       layers = [
482           keras.layers.Dense(
483               10, activation='relu', activity_regularizer=reg,
484               kernel_initializer='ones', use_bias=False),
485           keras.layers.Dense(
486               1, activation='sigmoid', kernel_initializer='ones',
487               use_bias=False),
488       ]
489 
490       model = testing_utils.get_model_from_layers(
491           layers, input_shape=(10,))
492 
493       x = np.ones((10, 10), 'float32')
494       y = np.ones((10, 1), 'float32')
495 
496       optimizer = RMSPropOptimizer(learning_rate=0.001)
497       model.compile(optimizer, 'binary_crossentropy',
498                     run_eagerly=testing_utils.should_run_eagerly())
499       model.fit(x, y, batch_size=2, epochs=5)
500       loss[reg] = model.evaluate(x, y)
501     self.assertLess(loss[None], loss['l2'])
502 
503   @keras_parameterized.run_all_keras_modes
504   @keras_parameterized.run_with_all_model_types
505   def test_activity_regularizer_loss_value(self):
506     layer = keras.layers.Dense(
507         1, kernel_initializer=keras.initializers.zeros(),
508         bias_initializer=keras.initializers.ones(), activity_regularizer='l2')
509 
510     model = testing_utils.get_model_from_layers([layer], input_shape=(10,))
511 
512     x = np.ones((10, 10), 'float32')
513     y = np.ones((10, 1), 'float32')
514     optimizer = RMSPropOptimizer(learning_rate=0.001)
515     model.compile(optimizer, 'binary_crossentropy',
516                   run_eagerly=testing_utils.should_run_eagerly())
517     loss = model.test_on_batch(x, y)
518     self.assertAlmostEqual(0.01, loss, places=4)
519 
520   @keras_parameterized.run_all_keras_modes
521   def test_activity_regularizer_batch_independent(self):
522     inputs = keras.layers.Input(shape=(10,))
523     x = keras.layers.Dense(
524         10, activation='relu', activity_regularizer='l2')(
525             inputs)
526     outputs = keras.layers.Dense(1, activation='sigmoid')(x)
527     model = keras.Model(inputs, outputs)
528 
529     optimizer = RMSPropOptimizer(learning_rate=0.001)
530     model.compile(optimizer, 'binary_crossentropy',
531                   run_eagerly=testing_utils.should_run_eagerly())
532 
533     x = np.ones((10, 10), 'float32')
534     y = np.ones((10, 1), 'float32')
535     loss_small_batch = model.test_on_batch(x, y)
536 
537     x2 = np.ones((20, 10), 'float32')
538     y2 = np.ones((20, 1), 'float32')
539     loss_big_batch = model.test_on_batch(x2, y2)
540 
541     self.assertAlmostEqual(loss_small_batch, loss_big_batch, places=4)
542 
543   @keras_parameterized.run_all_keras_modes
544   def test_activity_regularizer_in_model_call(self):
545 
546     class MyModel(keras.Model):
547 
548       def call(self, inputs):
549         self.add_loss(inputs)
550         return inputs
551 
552     x = ops.convert_to_tensor(1.)
553     model = MyModel()
554     _ = model(x)
555     self.assertEqual(1, len(model.losses))
556 
557   @keras_parameterized.run_all_keras_modes
558   def test_custom_mapping_in_config(self):
559 
560     class MyModel(keras.Model):
561 
562       def call(self, inputs):
563         return inputs
564 
565       def get_config(self):
566         self.a = {}
567         return {'a': self.a}
568 
569     model = MyModel()
570     self.assertIn('{"a": {}}', model.to_json())
571 
572   @keras_parameterized.run_all_keras_modes
573   def test_training_on_sparse_data_with_dense_placeholders(self):
574     # TODO(kaftan) Test seems to not work, file ticket
575     if testing_utils.should_run_eagerly() and context.executing_eagerly():
576       self.skipTest('Skipping running model eagerly.')
577 
578     if scipy_sparse is None:
579       return
580 
581     test_inputs = [
582         scipy_sparse.random(6, 3, density=0.25).tocsr() for _ in range(2)
583     ]
584     test_outputs = [
585         scipy_sparse.random(6, i, density=0.25).tocsr() for i in range(3, 5)
586     ]
587     in1 = keras.layers.Input(shape=(3,))
588     in2 = keras.layers.Input(shape=(3,))
589     out1 = keras.layers.Dropout(0.5, name='dropout')(in1)
590     out2 = keras.layers.Dense(4, name='dense_1')(in2)
591     model = keras.Model([in1, in2], [out1, out2])
592     model.predict(test_inputs, batch_size=2)
593     optimizer = RMSPropOptimizer(learning_rate=0.001)
594     model.compile(
595         optimizer,
596         'mse',
597         metrics=['mae', metrics_module.CategoricalAccuracy()],
598         run_eagerly=testing_utils.should_run_eagerly())
599     model.fit(test_inputs, test_outputs,
600               epochs=1, batch_size=2, validation_split=0.5)
601     model.evaluate(test_inputs, test_outputs, batch_size=2)
602 
603   @keras_parameterized.run_all_keras_modes
604   def test_compile_with_sparse_placeholders(self):
605     # TODO(kaftan) Test seems to not work, file ticket
606     if testing_utils.should_run_eagerly() and context.executing_eagerly():
607       self.skipTest('Skipping running model eagerly.')
608 
609     input_layer = keras.layers.Input(shape=(10,), sparse=True)
610     weights = variables_lib.Variable(
611         np.ones((10, 1)).astype(np.float32), name='weights')
612     weights_mult = lambda x: sparse_ops.sparse_tensor_dense_matmul(x, weights)
613     output_layer = keras.layers.Lambda(weights_mult)(input_layer)
614     model = keras.Model([input_layer], output_layer)
615     model.compile(
616         loss='binary_crossentropy',
617         optimizer=keras.optimizers.Adam(lr=0.0001),
618         metrics=['accuracy'],
619         run_eagerly=testing_utils.should_run_eagerly())
620 
621   def test_that_trainable_disables_updates(self):
622     val_a = np.random.random((10, 4))
623     val_out = np.random.random((10, 4))
624 
625     with self.cached_session():
626       a = keras.layers.Input(shape=(4,))
627       layer = keras.layers.BatchNormalization(input_shape=(4,))
628       b = layer(a)
629       model = keras.Model(a, b)
630 
631       model.trainable = False
632       assert not model.updates
633 
634       model.compile('sgd', 'mse')
635       assert not model.updates
636 
637       x1 = model.predict(val_a)
638       model.train_on_batch(val_a, val_out)
639       x2 = model.predict(val_a)
640       self.assertAllClose(x1, x2, atol=1e-7)
641 
642       model.trainable = True
643       model.compile('sgd', 'mse')
644       assert model.updates
645 
646       model.train_on_batch(val_a, val_out)
647       x2 = model.predict(val_a)
648       assert np.abs(np.sum(x1 - x2)) > 1e-5
649 
650       layer.trainable = False
651       model.compile('sgd', 'mse')
652       assert not model.updates
653 
654       x1 = model.predict(val_a)
655       model.train_on_batch(val_a, val_out)
656       x2 = model.predict(val_a)
657       self.assertAllClose(x1, x2, atol=1e-7)
658 
659   def test_logs_passed_to_callbacks(self):
660     with self.cached_session():
661       input_dim = 5
662       num_classes = 1
663 
664       class TestCallback(Callback):
665 
666         def __init__(self):
667           super(TestCallback, self).__init__()
668           self.epoch_end_logs = None
669           self.batch_end_logs = None
670           self.epoch_end_call_count = 0
671           self.batch_end_call_count = 0
672 
673         def on_epoch_end(self, epoch, logs=None):
674           self.epoch_end_logs = logs
675           self.epoch_end_call_count += 1
676 
677         def on_batch_end(self, batch, logs=None):
678           self.batch_end_logs = logs
679           self.batch_end_call_count += 1
680 
681       model = testing_utils.get_small_sequential_mlp(
682           num_hidden=10, num_classes=num_classes, input_dim=input_dim)
683       model.compile(
684           loss='binary_crossentropy',
685           metrics=['acc'],
686           weighted_metrics=['mae'],
687           optimizer=RMSPropOptimizer(learning_rate=0.01))
688 
689       np.random.seed(1337)
690       (x_train, y_train), (_, _) = testing_utils.get_test_data(
691           train_samples=10,
692           test_samples=10,
693           input_shape=(input_dim,),
694           num_classes=num_classes)
695 
696       test_callback = TestCallback()
697       model.fit(
698           x_train,
699           y_train,
700           batch_size=2,
701           epochs=2,
702           verbose=0,
703           callbacks=[test_callback],
704           validation_data=(x_train, y_train))
705       self.assertEqual(test_callback.batch_end_call_count, 10)
706       self.assertEqual(test_callback.epoch_end_call_count, 2)
707 
708       weighted_metric = ('mae'
709                          if tf2.enabled() else 'weighted_mean_absolute_error')
710       self.assertSetEqual(
711           set(test_callback.batch_end_logs.keys()),
712           set(['batch', 'size', 'acc', 'loss', weighted_metric]))
713       self.assertSetEqual(
714           set(test_callback.epoch_end_logs.keys()),
715           set([
716               'acc', 'loss', weighted_metric, 'val_acc', 'val_loss',
717               'val_' + weighted_metric
718           ]))
719 
720   @keras_parameterized.run_all_keras_modes
721   def test_mismatched_output_shape_and_target_shape(self):
722     model = keras.Sequential([
723         keras.layers.Dense(2, input_shape=(3, 4)),
724         keras.layers.Dense(5),
725     ])
726     model.compile(RMSPropOptimizer(learning_rate=0.001),
727                   loss='sparse_categorical_crossentropy',
728                   run_eagerly=testing_utils.should_run_eagerly())
729     # Test with Numpy data
730     x_train = np.random.random((10, 3, 4))
731     y_train = np.random.randint(0, 5, size=(10, 3))
732     model.fit(x_train, y_train, batch_size=5, epochs=1)
733 
734     # Test with iterator
735     dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train))
736     dataset = dataset.repeat(10)
737     dataset = dataset.batch(10)
738     iterator = dataset_ops.make_one_shot_iterator(dataset)
739     model.fit(iterator, epochs=1, steps_per_epoch=2)
740 
741     if context.executing_eagerly():
742       # Test with eager execution
743       model.compile(RMSPropOptimizer(learning_rate=0.001),
744                     loss='sparse_categorical_crossentropy',
745                     run_eagerly=True)
746       model.fit(x_train, y_train, batch_size=5, epochs=1)
747 
748       # Test with eager execution and iterator
749       model.fit(iterator, epochs=1, steps_per_epoch=2)
750 
751   def test_losses_in_defun(self):
752     with context.eager_mode():
753       layer = keras.layers.Dense(1, kernel_regularizer='l1')
754       layer(array_ops.ones([1, 10]))
755 
756       @function.defun
757       def get_losses():
758         return layer.losses
759 
760       self.assertAllEqual(
761           self.evaluate(layer.losses), self.evaluate(get_losses()))
762 
763   @keras_parameterized.run_all_keras_modes
764   def test_logging(self):
765     mock_stdout = io.BytesIO() if six.PY2 else io.StringIO()
766     model = keras.models.Sequential()
767     model.add(keras.layers.Dense(10, activation='relu'))
768     model.add(keras.layers.Dense(1, activation='sigmoid'))
769     model.compile(
770         RMSPropOptimizer(learning_rate=0.001), loss='binary_crossentropy',
771         run_eagerly=testing_utils.should_run_eagerly())
772     with test.mock.patch.object(sys, 'stdout', mock_stdout):
773       model.fit(
774           np.ones((10, 10), 'float32'), np.ones((10, 1), 'float32'), epochs=10)
775     self.assertTrue('Epoch 5/10' in mock_stdout.getvalue())
776 
777   @tf_test_util.run_in_graph_and_eager_modes
778   def test_training_with_loss_instance(self):
779     a = keras.layers.Input(shape=(3,), name='input_a')
780     b = keras.layers.Input(shape=(3,), name='input_b')
781 
782     dense = keras.layers.Dense(4, name='dense')
783     c = dense(a)
784     d = dense(b)
785     e = keras.layers.Dropout(0.5, name='dropout')(c)
786 
787     model = keras.models.Model([a, b], [d, e])
788     loss_weights = [1., 0.5]
789     model.compile(
790         RMSPropOptimizer(learning_rate=0.001),
791         loss=keras.losses.MeanSquaredError(),
792         metrics=[metrics_module.CategoricalAccuracy(), 'mae'],
793         loss_weights=loss_weights)
794 
795     input_a_np = np.random.random((10, 3))
796     input_b_np = np.random.random((10, 3))
797 
798     output_d_np = np.random.random((10, 4))
799     output_e_np = np.random.random((10, 4))
800 
801     model.fit([input_a_np, input_b_np], [output_d_np, output_e_np],
802               epochs=1,
803               batch_size=5)
804 
805   @tf_test_util.run_in_graph_and_eager_modes
806   def test_static_batch_in_input_layer(self):
807 
808     class Counter(keras.callbacks.Callback):
809 
810       def __init__(self):
811         self.batches = 0
812 
813       def on_batch_end(self, batch, logs=None):
814         self.batches += 1
815 
816     x, y = np.ones((64, 10), 'float32'), np.ones((64, 1), 'float32')
817 
818     for batch_size, expected_batches in [(None, 2), (4, 16)]:
819       inputs = keras.Input(batch_size=batch_size, shape=(10,))
820       outputs = keras.layers.Dense(1, activation='sigmoid')(inputs)
821       model = keras.Model(inputs, outputs)
822 
823       model.compile(keras.optimizer_v2.adam.Adam(0.001), 'binary_crossentropy')
824       counter = Counter()
825       model.fit(x, y, callbacks=[counter])
826       self.assertEqual(counter.batches, expected_batches)
827 
828       model = keras.Sequential(
829           [keras.layers.Dense(1, batch_input_shape=(batch_size, 10))])
830       model.compile(keras.optimizer_v2.adam.Adam(0.001), 'binary_crossentropy')
831       counter = Counter()
832       model.fit(x, y, callbacks=[counter])
833       self.assertEqual(counter.batches, expected_batches)
834 
835   @tf_test_util.run_in_graph_and_eager_modes
836   def test_static_batch_in_input_layer_consistency_checks(self):
837     x, y = np.ones((64, 10), 'float32'), np.ones((64, 1), 'float32')
838 
839     inputs = keras.Input(batch_size=2, shape=(10,))
840     outputs = keras.layers.Dense(1, activation='sigmoid')(inputs)
841     model = keras.Model(inputs, outputs)
842     model.compile(keras.optimizer_v2.adam.Adam(0.001), 'binary_crossentropy')
843     with self.assertRaisesRegexp(ValueError,
844                                  'incompatible with the specified batch size'):
845       model.fit(x, y, batch_size=4)
846 
847     data = dataset_ops.DatasetV2.from_tensor_slices((x, y))
848     data = data.batch(4, drop_remainder=True)
849     with self.assertRaisesRegexp(ValueError,
850                                  'incompatible with the specified batch size'):
851       model.fit(data, steps_per_epoch=16)
852 
853   @tf_test_util.run_in_graph_and_eager_modes
854   def test_compatible_batch_size_functional_model(self):
855 
856     class MyLayer(keras.layers.Layer):
857 
858       def call(self, inputs):
859         return array_ops.concat(inputs, axis=0)
860 
861     input1 = keras.Input(batch_size=2, shape=(10,))
862     input2 = keras.Input(batch_size=3, shape=(10,))
863     outputs = MyLayer()([input1, input2])
864     with self.assertRaisesRegexp(ValueError,
865                                  'specified batch sizes of the Input Layers'):
866       keras.Model([input1, input2], outputs)
867 
868   @tf_test_util.run_in_graph_and_eager_modes
869   def test_calling_subclass_model_on_different_datasets(self):
870 
871     class SubclassedModel(keras.models.Model):
872 
873       def call(self, inputs):
874         return inputs * 2
875 
876     model = SubclassedModel()
877     dataset_one = dataset_ops.Dataset.range(2).batch(2)
878     dataset_two = dataset_ops.Dataset.range(3, 10).batch(2)
879     self.assertAllEqual([[0], [2]], model.predict(dataset_one, steps=1))
880     self.assertAllEqual([[6], [8], [10], [12]],
881                         model.predict(dataset_two, steps=2))
882 
883   def test_training_on_sparse_categorical_crossentropy_loss_with_softmax(self):
884     with context.eager_mode():
885       np.random.seed(1337)
886       train_x = np.ones((100, 4))
887       train_y = np.random.randint(0, 1, size=(100, 1))
888 
889       reference_model = testing_utils.get_small_sequential_mlp(16, 2,
890                                                                input_dim=4)
891       reference_model.compile(loss='sparse_categorical_crossentropy',
892                               optimizer=RMSPropOptimizer(learning_rate=0.001),
893                               run_eagerly=True)
894       fixed_weights = reference_model.get_weights()
895       reference_model_loss = reference_model.train_on_batch(train_x, train_y)
896 
897       test_model = testing_utils.get_small_sequential_mlp(16, 2, input_dim=4)
898       test_model.compile(loss='sparse_categorical_crossentropy',
899                          optimizer=RMSPropOptimizer(learning_rate=0.001),
900                          run_eagerly=False)
901       test_model.set_weights(fixed_weights)
902       test_model_loss = test_model.train_on_batch(train_x, train_y)
903       self.assertAlmostEqual(test_model_loss, reference_model_loss, places=4)
904 
905   def test_training_on_categorical_crossentropy_loss_with_softmax(self):
906     with context.eager_mode():
907       np.random.seed(1337)
908       train_x = np.ones((100, 4))
909       train_y = keras.utils.to_categorical(np.random.randint(0, 1,
910                                                              size=(100, 1)), 2)
911 
912       reference_model = testing_utils.get_small_sequential_mlp(16, 2,
913                                                                input_dim=4)
914       reference_model.compile(loss='categorical_crossentropy',
915                               optimizer=RMSPropOptimizer(learning_rate=0.001),
916                               run_eagerly=True)
917       fixed_weights = reference_model.get_weights()
918       reference_model_loss = reference_model.train_on_batch(train_x, train_y)
919 
920       test_model = testing_utils.get_small_sequential_mlp(16, 2, input_dim=4)
921       test_model.compile(loss='categorical_crossentropy',
922                          optimizer=RMSPropOptimizer(learning_rate=0.001),
923                          run_eagerly=False)
924       test_model.set_weights(fixed_weights)
925       test_model_loss = test_model.train_on_batch(train_x, train_y)
926       self.assertAlmostEqual(test_model_loss, reference_model_loss, places=4)
927 
928   def test_training_on_binary_crossentropy_loss(self):
929     with context.eager_mode():
930       train_x = np.ones((100, 4), dtype=np.float32)
931       train_y = np.ones((100, 1), dtype=np.float32)
932       reference_model = testing_utils.get_small_sequential_mlp(16, 1,
933                                                                input_dim=4)
934       reference_model.compile(loss='binary_crossentropy',
935                               optimizer=RMSPropOptimizer(learning_rate=0.001),
936                               run_eagerly=True)
937       fixed_weights = reference_model.get_weights()
938       reference_model_loss = reference_model.train_on_batch(train_x, train_y)
939 
940       test_model = testing_utils.get_small_sequential_mlp(16, 1, input_dim=4)
941       test_model.compile(loss='binary_crossentropy',
942                          optimizer=RMSPropOptimizer(learning_rate=0.001),
943                          run_eagerly=False)
944       test_model.set_weights(fixed_weights)
945       test_model_loss = test_model.train_on_batch(train_x, train_y)
946       self.assertAlmostEqual(test_model_loss, reference_model_loss, places=4)
947 
948   @keras_parameterized.run_with_all_model_types
949   @keras_parameterized.run_all_keras_modes
950   @parameterized.named_parameters(
951       ('default', 1, 4), ('integer_two', 2, 2), ('integer_four', 4, 1),
952       ('simple_list', [1, 3, 4], 3), ('duplicated_list', [4, 2, 2], 2))
953   def test_validation_freq(self, validation_freq, expected_runs):
954     x, y = np.ones((10, 10)), np.ones((10, 1))
955     model = testing_utils.get_small_mlp(2, 1, 10)
956     model.compile('sgd', 'mse')
957 
958     class ValCounter(keras.callbacks.Callback):
959 
960       def __init__(self):
961         self.val_runs = 0
962 
963       def on_test_begin(self, logs=None):
964         self.val_runs += 1
965 
966     val_counter = ValCounter()
967     model.fit(
968         x,
969         y,
970         epochs=4,
971         validation_data=(x, y),
972         validation_freq=validation_freq,
973         callbacks=[val_counter])
974     self.assertEqual(val_counter.val_runs, expected_runs)
975 
976   @keras_parameterized.run_all_keras_modes
977   def test_add_loss_correctness(self):
978     class Bias(keras.layers.Layer):
979 
980       def build(self, input_shape):
981         self.bias = self.add_variable('bias', (1,), initializer='zeros')
982 
983       def call(self, inputs):
984         return inputs + self.bias
985 
986     inputs = keras.Input(shape=(1,))
987     outputs = Bias()(inputs)
988     model = keras.Model(inputs, outputs)
989     targets = keras.Input(shape=(1,))
990 
991     model.add_loss(
992         math_ops.reduce_mean(
993             keras.losses.mean_absolute_error(targets, outputs)))
994 
995     # If we want to use the loss class instance as shown below, we will need to
996     # add graph scope as the reduction logic involves some eager mode checks.
997     with keras.backend.get_graph().as_default():
998       model.add_loss(keras.losses.MeanAbsoluteError()(targets, outputs))
999 
1000     if testing_utils.should_run_eagerly():
1001       with self.assertRaisesRegex(
1002           ValueError,
1003           'We currently do not support enabling `run_eagerly` on compile if '
1004           r'`model.add_loss\(tensor\)` or `model.add_metric\(tensor\)` '
1005           'has been called.'):
1006         model.compile('sgd', run_eagerly=True)
1007       return
1008     else:
1009       model.compile(
1010           keras.optimizer_v2.gradient_descent.SGD(0.033333),
1011           loss=keras.losses.MeanAbsoluteError(),
1012           target_tensors=[targets],
1013           run_eagerly=False)
1014 
1015       x = np.array([[0.], [1.], [2.]])
1016       y = np.array([[0.5], [2.], [3.5]])
1017       history = model.fit(x, y, batch_size=3, epochs=5)
1018       self.assertAllClose(history.history['loss'], [3., 2.7, 2.4, 2.1, 1.8],
1019                           1e-3)
1020 
1021   @keras_parameterized.run_all_keras_modes
1022   def test_clear_losses(self):
1023 
1024     class LayerWithSharedNestedLossLayer(keras.layers.Layer):
1025 
1026       def __init__(self):
1027         super(LayerWithSharedNestedLossLayer, self).__init__()
1028         self.loss_layer = keras.layers.ActivityRegularization()
1029         self.add_weight(shape=(1,), regularizer='l2')
1030 
1031       def call(self, x):
1032         x = self.loss_layer(x)
1033         return self.loss_layer(x)
1034 
1035     inputs = keras.Input(shape=(1,))
1036     outputs = LayerWithSharedNestedLossLayer()(inputs)
1037     model = keras.Model(inputs, outputs)
1038 
1039     model(array_ops.ones((1, 1)))
1040     self.assertEqual(len(model.losses), 3)  # Weight loss + 2 activity losses.
1041 
1042     model(array_ops.ones((1, 1)))
1043     self.assertEqual(len(model.losses), 3)  # Losses are reset upon __call__.
1044 
1045   @keras_parameterized.run_with_all_model_types
1046   @keras_parameterized.run_all_keras_modes
1047   def test_layer_with_variable_output(self):
1048 
1049     class VariableOutputLayer(keras.layers.Layer):
1050 
1051       def build(self, input_shape):
1052         self.v = self.add_weight('output_var', shape=(2, 5), initializer='ones')
1053 
1054       def call(self, inputs):
1055         return self.v
1056 
1057     model = testing_utils.get_model_from_layers(
1058         [VariableOutputLayer(), keras.layers.Dense(1)], input_shape=(10,))
1059     # TODO(omalleyt): Make this work with `run_eagerly=True`.
1060     model.compile('sgd', 'mse', run_eagerly=False)
1061     model.fit(np.ones((10, 10)), np.ones((10, 1)), batch_size=2, epochs=5)
1062 
1063     self.assertLen(model.trainable_variables, 3)
1064 
1065 
1066 class TestExceptionsAndWarnings(keras_parameterized.TestCase):
1067 
1068   @keras_parameterized.run_all_keras_modes
1069   def test_invalid_loss(self):
1070     num_classes = 5
1071     train_samples = 1000
1072     test_samples = 1000
1073     input_dim = 5
1074 
1075     model = testing_utils.get_small_sequential_mlp(
1076         num_hidden=10, num_classes=num_classes, input_dim=input_dim)
1077     optimizer = RMSPropOptimizer(learning_rate=0.001)
1078     model.compile(optimizer, loss='categorical_crossentropy')
1079     np.random.seed(1337)
1080     (x_train, y_train), (_, _) = testing_utils.get_test_data(
1081         train_samples=train_samples,
1082         test_samples=test_samples,
1083         input_shape=(input_dim,),
1084         num_classes=num_classes)
1085 
1086     with self.assertRaises(ValueError):
1087       model.fit(x_train, np.concatenate([y_train, y_train], axis=-1))
1088 
1089     if not context.executing_eagerly():
1090       # TODO(psv): Investigate these use cases in eager mode.
1091       with self.assertRaises(ValueError):
1092         model.fit(x_train, y_train)
1093 
1094       with self.assertRaises(ValueError):
1095         model.compile(optimizer, loss=None,
1096                       run_eagerly=testing_utils.should_run_eagerly())
1097 
1098   @keras_parameterized.run_all_keras_modes
1099   def test_compile_warning_for_loss_missing_output(self):
1100     with self.cached_session():
1101       inp = keras.layers.Input(shape=(16,), name='input_a')
1102       out_1 = keras.layers.Dense(8, name='dense_1')(inp)
1103       out_2 = keras.layers.Dense(3, activation='softmax', name='dense_2')(out_1)
1104       model = keras.models.Model(inputs=[inp], outputs=[out_1, out_2])
1105       optimizer = RMSPropOptimizer(learning_rate=0.001)
1106 
1107       with test.mock.patch.object(logging, 'warning') as mock_log:
1108         model.compile(
1109             optimizer,
1110             loss={
1111                 'dense_2': 'categorical_crossentropy',
1112             },
1113             metrics={
1114                 'dense_2': 'categorical_accuracy',
1115                 'dense_1': metrics_module.CategoricalAccuracy(),
1116             },
1117             run_eagerly=testing_utils.should_run_eagerly())
1118         msg = ('Output dense_1 missing from loss dictionary. We assume this '
1119                'was done on purpose. The fit and evaluate APIs will not be '
1120                'expecting any data to be passed to dense_1.')
1121         self.assertRegexpMatches(str(mock_log.call_args), msg)
1122 
1123 
1124 class LossWeightingTest(keras_parameterized.TestCase):
1125 
1126   @keras_parameterized.run_all_keras_modes
1127   def test_class_weights(self):
1128     num_classes = 5
1129     batch_size = 5
1130     epochs = 10
1131     weighted_class = 3
1132     weight = 10.
1133     train_samples = 1000
1134     test_samples = 1000
1135     input_dim = 5
1136     learning_rate = 0.001
1137 
1138     model = testing_utils.get_small_sequential_mlp(
1139         num_hidden=10, num_classes=num_classes, input_dim=input_dim)
1140     model.compile(
1141         loss='categorical_crossentropy',
1142         metrics=['acc', metrics_module.CategoricalAccuracy()],
1143         weighted_metrics=['mae', metrics_module.CategoricalAccuracy()],
1144         optimizer=RMSPropOptimizer(learning_rate=learning_rate),
1145         run_eagerly=testing_utils.should_run_eagerly())
1146 
1147     np.random.seed(1337)
1148     (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
1149         train_samples=train_samples,
1150         test_samples=test_samples,
1151         input_shape=(input_dim,),
1152         num_classes=num_classes)
1153     int_y_test = y_test.copy()
1154     int_y_train = y_train.copy()
1155     # convert class vectors to binary class matrices
1156     y_train = keras.utils.to_categorical(y_train, num_classes)
1157     y_test = keras.utils.to_categorical(y_test, num_classes)
1158     test_ids = np.where(int_y_test == np.array(weighted_class))[0]
1159 
1160     class_weight = dict([(i, 1.) for i in range(num_classes)])
1161     class_weight[weighted_class] = weight
1162 
1163     model.fit(
1164         x_train,
1165         y_train,
1166         batch_size=batch_size,
1167         epochs=epochs // 3,
1168         verbose=0,
1169         class_weight=class_weight,
1170         validation_data=(x_train, y_train))
1171     model.fit(
1172         x_train,
1173         y_train,
1174         batch_size=batch_size,
1175         epochs=epochs // 2,
1176         verbose=0,
1177         class_weight=class_weight)
1178     model.fit(
1179         x_train,
1180         y_train,
1181         batch_size=batch_size,
1182         epochs=epochs // 2,
1183         verbose=0,
1184         class_weight=class_weight,
1185         validation_split=0.1)
1186 
1187     model.train_on_batch(
1188         x_train[:batch_size], y_train[:batch_size], class_weight=class_weight)
1189     ref_score = model.evaluate(x_test, y_test, verbose=0)
1190     score = model.evaluate(
1191         x_test[test_ids, :], y_test[test_ids, :], verbose=0)
1192     self.assertLess(score[0], ref_score[0])
1193 
1194   @keras_parameterized.run_all_keras_modes
1195   def test_sample_weights(self):
1196     num_classes = 5
1197     batch_size = 5
1198     epochs = 10
1199     weighted_class = 3
1200     weight = 10.
1201     train_samples = 1000
1202     test_samples = 1000
1203     input_dim = 5
1204     learning_rate = 0.001
1205 
1206     model = testing_utils.get_small_sequential_mlp(
1207         num_hidden=10, num_classes=num_classes, input_dim=input_dim)
1208     model.compile(
1209         RMSPropOptimizer(learning_rate=learning_rate),
1210         metrics=['acc', metrics_module.CategoricalAccuracy()],
1211         weighted_metrics=['mae', metrics_module.CategoricalAccuracy()],
1212         loss='categorical_crossentropy',
1213         run_eagerly=testing_utils.should_run_eagerly())
1214 
1215     np.random.seed(43)
1216     (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
1217         train_samples=train_samples,
1218         test_samples=test_samples,
1219         input_shape=(input_dim,),
1220         num_classes=num_classes)
1221     int_y_test = y_test.copy()
1222     int_y_train = y_train.copy()
1223     # convert class vectors to binary class matrices
1224     y_train = keras.utils.to_categorical(y_train, num_classes)
1225     y_test = keras.utils.to_categorical(y_test, num_classes)
1226     test_ids = np.where(int_y_test == np.array(weighted_class))[0]
1227 
1228     sample_weight = np.ones((y_train.shape[0]))
1229     sample_weight[int_y_train == weighted_class] = weight
1230 
1231     model.fit(
1232         x_train,
1233         y_train,
1234         batch_size=batch_size,
1235         epochs=epochs // 3,
1236         verbose=0,
1237         sample_weight=sample_weight)
1238     model.fit(
1239         x_train,
1240         y_train,
1241         batch_size=batch_size,
1242         epochs=epochs // 3,
1243         verbose=0,
1244         sample_weight=sample_weight,
1245         validation_split=0.1)
1246 
1247     model.train_on_batch(
1248         x_train[:batch_size],
1249         y_train[:batch_size],
1250         sample_weight=sample_weight[:batch_size])
1251     model.test_on_batch(
1252         x_train[:batch_size],
1253         y_train[:batch_size],
1254         sample_weight=sample_weight[:batch_size])
1255     ref_score = model.evaluate(x_test, y_test, verbose=0)
1256     if not context.executing_eagerly():
1257       score = model.evaluate(
1258           x_test[test_ids, :], y_test[test_ids, :], verbose=0)
1259       self.assertLess(score[0], ref_score[0])
1260 
1261   @keras_parameterized.run_all_keras_modes
1262   def test_temporal_sample_weights(self):
1263     num_classes = 5
1264     batch_size = 5
1265     epochs = 10
1266     weighted_class = 3
1267     weight = 10.
1268     train_samples = 1000
1269     test_samples = 1000
1270     input_dim = 5
1271     timesteps = 3
1272     learning_rate = 0.001
1273 
1274     with self.cached_session():
1275       model = keras.models.Sequential()
1276       model.add(
1277           keras.layers.TimeDistributed(
1278               keras.layers.Dense(num_classes),
1279               input_shape=(timesteps, input_dim)))
1280       model.add(keras.layers.Activation('softmax'))
1281 
1282       np.random.seed(1337)
1283       (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
1284           train_samples=train_samples,
1285           test_samples=test_samples,
1286           input_shape=(input_dim,),
1287           num_classes=num_classes)
1288       int_y_test = y_test.copy()
1289       int_y_train = y_train.copy()
1290       # convert class vectors to binary class matrices
1291       y_train = keras.utils.to_categorical(y_train, num_classes)
1292       y_test = keras.utils.to_categorical(y_test, num_classes)
1293       test_ids = np.where(int_y_test == np.array(weighted_class))[0]
1294 
1295       sample_weight = np.ones((y_train.shape[0]))
1296       sample_weight[int_y_train == weighted_class] = weight
1297 
1298       temporal_x_train = np.reshape(x_train, (len(x_train), 1,
1299                                               x_train.shape[1]))
1300       temporal_x_train = np.repeat(temporal_x_train, timesteps, axis=1)
1301       temporal_x_test = np.reshape(x_test, (len(x_test), 1, x_test.shape[1]))
1302       temporal_x_test = np.repeat(temporal_x_test, timesteps, axis=1)
1303 
1304       temporal_y_train = np.reshape(y_train, (len(y_train), 1,
1305                                               y_train.shape[1]))
1306       temporal_y_train = np.repeat(temporal_y_train, timesteps, axis=1)
1307       temporal_y_test = np.reshape(y_test, (len(y_test), 1, y_test.shape[1]))
1308       temporal_y_test = np.repeat(temporal_y_test, timesteps, axis=1)
1309 
1310       temporal_sample_weight = np.reshape(sample_weight, (len(sample_weight),
1311                                                           1))
1312       temporal_sample_weight = np.repeat(
1313           temporal_sample_weight, timesteps, axis=1)
1314 
1315       model.compile(
1316           RMSPropOptimizer(learning_rate=learning_rate),
1317           loss='categorical_crossentropy',
1318           metrics=['acc', metrics_module.CategoricalAccuracy()],
1319           weighted_metrics=['mae', metrics_module.CategoricalAccuracy()],
1320           sample_weight_mode='temporal',
1321           run_eagerly=testing_utils.should_run_eagerly())
1322 
1323       model.fit(
1324           temporal_x_train,
1325           temporal_y_train,
1326           batch_size=batch_size,
1327           epochs=epochs // 3,
1328           verbose=0,
1329           sample_weight=temporal_sample_weight)
1330       model.fit(
1331           temporal_x_train,
1332           temporal_y_train,
1333           batch_size=batch_size,
1334           epochs=epochs // 3,
1335           verbose=0,
1336           sample_weight=temporal_sample_weight,
1337           validation_split=0.1)
1338 
1339       model.train_on_batch(
1340           temporal_x_train[:batch_size],
1341           temporal_y_train[:batch_size],
1342           sample_weight=temporal_sample_weight[:batch_size])
1343       model.test_on_batch(
1344           temporal_x_train[:batch_size],
1345           temporal_y_train[:batch_size],
1346           sample_weight=temporal_sample_weight[:batch_size])
1347       ref_score = model.evaluate(temporal_x_test, temporal_y_test, verbose=0)
1348       if not context.executing_eagerly():
1349         score = model.evaluate(
1350             temporal_x_test[test_ids], temporal_y_test[test_ids], verbose=0)
1351         self.assertLess(score[0], ref_score[0])
1352 
1353   @keras_parameterized.run_all_keras_modes
1354   def test_class_weight_invalid_use_case(self):
1355     num_classes = 5
1356     train_samples = 1000
1357     test_samples = 1000
1358     input_dim = 5
1359     timesteps = 3
1360     learning_rate = 0.001
1361 
1362     with self.cached_session():
1363       model = keras.models.Sequential()
1364       model.add(
1365           keras.layers.TimeDistributed(
1366               keras.layers.Dense(num_classes),
1367               input_shape=(timesteps, input_dim)))
1368       model.add(keras.layers.Activation('softmax'))
1369       optimizer = RMSPropOptimizer(learning_rate=learning_rate)
1370       model.compile(optimizer, loss='binary_crossentropy',
1371                     run_eagerly=testing_utils.should_run_eagerly())
1372 
1373       (x_train, y_train), _ = testing_utils.get_test_data(
1374           train_samples=train_samples,
1375           test_samples=test_samples,
1376           input_shape=(input_dim,),
1377           num_classes=num_classes)
1378       # convert class vectors to binary class matrices
1379       y_train = keras.utils.to_categorical(y_train, num_classes)
1380       class_weight = dict([(i, 1.) for i in range(num_classes)])
1381 
1382       del class_weight[1]
1383       with self.assertRaises(ValueError):
1384         model.fit(x_train, y_train,
1385                   epochs=0, verbose=0, class_weight=class_weight)
1386 
1387       with self.assertRaises(ValueError):
1388         model.compile(
1389             optimizer, loss='binary_crossentropy', sample_weight_mode=[],
1390             run_eagerly=testing_utils.should_run_eagerly())
1391 
1392       # Build multi-output model
1393       x = keras.Input((3,))
1394       y1 = keras.layers.Dense(4, name='1')(x)
1395       y2 = keras.layers.Dense(4, name='2')(x)
1396       model = keras.models.Model(x, [y1, y2])
1397       model.compile(optimizer, loss='mse',
1398                     run_eagerly=testing_utils.should_run_eagerly())
1399       x_np = np.random.random((10, 3))
1400       y_np = np.random.random((10, 4))
1401       w_np = np.random.random((10,))
1402       # This will work
1403       model.fit(x_np, [y_np, y_np], epochs=1,
1404                 sample_weight={'1': w_np})
1405       # These will not
1406       with self.assertRaises(ValueError):
1407         model.fit(x_np, [y_np, y_np], epochs=1,
1408                   sample_weight=[w_np])
1409       with self.assertRaises(TypeError):
1410         model.fit(x_np, [y_np, y_np], epochs=1,
1411                   sample_weight=w_np)
1412       with self.assertRaises(ValueError):
1413         bad_w_np = np.random.random((11,))
1414         model.fit(x_np, [y_np, y_np], epochs=1,
1415                   sample_weight={'1': bad_w_np})
1416       with self.assertRaises(ValueError):
1417         bad_w_np = np.random.random((10, 2))
1418         model.fit(x_np, [y_np, y_np], epochs=1,
1419                   sample_weight={'1': bad_w_np})
1420       with self.assertRaises(ValueError):
1421         bad_w_np = np.random.random((10, 2, 2))
1422         model.fit(x_np, [y_np, y_np], epochs=1,
1423                   sample_weight={'1': bad_w_np})
1424 
1425   @keras_parameterized.run_all_keras_modes
1426   def test_default_sample_weight(self):
1427     """Verifies that fit works without having to set sample_weight."""
1428 
1429     num_classes = 5
1430     input_dim = 5
1431     timesteps = 3
1432     learning_rate = 0.001
1433 
1434     with self.cached_session():
1435       model = keras.models.Sequential()
1436       model.add(
1437           keras.layers.TimeDistributed(
1438               keras.layers.Dense(num_classes),
1439               input_shape=(timesteps, input_dim)))
1440 
1441       x = np.random.random((10, timesteps, input_dim))
1442       y = np.random.random((10, timesteps, num_classes))
1443       optimizer = RMSPropOptimizer(learning_rate=learning_rate)
1444 
1445       # sample_weight_mode is a list and mode value is None
1446       model.compile(optimizer, loss='mse', sample_weight_mode=[None],
1447                     run_eagerly=testing_utils.should_run_eagerly())
1448       model.fit(x, y, epochs=1, batch_size=10)
1449 
1450       # sample_weight_mode is a list and mode value is `temporal`
1451       model.compile(optimizer, loss='mse', sample_weight_mode=['temporal'],
1452                     run_eagerly=testing_utils.should_run_eagerly())
1453       model.fit(x, y, epochs=1, batch_size=10)
1454 
1455       # sample_weight_mode is a dict and mode value is None
1456       model.compile(
1457           optimizer, loss='mse', sample_weight_mode={'time_distributed': None},
1458           run_eagerly=testing_utils.should_run_eagerly())
1459       model.fit(x, y, epochs=1, batch_size=10)
1460 
1461       # sample_weight_mode is a dict and mode value is `temporal`
1462       model.compile(
1463           optimizer,
1464           loss='mse',
1465           sample_weight_mode={'time_distributed': 'temporal'},
1466           run_eagerly=testing_utils.should_run_eagerly())
1467       model.fit(x, y, epochs=1, batch_size=10)
1468 
1469       # sample_weight_mode is a not a list/dict and mode value is None
1470       model.compile(optimizer, loss='mse', sample_weight_mode=None,
1471                     run_eagerly=testing_utils.should_run_eagerly())
1472       model.fit(x, y, epochs=1, batch_size=10)
1473 
1474       # sample_weight_mode is a not a list/dict and mode value is `temporal`
1475       model.compile(optimizer, loss='mse', sample_weight_mode='temporal',
1476                     run_eagerly=testing_utils.should_run_eagerly())
1477       model.fit(x, y, epochs=1, batch_size=10)
1478 
1479 
1480 @keras_parameterized.run_all_keras_modes
1481 class MaskingTest(keras_parameterized.TestCase):
1482 
1483   def _get_model(self, input_shape=None):
1484     layers = [
1485         keras.layers.Masking(mask_value=0),
1486         keras.layers.TimeDistributed(
1487             keras.layers.Dense(1, kernel_initializer='one'))
1488     ]
1489     model = testing_utils.get_model_from_layers(layers, input_shape)
1490     model.compile(
1491         loss='mse',
1492         optimizer=RMSPropOptimizer(learning_rate=0.001),
1493         run_eagerly=testing_utils.should_run_eagerly())
1494     return model
1495 
1496   @keras_parameterized.run_with_all_model_types
1497   def test_masking(self):
1498     model = self._get_model(input_shape=(2, 1))
1499     x = np.array([[[1], [1]], [[0], [0]]])
1500     y = np.array([[[1], [1]], [[1], [1]]])
1501     loss = model.train_on_batch(x, y)
1502     self.assertEqual(loss, 0)
1503 
1504   @keras_parameterized.run_with_all_model_types(exclude_models='functional')
1505   def test_masking_deferred(self):
1506     model = self._get_model()
1507     x = np.array([[[1], [1]], [[0], [0]]])
1508     y = np.array([[[1], [1]], [[1], [1]]])
1509     loss = model.train_on_batch(x, y)
1510     self.assertEqual(loss, 0)
1511 
1512   def test_mask_argument_in_layer(self):
1513     # Test that the mask argument gets correctly passed to a layer in the
1514     # functional API.
1515 
1516     class CustomMaskedLayer(keras.layers.Layer):
1517 
1518       def __init__(self):
1519         super(CustomMaskedLayer, self).__init__()
1520         self.supports_masking = True
1521 
1522       def call(self, inputs, mask=None):
1523         assert mask is not None
1524         return inputs
1525 
1526       def compute_output_shape(self, input_shape):
1527         return input_shape
1528 
1529     x = np.random.random((5, 3))
1530     inputs = keras.layers.Input((3,))
1531     masked = keras.layers.Masking(mask_value=0)(inputs)
1532     outputs = CustomMaskedLayer()(masked)
1533 
1534     model = keras.Model(inputs, outputs)
1535     model.compile(
1536         loss='mse',
1537         optimizer=RMSPropOptimizer(learning_rate=0.001),
1538         run_eagerly=testing_utils.should_run_eagerly())
1539     y = np.random.random((5, 3))
1540     model.train_on_batch(x, y)
1541 
1542 
1543 class TestDynamicTrainability(keras_parameterized.TestCase):
1544 
1545   def test_trainable_warning(self):
1546     with self.cached_session():
1547       x = np.random.random((5, 3))
1548       y = np.random.random((5, 2))
1549 
1550       model = keras.models.Sequential()
1551       model.add(keras.layers.Dense(2, input_dim=3))
1552       model.trainable = False
1553       model.compile('rmsprop', 'mse')
1554       model.trainable = True
1555       model.train_on_batch(x, y)
1556       self.assertRaises(Warning)
1557 
1558   def test_trainable_argument(self):
1559     with self.cached_session():
1560       x = np.random.random((5, 3))
1561       y = np.random.random((5, 2))
1562 
1563       model = keras.models.Sequential()
1564       model.add(keras.layers.Dense(2, input_dim=3, trainable=False))
1565       model.compile('rmsprop', 'mse')
1566       out = model.predict(x)
1567       model.train_on_batch(x, y)
1568       out_2 = model.predict(x)
1569       self.assertAllClose(out, out_2)
1570 
1571       # test with nesting
1572       inputs = keras.layers.Input(shape=(3,))
1573       output = model(inputs)
1574       model = keras.models.Model(inputs, output)
1575       model.compile('rmsprop', 'mse')
1576       out = model.predict(x)
1577       model.train_on_batch(x, y)
1578       out_2 = model.predict(x)
1579       self.assertAllClose(out, out_2)
1580 
1581   def test_layer_trainability_switch(self):
1582     with self.cached_session():
1583       # with constructor argument, in Sequential
1584       model = keras.models.Sequential()
1585       model.add(keras.layers.Dense(2, trainable=False, input_dim=1))
1586       self.assertListEqual(model.trainable_weights, [])
1587 
1588       # by setting the `trainable` argument, in Sequential
1589       model = keras.models.Sequential()
1590       layer = keras.layers.Dense(2, input_dim=1)
1591       model.add(layer)
1592       self.assertListEqual(model.trainable_weights, layer.trainable_weights)
1593       layer.trainable = False
1594       self.assertListEqual(model.trainable_weights, [])
1595 
1596       # with constructor argument, in Model
1597       x = keras.layers.Input(shape=(1,))
1598       y = keras.layers.Dense(2, trainable=False)(x)
1599       model = keras.models.Model(x, y)
1600       self.assertListEqual(model.trainable_weights, [])
1601 
1602       # by setting the `trainable` argument, in Model
1603       x = keras.layers.Input(shape=(1,))
1604       layer = keras.layers.Dense(2)
1605       y = layer(x)
1606       model = keras.models.Model(x, y)
1607       self.assertListEqual(model.trainable_weights, layer.trainable_weights)
1608       layer.trainable = False
1609       self.assertListEqual(model.trainable_weights, [])
1610 
1611   def test_model_trainability_switch(self):
1612     with self.cached_session():
1613       # a non-trainable model has no trainable weights
1614       x = keras.layers.Input(shape=(1,))
1615       y = keras.layers.Dense(2)(x)
1616       model = keras.models.Model(x, y)
1617       model.trainable = False
1618       self.assertListEqual(model.trainable_weights, [])
1619 
1620       # same for Sequential
1621       model = keras.models.Sequential()
1622       model.add(keras.layers.Dense(2, input_dim=1))
1623       model.trainable = False
1624       self.assertListEqual(model.trainable_weights, [])
1625 
1626   def test_nested_model_trainability(self):
1627     with self.cached_session():
1628       # a Sequential inside a Model
1629       inner_model = keras.models.Sequential()
1630       inner_model.add(keras.layers.Dense(2, input_dim=1))
1631 
1632       x = keras.layers.Input(shape=(1,))
1633       y = inner_model(x)
1634       outer_model = keras.models.Model(x, y)
1635       self.assertListEqual(outer_model.trainable_weights,
1636                            inner_model.trainable_weights)
1637       inner_model.trainable = False
1638       self.assertListEqual(outer_model.trainable_weights, [])
1639       inner_model.trainable = True
1640       inner_model.layers[-1].trainable = False
1641       self.assertListEqual(outer_model.trainable_weights, [])
1642 
1643       # a Sequential inside a Sequential
1644       inner_model = keras.models.Sequential()
1645       inner_model.add(keras.layers.Dense(2, input_dim=1))
1646       outer_model = keras.models.Sequential()
1647       outer_model.add(inner_model)
1648       self.assertListEqual(outer_model.trainable_weights,
1649                            inner_model.trainable_weights)
1650       inner_model.trainable = False
1651       self.assertListEqual(outer_model.trainable_weights, [])
1652       inner_model.trainable = True
1653       inner_model.layers[-1].trainable = False
1654       self.assertListEqual(outer_model.trainable_weights, [])
1655 
1656       # a Model inside a Model
1657       x = keras.layers.Input(shape=(1,))
1658       y = keras.layers.Dense(2)(x)
1659       inner_model = keras.models.Model(x, y)
1660       x = keras.layers.Input(shape=(1,))
1661       y = inner_model(x)
1662       outer_model = keras.models.Model(x, y)
1663       self.assertListEqual(outer_model.trainable_weights,
1664                            inner_model.trainable_weights)
1665       inner_model.trainable = False
1666       self.assertListEqual(outer_model.trainable_weights, [])
1667       inner_model.trainable = True
1668       inner_model.layers[-1].trainable = False
1669       self.assertListEqual(outer_model.trainable_weights, [])
1670 
1671       # a Model inside a Sequential
1672       x = keras.layers.Input(shape=(1,))
1673       y = keras.layers.Dense(2)(x)
1674       inner_model = keras.models.Model(x, y)
1675       outer_model = keras.models.Sequential()
1676       outer_model.add(inner_model)
1677       self.assertListEqual(outer_model.trainable_weights,
1678                            inner_model.trainable_weights)
1679       inner_model.trainable = False
1680       self.assertListEqual(outer_model.trainable_weights, [])
1681       inner_model.trainable = True
1682       inner_model.layers[-1].trainable = False
1683       self.assertListEqual(outer_model.trainable_weights, [])
1684 
1685 
1686 class TestTrainingWithDataTensors(keras_parameterized.TestCase):
1687 
1688   @keras_parameterized.run_all_keras_modes
1689   def test_training_and_eval_methods_on_symbolic_tensors_single_io(self):
1690     # TODO(kaftan) Test seems to not work, file ticket
1691     if  context.executing_eagerly():
1692       self.skipTest('Skipping eager execution.')
1693 
1694     x = keras.layers.Input(shape=(3,), name='input')
1695     y = keras.layers.Dense(4, name='dense')(x)
1696     model = keras.Model(x, y)
1697 
1698     optimizer = RMSPropOptimizer(learning_rate=0.001)
1699     loss = 'mse'
1700     model.compile(
1701         optimizer,
1702         loss,
1703         metrics=['mae', metrics_module.CategoricalAccuracy()],
1704         run_eagerly=testing_utils.should_run_eagerly())
1705 
1706     inputs = keras.backend.zeros(shape=(10, 3))
1707     targets = keras.backend.zeros(shape=(10, 4))
1708 
1709     model.fit(inputs, targets, epochs=1, steps_per_epoch=2, verbose=0)
1710     model.evaluate(inputs, targets, steps=2, verbose=0)
1711     model.predict(inputs, steps=2)
1712     model.train_on_batch(inputs, targets)
1713     model.test_on_batch(inputs, targets)
1714     model.fit(inputs, targets,
1715               epochs=1, steps_per_epoch=2, verbose=0,
1716               validation_data=(inputs, targets), validation_steps=2)
1717 
1718     # Test with dynamic shape
1719     inputs = array_ops.placeholder_with_default(
1720         np.zeros((2, 3)), shape=tensor_shape.TensorShape([None, 3]))
1721     targets = array_ops.placeholder_with_default(
1722         np.zeros((2, 4)), shape=tensor_shape.TensorShape([None, 4]))
1723     self.assertEqual(inputs.shape.dims[0].value, None)
1724     model.fit(inputs, targets, epochs=1, steps_per_epoch=2, verbose=0)
1725     model.evaluate(inputs, targets, steps=2, verbose=0)
1726     model.predict(inputs, steps=2)
1727     model.train_on_batch(inputs, targets)
1728     model.test_on_batch(inputs, targets)
1729     model.fit(inputs, targets,
1730               epochs=1, steps_per_epoch=2, verbose=0,
1731               validation_data=(inputs, targets), validation_steps=2)
1732 
1733   @keras_parameterized.run_all_keras_modes
1734   def test_training_and_eval_methods_on_symbolic_tensors_multi_io(self):
1735     # TODO(kaftan) Test seems to not work, file ticket
1736     if context.executing_eagerly():
1737       self.skipTest('Skipping eager execution.')
1738 
1739     a = keras.layers.Input(shape=(3,), name='input_a')
1740     b = keras.layers.Input(shape=(3,), name='input_b')
1741 
1742     dense = keras.layers.Dense(4, name='dense')
1743     c = dense(a)
1744     d = dense(b)
1745     e = keras.layers.Dropout(0.5, name='dropout')(c)
1746 
1747     model = keras.models.Model([a, b], [d, e])
1748 
1749     optimizer = 'rmsprop'
1750     loss = 'mse'
1751     loss_weights = [1., 0.5]
1752     model.compile(
1753         optimizer,
1754         loss,
1755         metrics=['mae', metrics_module.CategoricalAccuracy()],
1756         loss_weights=loss_weights,
1757         run_eagerly=testing_utils.should_run_eagerly())
1758 
1759     input_a_tf = keras.backend.zeros(shape=(10, 3))
1760     input_b_tf = keras.backend.zeros(shape=(10, 3))
1761 
1762     output_d_tf = keras.backend.zeros(shape=(10, 4))
1763     output_e_tf = keras.backend.zeros(shape=(10, 4))
1764 
1765     model.fit(
1766         [input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
1767         epochs=1,
1768         steps_per_epoch=2,
1769         verbose=0)
1770     with self.assertRaisesRegexp(ValueError,
1771                                  'should specify the `steps_per_epoch`'):
1772       model.fit(
1773           [input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
1774           epochs=1,
1775           batch_size=5,
1776           verbose=0)
1777     model.train_on_batch([input_a_tf, input_b_tf], [output_d_tf, output_e_tf])
1778 
1779     # Test with dictionary inputs
1780     model.fit(
1781         {'input_a': input_a_tf,
1782          'input_b': input_b_tf},
1783         {'dense': output_d_tf,
1784          'dropout': output_e_tf},
1785         epochs=1,
1786         steps_per_epoch=2,
1787         verbose=0)
1788     model.fit(
1789         {'input_a': input_a_tf,
1790          'input_b': input_b_tf},
1791         {'dense': output_d_tf,
1792          'dropout': output_e_tf},
1793         validation_data=({'input_a': input_a_tf,
1794                           'input_b': input_b_tf},
1795                          {'dense': output_d_tf,
1796                           'dropout': output_e_tf}),
1797         epochs=1,
1798         steps_per_epoch=2,
1799         validation_steps=2,
1800         verbose=0)
1801     model.train_on_batch(
1802         {'input_a': input_a_tf,
1803          'input_b': input_b_tf},
1804         {'dense': output_d_tf,
1805          'dropout': output_e_tf})
1806 
1807     # Test with validation data
1808     model.fit(
1809         [input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
1810         validation_data=([input_a_tf, input_b_tf],
1811                          [output_d_tf, output_e_tf]),
1812         epochs=1,
1813         steps_per_epoch=2,
1814         validation_steps=2,
1815         verbose=0)
1816     # Test with validation split
1817     with self.assertRaisesRegexp(ValueError,
1818                                  'you cannot use `validation_split`'):
1819       model.fit(
1820           [input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
1821           epochs=2,
1822           steps_per_epoch=2,
1823           verbose=0,
1824           validation_split=0.2,
1825           validation_steps=2)
1826 
1827     # Test evaluation / prediction methods
1828     model.evaluate([input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
1829                    steps=2, verbose=0)
1830     model.predict([input_a_tf, input_b_tf], steps=2)
1831     model.test_on_batch([input_a_tf, input_b_tf], [output_d_tf, output_e_tf])
1832 
1833   @tf_test_util.run_deprecated_v1
1834   def test_model_with_input_feed_tensor(self):
1835     """We test building a model with a TF variable as input.
1836 
1837     We should be able to call fit, evaluate, predict,
1838     by only passing them data for the placeholder inputs
1839     in the model.
1840     """
1841     with self.cached_session():
1842       input_a_np = np.random.random((10, 3))
1843       input_b_np = np.random.random((10, 3))
1844 
1845       output_a_np = np.random.random((10, 4))
1846       output_b_np = np.random.random((10, 3))
1847 
1848       input_v = keras.backend.variables_module.Variable(
1849           input_a_np, dtype='float32')
1850       self.evaluate(variables_lib.variables_initializer([input_v]))
1851       a = keras.Input(tensor=input_v)
1852       b = keras.Input(shape=(3,), name='input_b')
1853 
1854       a_2 = keras.layers.Dense(4, name='dense_1')(a)
1855       dp = keras.layers.Dropout(0.5, name='dropout')
1856       b_2 = dp(b)
1857 
1858       model = keras.models.Model([a, b], [a_2, b_2])
1859       model.summary()
1860 
1861       optimizer = 'rmsprop'
1862       loss = 'mse'
1863       loss_weights = [1., 0.5]
1864       model.compile(optimizer, loss, metrics=['mean_squared_error'],
1865                     loss_weights=loss_weights,
1866                     sample_weight_mode=None)
1867 
1868       # test train_on_batch
1869       out = model.train_on_batch(input_b_np,
1870                                  [output_a_np, output_b_np])
1871       out = model.train_on_batch({'input_b': input_b_np},
1872                                  [output_a_np, output_b_np])
1873       out = model.test_on_batch({'input_b': input_b_np},
1874                                 [output_a_np, output_b_np])
1875       out = model.predict_on_batch({'input_b': input_b_np})
1876 
1877       # test fit
1878       out = model.fit({'input_b': input_b_np},
1879                       [output_a_np, output_b_np], epochs=1, batch_size=10)
1880       out = model.fit(input_b_np,
1881                       [output_a_np, output_b_np], epochs=1, batch_size=10)
1882 
1883       # test evaluate
1884       out = model.evaluate({'input_b': input_b_np},
1885                            [output_a_np, output_b_np], batch_size=10)
1886       out = model.evaluate(input_b_np,
1887                            [output_a_np, output_b_np], batch_size=10)
1888 
1889       # test predict
1890       out = model.predict({'input_b': input_b_np}, batch_size=10)
1891       out = model.predict(input_b_np, batch_size=10)
1892       self.assertEqual(len(out), 2)
1893 
1894       # Now test a model with a single input
1895       # i.e. we don't pass any data to fit the model.
1896       self.evaluate(variables_lib.variables_initializer([input_v]))
1897       a = keras.Input(tensor=input_v)
1898       a_2 = keras.layers.Dense(4, name='dense_1')(a)
1899       a_2 = keras.layers.Dropout(0.5, name='dropout')(a_2)
1900       model = keras.models.Model(a, a_2)
1901       model.summary()
1902 
1903       optimizer = 'rmsprop'
1904       loss = 'mse'
1905       model.compile(optimizer, loss, metrics=['mean_squared_error'])
1906 
1907       # test train_on_batch
1908       out = model.train_on_batch(None,
1909                                  output_a_np)
1910       out = model.train_on_batch(None,
1911                                  output_a_np)
1912       out = model.test_on_batch(None,
1913                                 output_a_np)
1914       out = model.predict_on_batch(None)
1915       out = model.train_on_batch([],
1916                                  output_a_np)
1917       out = model.train_on_batch({},
1918                                  output_a_np)
1919 
1920       # test fit
1921       _ = model.fit(None, output_a_np, epochs=1, steps_per_epoch=3)
1922       _ = model.fit(None, output_a_np, epochs=1, steps_per_epoch=3)
1923 
1924       # test evaluate
1925       _ = model.evaluate(None, output_a_np, steps=3)
1926       _ = model.evaluate(None, output_a_np, steps=3)
1927 
1928       # test predict
1929       out = model.predict(None, steps=3)
1930       out = model.predict(None, steps=3)
1931       self.assertEqual(out.shape, (10 * 3, 4))
1932 
1933       # Same, without learning phase
1934       # i.e. we don't pass any data to fit the model.
1935       self.evaluate(variables_lib.variables_initializer([input_v]))
1936       a = keras.Input(tensor=input_v)
1937       a_2 = keras.layers.Dense(4, name='dense_1')(a)
1938       model = keras.models.Model(a, a_2)
1939       model.summary()
1940 
1941       optimizer = 'rmsprop'
1942       loss = 'mse'
1943       model.compile(optimizer, loss, metrics=['mean_squared_error'])
1944 
1945       # test train_on_batch
1946       out = model.train_on_batch(None,
1947                                  output_a_np)
1948       out = model.train_on_batch(None,
1949                                  output_a_np)
1950       out = model.test_on_batch(None,
1951                                 output_a_np)
1952       out = model.predict_on_batch(None)
1953       out = model.train_on_batch([],
1954                                  output_a_np)
1955       out = model.train_on_batch({},
1956                                  output_a_np)
1957 
1958       # test fit
1959       _ = model.fit(None, output_a_np, epochs=1, steps_per_epoch=10)
1960       _ = model.fit(None, output_a_np, epochs=1, steps_per_epoch=10)
1961 
1962       # test evaluate
1963       _ = model.evaluate(None, output_a_np, steps=10)
1964       _ = model.evaluate(None, output_a_np, steps=10)
1965 
1966       # test predict
1967       out = model.predict(None, steps=3)
1968       out = model.predict(None, steps=3)
1969       self.assertEqual(out.shape, (10 * 3, 4))
1970 
1971   def test_model_with_partial_loss(self):
1972     with self.cached_session():
1973       a = keras.Input(shape=(3,), name='input_a')
1974       a_2 = keras.layers.Dense(4, name='dense_1')(a)
1975       dp = keras.layers.Dropout(0.5, name='dropout')
1976       a_3 = dp(a_2)
1977       model = keras.models.Model(a, [a_2, a_3])
1978 
1979       optimizer = 'rmsprop'
1980       loss = {'dropout': 'mse'}
1981       model.compile(optimizer, loss, metrics=['mae'])
1982 
1983       input_a_np = np.random.random((10, 3))
1984       output_a_np = np.random.random((10, 4))
1985 
1986       # test train_on_batch
1987       _ = model.train_on_batch(input_a_np, output_a_np)
1988       _ = model.test_on_batch(input_a_np, output_a_np)
1989       # fit
1990       _ = model.fit(input_a_np, [output_a_np])
1991       # evaluate
1992       _ = model.evaluate(input_a_np, [output_a_np])
1993 
1994       # Same without dropout.
1995       a = keras.Input(shape=(3,), name='input_a')
1996       a_2 = keras.layers.Dense(4, name='dense_1')(a)
1997       a_3 = keras.layers.Dense(4, name='dense_2')(a_2)
1998       model = keras.models.Model(a, [a_2, a_3])
1999 
2000       optimizer = 'rmsprop'
2001       loss = {'dense_2': 'mse'}
2002       model.compile(optimizer, loss, metrics={'dense_1': 'mae'})
2003 
2004       # test train_on_batch
2005       _ = model.train_on_batch(input_a_np, output_a_np)
2006       _ = model.test_on_batch(input_a_np, output_a_np)
2007       # fit
2008       _ = model.fit(input_a_np, [output_a_np])
2009       # evaluate
2010       _ = model.evaluate(input_a_np, [output_a_np])
2011 
2012   @tf_test_util.run_deprecated_v1
2013   def test_model_with_external_loss(self):
2014     with self.cached_session():
2015       # None loss, only regularization loss.
2016       a = keras.Input(shape=(3,), name='input_a')
2017       a_2 = keras.layers.Dense(4, name='dense_1',
2018                                kernel_regularizer='l1',
2019                                bias_regularizer='l2')(a)
2020       dp = keras.layers.Dropout(0.5, name='dropout')
2021       a_3 = dp(a_2)
2022 
2023       model = keras.models.Model(a, [a_2, a_3])
2024 
2025       optimizer = 'rmsprop'
2026       loss = None
2027       model.compile(optimizer, loss, metrics=['mae'])
2028 
2029       input_a_np = np.random.random((10, 3))
2030 
2031       # test train_on_batch
2032       out = model.train_on_batch(input_a_np, None)
2033       out = model.test_on_batch(input_a_np, None)
2034       # fit
2035       out = model.fit(input_a_np, None)
2036       # evaluate
2037       out = model.evaluate(input_a_np, None)
2038 
2039       # No dropout, external loss.
2040       a = keras.Input(shape=(3,), name='input_a')
2041       a_2 = keras.layers.Dense(4, name='dense_1')(a)
2042       a_3 = keras.layers.Dense(4, name='dense_2')(a)
2043 
2044       model = keras.models.Model(a, [a_2, a_3])
2045       model.add_loss(keras.backend.mean(a_3 + a_2))
2046 
2047       optimizer = 'rmsprop'
2048       loss = None
2049       model.compile(optimizer, loss, metrics=['mae'])
2050 
2051       # test train_on_batch
2052       out = model.train_on_batch(input_a_np, None)
2053       out = model.test_on_batch(input_a_np, None)
2054       # fit
2055       out = model.fit(input_a_np, None)
2056       # evaluate
2057       out = model.evaluate(input_a_np, None)
2058 
2059       # Test model with no external data at all.
2060       input_v = keras.backend.variables_module.Variable(
2061           input_a_np, dtype='float32')
2062       self.evaluate(variables_lib.variables_initializer([input_v]))
2063       a = keras.Input(tensor=input_v)
2064       a_2 = keras.layers.Dense(4, name='dense_1')(a)
2065       a_2 = keras.layers.Dropout(0.5, name='dropout')(a_2)
2066       model = keras.models.Model(a, a_2)
2067       model.add_loss(keras.backend.mean(a_2))
2068 
2069       model.compile(optimizer='rmsprop',
2070                     loss=None,
2071                     metrics=['mean_squared_error'])
2072 
2073       # test train_on_batch
2074       out = model.train_on_batch(None, None)
2075       out = model.test_on_batch(None, None)
2076       out = model.predict_on_batch(None)
2077 
2078       # test fit
2079       with self.assertRaises(ValueError):
2080         out = model.fit(None, None, epochs=1, batch_size=10)
2081       out = model.fit(None, None, epochs=1, steps_per_epoch=1)
2082 
2083       # test fit with validation data
2084       with self.assertRaises(ValueError):
2085         out = model.fit(None, None, epochs=1,
2086                         steps_per_epoch=None,
2087                         validation_steps=2)
2088       out = model.fit(None, None, epochs=1,
2089                       steps_per_epoch=2,
2090                       validation_steps=2)
2091 
2092       # test evaluate
2093       with self.assertRaises(ValueError):
2094         out = model.evaluate(None, None, batch_size=10)
2095       out = model.evaluate(None, None, steps=3)
2096 
2097       # test predict
2098       with self.assertRaises(ValueError):
2099         out = model.predict(None, batch_size=10)
2100       out = model.predict(None, steps=3)
2101       self.assertEqual(out.shape, (10 * 3, 4))
2102 
2103       # Test multi-output model with no external data at all.
2104       self.evaluate(variables_lib.variables_initializer([input_v]))
2105       a = keras.Input(tensor=input_v)
2106       a_1 = keras.layers.Dense(4, name='dense_1')(a)
2107       a_2 = keras.layers.Dropout(0.5, name='dropout')(a_1)
2108       model = keras.models.Model(a, [a_1, a_2])
2109       model.add_loss(keras.backend.mean(a_2))
2110 
2111       model.compile(optimizer='rmsprop',
2112                     loss=None,
2113                     metrics=['mean_squared_error'])
2114 
2115       # test train_on_batch
2116       out = model.train_on_batch(None, None)
2117       out = model.test_on_batch(None, None)
2118       out = model.predict_on_batch(None)
2119 
2120       # test fit
2121       with self.assertRaises(ValueError):
2122         out = model.fit(None, None, epochs=1, batch_size=10)
2123       out = model.fit(None, None, epochs=1, steps_per_epoch=1)
2124 
2125       # test fit with validation data
2126       out = model.fit(None, None, epochs=1,
2127                       steps_per_epoch=2,
2128                       validation_steps=2)
2129 
2130       # test evaluate
2131       with self.assertRaises(ValueError):
2132         out = model.evaluate(None, None, batch_size=10)
2133       out = model.evaluate(None, None, steps=3)
2134 
2135       # test predict
2136       with self.assertRaises(ValueError):
2137         out = model.predict(None, batch_size=10, verbose=1)
2138       out = model.predict(None, steps=3)
2139       self.assertEqual(len(out), 2)
2140       self.assertEqual(out[0].shape, (10 * 3, 4))
2141       self.assertEqual(out[1].shape, (10 * 3, 4))
2142 
2143   def test_target_tensors(self):
2144     with self.cached_session():
2145       # single-output, as list
2146       model = keras.models.Sequential()
2147       model.add(keras.layers.Dense(4, input_shape=(4,), name='dense'))
2148       input_val = np.random.random((10, 4))
2149       target_val = np.random.random((10, 4))
2150       target = keras.backend.variable(target_val)
2151       model.compile(optimizer='rmsprop', loss='mse', target_tensors=[target])
2152       model.train_on_batch(input_val, None)
2153 
2154       # single-output, as single tensor
2155       model.compile(optimizer='rmsprop', loss='mse', target_tensors=target)
2156       model.train_on_batch(input_val, None)
2157 
2158       # single-output, as dict
2159       model.compile(optimizer='rmsprop', loss='mse',
2160                     target_tensors={'dense': target})
2161       model.train_on_batch(input_val, None)
2162 
2163       # test invalid arguments
2164       with self.assertRaises(TypeError):
2165         model.compile(optimizer='rmsprop', loss='mse',
2166                       target_tensors=set())
2167       with self.assertRaises(ValueError):
2168         model.compile(optimizer='rmsprop', loss='mse',
2169                       target_tensors=[target, target])
2170       with self.assertRaises(ValueError):
2171         model.compile(optimizer='rmsprop', loss='mse',
2172                       target_tensors={'dense2': None})
2173       with self.assertRaises(ValueError):
2174         model.compile(optimizer='rmsprop', loss='mse',
2175                       target_tensors=[target])
2176         model.train_on_batch(input_val, target_val)
2177 
2178       # multi-output, as list
2179       input_val = np.random.random((10, 4))
2180       target_val_a = np.random.random((10, 4))
2181       target_val_b = np.random.random((10, 4))
2182       target_a = keras.backend.variable(target_val_a)
2183       target_b = keras.backend.variable(target_val_b)
2184 
2185       inputs = keras.layers.Input(shape=(4,))
2186       output_a = keras.layers.Dense(4, name='dense_a')(inputs)
2187       output_b = keras.layers.Dense(4, name='dense_b')(inputs)
2188       model = keras.models.Model(inputs, [output_a, output_b])
2189       model.compile(optimizer='rmsprop', loss='mse',
2190                     target_tensors=[target_a, target_b])
2191       model.train_on_batch(input_val, None)
2192 
2193       # multi-output, as dict
2194       model.compile(optimizer='rmsprop', loss='mse',
2195                     target_tensors={'dense_a': target_a,
2196                                     'dense_b': target_b})
2197       model.train_on_batch(input_val, None)
2198 
2199       # test with sample weights
2200       model.compile(
2201           optimizer='rmsprop',
2202           loss='mse',
2203           metrics=['mae', metrics_module.CategoricalAccuracy()],
2204           target_tensors=[target_a, target_b])
2205       model.train_on_batch(input_val, None,
2206                            sample_weight={'dense_a': np.random.random((10,))})
2207 
2208   @tf_test_util.run_deprecated_v1
2209   def test_model_custom_target_tensors(self):
2210     with self.cached_session():
2211       a = keras.Input(shape=(3,), name='input_a')
2212       b = keras.Input(shape=(3,), name='input_b')
2213 
2214       a_2 = keras.layers.Dense(4, name='dense_1')(a)
2215       dp = keras.layers.Dropout(0.5, name='dropout')
2216       b_2 = dp(b)
2217 
2218       y = keras.backend.placeholder([10, 4], name='y')
2219       y1 = keras.backend.placeholder([10, 3], name='y1')
2220       y2 = keras.backend.placeholder([7, 5], name='y2')
2221       model = keras.models.Model([a, b], [a_2, b_2])
2222 
2223       optimizer = 'rmsprop'
2224       loss = 'mse'
2225       loss_weights = [1., 0.5]
2226 
2227       # test list of target tensors
2228       with self.assertRaises(ValueError):
2229         model.compile(optimizer, loss, metrics=[], loss_weights=loss_weights,
2230                       sample_weight_mode=None, target_tensors=[y, y1, y2])
2231       model.compile(optimizer, loss, metrics=[], loss_weights=loss_weights,
2232                     sample_weight_mode=None, target_tensors=[y, y1])
2233       input_a_np = np.random.random((10, 3))
2234       input_b_np = np.random.random((10, 3))
2235 
2236       output_a_np = np.random.random((10, 4))
2237       output_b_np = np.random.random((10, 3))
2238 
2239       _ = model.train_on_batch([input_a_np, input_b_np],
2240                                [output_a_np, output_b_np],
2241                                {y: np.random.random((10, 4)),
2242                                 y1: np.random.random((10, 3))})
2243       # test dictionary of target_tensors
2244       with self.assertRaises(ValueError):
2245         model.compile(optimizer, loss,
2246                       metrics=[],
2247                       loss_weights=loss_weights,
2248                       sample_weight_mode=None,
2249                       target_tensors={'does_not_exist': y2})
2250       # test dictionary of target_tensors
2251       model.compile(optimizer, loss,
2252                     metrics=[],
2253                     loss_weights=loss_weights,
2254                     sample_weight_mode=None,
2255                     target_tensors={'dense_1': y, 'dropout': y1})
2256       _ = model.train_on_batch([input_a_np, input_b_np],
2257                                [output_a_np, output_b_np],
2258                                {y: np.random.random((10, 4)),
2259                                 y1: np.random.random((10, 3))})
2260 
2261       # test with custom TF placeholder as target
2262       pl_target_a = keras.backend.array_ops.placeholder('float32',
2263                                                         shape=(None, 4))
2264       model.compile(optimizer='rmsprop', loss='mse',
2265                     target_tensors={'dense_1': pl_target_a})
2266       model.train_on_batch([input_a_np, input_b_np],
2267                            [output_a_np, output_b_np])
2268 
2269 
2270 class TestTrainingWithMetrics(keras_parameterized.TestCase):
2271   """Training tests related to metrics."""
2272 
2273   @keras_parameterized.run_all_keras_modes
2274   def test_metrics_names(self):
2275     a = keras.layers.Input(shape=(3,), name='input_a')
2276     b = keras.layers.Input(shape=(3,), name='input_b')
2277 
2278     dense = keras.layers.Dense(4, name='dense')
2279     c = dense(a)
2280     d = dense(b)
2281     e = keras.layers.Dropout(0.5, name='dropout')(c)
2282 
2283     model = keras.models.Model([a, b], [d, e])
2284 
2285     optimizer = RMSPropOptimizer(learning_rate=0.001)
2286     metrics = ['mse', metrics_module.BinaryAccuracy()]
2287     model.compile(optimizer, loss='mae', metrics=metrics,
2288                   run_eagerly=testing_utils.should_run_eagerly())
2289 
2290     mse_metric = 'mse' if tf2.enabled() else 'mean_squared_error'
2291     reference_metric_names = [
2292         'loss', 'dense_loss', 'dropout_loss', 'dense_' + mse_metric,
2293         'dense_binary_accuracy', 'dropout_' + mse_metric,
2294         'dropout_binary_accuracy'
2295     ]
2296     self.assertEqual(reference_metric_names, model.metrics_names)
2297 
2298     # Verify that model metric names are not altered during training.
2299     input_a_np = np.random.random((10, 3))
2300     input_b_np = np.random.random((10, 3))
2301 
2302     output_d_np = np.random.random((10, 4))
2303     output_e_np = np.random.random((10, 4))
2304 
2305     model.fit([input_a_np, input_b_np], [output_d_np, output_e_np],
2306               epochs=1,
2307               batch_size=5)
2308     self.assertEqual(reference_metric_names, model.metrics_names)
2309 
2310   @keras_parameterized.run_all_keras_modes
2311   def test_metric_state_reset_between_fit_and_evaluate(self):
2312     model = keras.Sequential()
2313     model.add(keras.layers.Dense(3, activation='relu', input_dim=4))
2314     model.add(keras.layers.Dense(1, activation='sigmoid'))
2315     acc_obj = metrics_module.BinaryAccuracy()
2316     model.compile(
2317         loss='mae',
2318         metrics=[acc_obj],
2319         optimizer=RMSPropOptimizer(learning_rate=0.001),
2320         run_eagerly=testing_utils.should_run_eagerly())
2321 
2322     x_train = np.random.random((100, 4))
2323     y_train = np.random.random((100, 1))
2324     model.fit(x_train, y_train, batch_size=5, epochs=2)
2325     self.assertEqual(self.evaluate(acc_obj.count), 100)
2326 
2327     x_test = np.random.random((10, 4))
2328     y_test = np.random.random((10, 1))
2329     model.evaluate(x_test, y_test, batch_size=5)
2330     self.assertEqual(self.evaluate(acc_obj.count), 10)
2331 
2332   @keras_parameterized.run_with_all_model_types(exclude_models=['sequential'])
2333   @keras_parameterized.run_all_keras_modes
2334   def test_metrics_valid_compile_input_formats(self):
2335     inp_1 = keras.layers.Input(shape=(1,), name='input_1')
2336     inp_2 = keras.layers.Input(shape=(1,), name='input_2')
2337     x = keras.layers.Dense(3, kernel_initializer='ones', trainable=False)
2338     out_1 = keras.layers.Dense(
2339         1, kernel_initializer='ones', name='output_1', trainable=False)
2340     out_2 = keras.layers.Dense(
2341         1, kernel_initializer='ones', name='output_2', trainable=False)
2342 
2343     branch_a = [inp_1, x, out_1]
2344     branch_b = [inp_2, x, out_2]
2345     model = testing_utils.get_multi_io_model(branch_a, branch_b)
2346 
2347     # list of metrics.
2348     model.compile(
2349         optimizer='rmsprop',
2350         loss='mse',
2351         metrics=[keras.metrics.MeanSquaredError()],
2352         weighted_metrics=[keras.metrics.MeanSquaredError()],
2353         run_eagerly=testing_utils.should_run_eagerly())
2354 
2355     # list of list of metrics.
2356     model.compile(
2357         optimizer='rmsprop',
2358         loss='mse',
2359         metrics=[
2360             keras.metrics.MeanSquaredError(),
2361             [keras.metrics.MeanSquaredError(),
2362              keras.metrics.Accuracy()]
2363         ],
2364         weighted_metrics=[
2365             keras.metrics.MeanSquaredError(),
2366             [keras.metrics.MeanSquaredError(),
2367              keras.metrics.Accuracy()]
2368         ],
2369         run_eagerly=testing_utils.should_run_eagerly())
2370 
2371     # dict of metrics.
2372     model.compile(
2373         optimizer='rmsprop',
2374         loss='mse',
2375         metrics={
2376             'output_1':
2377                 keras.metrics.MeanSquaredError(),
2378             'output_2': [
2379                 keras.metrics.MeanSquaredError(),
2380                 keras.metrics.Accuracy()
2381             ],
2382         },
2383         weighted_metrics={
2384             'output_1':
2385                 keras.metrics.MeanSquaredError(),
2386             'output_2': [
2387                 keras.metrics.MeanSquaredError(),
2388                 keras.metrics.Accuracy()
2389             ],
2390         },
2391         run_eagerly=testing_utils.should_run_eagerly())
2392 
2393   @keras_parameterized.run_all_keras_modes
2394   def test_invalid_metrics(self):
2395     num_classes = 5
2396     input_dim = 5
2397 
2398     model = testing_utils.get_small_sequential_mlp(
2399         num_hidden=10, num_classes=num_classes, input_dim=input_dim)
2400 
2401     with self.assertRaisesRegexp(
2402         TypeError, 'Type of `metrics` argument not understood. '
2403         'Expected a list or dictionary, found: '):
2404       model.compile(
2405           RMSPropOptimizer(learning_rate=0.001),
2406           loss='categorical_crossentropy',
2407           metrics=metrics_module.CategoricalAccuracy(),
2408           run_eagerly=testing_utils.should_run_eagerly())
2409 
2410     inp = keras.layers.Input(shape=(1,))
2411     x = keras.layers.Dense(3, activation='relu')(inp)
2412     out_1 = keras.layers.Dense(1, activation='sigmoid', name='output_1')(x)
2413     out_2 = keras.layers.Dense(1, activation='sigmoid', name='output_2')(x)
2414     model = keras.models.Model(inp, [out_1, out_2])
2415     with self.assertRaisesRegex(
2416         ValueError, 'When passing a list of lists as `metrics`, '
2417         'it should have one entry per model output. '
2418         'The model has 2 outputs, but you passed metrics='):
2419       model.compile('rmsprop', loss='mse', metrics=[['mse']])
2420 
2421   @keras_parameterized.run_all_keras_modes
2422   def test_metrics_masking(self):
2423     if testing_utils.should_run_eagerly():
2424       self.skipTest('b/120495761')
2425     with self.cached_session():
2426       np.random.seed(1337)
2427       model = keras.models.Sequential()
2428       model.add(keras.layers.Masking(mask_value=0, input_shape=(2, 1)))
2429       model.add(
2430           keras.layers.TimeDistributed(
2431               keras.layers.Dense(1, kernel_initializer='ones')))
2432       model.compile(
2433           RMSPropOptimizer(learning_rate=0.001),
2434           loss='mse',
2435           weighted_metrics=['accuracy'],
2436           run_eagerly=testing_utils.should_run_eagerly())
2437 
2438       # verify that masking is applied.
2439       x = np.array([[[1], [1]], [[1], [1]], [[0], [0]]])
2440       y = np.array([[[1], [1]], [[0], [1]], [[1], [1]]])
2441       scores = model.train_on_batch(x, y)
2442       self.assertArrayNear(scores, [0.25, 0.75], 0.1)
2443 
2444       # verify that masking is combined with sample weights.
2445       w = np.array([3, 2, 4])
2446       scores = model.train_on_batch(x, y, sample_weight=w)
2447       self.assertArrayNear(scores, [0.3328, 0.8], 0.001)
2448 
2449   @keras_parameterized.run_all_keras_modes
2450   def test_add_metric_with_tensor_on_model(self):
2451     x = keras.layers.Input(shape=(1,))
2452     y = keras.layers.Dense(1, kernel_initializer='ones')(x)
2453     model = keras.models.Model(x, y)
2454     model.add_metric(
2455         math_ops.reduce_sum(y), name='metric_1', aggregation='mean')
2456 
2457     # test with a metric which does not have the standard signature:
2458     # (y_true, y_pred, sample_Weight)
2459     with keras.backend.get_graph().as_default():
2460       model.add_metric(metrics_module.Mean(name='metric_2')(y))
2461 
2462     if testing_utils.should_run_eagerly():
2463       with self.assertRaisesRegex(
2464           ValueError,
2465           'We currently do not support enabling `run_eagerly` on compile if '
2466           r'`model.add_loss\(tensor\)` or `model.add_metric\(tensor\)` '
2467           'has been called.'):
2468         model.compile('sgd', run_eagerly=True)
2469       return
2470     else:
2471       model.compile('sgd', loss='mse', run_eagerly=False)
2472 
2473     inputs = np.ones(shape=(10, 1))
2474     targets = np.ones(shape=(10, 1))
2475     history = model.fit(
2476         inputs,
2477         targets,
2478         epochs=2,
2479         batch_size=5,
2480         validation_data=(inputs, targets))
2481     self.assertEqual(history.history['metric_1'][-1], 5)
2482     self.assertEqual(history.history['metric_2'][-1], 1)
2483     self.assertEqual(history.history['val_metric_1'][-1], 5)
2484     self.assertEqual(history.history['val_metric_2'][-1], 1)
2485 
2486     eval_results = model.evaluate(inputs, targets, batch_size=5)
2487     self.assertEqual(eval_results[-1], 1)
2488     self.assertEqual(eval_results[-2], 5)
2489 
2490     model.predict(inputs, batch_size=5)
2491     model.train_on_batch(inputs, targets)
2492     model.test_on_batch(inputs, targets)
2493 
2494   @keras_parameterized.run_all_keras_modes
2495   def test_add_metric_in_model_call(self):
2496 
2497     class TestModel(keras.Model):
2498 
2499       def __init__(self):
2500         super(TestModel, self).__init__(name='test_model')
2501         self.dense1 = keras.layers.Dense(2, kernel_initializer='ones')
2502         self.mean = metrics_module.Mean(name='metric_1')
2503 
2504       def call(self, x):
2505         self.add_metric(
2506             math_ops.reduce_sum(x), name='metric_2', aggregation='mean')
2507         # Provide same name as in the instance created in __init__
2508         # for eager mode
2509         self.add_metric(self.mean(x), name='metric_1')
2510         return self.dense1(x)
2511 
2512     model = TestModel()
2513     model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01),
2514                   run_eagerly=testing_utils.should_run_eagerly())
2515 
2516     x = np.ones(shape=(10, 1))
2517     y = np.ones(shape=(10, 2))
2518     history = model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
2519     self.assertAlmostEqual(history.history['metric_1'][-1], 1, 0)
2520     self.assertAlmostEqual(history.history['val_metric_1'][-1], 1, 0)
2521     self.assertAlmostEqual(history.history['metric_2'][-1], 5, 0)
2522     self.assertAlmostEqual(history.history['val_metric_2'][-1], 5, 0)
2523 
2524     eval_results = model.evaluate(x, y, batch_size=5)
2525     self.assertAlmostEqual(eval_results[1], 1, 0)
2526     self.assertAlmostEqual(eval_results[2], 5, 0)
2527 
2528     model.predict(x, batch_size=5)
2529     model.train_on_batch(x, y)
2530     model.test_on_batch(x, y)
2531 
2532   @keras_parameterized.run_with_all_model_types
2533   @keras_parameterized.run_all_keras_modes
2534   def test_add_metric_in_layer_call(self):
2535 
2536     class TestLayer(keras.layers.Layer):
2537 
2538       def build(self, input_shape):
2539         self.a = self.add_variable(
2540             'a', (1, 1), initializer='ones', trainable=False)
2541         self.built = True
2542 
2543       def call(self, inputs):
2544         self.add_metric(
2545             math_ops.reduce_sum(inputs), name='metric_1', aggregation='mean')
2546         return inputs + 1
2547 
2548     layers = [
2549         TestLayer(input_shape=(1,)),
2550         keras.layers.Dense(2, kernel_initializer='ones')
2551     ]
2552     model = testing_utils.get_model_from_layers(layers, input_shape=(1,))
2553     model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01),
2554                   run_eagerly=testing_utils.should_run_eagerly())
2555 
2556     x = np.ones(shape=(10, 1))
2557     y = np.ones(shape=(10, 2))
2558     history = model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
2559     self.assertEqual(history.history['metric_1'][-1], 5)
2560     self.assertAlmostEqual(history.history['val_metric_1'][-1], 5, 0)
2561 
2562   @keras_parameterized.run_all_keras_modes
2563   def test_model_metrics_list(self):
2564     x = keras.layers.Input(shape=(1,))
2565     y = keras.layers.Dense(1, kernel_initializer='ones')(x)
2566     model = keras.models.Model(x, y)
2567     model.add_metric(
2568         math_ops.reduce_sum(y), name='metric_1', aggregation='mean')
2569     with keras.backend.get_graph().as_default():
2570       model.add_metric(metrics_module.Mean(name='metric_2')(y))
2571 
2572     if testing_utils.should_run_eagerly():
2573       with self.assertRaisesRegex(
2574           ValueError,
2575           'We currently do not support enabling `run_eagerly` on compile if '
2576           r'`model.add_loss\(tensor\)` or `model.add_metric\(tensor\)` '
2577           'has been called.'):
2578         model.compile('sgd', run_eagerly=True)
2579       return
2580     else:
2581       model.compile(
2582           'sgd',
2583           loss='mse',
2584           metrics=[metrics_module.Accuracy('acc')],
2585           run_eagerly=False)
2586 
2587     # Verify that the metrics added using `compile` and `add_metric` API are
2588     # included
2589     self.assertEqual([m.name for m in model._compile_metrics], ['acc'])
2590     self.assertEqual([m.name for m in model.metrics],
2591                      ['acc', 'metric_1', 'metric_2'])
2592 
2593   @keras_parameterized.run_all_keras_modes
2594   def test_model_metrics_list_in_call(self):
2595 
2596     class TestModel(keras.Model):
2597 
2598       def __init__(self):
2599         super(TestModel, self).__init__(name='test_model')
2600         self.dense1 = keras.layers.Dense(2, kernel_initializer='ones')
2601 
2602       def call(self, x):
2603         self.add_metric(
2604             math_ops.reduce_sum(x), name='metric_1', aggregation='mean')
2605         return self.dense1(x)
2606 
2607     model = TestModel()
2608     model.compile(
2609         loss='mse',
2610         optimizer=RMSPropOptimizer(0.01),
2611         metrics=[metrics_module.Accuracy('acc')],
2612         run_eagerly=testing_utils.should_run_eagerly())
2613     x = np.ones(shape=(10, 1))
2614     y = np.ones(shape=(10, 2))
2615     model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
2616 
2617     self.assertEqual([m.name for m in model._compile_metrics], ['acc'])
2618     self.assertEqual([m.name for m in model.metrics], ['acc', 'metric_1'])
2619 
2620   @keras_parameterized.run_all_keras_modes
2621   def test_multiple_add_metric_calls(self):
2622 
2623     class TestModel(keras.Model):
2624 
2625       def __init__(self):
2626         super(TestModel, self).__init__(name='test_model')
2627         self.dense1 = keras.layers.Dense(2, kernel_initializer='ones')
2628         self.mean1 = metrics_module.Mean(name='metric_1')
2629         self.mean2 = metrics_module.Mean(name='metric_2')
2630 
2631       def call(self, x):
2632         self.add_metric(self.mean2(x), name='metric_2')
2633         self.add_metric(self.mean1(x), name='metric_1')
2634         self.add_metric(
2635             math_ops.reduce_sum(x), name='metric_3', aggregation='mean')
2636         return self.dense1(x)
2637 
2638     model = TestModel()
2639     model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01),
2640                   run_eagerly=testing_utils.should_run_eagerly())
2641 
2642     x = np.ones(shape=(10, 1))
2643     y = np.ones(shape=(10, 2))
2644     history = model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
2645     self.assertAlmostEqual(history.history['metric_1'][-1], 1, 0)
2646     self.assertAlmostEqual(history.history['metric_2'][-1], 1, 0)
2647     self.assertAlmostEqual(history.history['metric_3'][-1], 5, 0)
2648 
2649     eval_results = model.evaluate(x, y, batch_size=5)
2650     self.assertArrayNear(eval_results[1:4], [1, 1, 5], 0.1)
2651 
2652     model.predict(x, batch_size=5)
2653     model.train_on_batch(x, y)
2654     model.test_on_batch(x, y)
2655 
2656   @keras_parameterized.run_with_all_model_types
2657   @keras_parameterized.run_all_keras_modes
2658   def test_invalid_metric_tensor(self):
2659 
2660     class TestLayer(keras.layers.Layer):
2661 
2662       def build(self, input_shape):
2663         self.built = True
2664 
2665       def call(self, inputs):
2666         self.add_metric(math_ops.reduce_mean(inputs), name='metric_1')
2667         return inputs + 1
2668 
2669     layers = [TestLayer(input_shape=(1,))]
2670     layers.append(keras.layers.Dense(2, kernel_initializer='ones'))
2671     x = np.ones(shape=(10, 1))
2672     y = np.ones(shape=(10, 2))
2673 
2674     with self.assertRaisesRegexp(
2675         ValueError,
2676         'We do not support adding an aggregated metric result tensor that is '
2677         'not the output of a `tf.keras.metrics.Metric` metric instance.'):
2678       model = testing_utils.get_model_from_layers(layers, input_shape=(1,))
2679       model.compile(
2680           loss='mse',
2681           optimizer=RMSPropOptimizer(0.01),
2682           run_eagerly=testing_utils.should_run_eagerly())
2683       model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
2684 
2685   @keras_parameterized.run_all_keras_modes
2686   def test_duplicate_metric_name_in_add_metric(self):
2687 
2688     class TestModel(keras.Model):
2689 
2690       def __init__(self):
2691         super(TestModel, self).__init__(name='test_model')
2692         self.dense1 = keras.layers.Dense(2, kernel_initializer='ones')
2693         self.mean = metrics_module.Mean(name='metric_1')
2694         self.mean2 = metrics_module.Mean(name='metric_1')
2695 
2696       def call(self, x):
2697         self.add_metric(self.mean(x), name='metric_1')
2698         return self.dense1(x)
2699 
2700     model = TestModel()
2701     model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01),
2702                   run_eagerly=testing_utils.should_run_eagerly())
2703 
2704     x = np.ones(shape=(10, 1))
2705     y = np.ones(shape=(10, 2))
2706     with self.assertRaisesRegexp(
2707         ValueError,
2708         'Please provide different names for the metrics you have added. '
2709         'We found 2 metrics with the name: "metric_1"'):
2710       model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
2711 
2712   @keras_parameterized.run_all_keras_modes
2713   def test_add_metric_without_name(self):
2714 
2715     class TestModel(keras.Model):
2716 
2717       def __init__(self):
2718         super(TestModel, self).__init__(name='test_model')
2719         self.dense1 = keras.layers.Dense(2, kernel_initializer='ones')
2720 
2721       def call(self, x):
2722         self.add_metric(math_ops.reduce_sum(x), aggregation='mean')
2723         return self.dense1(x)
2724 
2725     model = TestModel()
2726     model.compile(loss='mse', optimizer=RMSPropOptimizer(0.01),
2727                   run_eagerly=testing_utils.should_run_eagerly())
2728     x = np.ones(shape=(10, 1))
2729     y = np.ones(shape=(10, 2))
2730 
2731     with self.assertRaisesRegex(ValueError,
2732                                 'Please provide a name for your metric like'):
2733       model.fit(x, y, epochs=2, batch_size=5, validation_data=(x, y))
2734 
2735   @keras_parameterized.run_all_keras_modes
2736   def test_add_metric_correctness(self):
2737     inputs = keras.Input(shape=(1,))
2738     targets = keras.Input(shape=(1,))
2739 
2740     class Bias(keras.layers.Layer):
2741 
2742       def build(self, input_shape):
2743         self.bias = self.add_variable('bias', (1,), initializer='zeros')
2744         self.mae = metrics_module.MeanAbsoluteError(name='mae_1')
2745 
2746       def call(self, inputs):
2747         outputs = inputs + self.bias
2748         self.add_metric(self.mae(targets, outputs), name='mae_1')
2749         return outputs
2750 
2751     outputs = Bias()(inputs)
2752     model = keras.Model(inputs, outputs)
2753 
2754     model.add_metric(
2755         metrics_module.mean_absolute_error(targets, outputs),
2756         name='mae_2',
2757         aggregation='mean')
2758 
2759     # If we want to use the metric class instance as shown below, we will need
2760     # to add graph scope as the reduction logic involves some eager mode checks.
2761     with keras.backend.get_graph().as_default():
2762       model.add_metric(
2763           metrics_module.MeanAbsoluteError(name='mae_3')(targets, outputs))
2764 
2765     if testing_utils.should_run_eagerly():
2766       with self.assertRaisesRegex(
2767           ValueError,
2768           'We currently do not support enabling `run_eagerly` on compile if '
2769           r'`model.add_loss\(tensor\)` or `model.add_metric\(tensor\)` '
2770           'has been called.'):
2771         model.compile('sgd', run_eagerly=True)
2772       return
2773     else:
2774       model.compile(
2775           loss='mae',
2776           optimizer=keras.optimizer_v2.gradient_descent.SGD(0.1),
2777           metrics=[metrics_module.MeanAbsoluteError(name='mae_4')],
2778           target_tensors=[targets],
2779           run_eagerly=False)
2780 
2781     x = np.array([[0.], [1.], [2.]])
2782     y = np.array([[0.5], [2.], [3.5]])
2783     history = model.fit(x, y, batch_size=3, epochs=5)
2784 
2785     expected_val = [1., 0.9, 0.8, 0.7, 0.6]
2786     for key in ['loss', 'mae_1', 'mae_2', 'mae_3', 'mae_4']:
2787       self.assertAllClose(history.history[key], expected_val, 1e-3)
2788 
2789 
2790 class BareUpdateLayer(keras.layers.Layer):
2791 
2792   def build(self, input_shape):
2793     self.counter = self.add_weight(
2794         'counter',
2795         dtype='int32',
2796         shape=(),
2797         initializer='zeros',
2798         trainable=False)
2799 
2800   def call(self, inputs):
2801     state_ops.assign_add(self.counter, 1)
2802     return math_ops.cast(self.counter, inputs.dtype) * inputs
2803 
2804 
2805 class AddUpdateLayer(keras.layers.Layer):
2806 
2807   def build(self, input_shape):
2808     self.counter = self.add_weight(
2809         'counter',
2810         dtype='int32',
2811         shape=(),
2812         initializer='zeros',
2813         trainable=False)
2814 
2815   def call(self, inputs):
2816     # Make sure update isn't run twice.
2817     self.add_update(state_ops.assign_add(self.counter, 1))
2818     return math_ops.cast(self.counter, inputs.dtype) * inputs
2819 
2820 
2821 class NestedUpdateLayer(keras.layers.Layer):
2822 
2823   def build(self, input_shape):
2824     self.layer = BareUpdateLayer()
2825     self.layer.build(input_shape)
2826 
2827   @property
2828   def counter(self):
2829     return self.layer.counter
2830 
2831   def call(self, inputs):
2832     return self.layer(inputs)
2833 
2834 
2835 @keras_parameterized.run_all_keras_modes(always_skip_v1=True)
2836 class TestAutoUpdates(keras_parameterized.TestCase):
2837 
2838   @keras_parameterized.run_with_all_model_types
2839   @parameterized.named_parameters(('bare_update', BareUpdateLayer()),
2840                                   ('add_update', AddUpdateLayer()),
2841                                   ('nested_update', NestedUpdateLayer()))
2842   def test_updates_in_model(self, layer):
2843     x, y = np.ones((10, 10)), np.ones((10, 1))
2844     model = testing_utils.get_model_from_layers(
2845         [layer, keras.layers.Dense(1)], input_shape=(10,))
2846     model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly())
2847     model.fit(x, y, batch_size=2, epochs=1)
2848     if not testing_utils.should_run_eagerly():
2849       # Check that `trainable=False` disables updates.
2850       layer.trainable = False
2851       model.compile(
2852           'sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly())
2853       model.fit(x, y, batch_size=2, epochs=1)
2854     self.assertEqual(self.evaluate(layer.counter), 5)
2855 
2856   @parameterized.named_parameters(('bare_update', BareUpdateLayer()),
2857                                   ('add_update', AddUpdateLayer()),
2858                                   ('nested_update', NestedUpdateLayer()))
2859   def test_updates_standalone_layer(self, layer):
2860     y = layer(np.ones((10, 10)))
2861     self.evaluate(layer.counter.initializer)
2862     self.evaluate(y)
2863     self.assertEqual(self.evaluate(layer.counter), 1)
2864 
2865   def test_trainable_false(self):
2866     x = keras.backend.placeholder(shape=(10, 10), dtype='float32')
2867     layer = NestedUpdateLayer()
2868     layer.trainable = False
2869     y = layer(x)
2870     func = keras.backend.function([x], [y])
2871     x_val = np.ones((10, 10))
2872     func(x_val)
2873     counter = keras.backend.get_value(layer.counter)
2874     self.assertEqual(counter, 0)
2875 
2876   @keras_parameterized.run_with_all_model_types
2877   def test_batchnorm_trainable_false(self):
2878     bn = keras.layers.BatchNormalization()
2879     bn.trainable = False
2880     model = testing_utils.get_model_from_layers([bn, keras.layers.Dense(1)],
2881                                                 input_shape=(10,))
2882     model.compile('sgd', 'mse', run_eagerly=testing_utils.should_run_eagerly())
2883     x, y = np.ones((10, 10)), np.ones((10, 1))
2884     model.fit(x, y, batch_size=2, epochs=1)
2885     self.assertAllEqual(self.evaluate(bn.moving_mean), np.zeros((10,)))
2886     self.assertAllEqual(self.evaluate(bn.moving_variance), np.ones((10,)))
2887 
2888 
2889 if __name__ == '__main__':
2890   test.main()
2891