1# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""Tests for kernelized.py."""
16
17from __future__ import absolute_import
18from __future__ import division
19from __future__ import print_function
20
21import functools
22import math
23
24from absl.testing import parameterized
25import numpy as np
26
27from tensorflow.python.eager import context
28from tensorflow.python.framework import constant_op
29from tensorflow.python.framework import dtypes
30from tensorflow.python.framework import ops
31from tensorflow.python.framework import random_seed
32from tensorflow.python.framework import tensor_shape
33from tensorflow.python.framework import test_util
34from tensorflow.python.keras import backend as keras_backend
35from tensorflow.python.keras import initializers
36from tensorflow.python.keras.layers import kernelized as kernel_layers
37from tensorflow.python.keras.utils import kernelized_utils
38from tensorflow.python.ops import array_ops
39from tensorflow.python.ops import init_ops
40from tensorflow.python.ops import math_ops
41from tensorflow.python.ops import random_ops
42from tensorflow.python.platform import test
43
44
45def _exact_gaussian(stddev):
46  return functools.partial(
47      kernelized_utils.exact_gaussian_kernel, stddev=stddev)
48
49
50def _exact_laplacian(stddev):
51  return functools.partial(
52      kernelized_utils.exact_laplacian_kernel, stddev=stddev)
53
54
55class RandomFourierFeaturesTest(test.TestCase, parameterized.TestCase):
56
57  def _assert_all_close(self, expected, actual, atol=0.001):
58    if not context.executing_eagerly():
59      with self.cached_session() as sess:
60        keras_backend._initialize_variables(sess)
61        self.assertAllClose(expected, actual, atol=atol)
62    else:
63      self.assertAllClose(expected, actual, atol=atol)
64
65  @test_util.run_in_graph_and_eager_modes()
66  def test_invalid_output_dim(self):
67    with self.assertRaisesRegexp(
68        ValueError, r'`output_dim` should be a positive integer. Given: -3.'):
69      _ = kernel_layers.RandomFourierFeatures(output_dim=-3, scale=2.0)
70
71  @test_util.run_in_graph_and_eager_modes()
72  def test_unsupported_kernel_type(self):
73    with self.assertRaisesRegexp(
74        ValueError, r'Unsupported kernel type: \'unsupported_kernel\'.'):
75      _ = kernel_layers.RandomFourierFeatures(
76          3, 'unsupported_kernel', stddev=2.0)
77
78  @test_util.run_in_graph_and_eager_modes()
79  def test_invalid_scale(self):
80    with self.assertRaisesRegexp(
81        ValueError,
82        r'When provided, `scale` should be a positive float. Given: 0.0.'):
83      _ = kernel_layers.RandomFourierFeatures(output_dim=10, scale=0.0)
84
85  @test_util.run_in_graph_and_eager_modes()
86  def test_invalid_input_shape(self):
87    inputs = random_ops.random_uniform((3, 2, 4), seed=1)
88    rff_layer = kernel_layers.RandomFourierFeatures(output_dim=10, scale=3.0)
89    with self.assertRaisesRegexp(
90        ValueError,
91        r'The rank of the input tensor should be 2. Got 3 instead.'):
92      _ = rff_layer.apply(inputs)
93
94  @parameterized.named_parameters(
95      ('gaussian', 'gaussian', 10.0, False),
96      ('random', init_ops.random_uniform_initializer, 1.0, True))
97  @test_util.run_in_graph_and_eager_modes()
98  def test_random_features_properties(self, initializer, scale, trainable):
99    rff_layer = kernel_layers.RandomFourierFeatures(
100        output_dim=10,
101        kernel_initializer=initializer,
102        scale=scale,
103        trainable=trainable)
104    self.assertEqual(rff_layer.output_dim, 10)
105    self.assertEqual(rff_layer.kernel_initializer, initializer)
106    self.assertEqual(rff_layer.scale, scale)
107    self.assertEqual(rff_layer.trainable, trainable)
108
109  @parameterized.named_parameters(('gaussian', 'gaussian', False),
110                                  ('laplacian', 'laplacian', True),
111                                  ('other', init_ops.ones_initializer, True))
112  @test_util.run_in_graph_and_eager_modes()
113  def test_call(self, initializer, trainable):
114    rff_layer = kernel_layers.RandomFourierFeatures(
115        output_dim=10,
116        kernel_initializer=initializer,
117        scale=1.0,
118        trainable=trainable,
119        name='random_fourier_features')
120    inputs = random_ops.random_uniform((3, 2), seed=1)
121    outputs = rff_layer(inputs)
122    self.assertListEqual([3, 10], outputs.get_shape().as_list())
123    num_trainable_vars = 1 if trainable else 0
124    self.assertLen(rff_layer.non_trainable_variables, 3 - num_trainable_vars)
125    if not context.executing_eagerly():
126      self.assertLen(
127          ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES),
128          num_trainable_vars)
129
130  @test_util.assert_no_new_pyobjects_executing_eagerly
131  def test_no_eager_Leak(self):
132    # Tests that repeatedly constructing and building a Layer does not leak
133    # Python objects.
134    inputs = random_ops.random_uniform((5, 4), seed=1)
135    kernel_layers.RandomFourierFeatures(output_dim=4, name='rff')(inputs)
136    kernel_layers.RandomFourierFeatures(output_dim=10, scale=2.0)(inputs)
137
138  @test_util.run_in_graph_and_eager_modes()
139  def test_output_shape(self):
140    inputs = random_ops.random_uniform((3, 2), seed=1)
141    rff_layer = kernel_layers.RandomFourierFeatures(
142        output_dim=7, name='random_fourier_features', trainable=True)
143    outputs = rff_layer(inputs)
144    self.assertEqual([3, 7], outputs.get_shape().as_list())
145
146  @parameterized.named_parameters(
147      ('gaussian', 'gaussian'), ('laplacian', 'laplacian'),
148      ('other', init_ops.random_uniform_initializer))
149  @test_util.run_deprecated_v1
150  def test_call_on_placeholder(self, initializer):
151    inputs = array_ops.placeholder(dtype=dtypes.float32, shape=[None, None])
152    rff_layer = kernel_layers.RandomFourierFeatures(
153        output_dim=5,
154        kernel_initializer=initializer,
155        name='random_fourier_features')
156    with self.assertRaisesRegexp(
157        ValueError, r'The last dimension of the inputs to '
158        '`RandomFourierFeatures` should be defined. Found `None`.'):
159      rff_layer(inputs)
160
161    inputs = array_ops.placeholder(dtype=dtypes.float32, shape=[2, None])
162    rff_layer = kernel_layers.RandomFourierFeatures(
163        output_dim=5,
164        kernel_initializer=initializer,
165        name='random_fourier_features')
166    with self.assertRaisesRegexp(
167        ValueError, r'The last dimension of the inputs to '
168        '`RandomFourierFeatures` should be defined. Found `None`.'):
169      rff_layer(inputs)
170
171    inputs = array_ops.placeholder(dtype=dtypes.float32, shape=[None, 3])
172    rff_layer = kernel_layers.RandomFourierFeatures(
173        output_dim=5, name='random_fourier_features')
174    rff_layer(inputs)
175
176  @parameterized.named_parameters(('gaussian', 10, 'gaussian', 2.0),
177                                  ('laplacian', 5, 'laplacian', None),
178                                  ('other', 10, init_ops.ones_initializer, 1.0))
179  @test_util.run_in_graph_and_eager_modes()
180  def test_compute_output_shape(self, output_dim, initializer, scale):
181    rff_layer = kernel_layers.RandomFourierFeatures(
182        output_dim, initializer, scale=scale, name='rff')
183    with self.assertRaises(ValueError):
184      rff_layer.compute_output_shape(tensor_shape.TensorShape(None))
185    with self.assertRaises(ValueError):
186      rff_layer.compute_output_shape(tensor_shape.TensorShape([]))
187    with self.assertRaises(ValueError):
188      rff_layer.compute_output_shape(tensor_shape.TensorShape([3]))
189    with self.assertRaises(ValueError):
190      rff_layer.compute_output_shape(tensor_shape.TensorShape([3, 2, 3]))
191
192    with self.assertRaisesRegexp(
193        ValueError, r'The innermost dimension of input shape must be defined.'):
194      rff_layer.compute_output_shape(tensor_shape.TensorShape([3, None]))
195
196    self.assertEqual([None, output_dim],
197                     rff_layer.compute_output_shape((None, 3)).as_list())
198    self.assertEqual([None, output_dim],
199                     rff_layer.compute_output_shape(
200                         tensor_shape.TensorShape([None, 2])).as_list())
201    self.assertEqual([4, output_dim],
202                     rff_layer.compute_output_shape((4, 1)).as_list())
203
204  @parameterized.named_parameters(
205      ('gaussian', 10, 'gaussian', 3.0, False),
206      ('laplacian', 5, 'laplacian', 5.5, True),
207      ('other', 7, init_ops.random_uniform_initializer(), None, True))
208  @test_util.run_in_graph_and_eager_modes()
209  def test_get_config(self, output_dim, initializer, scale, trainable):
210    rff_layer = kernel_layers.RandomFourierFeatures(
211        output_dim,
212        initializer,
213        scale=scale,
214        trainable=trainable,
215        name='random_fourier_features',
216    )
217    expected_initializer = initializer
218    if isinstance(initializer, init_ops.Initializer):
219      expected_initializer = initializers.serialize(initializer)
220
221    expected_config = {
222        'output_dim': output_dim,
223        'kernel_initializer': expected_initializer,
224        'scale': scale,
225        'name': 'random_fourier_features',
226        'trainable': trainable,
227        'dtype': None,
228    }
229    self.assertLen(expected_config, len(rff_layer.get_config()))
230    self.assertSameElements(
231        list(expected_config.items()), list(rff_layer.get_config().items()))
232
233  @parameterized.named_parameters(
234      ('gaussian', 5, 'gaussian', None, True),
235      ('laplacian', 5, 'laplacian', 5.5, False),
236      ('other', 7, init_ops.ones_initializer(), 2.0, True))
237  @test_util.run_in_graph_and_eager_modes()
238  def test_from_config(self, output_dim, initializer, scale, trainable):
239    model_config = {
240        'output_dim': output_dim,
241        'kernel_initializer': initializer,
242        'scale': scale,
243        'trainable': trainable,
244        'name': 'random_fourier_features',
245    }
246    rff_layer = kernel_layers.RandomFourierFeatures.from_config(model_config)
247    self.assertEqual(rff_layer.output_dim, output_dim)
248    self.assertEqual(rff_layer.kernel_initializer, initializer)
249    self.assertEqual(rff_layer.scale, scale)
250    self.assertEqual(rff_layer.trainable, trainable)
251
252    inputs = random_ops.random_uniform((3, 2), seed=1)
253    outputs = rff_layer(inputs)
254    self.assertListEqual([3, output_dim], outputs.get_shape().as_list())
255    num_trainable_vars = 1 if trainable else 0
256    self.assertLen(rff_layer.trainable_variables, num_trainable_vars)
257    if trainable:
258      self.assertEqual('random_fourier_features/random_features_scale:0',
259                       rff_layer.trainable_variables[0].name)
260    self.assertLen(rff_layer.non_trainable_variables, 3 - num_trainable_vars)
261    if not context.executing_eagerly():
262      self.assertLen(
263          ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES),
264          num_trainable_vars)
265
266  @parameterized.named_parameters(
267      ('gaussian', 10, 'gaussian', 3.0, True),
268      ('laplacian', 5, 'laplacian', 5.5, False),
269      ('other', 10, init_ops.random_uniform_initializer(), None, True))
270  @test_util.run_in_graph_and_eager_modes()
271  def test_same_random_features_params_reused(self, output_dim, initializer,
272                                              scale, trainable):
273    """Applying the layer on the same input twice gives the same output."""
274    rff_layer = kernel_layers.RandomFourierFeatures(
275        output_dim=output_dim,
276        kernel_initializer=initializer,
277        scale=scale,
278        trainable=trainable,
279        name='random_fourier_features')
280    inputs = constant_op.constant(
281        np.random.uniform(low=-1.0, high=1.0, size=(2, 4)))
282    output1 = rff_layer.apply(inputs)
283    output2 = rff_layer.apply(inputs)
284    self._assert_all_close(output1, output2)
285
286  @parameterized.named_parameters(
287      ('gaussian', 'gaussian', 5.0), ('laplacian', 'laplacian', 3.0),
288      ('other', init_ops.random_uniform_initializer(), 5.0))
289  @test_util.run_in_graph_and_eager_modes()
290  def test_different_params_similar_approximation(self, initializer, scale):
291    random_seed.set_random_seed(12345)
292    rff_layer1 = kernel_layers.RandomFourierFeatures(
293        output_dim=3000,
294        kernel_initializer=initializer,
295        scale=scale,
296        name='rff1')
297    rff_layer2 = kernel_layers.RandomFourierFeatures(
298        output_dim=2000,
299        kernel_initializer=initializer,
300        scale=scale,
301        name='rff2')
302    # Two distinct inputs.
303    x = constant_op.constant([[1.0, -1.0, 0.5]])
304    y = constant_op.constant([[-1.0, 1.0, 1.0]])
305
306    # Apply both layers to both inputs.
307    output_x1 = math.sqrt(2.0 / 3000.0) * rff_layer1.apply(x)
308    output_y1 = math.sqrt(2.0 / 3000.0) * rff_layer1.apply(y)
309    output_x2 = math.sqrt(2.0 / 2000.0) * rff_layer2.apply(x)
310    output_y2 = math.sqrt(2.0 / 2000.0) * rff_layer2.apply(y)
311
312    # Compute the inner products of the outputs (on inputs x and y) for both
313    # layers. For any fixed random features layer rff_layer, and inputs x, y,
314    # rff_layer(x)^T * rff_layer(y) ~= K(x,y) up to a normalization factor.
315    approx_kernel1 = kernelized_utils.inner_product(output_x1, output_y1)
316    approx_kernel2 = kernelized_utils.inner_product(output_x2, output_y2)
317    self._assert_all_close(approx_kernel1, approx_kernel2, atol=0.08)
318
319  @parameterized.named_parameters(
320      ('gaussian', 'gaussian', 5.0, _exact_gaussian(stddev=5.0)),
321      ('laplacian', 'laplacian', 20.0, _exact_laplacian(stddev=20.0)))
322  @test_util.run_in_graph_and_eager_modes()
323  def test_bad_kernel_approximation(self, initializer, scale, exact_kernel_fn):
324    """Approximation is bad when output dimension is small."""
325    # Two distinct inputs.
326    x = constant_op.constant([[1.0, -1.0, 0.5]])
327    y = constant_op.constant([[-1.0, 1.0, 1.0]])
328
329    small_output_dim = 10
330    random_seed.set_random_seed(1234)
331    # Initialize layer.
332    rff_layer = kernel_layers.RandomFourierFeatures(
333        output_dim=small_output_dim,
334        kernel_initializer=initializer,
335        scale=scale,
336        name='random_fourier_features')
337
338    # Apply layer to both inputs.
339    output_x = math.sqrt(2.0 / small_output_dim) * rff_layer.apply(x)
340    output_y = math.sqrt(2.0 / small_output_dim) * rff_layer.apply(y)
341
342    # The inner products of the outputs (on inputs x and y) approximates the
343    # real value of the RBF kernel but poorly since the output dimension of the
344    # layer is small.
345    exact_kernel_value = exact_kernel_fn(x, y)
346    approx_kernel_value = kernelized_utils.inner_product(output_x, output_y)
347    abs_error = math_ops.abs(exact_kernel_value - approx_kernel_value)
348    if not context.executing_eagerly():
349      with self.cached_session() as sess:
350        keras_backend._initialize_variables(sess)
351        abs_error_eval = sess.run([abs_error])
352        self.assertGreater(abs_error_eval[0][0], 0.05)
353        self.assertLess(abs_error_eval[0][0], 0.5)
354    else:
355      self.assertGreater(abs_error, 0.05)
356      self.assertLess(abs_error, 0.5)
357
358  @parameterized.named_parameters(
359      ('gaussian', 'gaussian', 10.0, _exact_gaussian(stddev=10.0)),
360      ('laplacian', 'laplacian', 50.0, _exact_laplacian(stddev=50.0)))
361  @test_util.run_in_graph_and_eager_modes()
362  def test_good_kernel_approximation_multiple_inputs(self, initializer, scale,
363                                                     exact_kernel_fn):
364    # Parameters.
365    input_dim = 5
366    output_dim = 5000
367    x_rows = 20
368    y_rows = 30
369
370    random_seed.set_random_seed(1234)
371    x = random_ops.random_uniform(shape=(x_rows, input_dim), maxval=1.0)
372    y = random_ops.random_uniform(shape=(y_rows, input_dim), maxval=1.0)
373
374    rff_layer = kernel_layers.RandomFourierFeatures(
375        output_dim=output_dim,
376        kernel_initializer=initializer,
377        scale=scale,
378        name='random_fourier_features')
379
380    # The shapes of output_x and output_y are (x_rows, output_dim) and
381    # (y_rows, output_dim) respectively.
382    output_x = math.sqrt(2.0 / output_dim) * rff_layer.apply(x)
383    output_y = math.sqrt(2.0 / output_dim) * rff_layer.apply(y)
384
385    approx_kernel_matrix = kernelized_utils.inner_product(output_x, output_y)
386    exact_kernel_matrix = exact_kernel_fn(x, y)
387    self._assert_all_close(approx_kernel_matrix, exact_kernel_matrix, atol=0.1)
388
389
390if __name__ == '__main__':
391  test.main()
392