1# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""Layers that act as activation functions.
16"""
17from __future__ import absolute_import
18from __future__ import division
19from __future__ import print_function
20
21from tensorflow.python.framework import dtypes
22from tensorflow.python.keras import backend as K
23from tensorflow.python.keras import constraints
24from tensorflow.python.keras import initializers
25from tensorflow.python.keras import regularizers
26from tensorflow.python.keras.engine.base_layer import Layer
27from tensorflow.python.keras.engine.input_spec import InputSpec
28from tensorflow.python.keras.utils import tf_utils
29from tensorflow.python.ops import math_ops
30from tensorflow.python.util.tf_export import keras_export
31
32
33def get_globals():
34  return globals()
35
36
37@keras_export('keras.layers.LeakyReLU')
38class LeakyReLU(Layer):
39  """Leaky version of a Rectified Linear Unit.
40
41  It allows a small gradient when the unit is not active:
42
43  ```
44    f(x) = alpha * x if x < 0
45    f(x) = x if x >= 0
46  ```
47
48  Usage:
49
50  >>> layer = tf.keras.layers.LeakyReLU()
51  >>> output = layer([-3.0, -1.0, 0.0, 2.0])
52  >>> list(output.numpy())
53  [-0.9, -0.3, 0.0, 2.0]
54  >>> layer = tf.keras.layers.LeakyReLU(alpha=0.1)
55  >>> output = layer([-3.0, -1.0, 0.0, 2.0])
56  >>> list(output.numpy())
57  [-0.3, -0.1, 0.0, 2.0]
58
59  Input shape:
60    Arbitrary. Use the keyword argument `input_shape`
61    (tuple of integers, does not include the batch axis)
62    when using this layer as the first layer in a model.
63
64  Output shape:
65    Same shape as the input.
66
67  Args:
68    alpha: Float >= 0. Negative slope coefficient. Default to 0.3.
69
70  """
71
72  def __init__(self, alpha=0.3, **kwargs):
73    super(LeakyReLU, self).__init__(**kwargs)
74    if alpha is None:
75      raise ValueError('alpha of leaky Relu layer '
76                       'cannot be None. Required a float')
77    self.supports_masking = True
78    self.alpha = K.cast_to_floatx(alpha)
79
80  def call(self, inputs):
81    return K.relu(inputs, alpha=self.alpha)
82
83  def get_config(self):
84    config = {'alpha': float(self.alpha)}
85    base_config = super(LeakyReLU, self).get_config()
86    return dict(list(base_config.items()) + list(config.items()))
87
88  @tf_utils.shape_type_conversion
89  def compute_output_shape(self, input_shape):
90    return input_shape
91
92
93@keras_export('keras.layers.PReLU')
94class PReLU(Layer):
95  """Parametric Rectified Linear Unit.
96
97  It follows:
98
99  ```
100    f(x) = alpha * x for x < 0
101    f(x) = x for x >= 0
102  ```
103
104  where `alpha` is a learned array with the same shape as x.
105
106  Input shape:
107    Arbitrary. Use the keyword argument `input_shape`
108    (tuple of integers, does not include the samples axis)
109    when using this layer as the first layer in a model.
110
111  Output shape:
112    Same shape as the input.
113
114  Args:
115    alpha_initializer: Initializer function for the weights.
116    alpha_regularizer: Regularizer for the weights.
117    alpha_constraint: Constraint for the weights.
118    shared_axes: The axes along which to share learnable
119      parameters for the activation function.
120      For example, if the incoming feature maps
121      are from a 2D convolution
122      with output shape `(batch, height, width, channels)`,
123      and you wish to share parameters across space
124      so that each filter only has one set of parameters,
125      set `shared_axes=[1, 2]`.
126  """
127
128  def __init__(self,
129               alpha_initializer='zeros',
130               alpha_regularizer=None,
131               alpha_constraint=None,
132               shared_axes=None,
133               **kwargs):
134    super(PReLU, self).__init__(**kwargs)
135    self.supports_masking = True
136    self.alpha_initializer = initializers.get(alpha_initializer)
137    self.alpha_regularizer = regularizers.get(alpha_regularizer)
138    self.alpha_constraint = constraints.get(alpha_constraint)
139    if shared_axes is None:
140      self.shared_axes = None
141    elif not isinstance(shared_axes, (list, tuple)):
142      self.shared_axes = [shared_axes]
143    else:
144      self.shared_axes = list(shared_axes)
145
146  @tf_utils.shape_type_conversion
147  def build(self, input_shape):
148    param_shape = list(input_shape[1:])
149    if self.shared_axes is not None:
150      for i in self.shared_axes:
151        param_shape[i - 1] = 1
152    self.alpha = self.add_weight(
153        shape=param_shape,
154        name='alpha',
155        initializer=self.alpha_initializer,
156        regularizer=self.alpha_regularizer,
157        constraint=self.alpha_constraint)
158    # Set input spec
159    axes = {}
160    if self.shared_axes:
161      for i in range(1, len(input_shape)):
162        if i not in self.shared_axes:
163          axes[i] = input_shape[i]
164    self.input_spec = InputSpec(ndim=len(input_shape), axes=axes)
165    self.built = True
166
167  def call(self, inputs):
168    pos = K.relu(inputs)
169    neg = -self.alpha * K.relu(-inputs)
170    return pos + neg
171
172  def get_config(self):
173    config = {
174        'alpha_initializer': initializers.serialize(self.alpha_initializer),
175        'alpha_regularizer': regularizers.serialize(self.alpha_regularizer),
176        'alpha_constraint': constraints.serialize(self.alpha_constraint),
177        'shared_axes': self.shared_axes
178    }
179    base_config = super(PReLU, self).get_config()
180    return dict(list(base_config.items()) + list(config.items()))
181
182  @tf_utils.shape_type_conversion
183  def compute_output_shape(self, input_shape):
184    return input_shape
185
186
187@keras_export('keras.layers.ELU')
188class ELU(Layer):
189  """Exponential Linear Unit.
190
191  It follows:
192
193  ```
194    f(x) =  alpha * (exp(x) - 1.) for x < 0
195    f(x) = x for x >= 0
196  ```
197
198  Input shape:
199    Arbitrary. Use the keyword argument `input_shape`
200    (tuple of integers, does not include the samples axis)
201    when using this layer as the first layer in a model.
202
203  Output shape:
204    Same shape as the input.
205
206  Args:
207    alpha: Scale for the negative factor.
208  """
209
210  def __init__(self, alpha=1.0, **kwargs):
211    super(ELU, self).__init__(**kwargs)
212    if alpha is None:
213      raise ValueError('alpha of ELU layer ' 'cannot be None. Required a float')
214    self.supports_masking = True
215    self.alpha = K.cast_to_floatx(alpha)
216
217  def call(self, inputs):
218    return K.elu(inputs, self.alpha)
219
220  def get_config(self):
221    config = {'alpha': float(self.alpha)}
222    base_config = super(ELU, self).get_config()
223    return dict(list(base_config.items()) + list(config.items()))
224
225  @tf_utils.shape_type_conversion
226  def compute_output_shape(self, input_shape):
227    return input_shape
228
229
230@keras_export('keras.layers.ThresholdedReLU')
231class ThresholdedReLU(Layer):
232  """Thresholded Rectified Linear Unit.
233
234  It follows:
235
236  ```
237    f(x) = x for x > theta
238    f(x) = 0 otherwise`
239  ```
240
241  Input shape:
242    Arbitrary. Use the keyword argument `input_shape`
243    (tuple of integers, does not include the samples axis)
244    when using this layer as the first layer in a model.
245
246  Output shape:
247    Same shape as the input.
248
249  Args:
250    theta: Float >= 0. Threshold location of activation.
251  """
252
253  def __init__(self, theta=1.0, **kwargs):
254    super(ThresholdedReLU, self).__init__(**kwargs)
255    self.supports_masking = True
256    self.theta = K.cast_to_floatx(theta)
257
258  def call(self, inputs):
259    theta = math_ops.cast(self.theta, inputs.dtype)
260    return inputs * math_ops.cast(math_ops.greater(inputs, theta), inputs.dtype)
261
262  def get_config(self):
263    config = {'theta': float(self.theta)}
264    base_config = super(ThresholdedReLU, self).get_config()
265    return dict(list(base_config.items()) + list(config.items()))
266
267  @tf_utils.shape_type_conversion
268  def compute_output_shape(self, input_shape):
269    return input_shape
270
271
272def _large_compatible_negative(tensor_type):
273  """Large negative number as Tensor.
274
275  This function is necessary because the standard value for epsilon
276  in this module (-1e9) cannot be represented using tf.float16
277
278  Args:
279    tensor_type: a dtype to determine the type.
280
281  Returns:
282    a large negative number.
283  """
284  if tensor_type == dtypes.float16:
285    return dtypes.float16.min
286  return -1e9
287
288
289@keras_export('keras.layers.Softmax')
290class Softmax(Layer):
291  """Softmax activation function.
292
293  Example without mask:
294
295  >>> inp = np.asarray([1., 2., 1.])
296  >>> layer = tf.keras.layers.Softmax()
297  >>> layer(inp).numpy()
298  array([0.21194157, 0.5761169 , 0.21194157], dtype=float32)
299  >>> mask = np.asarray([True, False, True], dtype=bool)
300  >>> layer(inp, mask).numpy()
301  array([0.5, 0. , 0.5], dtype=float32)
302
303  Input shape:
304    Arbitrary. Use the keyword argument `input_shape`
305    (tuple of integers, does not include the samples axis)
306    when using this layer as the first layer in a model.
307
308  Output shape:
309    Same shape as the input.
310
311  Args:
312    axis: Integer, or list of Integers, axis along which the softmax
313      normalization is applied.
314  Call arguments:
315    inputs: The inputs, or logits to the softmax layer.
316    mask: A boolean mask of the same shape as `inputs`. Defaults to `None`. The
317      mask specifies 1 to keep and 0 to mask.
318
319  Returns:
320    softmaxed output with the same shape as `inputs`.
321  """
322
323  def __init__(self, axis=-1, **kwargs):
324    super(Softmax, self).__init__(**kwargs)
325    self.supports_masking = True
326    self.axis = axis
327
328  def call(self, inputs, mask=None):
329    if mask is not None:
330      # Since mask is 1.0 for positions we want to keep and 0.0 for
331      # masked positions, this operation will create a tensor which is 0.0 for
332      # positions we want to attend and -1e.9 for masked positions.
333      adder = (1.0 - math_ops.cast(mask, inputs.dtype)) * (
334          _large_compatible_negative(inputs.dtype))
335
336      # Since we are adding it to the raw scores before the softmax, this is
337      # effectively the same as removing these entirely.
338      inputs += adder
339    if isinstance(self.axis, (tuple, list)):
340      if len(self.axis) > 1:
341        return math_ops.exp(inputs - math_ops.reduce_logsumexp(
342            inputs, axis=self.axis, keepdims=True))
343      else:
344        return K.softmax(inputs, axis=self.axis[0])
345    return K.softmax(inputs, axis=self.axis)
346
347  def get_config(self):
348    config = {'axis': self.axis}
349    base_config = super(Softmax, self).get_config()
350    return dict(list(base_config.items()) + list(config.items()))
351
352  @tf_utils.shape_type_conversion
353  def compute_output_shape(self, input_shape):
354    return input_shape
355
356
357@keras_export('keras.layers.ReLU')
358class ReLU(Layer):
359  """Rectified Linear Unit activation function.
360
361  With default values, it returns element-wise `max(x, 0)`.
362
363  Otherwise, it follows:
364
365  ```
366    f(x) = max_value if x >= max_value
367    f(x) = x if threshold <= x < max_value
368    f(x) = negative_slope * (x - threshold) otherwise
369  ```
370
371  Usage:
372
373  >>> layer = tf.keras.layers.ReLU()
374  >>> output = layer([-3.0, -1.0, 0.0, 2.0])
375  >>> list(output.numpy())
376  [0.0, 0.0, 0.0, 2.0]
377  >>> layer = tf.keras.layers.ReLU(max_value=1.0)
378  >>> output = layer([-3.0, -1.0, 0.0, 2.0])
379  >>> list(output.numpy())
380  [0.0, 0.0, 0.0, 1.0]
381  >>> layer = tf.keras.layers.ReLU(negative_slope=1.0)
382  >>> output = layer([-3.0, -1.0, 0.0, 2.0])
383  >>> list(output.numpy())
384  [-3.0, -1.0, 0.0, 2.0]
385  >>> layer = tf.keras.layers.ReLU(threshold=1.5)
386  >>> output = layer([-3.0, -1.0, 1.0, 2.0])
387  >>> list(output.numpy())
388  [0.0, 0.0, 0.0, 2.0]
389
390  Input shape:
391    Arbitrary. Use the keyword argument `input_shape`
392    (tuple of integers, does not include the batch axis)
393    when using this layer as the first layer in a model.
394
395  Output shape:
396    Same shape as the input.
397
398  Args:
399    max_value: Float >= 0. Maximum activation value. Default to None, which
400      means unlimited.
401    negative_slope: Float >= 0. Negative slope coefficient. Default to 0.
402    threshold: Float. Threshold value for thresholded activation. Default to 0.
403  """
404
405  def __init__(self, max_value=None, negative_slope=0, threshold=0, **kwargs):
406    super(ReLU, self).__init__(**kwargs)
407    if max_value is not None and max_value < 0.:
408      raise ValueError('max_value of Relu layer '
409                       'cannot be negative value: ' + str(max_value))
410    if negative_slope < 0.:
411      raise ValueError('negative_slope of Relu layer '
412                       'cannot be negative value: ' + str(negative_slope))
413    if threshold is None:
414      raise ValueError('threshold of Relu layer '
415                       'cannot be None. Required a float')
416
417    self.supports_masking = True
418    if max_value is not None:
419      max_value = K.cast_to_floatx(max_value)
420    self.max_value = max_value
421    self.negative_slope = K.cast_to_floatx(negative_slope)
422    self.threshold = K.cast_to_floatx(threshold)
423
424  def call(self, inputs):
425    # alpha is used for leaky relu slope in activations instead of
426    # negative_slope.
427    return K.relu(inputs,
428                  alpha=self.negative_slope,
429                  max_value=self.max_value,
430                  threshold=self.threshold)
431
432  def get_config(self):
433    config = {
434        'max_value': self.max_value,
435        'negative_slope': self.negative_slope,
436        'threshold': self.threshold
437    }
438    base_config = super(ReLU, self).get_config()
439    return dict(list(base_config.items()) + list(config.items()))
440
441  @tf_utils.shape_type_conversion
442  def compute_output_shape(self, input_shape):
443    return input_shape
444