1# Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Tests for miscellaneous functionality in tensorflow.ops.nn.""" 16 17from __future__ import absolute_import 18from __future__ import division 19from __future__ import print_function 20 21import math 22 23from absl.testing import parameterized 24import numpy as np 25from six.moves import xrange # pylint: disable=redefined-builtin 26 27from tensorflow.python.eager import def_function 28from tensorflow.python.framework import constant_op 29from tensorflow.python.framework import dtypes 30from tensorflow.python.framework import errors 31from tensorflow.python.framework import ops 32from tensorflow.python.framework import tensor_spec 33from tensorflow.python.framework import test_util 34from tensorflow.python.ops import array_ops 35from tensorflow.python.ops import gradient_checker 36from tensorflow.python.ops import gradient_checker_v2 37from tensorflow.python.ops import math_ops 38from tensorflow.python.ops import nn 39from tensorflow.python.ops import nn_impl 40from tensorflow.python.ops import nn_ops 41from tensorflow.python.ops import partitioned_variables 42from tensorflow.python.ops import variable_scope 43from tensorflow.python.ops import variables 44import tensorflow.python.ops.nn_grad # pylint: disable=unused-import 45from tensorflow.python.ops.nn_impl import _compute_sampled_logits 46from tensorflow.python.ops.ragged import ragged_factory_ops 47from tensorflow.python.platform import test as test_lib 48 49 50class ZeroFractionTest(test_lib.TestCase): 51 52 def _ZeroFraction(self, x): 53 assert x.shape 54 total_elements = np.prod(x.shape) 55 nonzeros = np.count_nonzero(x.flatten()) 56 return 1.0 - nonzeros / total_elements 57 58 @test_util.run_deprecated_v1 59 def testZeroFraction(self): 60 x_shape = [5, 17] 61 x_np = np.random.randint(0, 2, size=x_shape).astype(np.float32) 62 y_np = self._ZeroFraction(x_np) 63 64 x_tf = constant_op.constant(x_np) 65 x_tf.set_shape(x_shape) 66 y_tf = nn_impl.zero_fraction(x_tf) 67 y_tf_np = self.evaluate(y_tf) 68 69 eps = 1e-8 70 self.assertAllClose(y_tf_np, y_np, eps) 71 72 @test_util.run_deprecated_v1 73 def testZeroFractionEmpty(self): 74 x = np.zeros(0) 75 y = self.evaluate(nn_impl.zero_fraction(x)) 76 self.assertTrue(np.isnan(y)) 77 78 @test_util.run_deprecated_v1 79 def testZeroFraction2_27Zeros(self): 80 sparsity = nn_impl.zero_fraction( 81 array_ops.zeros([int(2**27 * 1.01)], dtype=dtypes.int8)) 82 self.assertAllClose(1.0, self.evaluate(sparsity)) 83 84 @test_util.run_deprecated_v1 85 def testZeroFraction2_27Ones(self): 86 sparsity = nn_impl.zero_fraction( 87 array_ops.ones([int(2**27 * 1.01)], dtype=dtypes.int8)) 88 self.assertAllClose(0.0, self.evaluate(sparsity)) 89 90 @test_util.run_deprecated_v1 91 def testUnknownSize(self): 92 value = array_ops.placeholder(dtype=dtypes.float32) 93 sparsity = nn_impl.zero_fraction(value) 94 with self.cached_session() as sess: 95 self.assertAllClose( 96 0.25, 97 sess.run(sparsity, {value: [[0., 1.], [0.3, 2.]]})) 98 99 100class SoftmaxTest(test_lib.TestCase, parameterized.TestCase): 101 102 def _softmax(self, x): 103 assert len(x.shape) == 2 104 m = x.max(1)[:, np.newaxis] 105 u = np.exp(x - m) 106 z = u.sum(1)[:, np.newaxis] 107 return u / z 108 109 @test_util.run_in_graph_and_eager_modes 110 def testSoftmax(self): 111 x_shape = [5, 10] 112 x_np = np.random.randn(*x_shape).astype(np.float32) 113 y_np = self._softmax(x_np) 114 x_tf = constant_op.constant(x_np) 115 y_tf = nn_ops.softmax_v2(x_tf) 116 y_tf_last_dim = nn_ops.softmax_v2(x_tf, 1) 117 y_tf_np = self.evaluate(y_tf) 118 y_tf_last_dim_np = self.evaluate(y_tf_last_dim) 119 eps = 1e-3 120 self.assertAllClose(y_tf_np, y_np, eps) 121 self.assertAllClose(y_tf_last_dim_np, y_np, eps) 122 123 def testSoftmaxAxes(self): 124 arr = np.linspace(0., 1, 12).reshape(3, 4) 125 x_neg_axis = nn_ops.softmax_v2(arr, axis=-2) 126 y_pos_axis = nn_ops.softmax_v2(arr, axis=0) 127 z_gt_axis = nn_ops.softmax_v2(arr, axis=0) 128 x_neg_axis_tf = self.evaluate(x_neg_axis) 129 y_pos_axis_tf = self.evaluate(y_pos_axis) 130 z_gt_axis_tf = self.evaluate(z_gt_axis) 131 eps = 1e-3 132 self.assertAllClose(x_neg_axis_tf, y_pos_axis_tf, eps) 133 self.assertAllClose(y_pos_axis_tf, z_gt_axis_tf, eps) 134 135 def testSoftmaxExtendType(self): 136 x_shape = [5, 10] 137 x_np = np.random.randn(*x_shape).astype(np.float32) 138 139 x_f32_tf = constant_op.constant(x_np) 140 x_bf16_tf = math_ops.cast(x_f32_tf, dtypes.bfloat16) 141 y_f32_tf = self.evaluate(nn_ops.softmax(x_f32_tf)) 142 y_bf16_tf = self.evaluate(nn_ops.softmax(x_bf16_tf)) 143 expected = math_ops.cast(y_f32_tf, dtypes.bfloat16) 144 tol = x_shape[1] * 1e-3 145 self.assertAllClose(y_bf16_tf, expected, rtol=tol, atol=tol) 146 147 @parameterized.parameters(((5, 10),), ((2, 3, 4),)) 148 @test_util.run_deprecated_v1 149 def testGradient(self, x_shape): 150 x_np = np.random.randn(*x_shape).astype(np.float64) 151 with self.cached_session(): 152 x_tf = constant_op.constant(x_np) 153 y_tf = nn_ops.softmax_v2(x_tf) 154 err = gradient_checker.compute_gradient_error(x_tf, x_shape, y_tf, 155 x_shape) 156 eps = 2e-8 157 self.assertLess(err, eps) 158 159 160class LogPoissonLossTest(test_lib.TestCase): 161 162 def _log_poisson_loss(self, x, z, compute_full_loss=False): 163 lpl = np.exp(x) - z * x 164 if compute_full_loss: 165 stirling_approx = z * np.log(z) - z + 0.5 * np.log(2. * np.pi * z) 166 lpl += np.ma.masked_array(stirling_approx, mask=(z <= 1)).filled(0.) 167 return lpl 168 169 @test_util.run_in_graph_and_eager_modes 170 def testLogPoissonLoss(self): 171 x_shape = [5, 10] 172 x_np = np.random.randn(*x_shape).astype(np.float32) 173 z_np = np.random.randint(0, 5, size=x_shape).astype(np.float32) 174 y_np = self._log_poisson_loss(x_np, z_np, compute_full_loss=False) 175 y_np_stirling = self._log_poisson_loss(x_np, z_np, compute_full_loss=True) 176 y_tf = nn_impl.log_poisson_loss(z_np, x_np, compute_full_loss=False) 177 y_tf_stirling = nn_impl.log_poisson_loss(z_np, x_np, compute_full_loss=True) 178 y_tf_np = self.evaluate(y_tf) 179 y_tf_np_stirling = self.evaluate(y_tf_stirling) 180 eps = 1e-3 181 self.assertAllClose(y_tf_np, y_np, eps) 182 self.assertAllClose(y_tf_np_stirling, y_np_stirling, eps) 183 184 @test_util.run_deprecated_v1 185 def testGradient(self): 186 x_shape = [5, 10] 187 x_np = np.random.randn(*x_shape).astype(np.float64) 188 z_np = np.random.randint(0, 5, size=x_shape).astype(np.float64) 189 with self.cached_session(): 190 x_tf = constant_op.constant(x_np) 191 y_tf = nn_impl.log_poisson_loss(z_np, x_tf, compute_full_loss=False) 192 y_tf_stirling = nn_impl.log_poisson_loss( 193 z_np, x_tf, compute_full_loss=True) 194 err = gradient_checker.compute_gradient_error(x_tf, x_shape, y_tf, 195 x_shape) 196 err_stirling = gradient_checker.compute_gradient_error( 197 x_tf, x_shape, y_tf_stirling, x_shape) 198 eps = 1e-6 199 self.assertLess(err, eps) 200 self.assertLess(err_stirling, eps) 201 202 203class LogSoftmaxTest(test_lib.TestCase, parameterized.TestCase): 204 205 def _log_softmax(self, x): 206 assert len(x.shape) == 2 207 m = x.max(1)[:, np.newaxis] 208 u = x - m 209 return u - np.log(np.sum(np.exp(u), 1, keepdims=True)) 210 211 @test_util.run_in_graph_and_eager_modes 212 def testLogSoftmax(self): 213 x_shape = [5, 10] 214 x_np = np.random.randn(*x_shape).astype(np.float32) 215 y_np = self._log_softmax(x_np) 216 x_tf = constant_op.constant(x_np) 217 y_tf = nn_ops.log_softmax_v2(x_tf) 218 y_tf_np = self.evaluate(y_tf) 219 eps = 1e-3 220 self.assertAllClose(y_tf_np, y_np, eps) 221 222 def testLogSoftmaxAxes(self): 223 arr = np.linspace(0., 1, 12).reshape(3, 4) 224 x_neg_axis = nn_ops.log_softmax_v2(arr, axis=-2) 225 y_pos_axis = nn_ops.log_softmax_v2(arr, axis=0) 226 z_gt_axis = nn_ops.log_softmax_v2(arr, axis=0) 227 x_neg_axis_tf = self.evaluate(x_neg_axis) 228 y_pos_axis_tf = self.evaluate(y_pos_axis) 229 z_gt_axis_tf = self.evaluate(z_gt_axis) 230 eps = 1e-3 231 self.assertAllClose(x_neg_axis_tf, y_pos_axis_tf, eps) 232 self.assertAllClose(y_pos_axis_tf, z_gt_axis_tf, eps) 233 234 @parameterized.parameters(((5, 10),), ((2, 3, 4),)) 235 @test_util.run_deprecated_v1 236 def testGradient(self, x_shape): 237 x_np = np.random.randn(*x_shape).astype(np.float64) 238 with self.cached_session(): 239 x_tf = constant_op.constant(x_np) 240 y_tf = nn_ops.log_softmax_v2(x_tf) 241 err = gradient_checker.compute_gradient_error(x_tf, x_shape, y_tf, 242 x_shape) 243 eps = 1e-7 244 self.assertLess(err, eps) 245 246 247class L2LossTest(test_lib.TestCase): 248 249 @test_util.run_in_graph_and_eager_modes 250 def testL2Loss(self): 251 for dtype in [dtypes.float32, dtypes.float64]: 252 x = constant_op.constant( 253 [1.0, 0.0, 3.0, 2.0], shape=[2, 2], name="x", dtype=dtype) 254 l2loss = nn_ops.l2_loss(x) 255 value = self.evaluate(l2loss) 256 self.assertAllClose(7.0, value) 257 258 @test_util.run_deprecated_v1 259 def testGradient(self): 260 x_shape = [20, 7, 3] 261 np.random.seed(1) # Make it reproducible. 262 x_val = np.random.random_sample(x_shape).astype(np.float64) 263 with self.cached_session(): 264 x = constant_op.constant(x_val, name="x") 265 output = nn_ops.l2_loss(x) 266 err = gradient_checker.compute_gradient_error(x, x_shape, output, [1]) 267 print("L2Loss gradient err = %g " % err) 268 err_tolerance = 1e-10 269 self.assertLess(err, err_tolerance) 270 271 272class L2NormalizeTest(test_lib.TestCase): 273 274 def _l2Normalize(self, x, dim): 275 if isinstance(dim, list): 276 norm = np.linalg.norm(x, axis=tuple(dim)) 277 for d in dim: 278 norm = np.expand_dims(norm, d) 279 return x / norm 280 else: 281 norm = np.apply_along_axis(np.linalg.norm, dim, x) 282 return x / np.expand_dims(norm, dim) 283 284 @test_util.run_in_graph_and_eager_modes 285 def testL2Normalize(self): 286 x_shape = [20, 7, 3] 287 np.random.seed(1) 288 x_np = np.random.random_sample(x_shape).astype(np.float32) 289 for dim in range(len(x_shape)): 290 y_np = self._l2Normalize(x_np, dim) 291 x_tf = constant_op.constant(x_np, name="x") 292 y_tf = nn_impl.l2_normalize_v2(x_tf, dim) 293 self.assertAllClose(y_np, self.evaluate(y_tf)) 294 295 @test_util.run_in_graph_and_eager_modes 296 def testL2NormalizeDimArray(self): 297 x_shape = [20, 7, 3] 298 np.random.seed(1) 299 x_np = np.random.random_sample(x_shape).astype(np.float32) 300 dim = [1, 2] 301 y_np = self._l2Normalize(x_np, dim) 302 x_tf = constant_op.constant(x_np, name="x") 303 y_tf = nn_impl.l2_normalize_v2(x_tf, dim) 304 self.assertAllClose(y_np, self.evaluate(y_tf)) 305 306 @test_util.run_deprecated_v1 307 def testL2NormalizeGradient(self): 308 x_shape = [20, 7, 3] 309 np.random.seed(1) 310 x_np = np.random.random_sample(x_shape).astype(np.float64) 311 for dim in range(len(x_shape)): 312 with self.cached_session(): 313 x_tf = constant_op.constant(x_np, name="x") 314 y_tf = nn_impl.l2_normalize_v2(x_tf, dim) 315 err = gradient_checker.compute_gradient_error(x_tf, x_shape, y_tf, 316 x_shape) 317 print("L2Normalize gradient err = %g " % err) 318 self.assertLess(err, 1e-4) 319 320 @test_util.run_in_graph_and_eager_modes 321 def testL2NormalizeComplex(self): 322 x_shape = [20, 7, 3] 323 for dtype in [np.complex64, np.complex128]: 324 np.random.seed(1) 325 x_np = ( 326 np.random.random_sample(x_shape).astype(dtype) + 327 np.random.random_sample(x_shape).astype(dtype) * 1j) 328 for dim in range(len(x_shape)): 329 y_np = self._l2Normalize(x_np, dim) 330 x_tf = constant_op.constant(x_np, name="x") 331 y_tf = nn_impl.l2_normalize_v2(x_tf, dim) 332 self.assertAllClose(y_np, self.evaluate(y_tf)) 333 334 335class DropoutTest(test_lib.TestCase): 336 337 def testDropout(self): 338 # Runs dropout with 0-1 tensor 10 times, sum the number of ones and validate 339 # that it is producing approximately the right number of ones over a large 340 # number of samples, based on the keep probability. 341 x_dim = 40 342 y_dim = 30 343 num_iter = 10 344 for keep_prob in [0.1, 0.5, 0.8]: 345 t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) 346 dropout = nn_ops.dropout(t, rate=(1 - keep_prob)) 347 final_count = 0 348 self.assertEqual([x_dim, y_dim], dropout.get_shape()) 349 for _ in xrange(0, num_iter): 350 value = self.evaluate(dropout) 351 final_count += np.count_nonzero(value) 352 # Verifies that there are only two values: 0 and 1/keep_prob. 353 sorted_value = np.unique(np.sort(value)) 354 self.assertEqual(0, sorted_value[0]) 355 self.assertAllClose(1 / keep_prob, sorted_value[1]) 356 357 # Check that we are in the 15% error range 358 expected_count = x_dim * y_dim * keep_prob * num_iter 359 rel_error = math.fabs(final_count - expected_count) / expected_count 360 print(rel_error) 361 self.assertTrue(rel_error < 0.15) 362 363 def testShapedDropout(self): 364 # Runs dropout with 0-1 tensor 10 times, sum the number of ones and validate 365 # that it is producing approximately the right number of ones over a large 366 # number of samples, based on the keep probability. This time with shaped 367 # noise. 368 x_dim = 40 * 30 369 y_dim = 3 370 num_iter = 10 371 for keep_prob in [0.1, 0.5, 0.8]: 372 t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) 373 dropout = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim, 1]) 374 self.assertEqual([x_dim, y_dim], dropout.get_shape()) 375 final_count = 0 376 for _ in xrange(0, num_iter): 377 value = self.evaluate(dropout) 378 final_count += np.count_nonzero(value) 379 # Verifies that there are only two values: 0 and 1/keep_prob. 380 sorted_value = np.unique(np.sort(value)) 381 self.assertEqual(0, sorted_value[0]) 382 self.assertAllClose(1 / keep_prob, sorted_value[1]) 383 384 # Check that we are in the 15% error range 385 expected_count = x_dim * y_dim * keep_prob * num_iter 386 rel_error = math.fabs(final_count - expected_count) / expected_count 387 print(rel_error) 388 self.assertTrue(rel_error < 0.15) 389 390 def testShapedDropoutCorrelation(self): 391 # Runs a shaped dropout and tests that the correlations are correct. 392 x_dim = 40 393 y_dim = 30 394 num_iter = 10 395 for keep_prob in [0.1, 0.5, 0.8]: 396 t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) 397 dropout = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim, 1]) 398 self.assertEqual([x_dim, y_dim], dropout.get_shape()) 399 for _ in xrange(0, num_iter): 400 value = self.evaluate(dropout) 401 # Verifies that each y column as only one type of activation. 402 for i in xrange(x_dim): 403 sorted_value = np.unique(np.sort(value[i, :])) 404 self.assertEqual(sorted_value.size, 1) 405 406 @test_util.run_deprecated_v1 407 def testDropoutPlaceholderKeepProb(self): 408 # Runs dropout with 0-1 tensor 10 times, sum the number of ones and validate 409 # that it is producing approximately the right number of ones over a large 410 # number of samples, based on the keep probability. 411 x_dim = 40 412 y_dim = 30 413 num_iter = 10 414 for keep_prob in [0.1, 0.5, 0.8]: 415 with self.cached_session(): 416 t = constant_op.constant( 417 1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) 418 keep_prob_placeholder = array_ops.placeholder(dtypes.float32) 419 dropout = nn_ops.dropout(t, keep_prob_placeholder) 420 final_count = 0 421 self.assertEqual([x_dim, y_dim], dropout.get_shape()) 422 for _ in xrange(0, num_iter): 423 value = dropout.eval(feed_dict={keep_prob_placeholder: keep_prob}) 424 final_count += np.count_nonzero(value) 425 # Verifies that there are only two values: 0 and 1/keep_prob. 426 sorted_value = np.unique(np.sort(value)) 427 self.assertEqual(0, sorted_value[0]) 428 self.assertAllClose(1 / keep_prob, sorted_value[1]) 429 # Check that we are in the 15% error range 430 expected_count = x_dim * y_dim * keep_prob * num_iter 431 rel_error = math.fabs(final_count - expected_count) / expected_count 432 print(rel_error) 433 self.assertTrue(rel_error < 0.15) 434 435 @test_util.run_deprecated_v1 436 def testShapedDropoutUnknownShape(self): 437 x_dim = 40 438 y_dim = 30 439 keep_prob = 0.5 440 x = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) 441 dropout_x = nn_ops.dropout( 442 x, 443 rate=(1 - keep_prob), 444 noise_shape=array_ops.placeholder(dtypes.int32)) 445 self.assertEqual(x.get_shape(), dropout_x.get_shape()) 446 447 def testPartialShapedDropout(self): 448 x_dim = 40 * 30 449 y_dim = 3 450 num_iter = 10 451 for keep_prob in [0.1, 0.5, 0.8]: 452 t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) 453 # Set noise_shape=[None, 1] which means [x_dim, 1]. 454 dropout = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[None, 1]) 455 self.assertEqual([x_dim, y_dim], dropout.get_shape()) 456 final_count = 0 457 for _ in xrange(0, num_iter): 458 value = self.evaluate(dropout) 459 final_count += np.count_nonzero(value) 460 # Verifies that there are only two values: 0 and 1/keep_prob. 461 sorted_value = np.unique(np.sort(value)) 462 self.assertEqual(0, sorted_value[0]) 463 self.assertAllClose(1 / keep_prob, sorted_value[1]) 464 465 # Check that we are in the 15% error range 466 expected_count = x_dim * y_dim * keep_prob * num_iter 467 rel_error = math.fabs(final_count - expected_count) / expected_count 468 print(rel_error) 469 self.assertTrue(rel_error < 0.15) 470 471 @test_util.run_deprecated_v1 472 def testInvalidKeepProb(self): 473 x_dim = 40 474 y_dim = 30 475 t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) 476 with self.assertRaises(ValueError): 477 nn_ops.dropout(t, -1.0) 478 with self.assertRaises(ValueError): 479 nn_ops.dropout(t, 1.1) 480 with self.assertRaises(ValueError): 481 nn_ops.dropout(t, [0.0, 1.0]) 482 with self.assertRaises(ValueError): 483 nn_ops.dropout(t, array_ops.placeholder(dtypes.float64)) 484 with self.assertRaises(ValueError): 485 nn_ops.dropout(t, array_ops.placeholder(dtypes.float32, shape=[2])) 486 487 @test_util.run_deprecated_v1 488 def testInvalidRate(self): 489 x_dim = 40 490 y_dim = 30 491 t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) 492 with self.assertRaises(ValueError): 493 nn_ops.dropout_v2(t, -1.0) 494 with self.assertRaises(ValueError): 495 nn_ops.dropout_v2(t, 1.1) 496 with self.assertRaises(ValueError): 497 nn_ops.dropout_v2(t, [0.0, 1.0]) 498 499 def testLargeRate(self): 500 x_dim = 40 501 y_dim = 30 502 t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) 503 _ = nn_ops.dropout_v2(t, 0.9) 504 505 def testVariableRef(self): 506 x = variable_scope.get_variable("x", shape=[10, 10], dtype=dtypes.float32) 507 _ = nn_ops.dropout(x, keep_prob=0.1) 508 509 @test_util.run_deprecated_v1 510 def testShapedDropoutShapeError(self): 511 # Runs shaped dropout and verifies an error is thrown on misshapen noise. 512 x_dim = 40 513 y_dim = 30 514 keep_prob = 0.5 515 t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) 516 with self.assertRaises(ValueError): 517 _ = nn_ops.dropout( 518 t, rate=(1 - keep_prob), noise_shape=[x_dim, y_dim + 10]) 519 with self.assertRaises(ValueError): 520 _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim, y_dim, 5]) 521 with self.assertRaises(ValueError): 522 _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim + 3]) 523 with self.assertRaises(ValueError): 524 _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim]) 525 # test that broadcasting proceeds 526 _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[y_dim]) 527 _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[1, y_dim]) 528 _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim, 1]) 529 _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[1, 1]) 530 531 def testNoDropout(self): 532 x = array_ops.zeros((5,)) 533 y = nn_ops.dropout(x, rate=0) 534 self.assertAllEqual(x, y) 535 536 y = nn_ops.dropout_v2(x, rate=0) 537 self.assertAllEqual(x, y) 538 539 def testDropoutWithIntegerInputs(self): 540 x = constant_op.constant([1, 1, 1, 1, 1]) 541 with self.assertRaises(ValueError): 542 _ = nn_ops.dropout(x, 0.5) 543 544 545@test_util.run_all_without_tensor_float_32( 546 "Tests _compute_sampled_logits and related functions, which call matmul") 547class ComputeSampledLogitsTest(test_lib.TestCase): 548 549 def setUp(self): 550 self._eps = 1e-3 551 552 def _GenerateTestData(self, num_classes, dim, batch_size, num_true, labels, 553 sampled, subtract_log_q): 554 """Randomly generates input/output data for a single test case. 555 556 This function returns numpy constants for use in a test case. 557 558 Args: 559 num_classes: An int. The number of embedding classes in the test case. 560 dim: An int. The dimension of the embedding. 561 batch_size: An int. The batch size. 562 num_true: An int. The number of target classes per training example. 563 labels: A list of batch_size * num_true ints. The target classes. 564 sampled: A list of indices in [0, num_classes). 565 subtract_log_q: A bool corresponding to the parameter in 566 _compute_sampled_logits(). 567 568 Returns: 569 weights: Embedding weights to use as test input. It is a numpy array 570 of shape [num_classes, dim] 571 biases: Embedding biases to use as test input. It is a numpy array 572 of shape [num_classes]. 573 hidden_acts: Forward activations of the network to use as test input. 574 It is a numpy array of shape [batch_size, dim]. 575 sampled_vals: A tuple based on `sampled` to use as test input in the 576 format returned by a *_candidate_sampler function. 577 exp_logits: The output logits expected from _compute_sampled_logits(). 578 It is a numpy array of shape [batch_size, num_true + len(sampled)]. 579 exp_labels: The output labels expected from _compute_sampled_logits(). 580 It is a numpy array of shape [batch_size, num_true + len(sampled)]. 581 """ 582 weights = np.random.randn(num_classes, dim).astype(np.float32) 583 biases = np.random.randn(num_classes).astype(np.float32) 584 hidden_acts = np.random.randn(batch_size, dim).astype(np.float32) 585 586 true_exp = np.full([batch_size, 1], fill_value=0.5, dtype=np.float32) 587 sampled_exp = np.full([len(sampled)], fill_value=0.5, dtype=np.float32) 588 sampled_vals = (sampled, true_exp, sampled_exp) 589 590 sampled_w, sampled_b = weights[sampled], biases[sampled] 591 true_w, true_b = weights[labels], biases[labels] 592 593 true_logits = np.sum( 594 hidden_acts.reshape((batch_size, 1, dim)) * true_w.reshape( 595 (batch_size, num_true, dim)), 596 axis=2) 597 true_b = true_b.reshape((batch_size, num_true)) 598 true_logits += true_b 599 sampled_logits = np.dot(hidden_acts, sampled_w.T) + sampled_b 600 601 if subtract_log_q: 602 true_logits -= np.log(true_exp) 603 sampled_logits -= np.log(sampled_exp[np.newaxis, :]) 604 605 exp_logits = np.concatenate([true_logits, sampled_logits], axis=1) 606 exp_labels = np.hstack((np.ones_like(true_logits) / num_true, 607 np.zeros_like(sampled_logits))) 608 609 return weights, biases, hidden_acts, sampled_vals, exp_logits, exp_labels 610 611 def _ShardTestEmbeddings(self, weights, biases, num_shards): 612 """Shards the weights and biases returned by _GenerateTestData. 613 614 Args: 615 weights: The weights returned by _GenerateTestData. 616 biases: The biases returned by _GenerateTestData. 617 num_shards: The number of shards to create. 618 619 Returns: 620 sharded_weights: A list of size `num_shards` containing all the weights. 621 sharded_biases: A list of size `num_shards` containing all the biases. 622 """ 623 with ops.Graph().as_default() as g: 624 sharded_weights = variable_scope.get_variable( 625 "w", 626 partitioner=partitioned_variables.fixed_size_partitioner(num_shards), 627 initializer=constant_op.constant(weights)) 628 sharded_biases = variable_scope.get_variable( 629 "b", 630 partitioner=partitioned_variables.fixed_size_partitioner(num_shards), 631 initializer=constant_op.constant(biases)) 632 with self.session(graph=g) as sess: 633 self.evaluate(variables.global_variables_initializer()) 634 return self.evaluate([list(sharded_weights), list(sharded_biases)]) 635 636 def testShapes(self): 637 np.random.seed(0) 638 num_classes = 5 639 batch_size = 3 640 641 for num_true in range(1, 5): 642 labels = np.random.randint( 643 low=0, high=num_classes, size=batch_size * num_true) 644 (weights, biases, hidden_acts, sampled_vals, exp_logits, 645 exp_labels) = self._GenerateTestData( 646 num_classes=num_classes, 647 dim=10, 648 batch_size=batch_size, 649 num_true=num_true, 650 labels=labels, 651 sampled=[1, 0, 2, 3], 652 subtract_log_q=False) 653 logits_tensor, labels_tensor = _compute_sampled_logits( 654 weights=constant_op.constant(weights), 655 biases=constant_op.constant(biases), 656 labels=constant_op.constant( 657 labels, dtype=dtypes.int64, shape=(batch_size, num_true)), 658 inputs=constant_op.constant(hidden_acts), 659 num_sampled=4, 660 num_classes=num_classes, 661 num_true=num_true, 662 sampled_values=sampled_vals, 663 subtract_log_q=False, 664 remove_accidental_hits=False, 665 partition_strategy="div", 666 name="sampled_logits_basic_num_true_%d" % num_true) 667 got_logits, got_labels = self.evaluate([logits_tensor, labels_tensor]) 668 self.assertEqual(exp_logits.shape, got_logits.shape, self._eps) 669 self.assertEqual(exp_labels.shape, got_labels.shape, self._eps) 670 671 def testBasic(self): 672 """Without accidental hit removal or subtract_log_q.""" 673 np.random.seed(0) 674 num_classes = 5 675 batch_size = 3 676 677 for num_true in range(1, 5): 678 labels = np.random.randint( 679 low=0, high=num_classes, size=batch_size * num_true) 680 (weights, biases, hidden_acts, sampled_vals, exp_logits, 681 exp_labels) = self._GenerateTestData( 682 num_classes=num_classes, 683 dim=10, 684 batch_size=batch_size, 685 num_true=num_true, 686 labels=labels, 687 sampled=[1, 0, 2, 3], 688 subtract_log_q=False) 689 logits_tensor, labels_tensor = _compute_sampled_logits( 690 weights=constant_op.constant(weights), 691 biases=constant_op.constant(biases), 692 labels=constant_op.constant( 693 labels, dtype=dtypes.int64, shape=(batch_size, num_true)), 694 inputs=constant_op.constant(hidden_acts), 695 num_sampled=4, 696 num_classes=num_classes, 697 num_true=num_true, 698 sampled_values=sampled_vals, 699 subtract_log_q=False, 700 remove_accidental_hits=False, 701 partition_strategy="div", 702 name="sampled_logits_basic_num_true_%d" % num_true) 703 got_logits, got_labels = self.evaluate([logits_tensor, labels_tensor]) 704 self.assertAllClose(exp_logits, got_logits, self._eps) 705 self.assertAllClose(exp_labels, got_labels, self._eps) 706 707 def testAccidentalHitRemoval(self): 708 """With accidental hit removal, no subtract_log_q.""" 709 np.random.seed(0) 710 num_classes = 5 711 batch_size = 3 712 sampled = [1, 0, 2, 3] 713 714 for num_true in range(1, 5): 715 labels = np.random.randint( 716 low=0, high=num_classes, size=batch_size * num_true) 717 (weights, biases, hidden_acts, sampled_vals, _, 718 _) = self._GenerateTestData( 719 num_classes=num_classes, 720 dim=10, 721 batch_size=batch_size, 722 num_true=num_true, 723 labels=labels, 724 sampled=sampled, 725 subtract_log_q=False) 726 logits_tensor, _ = _compute_sampled_logits( 727 weights=constant_op.constant(weights), 728 biases=constant_op.constant(biases), 729 labels=constant_op.constant( 730 labels, dtype=dtypes.int64, shape=(batch_size, num_true)), 731 inputs=constant_op.constant(hidden_acts), 732 num_sampled=len(sampled), 733 num_classes=num_classes, 734 num_true=num_true, 735 sampled_values=sampled_vals, 736 subtract_log_q=False, 737 remove_accidental_hits=True, 738 partition_strategy="div", 739 name="sampled_logits_accidental_hit_removal_num_true_%d" % num_true) 740 # Test that the exponentiated logits of accidental hits are near 0. 741 # First we need to find the hits in this random test run: 742 labels_reshape = labels.reshape((batch_size, num_true)) 743 got_logits = self.evaluate(logits_tensor) 744 for row in xrange(batch_size): 745 row_labels = labels_reshape[row, :] 746 for col in xrange(len(sampled)): 747 if sampled[col] in row_labels: 748 # We need to add the num_true_test offset into logits_* 749 self.assertNear( 750 np.exp(got_logits[row, col + num_true]), 0., self._eps) 751 752 def testSubtractLogQ(self): 753 """With subtract_log_q, no accidental hit removal.""" 754 np.random.seed(0) 755 num_classes = 5 756 batch_size = 3 757 758 for num_true in range(1, 5): 759 labels = np.random.randint( 760 low=0, high=num_classes, size=batch_size * num_true) 761 (weights, biases, hidden_acts, sampled_vals, exp_logits, 762 exp_labels) = self._GenerateTestData( 763 num_classes=num_classes, 764 dim=10, 765 batch_size=batch_size, 766 num_true=num_true, 767 labels=labels, 768 sampled=[1, 0, 2, 3], 769 subtract_log_q=True) 770 logits_tensor, labels_tensor = _compute_sampled_logits( 771 weights=constant_op.constant(weights), 772 biases=constant_op.constant(biases), 773 labels=constant_op.constant( 774 labels, dtype=dtypes.int64, shape=(batch_size, num_true)), 775 inputs=constant_op.constant(hidden_acts), 776 num_sampled=4, 777 num_classes=num_classes, 778 num_true=num_true, 779 sampled_values=sampled_vals, 780 subtract_log_q=True, 781 remove_accidental_hits=False, 782 partition_strategy="div", 783 name="sampled_logits_subtract_log_q_num_true_%d" % num_true) 784 got_logits, got_labels = self.evaluate([logits_tensor, labels_tensor]) 785 self.assertAllClose(exp_logits, got_logits, self._eps) 786 self.assertAllClose(exp_labels, got_labels, self._eps) 787 788 def testSharded(self): 789 """With sharded weights and sharded biases.""" 790 np.random.seed(0) 791 num_classes = 5 792 batch_size = 3 793 794 for num_true in range(1, 5): 795 labels = np.random.randint( 796 low=0, high=num_classes, size=batch_size * num_true) 797 (weights, biases, hidden_acts, sampled_vals, exp_logits, 798 exp_labels) = self._GenerateTestData( 799 num_classes=num_classes, 800 dim=10, 801 batch_size=batch_size, 802 num_true=num_true, 803 labels=labels, 804 sampled=[1, 0, 2, 3], 805 subtract_log_q=False) 806 weight_shards, bias_shards = self._ShardTestEmbeddings( 807 weights, biases, num_shards=3) 808 logits_tensor, labels_tensor = _compute_sampled_logits( 809 weights=[constant_op.constant(shard) for shard in weight_shards], 810 biases=[constant_op.constant(shard) for shard in bias_shards], 811 labels=constant_op.constant( 812 labels, dtype=dtypes.int64, shape=(batch_size, num_true)), 813 inputs=constant_op.constant(hidden_acts), 814 num_sampled=4, 815 num_classes=num_classes, 816 num_true=num_true, 817 sampled_values=sampled_vals, 818 subtract_log_q=False, 819 remove_accidental_hits=False, 820 partition_strategy="div", 821 name="sampled_logits_sharded_num_true_%d" % num_true) 822 got_logits, got_labels = self.evaluate([logits_tensor, labels_tensor]) 823 self.assertAllClose(exp_logits, got_logits, self._eps) 824 self.assertAllClose(exp_labels, got_labels, self._eps) 825 826 def testNCELoss(self): 827 # A simple test to verify the numerics. 828 829 def _SigmoidCrossEntropyWithLogits(logits, targets): 830 # logits, targets: float arrays of the same shape. 831 assert logits.shape == targets.shape 832 pred = 1. / (1. + np.exp(-logits)) 833 eps = 0.0001 834 pred = np.minimum(np.maximum(pred, eps), 1 - eps) 835 return -targets * np.log(pred) - (1. - targets) * np.log(1. - pred) 836 837 np.random.seed(0) 838 num_classes = 5 839 batch_size = 3 840 labels = [0, 1, 2] 841 (weights, biases, hidden_acts, sampled_vals, exp_logits, 842 exp_labels) = self._GenerateTestData( 843 num_classes=num_classes, 844 dim=10, 845 batch_size=batch_size, 846 num_true=1, 847 labels=labels, 848 sampled=[1, 0, 2, 3], 849 subtract_log_q=True) 850 exp_nce_loss = np.sum( 851 _SigmoidCrossEntropyWithLogits(exp_logits, exp_labels), 1) 852 853 got_nce_loss = nn_impl.nce_loss_v2( 854 weights=constant_op.constant(weights), 855 biases=constant_op.constant(biases), 856 labels=constant_op.constant(labels, shape=(batch_size, 1)), 857 inputs=constant_op.constant(hidden_acts), 858 num_sampled=4, 859 num_classes=num_classes, 860 num_true=1, 861 sampled_values=sampled_vals) 862 863 self.assertAllClose(exp_nce_loss, self.evaluate(got_nce_loss), 1e-4) 864 865 # Test with sharded weights and sharded biases. 866 weight_shards, bias_shards = self._ShardTestEmbeddings( 867 weights, biases, num_shards=3) 868 got_nce_loss = nn_impl.nce_loss_v2( 869 weights=[constant_op.constant(shard) for shard in weight_shards], 870 biases=[constant_op.constant(shard) for shard in bias_shards], 871 labels=constant_op.constant(labels, shape=(batch_size, 1)), 872 inputs=constant_op.constant(hidden_acts), 873 num_sampled=4, 874 num_classes=num_classes, 875 num_true=1, 876 sampled_values=sampled_vals) 877 878 self.assertAllClose(exp_nce_loss, self.evaluate(got_nce_loss), 1e-4) 879 880 def testSampledSoftmaxLoss(self): 881 # A simple test to verify the numerics. 882 883 def _SoftmaxCrossEntropyWithLogits(logits, targets): 884 # logits, targets: float arrays of the same shape. 885 assert logits.shape == targets.shape 886 stable_exp_logits = np.exp( 887 logits - np.amax(logits, axis=1, keepdims=True)) 888 pred = stable_exp_logits / np.sum(stable_exp_logits, 1, keepdims=True) 889 return -np.sum(targets * np.log(pred + 1.0e-20), axis=1) 890 891 np.random.seed(0) 892 num_classes = 5 893 batch_size = 3 894 labels = [0, 1, 2] 895 (weights, biases, hidden_acts, sampled_vals, exp_logits, 896 exp_labels) = self._GenerateTestData( 897 num_classes=num_classes, 898 dim=10, 899 batch_size=batch_size, 900 num_true=1, 901 labels=labels, 902 sampled=[1, 0, 2, 3], 903 subtract_log_q=True) 904 exp_sampled_softmax_loss = _SoftmaxCrossEntropyWithLogits( 905 exp_logits, exp_labels) 906 907 got_sampled_softmax_loss = nn_impl.sampled_softmax_loss_v2( 908 weights=constant_op.constant(weights), 909 biases=constant_op.constant(biases), 910 labels=constant_op.constant(labels, shape=(batch_size, 1)), 911 inputs=constant_op.constant(hidden_acts), 912 num_sampled=4, 913 num_classes=num_classes, 914 num_true=1, 915 sampled_values=sampled_vals, 916 remove_accidental_hits=False) 917 918 self.assertAllClose(exp_sampled_softmax_loss, 919 self.evaluate(got_sampled_softmax_loss), 1e-4) 920 921 # Test with sharded weights and sharded biases. 922 weight_shards, bias_shards = self._ShardTestEmbeddings( 923 weights, biases, num_shards=3) 924 got_sampled_softmax_loss = nn_impl.sampled_softmax_loss_v2( 925 weights=[constant_op.constant(shard) for shard in weight_shards], 926 biases=[constant_op.constant(shard) for shard in bias_shards], 927 labels=constant_op.constant(labels, shape=(batch_size, 1)), 928 inputs=constant_op.constant(hidden_acts), 929 num_sampled=4, 930 num_classes=num_classes, 931 num_true=1, 932 sampled_values=sampled_vals, 933 remove_accidental_hits=False) 934 935 self.assertAllClose(exp_sampled_softmax_loss, 936 self.evaluate(got_sampled_softmax_loss), 1e-4) 937 938 def testSampledSoftmaxLossBf16(self): 939 # A simple test to verify the numerics for bfloat16. 940 def _SoftmaxCrossEntropyWithLogits(logits, targets): 941 # logits, targets: float arrays of the same shape. 942 assert logits.shape == targets.shape 943 stable_exp_logits = np.exp( 944 logits - np.amax(logits, axis=1, keepdims=True)) 945 pred = stable_exp_logits / np.sum(stable_exp_logits, 1, keepdims=True) 946 return -np.sum(targets * np.log(pred + 1.0e-20), axis=1) 947 948 np.random.seed(0) 949 num_classes = 5 950 batch_size = 3 951 labels = [0, 1, 2] 952 sampled = [1, 0, 2, 3] 953 (weights, biases, hidden_acts, _, exp_logits, 954 exp_labels) = self._GenerateTestData( 955 num_classes=num_classes, 956 dim=10, 957 batch_size=batch_size, 958 num_true=1, 959 labels=labels, 960 sampled=sampled, 961 subtract_log_q=True) 962 exp_sampled_softmax_loss = _SoftmaxCrossEntropyWithLogits( 963 exp_logits, exp_labels) 964 965 true_exp_bf16 = np.full([batch_size, 1], 966 fill_value=0.5, 967 dtype=dtypes.bfloat16.as_numpy_dtype) 968 sampled_exp_bf16 = np.full([len(sampled)], 969 fill_value=0.5, 970 dtype=dtypes.bfloat16.as_numpy_dtype) 971 sampled_vals_bf16 = (sampled, true_exp_bf16, sampled_exp_bf16) 972 973 got_sampled_softmax_loss = math_ops.cast( 974 nn_impl.sampled_softmax_loss_v2( 975 weights=constant_op.constant(weights, dtype=dtypes.bfloat16), 976 biases=constant_op.constant(biases, dtype=dtypes.bfloat16), 977 labels=constant_op.constant( 978 labels, shape=(batch_size, 1), dtype=dtypes.bfloat16), 979 inputs=constant_op.constant(hidden_acts, dtype=dtypes.bfloat16), 980 num_sampled=4, 981 num_classes=num_classes, 982 num_true=1, 983 sampled_values=sampled_vals_bf16, 984 remove_accidental_hits=False), dtypes.float32) 985 986 self.assertAllClose(exp_sampled_softmax_loss, 987 self.evaluate(got_sampled_softmax_loss), 1e-1) 988 989 990class CReluTest(test_lib.TestCase): 991 992 def test(self): 993 np.random.seed(1) # Make it reproducible. 994 x = np.random.randn(3, 4).astype(np.float32) 995 y = np.concatenate([x * (x > 0), -x * (x < 0)], axis=1) 996 997 z = self.evaluate(nn_ops.crelu(constant_op.constant(x))) 998 self.assertAllClose(y, z, 1e-4) 999 1000 1001class ReluTest(test_lib.TestCase): 1002 1003 def test(self): 1004 np.random.seed(1) # Make it reproducible. 1005 x = np.random.randn(3, 4).astype(np.float32) 1006 y = np.maximum(x, 0.0) 1007 1008 z = self.evaluate(nn_ops.relu(constant_op.constant(x))) 1009 self.assertAllEqual(y, z) 1010 1011 @test_util.disable_xla( 1012 "This test relies on undefined behavior that XLA does not replicate") 1013 @test_util.run_deprecated_v1 1014 def testNaNs(self): 1015 # Test that relu(nan) = nan for various sizes. 1016 for i in range(18): 1017 x = np.zeros(i) + np.nan 1018 # TODO(b/178335491): This is broken on GPU today. 1019 with self.cached_session(use_gpu=False): 1020 z = nn_ops.relu(constant_op.constant(x)).eval() 1021 self.assertTrue(np.isnan(z).all()) 1022 1023 1024class LeakyReluTest(test_lib.TestCase): 1025 1026 def testRange(self): 1027 batch_size = 3 1028 height, width = 4, 4 1029 np.random.seed(1) # Make it reproducible. 1030 inputs = np.random.uniform(size=(batch_size, height, width, 3)).astype( 1031 np.float32) 1032 inputs = constant_op.constant(inputs) 1033 1034 outputs = nn_ops.leaky_relu(inputs) 1035 self.assertEqual(inputs.shape, outputs.shape) 1036 1037 inputs, outputs = self.evaluate([inputs, outputs]) 1038 1039 self.assertGreaterEqual(outputs.min(), 0.0) 1040 self.assertLessEqual(outputs.max(), 1.0) 1041 self.assertAllClose(inputs, outputs) 1042 1043 @test_util.run_deprecated_v1 1044 def testValues(self): 1045 for dtype in [np.int32, np.int64, np.float16, np.float32, np.float64]: 1046 np_values = np.array([-2, -1, 0, 1, 2], dtype=dtype) 1047 outputs = nn_ops.leaky_relu(constant_op.constant(np_values)) 1048 1049 outputs = self.evaluate(outputs) 1050 1051 tol = 2e-3 if dtype == np.float16 else 1e-6 1052 self.assertAllClose( 1053 outputs, [-0.4, -0.2, 0.0, 1.0, 2.0], rtol=tol, atol=tol) 1054 1055 @test_util.run_deprecated_v1 1056 def testName(self): 1057 np_values = np.array([-2, -1, 0, 1, 2], dtype=np.float64) 1058 outputs_with_name_set = nn_ops.leaky_relu( 1059 constant_op.constant(np_values), 1060 name='test_relu_op') 1061 self.assertEqual(outputs_with_name_set.name, 'test_relu_op:0') 1062 outputs_without_name_set = nn_ops.leaky_relu( 1063 constant_op.constant(np_values)) 1064 self.assertEqual(outputs_without_name_set.name, 'LeakyRelu:0') 1065 1066 1067class GeluTest(test_lib.TestCase): 1068 1069 def test(self): 1070 1071 def gelu(x, approximate=False): 1072 if approximate: 1073 return 0.5 * x * (1.0 + np.tanh( 1074 np.sqrt(2.0 / np.pi) * (x + 0.044715 * np.power(x, 3)))) 1075 else: 1076 from scipy.stats import norm # pylint: disable=g-import-not-at-top 1077 return x * norm.cdf(x) 1078 1079 np.random.seed(1) # Make it reproducible. 1080 x = np.random.randn(3, 4).astype(np.float32) 1081 y = gelu(x) 1082 z = self.evaluate(nn_ops.gelu(x)) 1083 self.assertAllClose(y, z) 1084 1085 y = gelu(x, True) 1086 z = self.evaluate(nn_ops.gelu(x, True)) 1087 self.assertAllClose(y, z) 1088 1089 1090class SwishTest(test_lib.TestCase): 1091 1092 @test_util.run_deprecated_v1 1093 def testValues(self): 1094 np_values = np.array( 1095 [np.linspace(-7.0, 0.0, 100), 1096 np.linspace(0.0, 7.0, 100)], 1097 dtype=np.float32) 1098 tf_values = constant_op.constant(np_values) 1099 actual_tf_outputs = nn_impl.swish(tf_values) 1100 expected_tf_outputs = tf_values * math_ops.sigmoid(tf_values) 1101 1102 actual_outputs, expected_outputs = self.evaluate( 1103 [actual_tf_outputs, expected_tf_outputs]) 1104 1105 self.assertAllClose(actual_outputs, expected_outputs) 1106 1107 @test_util.run_deprecated_v1 1108 def testGradients(self): 1109 shape = [5, 3, 4] 1110 sigma = 5 1111 input_values = np.random.randn(*shape) * sigma 1112 x_tf = constant_op.constant(input_values) 1113 y_tf = nn_impl.swish(x_tf) 1114 with self.cached_session(): 1115 err = gradient_checker.compute_gradient_error(x_tf, shape, y_tf, shape) 1116 self.assertLess(err, 1e-4) 1117 1118 1119class MomentsTest(test_lib.TestCase): 1120 1121 def doOutputTest(self, 1122 input_shape, 1123 moments_axes, 1124 tol=1e-4, 1125 check_gradients=False): 1126 for mu in [0.0, 1.0, 1e3]: 1127 for sigma in [1.0, 0.1]: 1128 for keep_dims in [True, False]: 1129 input_values = np.random.rand(*input_shape) * sigma + mu 1130 expected_mean = np.mean( 1131 input_values, axis=moments_axes, keepdims=keep_dims) 1132 expected_var = np.var( 1133 input_values, axis=moments_axes, keepdims=keep_dims) 1134 with ops.Graph().as_default() as g: 1135 with self.session(graph=g) as sess: 1136 inputs = constant_op.constant( 1137 input_values, shape=input_shape, dtype=dtypes.float32) 1138 mean, variance = nn_impl.moments_v2( 1139 inputs, moments_axes, keepdims=keep_dims) 1140 1141 if check_gradients: 1142 err = gradient_checker.compute_gradient_error( 1143 inputs, input_shape, mean, mean.shape.as_list()) 1144 self.assertLess(err, 1e-3) 1145 err = gradient_checker.compute_gradient_error( 1146 inputs, input_shape, variance, variance.shape.as_list()) 1147 self.assertLess(err, 1e-3) 1148 1149 # Evaluate. 1150 [mean, variance] = self.evaluate([mean, variance]) 1151 # Make sure that there are no NaNs 1152 self.assertFalse(np.isnan(mean).any()) 1153 self.assertFalse(np.isnan(variance).any()) 1154 self.assertAllClose(mean, expected_mean, rtol=tol, atol=tol) 1155 self.assertAllClose(variance, expected_var, rtol=tol, atol=tol) 1156 1157 def testOutputAndGradient2DInput0(self): 1158 self.doOutputTest((10, 10), (0,), check_gradients=True) 1159 1160 def testOutputAndGradient2DInput01(self): 1161 self.doOutputTest((10, 10), (0, 1), check_gradients=True) 1162 1163 def testOutput2DInput0(self): 1164 self.doOutputTest((10, 300), (0,)) 1165 1166 def testOutput2DInput1(self): 1167 self.doOutputTest((10, 300), (1,)) 1168 1169 def testOutput2DInput01(self): 1170 self.doOutputTest((10, 300), (0, 1)) 1171 1172 def testOutput4DInput0(self): 1173 self.doOutputTest((10, 10, 10, 30), (0,)) 1174 1175 def testOutput4DInput1(self): 1176 self.doOutputTest((10, 10, 10, 30), (1,)) 1177 1178 def testOutput4DInput3(self): 1179 self.doOutputTest((10, 10, 10, 30), (3,)) 1180 1181 def testOutput4DInput012(self): 1182 self.doOutputTest((10, 10, 10, 30), (0, 1, 2)) 1183 1184 def testOutput4DInput123(self): 1185 self.doOutputTest((10, 10, 10, 30), (1, 2, 3)) 1186 1187 1188class DataFormatDimMapTest(test_lib.TestCase): 1189 1190 def _test(self, x_val, y_val_expected): 1191 x = constant_op.constant(x_val) 1192 y = nn_ops.data_format_dim_map(x) 1193 1194 y_val = self.evaluate(y) 1195 self.assertAllEqual(y_val, y_val_expected) 1196 1197 def test(self): 1198 self._test(0, 0) 1199 self._test(1, 2) 1200 self._test(2, 3) 1201 self._test(3, 1) 1202 self._test(-1, 1) 1203 self._test(-2, 3) 1204 self._test(-3, 2) 1205 self._test(-4, 0) 1206 self._test([1, 3], [2, 1]) 1207 self._test([1, 3, -2], [2, 1, 3]) 1208 self._test([1, -3, -2], [2, 2, 3]) 1209 self._test([[1, -3], [1, -1]], [[2, 2], [2, 1]]) 1210 1211 def testNHWCtoNCHW(self): 1212 x_val = [1, -3, -2] 1213 y_val_expected = [2, 2, 3] 1214 x = constant_op.constant(x_val) 1215 y = nn_ops.data_format_dim_map(x, src_format="NHWC", dst_format="NCHW") 1216 with test_util.use_gpu(): 1217 y_val = self.evaluate(y) 1218 self.assertAllEqual(y_val, y_val_expected) 1219 1220 def testNHWCtoHWNC(self): 1221 x_val = [-4, -3, -2, -1, 0, 1, 2, 3] 1222 y_val_expected = [2, 0, 1, 3, 2, 0, 1, 3] 1223 x = constant_op.constant(x_val) 1224 y = nn_ops.data_format_dim_map(x, src_format="NHWC", dst_format="HWNC") 1225 with test_util.use_gpu(): 1226 y_val = self.evaluate(y) 1227 self.assertAllEqual(y_val, y_val_expected) 1228 1229 def testNHWCtoWHCN(self): 1230 x_val = [-4, -3, -2, -1, 0, 1, 2, 3] 1231 y_val_expected = [3, 1, 0, 2, 3, 1, 0, 2] 1232 x = constant_op.constant(x_val) 1233 y = nn_ops.data_format_dim_map(x, src_format="NHWC", dst_format="WHCN") 1234 with test_util.use_gpu(): 1235 y_val = self.evaluate(y) 1236 self.assertAllEqual(y_val, y_val_expected) 1237 1238 def testNDHWCtoNCDHW(self): 1239 x_val = [1, -4, -3, -2] 1240 y_val_expected = [2, 2, 3, 4] 1241 x = constant_op.constant(x_val) 1242 y = nn_ops.data_format_dim_map(x, src_format="NDHWC", dst_format="NCDHW") 1243 with test_util.use_gpu(): 1244 y_val = self.evaluate(y) 1245 self.assertAllEqual(y_val, y_val_expected) 1246 1247 def testNDHWCtoDHWNC(self): 1248 x_val = [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4] 1249 y_val_expected = [3, 0, 1, 2, 4, 3, 0, 1, 2, 4] 1250 x = constant_op.constant(x_val) 1251 y = nn_ops.data_format_dim_map(x, src_format="NDHWC", dst_format="DHWNC") 1252 with test_util.use_gpu(): 1253 y_val = self.evaluate(y) 1254 self.assertAllEqual(y_val, y_val_expected) 1255 1256 def testDNHWCtoWHDCN(self): 1257 x_val = [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4] 1258 y_val_expected = [4, 2, 1, 0, 3, 4, 2, 1, 0, 3] 1259 x = constant_op.constant(x_val) 1260 y = nn_ops.data_format_dim_map(x, src_format="NDHWC", dst_format="WHDCN") 1261 with test_util.use_gpu(): 1262 y_val = self.evaluate(y) 1263 self.assertAllEqual(y_val, y_val_expected) 1264 1265 @test_util.disable_xla("XLA catches the error and rethrows as different one") 1266 def testArbitraryASCII(self): 1267 x_val = [-4, -3, -2, -1, 0, 1, 2, 3] 1268 y_val_expected = [3, 2, 1, 0, 3, 2, 1, 0] 1269 x = constant_op.constant(x_val) 1270 y = nn_ops.data_format_dim_map(x, src_format="qwer", dst_format="rewq") 1271 with test_util.use_gpu(): 1272 y_val = self.evaluate(y) 1273 self.assertAllEqual(y_val, y_val_expected) 1274 1275 @test_util.disable_xla("XLA catches the error and rethrows as different one") 1276 def testInvalidLength(self): 1277 x = [-4, -3, -2, -1, 0, 1, 2, 3] 1278 with self.assertRaisesRegex(errors.InvalidArgumentError, 1279 "Source format must be of length 4 or 5"): 1280 op = nn_ops.data_format_dim_map( 1281 x, src_format="12345678", dst_format="87654321") 1282 with test_util.use_gpu(): 1283 self.evaluate(op) 1284 1285 @test_util.disable_xla("XLA catches the error and rethrows as different one") 1286 def testDuplicateSrc(self): 1287 x = [-4, -3, -2, -1, 0, 1, 2, 3] 1288 with self.assertRaisesRegex( 1289 errors.InvalidArgumentError, 1290 "Destination and source format must determine a permutation"): 1291 op = nn_ops.data_format_dim_map(x, src_format="1233", dst_format="4321") 1292 with test_util.use_gpu(): 1293 self.evaluate(op) 1294 1295 @test_util.disable_xla("XLA catches the error and rethrows as different one") 1296 def testDuplicateDst(self): 1297 x = [-4, -3, -2, -1, 0, 1, 2, 3] 1298 with self.assertRaisesRegex( 1299 errors.InvalidArgumentError, 1300 "Destination and source format must determine a permutation"): 1301 op = nn_ops.data_format_dim_map(x, src_format="1234", dst_format="3321") 1302 with test_util.use_gpu(): 1303 self.evaluate(op) 1304 1305 @test_util.disable_xla("XLA catches the error and rethrows as different one") 1306 def testExtraSpecifiers(self): 1307 x = [-4, -3, -2, -1, 0, 1, 2, 3] 1308 with self.assertRaisesRegex( 1309 errors.InvalidArgumentError, 1310 "Destination and source format must determine a permutation"): 1311 op = nn_ops.data_format_dim_map(x, src_format="1234", dst_format="5321") 1312 with test_util.use_gpu(): 1313 self.evaluate(op) 1314 1315 1316class DataFormatVectorPermuteTest(test_lib.TestCase): 1317 1318 def testNHWCToNCHW(self): 1319 x_val = [7, 4, 9, 3] 1320 x = constant_op.constant(x_val) 1321 y = nn_ops.data_format_vec_permute(x) 1322 with test_util.use_gpu(): 1323 y_val = self.evaluate(y) 1324 self.assertAllEqual(y_val, [7, 3, 4, 9]) 1325 1326 def testNHWCToNCHW_Size2(self): 1327 x_val = [4, 9] 1328 x = constant_op.constant(x_val) 1329 y = nn_ops.data_format_vec_permute(x) 1330 with test_util.use_gpu(): 1331 y_val = self.evaluate(y) 1332 self.assertAllEqual(y_val, [4, 9]) 1333 1334 @test_util.disable_xla("unsupported data format") 1335 def testNHWCToWHCN(self): 1336 x_val = [7, 4, 9, 3] 1337 x = constant_op.constant(x_val) 1338 y = nn_ops.data_format_vec_permute(x, src_format="NHWC", dst_format="WHCN") 1339 with test_util.use_gpu(): 1340 y_val = self.evaluate(y) 1341 self.assertAllEqual(y_val, [9, 4, 3, 7]) 1342 1343 @test_util.disable_xla("unsupported data format") 1344 def testNHWCToWHCN_Size2(self): 1345 x_val = [4, 9] 1346 x = constant_op.constant(x_val) 1347 y = nn_ops.data_format_vec_permute(x, src_format="NHWC", dst_format="WHCN") 1348 with test_util.use_gpu(): 1349 y_val = self.evaluate(y) 1350 self.assertAllEqual(y_val, [9, 4]) 1351 1352 def testNCHWToNHWC(self): 1353 x_val = [7, 4, 9, 3] 1354 x = constant_op.constant(x_val) 1355 y = nn_ops.data_format_vec_permute(x, src_format="NCHW", dst_format="NHWC") 1356 with test_util.use_gpu(): 1357 y_val = self.evaluate(y) 1358 self.assertAllEqual(y_val, [7, 9, 3, 4]) 1359 1360 def testNCHWToNHWC_Size2(self): 1361 x_val = [9, 3] 1362 x = constant_op.constant(x_val) 1363 y = nn_ops.data_format_vec_permute(x) 1364 with test_util.use_gpu(): 1365 y_val = self.evaluate(y) 1366 self.assertAllEqual(y_val, [9, 3]) 1367 1368 def testNHWCToHWNC(self): 1369 x_val = [7, 4, 9, 3] 1370 x = constant_op.constant(x_val) 1371 y = nn_ops.data_format_vec_permute(x, src_format="NHWC", dst_format="HWNC") 1372 with test_util.use_gpu(): 1373 y_val = self.evaluate(y) 1374 self.assertAllEqual(y_val, [4, 9, 7, 3]) 1375 1376 def testHWNCToNHWC(self): 1377 x_val = [7, 4, 9, 3] 1378 x = constant_op.constant(x_val) 1379 y = nn_ops.data_format_vec_permute(x, src_format="HWNC", dst_format="NHWC") 1380 with test_util.use_gpu(): 1381 y_val = self.evaluate(y) 1382 self.assertAllEqual(y_val, [9, 7, 4, 3]) 1383 1384 def testNHWCToNCHW2D(self): 1385 x_val = [[7, 4], [9, 3], [4, 5], [5, 1]] 1386 x = constant_op.constant(x_val) 1387 y = nn_ops.data_format_vec_permute(x) 1388 with test_util.use_gpu(): 1389 y_val = self.evaluate(y) 1390 self.assertAllEqual(y_val, [[7, 4], [5, 1], [9, 3], [4, 5]]) 1391 1392 def testNHWCToHWNC2D(self): 1393 x_val = [[7, 4], [9, 3], [4, 5], [5, 1]] 1394 x = constant_op.constant(x_val) 1395 y = nn_ops.data_format_vec_permute(x, src_format="NHWC", dst_format="HWNC") 1396 with test_util.use_gpu(): 1397 y_val = self.evaluate(y) 1398 self.assertAllEqual(y_val, [[9, 3], [4, 5], [7, 4], [5, 1]]) 1399 1400 def testHWNCToNHWC2D(self): 1401 x_val = [[7, 4], [9, 3], [4, 5], [5, 1]] 1402 x = constant_op.constant(x_val) 1403 y = nn_ops.data_format_vec_permute(x, src_format="HWNC", dst_format="NHWC") 1404 with test_util.use_gpu(): 1405 y_val = self.evaluate(y) 1406 self.assertAllEqual(y_val, [[4, 5], [7, 4], [9, 3], [5, 1]]) 1407 1408 def testNCHWToNHWC2D(self): 1409 x_val = [[7, 4], [9, 3], [4, 5], [5, 1]] 1410 x = constant_op.constant(x_val) 1411 y = nn_ops.data_format_vec_permute(x, src_format="NCHW", dst_format="NHWC") 1412 with test_util.use_gpu(): 1413 y_val = self.evaluate(y) 1414 self.assertAllEqual(y_val, [[7, 4], [4, 5], [5, 1], [9, 3]]) 1415 1416 @test_util.disable_xla("XLA catches the error and rethrows as different one") 1417 def testInvalidLength(self): 1418 x = [0, 1, 2, 3] 1419 with self.assertRaisesRegex(errors.InvalidArgumentError, 1420 "Source format must be of length 4 or 5"): 1421 op = nn_ops.data_format_vec_permute( 1422 x, src_format="12345678", dst_format="87654321") 1423 with test_util.use_gpu(): 1424 self.evaluate(op) 1425 1426 @test_util.disable_xla("XLA catches the error and rethrows as different one") 1427 def testDuplicateSrc(self): 1428 x = [0, 1, 2, 3] 1429 with self.assertRaisesRegex( 1430 errors.InvalidArgumentError, 1431 "Destination and source format must determine a permutation"): 1432 op = nn_ops.data_format_vec_permute( 1433 x, src_format="1233", dst_format="4321") 1434 with test_util.use_gpu(): 1435 self.evaluate(op) 1436 1437 @test_util.disable_xla("XLA catches the error and rethrows as different one") 1438 def testDuplicateDst(self): 1439 x = [0, 1, 2, 3] 1440 with self.assertRaisesRegex( 1441 errors.InvalidArgumentError, 1442 "Destination and source format must determine a permutation"): 1443 op = nn_ops.data_format_vec_permute( 1444 x, src_format="1234", dst_format="3321") 1445 with test_util.use_gpu(): 1446 self.evaluate(op) 1447 1448 @test_util.disable_xla("XLA catches the error and rethrows as different one") 1449 def testExtraSpecifiers(self): 1450 x = [0, 1, 2, 3] 1451 with self.assertRaisesRegex( 1452 errors.InvalidArgumentError, 1453 "Destination and source format must determine a permutation"): 1454 op = nn_ops.data_format_vec_permute( 1455 x, src_format="1234", dst_format="5321") 1456 with test_util.use_gpu(): 1457 self.evaluate(op) 1458 1459 @test_util.disable_xla("XLA catches the error and rethrows as different one") 1460 def test2DNoWH(self): 1461 x = [[0, 1], [2, 3]] 1462 with self.assertRaisesRegex( 1463 errors.InvalidArgumentError, 1464 "Format specifier must contain H and W for 2D case"): 1465 op = nn_ops.data_format_vec_permute( 1466 x, src_format="1234", dst_format="4321") 1467 with test_util.use_gpu(): 1468 self.evaluate(op) 1469 1470 1471@test_util.run_all_in_graph_and_eager_modes 1472class AvgPoolTest(test_lib.TestCase): 1473 1474 def test1DTensor(self): 1475 x = array_ops.ones([3, 6, 5]) 1476 ksize = 2 1477 strides = 2 1478 1479 y1 = nn_ops.avg_pool_v2(x, ksize, strides, "SAME") 1480 y2 = nn_ops.avg_pool1d(x, ksize, strides, "SAME") 1481 1482 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1483 1484 def test1DNumpy(self): 1485 # explicitly use float32 for ROCm, as MIOpen does not yet support float64 1486 # np.ones defaults to using float64 when dtype is not explicitly specified 1487 dtype = np.float32 if test_lib.is_built_with_rocm() else np.float64 1488 x = np.ones([3, 6, 5], dtype=dtype) 1489 ksize = 2 1490 strides = 2 1491 1492 y1 = nn_ops.avg_pool_v2(x, ksize, strides, "SAME") 1493 y2 = nn_ops.avg_pool1d(x, ksize, strides, "SAME") 1494 1495 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1496 1497 def test1DNumpyWithGolden(self): 1498 dtype = np.float32 if test_lib.is_built_with_rocm() else np.float64 1499 x = np.array([[[3], [6], [5]], 1500 [[1], [0], [1]]], dtype=dtype) 1501 ksize = 2 1502 strides = 1 1503 y = nn_ops.avg_pool1d(x, ksize, strides, "SAME") 1504 expected_y = np.array([[[4.5], [5.5], [5.0]], 1505 [[0.5], [0.5], [1.0]]], dtype=dtype) 1506 self.assertAllEqual(self.evaluate(y), expected_y) 1507 1508 def test2DTensor(self): 1509 x = array_ops.ones([3, 6, 6, 5]) 1510 ksize = 2 1511 strides = 2 1512 1513 y1 = nn_ops.avg_pool_v2(x, ksize, strides, "SAME") 1514 y2 = nn_ops.avg_pool(x, ksize, strides, "SAME") 1515 1516 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1517 1518 def test2DNumpy(self): 1519 # explicitly use float32 for ROCm, as MIOpen does not yet support float64 1520 # np.ones defaults to using float64 when dtype is not explicitly specified 1521 dtype = np.float32 if test_lib.is_built_with_rocm() else np.float64 1522 x = np.ones([3, 6, 6, 5], dtype=dtype) 1523 ksize = 2 1524 strides = 2 1525 1526 y1 = nn_ops.avg_pool_v2(x, ksize, strides, "SAME") 1527 y2 = nn_ops.avg_pool(x, ksize, strides, "SAME") 1528 1529 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1530 1531 def test3DTensor(self): 1532 if test_lib.is_built_with_rocm(): 1533 self.skipTest("Pooling with 3D tensors is not supported in ROCm") 1534 x = array_ops.ones([3, 7, 6, 6, 5]) 1535 ksize = 2 1536 strides = 2 1537 1538 y1 = nn_ops.avg_pool_v2(x, ksize, strides, "SAME") 1539 y2 = nn_ops.avg_pool3d(x, ksize, strides, "SAME") 1540 1541 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1542 1543 def test3DNumpy(self): 1544 if test_lib.is_built_with_rocm(): 1545 self.skipTest("Pooling with 3D tensors is not supported in ROCm") 1546 x = np.ones([3, 7, 6, 6, 5], dtype=np.float32) 1547 ksize = 2 1548 strides = 2 1549 1550 y1 = nn_ops.avg_pool_v2(x, ksize, strides, "SAME") 1551 y2 = nn_ops.avg_pool3d(x, ksize, strides, "SAME") 1552 1553 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1554 1555 1556@test_util.run_all_in_graph_and_eager_modes 1557class MaxPoolTest(test_lib.TestCase): 1558 1559 def test1DTensor(self): 1560 x = array_ops.ones([3, 6, 5]) 1561 ksize = 2 1562 strides = 2 1563 1564 y1 = nn_ops.max_pool_v2(x, ksize, strides, "SAME") 1565 y2 = nn_ops.max_pool1d(x, ksize, strides, "SAME") 1566 1567 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1568 1569 def test1DNumpy(self): 1570 # explicitly use float32 for ROCm, as MIOpen does not yet support float64 1571 # np.ones defaults to using float64 when dtype is not explicitly specified 1572 dtype = np.float32 if test_lib.is_built_with_rocm() else np.float64 1573 x = np.ones([3, 6, 5], dtype=dtype) 1574 ksize = 2 1575 strides = 2 1576 1577 y1 = nn_ops.max_pool_v2(x, ksize, strides, "SAME") 1578 y2 = nn_ops.max_pool1d(x, ksize, strides, "SAME") 1579 1580 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1581 1582 def test1DNumpyWithGolden(self): 1583 dtype = np.float32 if test_lib.is_built_with_rocm() else np.float64 1584 x = np.array([[[3], [6], [5]], 1585 [[1], [0], [1]]], dtype=dtype) 1586 ksize = 2 1587 strides = 1 1588 y = nn_ops.max_pool1d(x, ksize, strides, "SAME") 1589 expected_y = np.array([[[6], [6], [5]], 1590 [[1], [1], [1]]], dtype=dtype) 1591 self.assertAllEqual(self.evaluate(y), expected_y) 1592 1593 def test2DTensor(self): 1594 x = array_ops.ones([3, 6, 6, 5]) 1595 ksize = 2 1596 strides = 2 1597 1598 y1 = nn_ops.max_pool_v2(x, ksize, strides, "SAME") 1599 y2 = nn_ops.max_pool(x, ksize, strides, "SAME") 1600 1601 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1602 1603 def test2DNumpy(self): 1604 # explicitly use float32 for ROCm, as MIOpen does not yet support float64 1605 # np.ones defaults to using float64 when dtype is not explicitly specified 1606 dtype = np.float32 if test_lib.is_built_with_rocm() else np.float64 1607 x = np.ones([3, 6, 6, 5], dtype=dtype) 1608 ksize = 2 1609 strides = 2 1610 1611 y1 = nn_ops.max_pool_v2(x, ksize, strides, "SAME") 1612 y2 = nn_ops.max_pool(x, ksize, strides, "SAME") 1613 1614 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1615 1616 def test3DTensor(self): 1617 if test_lib.is_built_with_rocm(): 1618 self.skipTest("Pooling with 3D tensors is not supported in ROCm") 1619 x = array_ops.ones([3, 7, 6, 6, 5]) 1620 ksize = 2 1621 strides = 2 1622 1623 y1 = nn_ops.max_pool_v2(x, ksize, strides, "SAME") 1624 y2 = nn_ops.max_pool3d(x, ksize, strides, "SAME") 1625 1626 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1627 1628 def test3DNumpy(self): 1629 if test_lib.is_built_with_rocm(): 1630 self.skipTest("Pooling with 3D tensors is not supported in ROCm") 1631 x = np.ones([3, 7, 6, 6, 5], dtype=np.float32) 1632 ksize = 2 1633 strides = 2 1634 1635 y1 = nn_ops.max_pool_v2(x, ksize, strides, "SAME") 1636 y2 = nn_ops.max_pool3d(x, ksize, strides, "SAME") 1637 1638 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1639 1640 def testIncorrectSizeInputSmall(self): 1641 x = array_ops.ones([3, 4]) 1642 with self.assertRaisesRegex( 1643 ValueError, "Input tensor must be of rank 3, 4 or 5 but was 2."): 1644 nn_ops.max_pool_v2(x, 2, 2, "SAME") 1645 1646 def testIncorrectSizeInput(self): 1647 x = array_ops.ones([3, 4, 1, 2, 1, 2]) 1648 with self.assertRaisesRegex( 1649 ValueError, "Input tensor must be of rank 3, 4 or 5 but was 6."): 1650 nn_ops.max_pool_v2(x, 2, 2, "SAME") 1651 1652 1653@test_util.run_all_in_graph_and_eager_modes 1654class ConvolutionTest(test_lib.TestCase): 1655 1656 def testUnknownSize(self): 1657 x = tensor_spec.TensorSpec(None, dtypes.float32, name="x") 1658 k = np.ones([3, 6, 6, 5], dtype=np.float32) 1659 1660 @def_function.function 1661 def F(value): 1662 return nn_ops.convolution(value, k, "SAME") 1663 1664 F.get_concrete_function(x) 1665 1666 1667class ConvTransposeTest(test_lib.TestCase): 1668 1669 def test1D(self): 1670 t = array_ops.ones([2, 4, 3]) 1671 v = array_ops.ones([2, 5, 3]) 1672 strides = 2 1673 1674 y1 = nn_ops.conv1d_transpose(t, v, [2, 8, 5], strides) 1675 y2 = nn_ops.conv_transpose(t, v, [2, 8, 5], strides) 1676 1677 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1678 1679 def test1DTensor(self): 1680 t = array_ops.ones([2, 4, 3]) 1681 v = array_ops.ones([2, 5, 3]) 1682 strides = 2 1683 1684 y1 = nn_ops.conv1d_transpose(t, v, [2, 8, 5], strides) 1685 y2 = nn_ops.conv_transpose(t, v, constant_op.constant([2, 8, 5]), strides) 1686 1687 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1688 1689 def test2D(self): 1690 t = array_ops.ones([2, 4, 4, 3]) 1691 v = array_ops.ones([2, 2, 5, 3]) 1692 strides = 2 1693 1694 y1 = nn_ops.conv2d_transpose_v2(t, v, [2, 8, 8, 5], strides) 1695 y2 = nn_ops.conv_transpose(t, v, [2, 8, 8, 5], strides) 1696 1697 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1698 1699 def test2DTensor(self): 1700 t = array_ops.ones([2, 4, 4, 3]) 1701 v = array_ops.ones([2, 2, 5, 3]) 1702 strides = 2 1703 1704 y1 = nn_ops.conv2d_transpose_v2(t, v, [2, 8, 8, 5], strides) 1705 y2 = nn_ops.conv_transpose(t, v, constant_op.constant([2, 8, 8, 5]), 1706 strides) 1707 1708 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1709 1710 def test3D(self): 1711 t = array_ops.ones([2, 4, 4, 4, 3]) 1712 v = array_ops.ones([2, 2, 2, 5, 3]) 1713 strides = 2 1714 1715 y1 = nn_ops.conv3d_transpose_v2(t, v, [2, 8, 8, 8, 5], strides) 1716 y2 = nn_ops.conv_transpose(t, v, [2, 8, 8, 8, 5], strides) 1717 1718 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1719 1720 def test3DTensor(self): 1721 t = array_ops.ones([2, 4, 4, 4, 3]) 1722 v = array_ops.ones([2, 2, 2, 5, 3]) 1723 strides = 2 1724 1725 y1 = nn_ops.conv3d_transpose_v2(t, v, [2, 8, 8, 8, 5], strides) 1726 y2 = nn_ops.conv_transpose(t, v, constant_op.constant([2, 8, 8, 8, 5]), 1727 strides) 1728 1729 self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) 1730 1731 def testIncorrectSizeInputSmall(self): 1732 with self.assertRaisesRegex( 1733 ValueError, "output_shape must be of length 3, 4 or 5 but was 2."): 1734 nn_ops.conv_transpose(None, 2, [2, 3], "SAME") 1735 1736 def testIncorrectSizeInput(self): 1737 with self.assertRaisesRegex( 1738 ValueError, "output_shape must be of length 3, 4 or 5 but was 6."): 1739 nn_ops.conv_transpose(None, 2, [2, 3, 4, 2, 5, 1], "SAME") 1740 1741 def testTensorsNoShape(self): 1742 with self.assertRaisesRegex( 1743 ValueError, 1744 "output_shape must be a tensor or sized collection."): 1745 nn_ops.conv_transpose(None, None, None, None) 1746 1747 1748class RaggedEmbeddingTest(test_lib.TestCase): 1749 1750 def testRaggedTensor(self): 1751 weights = constant_op.constant([[0, 0, 0], [1, 1, 1], [2, 2, 2], [3, 3, 3]]) 1752 ragged_ids = ragged_factory_ops.constant([[1, 2, 3], [0], [1, 2]], 1753 ragged_rank=1) 1754 1755 embedded_ragged = nn.embedding_lookup_ragged(weights, ragged_ids) 1756 expected_output = ragged_factory_ops.constant( 1757 [[[1, 1, 1], [2, 2, 2], [3, 3, 3]], [[0, 0, 0]], [[1, 1, 1], [2, 2, 2]] 1758 ], 1759 ragged_rank=1) 1760 1761 self.assertAllEqual(expected_output, embedded_ragged) 1762 1763 def testMultipleRaggedDimTensor(self): 1764 weights = constant_op.constant([[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], 1765 [5, 5], [6, 6]]) 1766 ragged_ids = ragged_factory_ops.constant( 1767 [[[[3, 4], [0, 6]], []], [[[2, 1], [1, 0]], [[2, 5], [2, 3]]], [[[1, 0]] 1768 ]], 1769 ragged_rank=2) 1770 1771 embedded_ragged = nn.embedding_lookup_ragged(weights, ragged_ids) 1772 expected_output = ragged_factory_ops.constant( 1773 [[[[[3, 3], [4, 4]], [[0, 0], [6, 6]]], []], 1774 [[[[2, 2], [1, 1]], [[1, 1], [0, 0]]], 1775 [[[2, 2], [5, 5]], [[2, 2], [3, 3]]]], [[[[1, 1], [0, 0]]]]], 1776 ragged_rank=2) 1777 1778 self.assertAllEqual(expected_output, embedded_ragged) 1779 1780 def testMissingWeights(self): 1781 ragged_ids = ragged_factory_ops.constant([[1, 2, 3], [0], [1, 2]]) 1782 1783 with self.assertRaisesRegex(ValueError, 1784 "The embedding weights must be specified.*"): 1785 nn.embedding_lookup_ragged(None, ragged_ids) 1786 1787 def testEmptyWeights(self): 1788 ragged_ids = ragged_factory_ops.constant([[1, 2, 3], [0], [1, 2]]) 1789 1790 with self.assertRaisesRegex(ValueError, 1791 "The embedding weights should not be empty.*"): 1792 nn.embedding_lookup_ragged([], ragged_ids) 1793 1794 def testInvalidIndicesType(self): 1795 weights = constant_op.constant([[0, 0, 0], [1, 1, 1], [2, 2, 2]]) 1796 ragged_ids = ragged_factory_ops.constant([[1., 2., 3.], [1., 2.]]) 1797 1798 with self.assertRaisesRegex( 1799 ValueError, "The values contained by the inputs have type*"): 1800 nn.embedding_lookup_ragged(weights, ragged_ids) 1801 1802 def testMaxNormForEmbeddings(self): 1803 weights = constant_op.constant([[0, 0, 0, 0], [1, 1, 1, 1], 1804 [2, 2, 2, 2], [3, 3, 3, 3]], 1805 dtype=dtypes.float32) 1806 ragged_ids = ragged_factory_ops.constant([[1, 2, 3], [0], [1, 2]], 1807 ragged_rank=1) 1808 1809 actual_embeddings = [ 1810 nn.embedding_lookup(weights, ragged_ids, max_norm=max_norm) 1811 for max_norm in [1, 2, 5]] 1812 1813 expected_embeddings = ( 1814 # max_norm = 1 1815 [[[.5, .5, .5, .5], [.5, .5, .5, .5], [.5, .5, .5, .5]], 1816 [[0, 0, 0, 0]], [[.5, .5, .5, .5], [.5, .5, .5, .5]]], 1817 # max_norm = 2 1818 [[[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]], 1819 [[0, 0, 0, 0]], [[1, 1, 1, 1], [1, 1, 1, 1]]], 1820 # max_norm = 5 1821 [[[1, 1, 1, 1], [2, 2, 2, 2], [2.5, 2.5, 2.5, 2.5]], 1822 [[0, 0, 0, 0]], [[1, 1, 1, 1], [2, 2, 2, 2]]], 1823 ) 1824 1825 for expected, actual in zip(expected_embeddings, actual_embeddings): 1826 self.assertAllClose( 1827 ragged_factory_ops.constant(expected, dtype=float, ragged_rank=1), 1828 actual) 1829 1830 1831class IsotonicTest(parameterized.TestCase, test_lib.TestCase): 1832 1833 @test_util.run_in_graph_and_eager_modes 1834 def test_increasing_and_decreasing(self): 1835 x = constant_op.constant([[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]], 1836 dtype=dtypes.float64) 1837 y, segments = nn_ops.isotonic_regression(x, decreasing=False) 1838 self.assertAllClose(y, x) 1839 self.assertAllClose(segments, [[0, 1, 2, 3, 4], [0, 1, 2, 3, 4]]) 1840 1841 y, segments = nn_ops.isotonic_regression(x, decreasing=True) 1842 self.assertAllClose( 1843 y, 1844 [ 1845 [2, 2, 2, 2, 2], # Average of the inputs. 1846 [7, 7, 7, 7, 7] 1847 ]) 1848 self.assertAllClose(segments, array_ops.zeros((2, 5))) 1849 1850 y, segments = nn_ops.isotonic_regression(-x, decreasing=True) 1851 self.assertAllClose(segments, [[0, 1, 2, 3, 4], [0, 1, 2, 3, 4]]) 1852 1853 self.assertAllClose(y, -x) 1854 y, segments = nn_ops.isotonic_regression(-x, decreasing=False) 1855 self.assertAllClose( 1856 -y, 1857 [ 1858 [2, 2, 2, 2, 2], # Average of the inputs. 1859 [7, 7, 7, 7, 7] 1860 ]) 1861 self.assertAllClose(segments, array_ops.zeros((2, 5))) 1862 1863 @test_util.run_in_graph_and_eager_modes 1864 def test_different_axis(self): 1865 x = constant_op.constant([[0, 6, 2, 8, 4], [5, 1, 7, 3, 9]], 1866 dtype=dtypes.float64) 1867 y, segments = nn_ops.isotonic_regression(x, decreasing=True, axis=0) 1868 self.assertAllClose( 1869 y, 1870 [ 1871 [2.5, 6, 4.5, 8, 6.5], # Either identity or average. 1872 [2.5, 1, 4.5, 3, 6.5] 1873 ]) 1874 self.assertAllClose(segments, [[0, 0, 0, 0, 0], [0, 1, 0, 1, 0]]) 1875 1876 @test_util.run_v2_only 1877 def testGradientV2(self, dtype=np.float64, batch_size=30, dimensions=50): 1878 1879 @def_function.function 1880 def ComputeIsotonicFn(x): 1881 y, _ = nn_ops.isotonic_regression(x) # No gradient wrt segments. 1882 return y 1883 1884 np.random.seed(0) 1885 x_init = np.random.randn(batch_size, dimensions).astype(dtype) 1886 grad_theoretical, grad_numerical = gradient_checker_v2.compute_gradient( 1887 ComputeIsotonicFn, [x_init], delta=1e-5) 1888 self.assertAllClose(grad_theoretical, grad_numerical) 1889 1890 @test_util.run_v1_only("compute_gradient_error is v1 only") 1891 def testGradientV1(self, dtype=np.float64, batch_size=30, dimensions=50): 1892 np.random.seed(0) 1893 x_init = np.random.randn(batch_size, dimensions).astype(dtype) 1894 with self.cached_session(): 1895 x = array_ops.placeholder(dtype, (batch_size, dimensions)) 1896 y, _ = nn_ops.isotonic_regression(x) # Segments have no gradient. 1897 max_error = gradient_checker.compute_gradient_error( 1898 x, (batch_size, dimensions), y, (batch_size, dimensions), x_init) 1899 self.assertAllClose(max_error, 0.) 1900 1901 @parameterized.parameters([[dtypes.half, dtypes.half], 1902 [dtypes.bfloat16, dtypes.bfloat16], 1903 [dtypes.float32, dtypes.float32], 1904 [dtypes.float64, dtypes.float64], 1905 [dtypes.int32, dtypes.float64], 1906 [dtypes.int16, dtypes.float32]]) 1907 def testTypePromotion(self, dtype_in, expected_dtype_out): 1908 x = constant_op.constant([[0, 6, 2, 8, 4], [5, 1, 7, 3, 9]], dtype=dtype_in) 1909 y, segments = nn_ops.isotonic_regression(x) 1910 self.assertEqual(y.dtype, expected_dtype_out) 1911 self.assertEqual(segments.dtype, dtypes.int32) 1912 1913 1914if __name__ == "__main__": 1915 test_lib.main() 1916