1# Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Tests for arbitrary expression evaluator.""" 16from __future__ import absolute_import 17from __future__ import division 18from __future__ import print_function 19 20import numpy as np 21 22from tensorflow.python.debug.cli import evaluator 23from tensorflow.python.debug.lib import debug_data 24from tensorflow.python.framework import test_util 25from tensorflow.python.platform import test 26 27 28class ParseDebugTensorNameTest(test_util.TensorFlowTestCase): 29 30 def testParseNamesWithoutPrefixOrSuffix(self): 31 device_name, node_name, output_slot, debug_op, exec_index = ( 32 evaluator._parse_debug_tensor_name("foo:1")) 33 self.assertIsNone(device_name) 34 self.assertEqual("foo", node_name) 35 self.assertEqual(1, output_slot) 36 self.assertEqual("DebugIdentity", debug_op) 37 self.assertEqual(0, exec_index) 38 39 device_name, node_name, output_slot, debug_op, exec_index = ( 40 evaluator._parse_debug_tensor_name("hidden_0/Weights:0")) 41 self.assertIsNone(device_name) 42 self.assertEqual("hidden_0/Weights", node_name) 43 self.assertEqual(0, output_slot) 44 self.assertEqual("DebugIdentity", debug_op) 45 self.assertEqual(0, exec_index) 46 47 def testParseNamesWithoutPrefixWithDebugOpSuffix(self): 48 device_name, node_name, output_slot, debug_op, exec_index = ( 49 evaluator._parse_debug_tensor_name("foo:1:DebugNanCount")) 50 self.assertIsNone(device_name) 51 self.assertEqual("foo", node_name) 52 self.assertEqual(1, output_slot) 53 self.assertEqual("DebugNanCount", debug_op) 54 self.assertEqual(0, exec_index) 55 56 device_name, node_name, output_slot, debug_op, exec_index = ( 57 evaluator._parse_debug_tensor_name( 58 "hidden_0/Weights:0:DebugNumericSummary")) 59 self.assertIsNone(device_name) 60 self.assertEqual("hidden_0/Weights", node_name) 61 self.assertEqual(0, output_slot) 62 self.assertEqual("DebugNumericSummary", debug_op) 63 self.assertEqual(0, exec_index) 64 65 def testParseNamesWithDeviceNamePrefixWithoutDebugOpSuffix(self): 66 device_name, node_name, output_slot, debug_op, exec_index = ( 67 evaluator._parse_debug_tensor_name( 68 "/job:ps/replica:0/task:2/cpu:0:foo:1")) 69 self.assertEqual("/job:ps/replica:0/task:2/cpu:0", device_name) 70 self.assertEqual("foo", node_name) 71 self.assertEqual(1, output_slot) 72 self.assertEqual("DebugIdentity", debug_op) 73 self.assertEqual(0, exec_index) 74 75 device_name, node_name, output_slot, debug_op, exec_index = ( 76 evaluator._parse_debug_tensor_name( 77 "/job:worker/replica:0/task:3/gpu:0:hidden_0/Weights:0")) 78 self.assertEqual("/job:worker/replica:0/task:3/gpu:0", device_name) 79 self.assertEqual("hidden_0/Weights", node_name) 80 self.assertEqual(0, output_slot) 81 self.assertEqual("DebugIdentity", debug_op) 82 self.assertEqual(0, exec_index) 83 84 def testParseNamesWithDeviceNamePrefixWithDebugOpSuffix(self): 85 device_name, node_name, output_slot, debug_op, exec_index = ( 86 evaluator._parse_debug_tensor_name( 87 "/job:ps/replica:0/task:2/cpu:0:foo:1:DebugNanCount")) 88 self.assertEqual("/job:ps/replica:0/task:2/cpu:0", device_name) 89 self.assertEqual("foo", node_name) 90 self.assertEqual(1, output_slot) 91 self.assertEqual("DebugNanCount", debug_op) 92 self.assertEqual(0, exec_index) 93 94 device_name, node_name, output_slot, debug_op, exec_index = ( 95 evaluator._parse_debug_tensor_name( 96 "/job:worker/replica:0/task:3/gpu:0:" 97 "hidden_0/Weights:0:DebugNumericSummary")) 98 self.assertEqual("/job:worker/replica:0/task:3/gpu:0", device_name) 99 self.assertEqual("hidden_0/Weights", node_name) 100 self.assertEqual(0, output_slot) 101 self.assertEqual("DebugNumericSummary", debug_op) 102 self.assertEqual(0, exec_index) 103 104 def testParseMalformedDebugTensorName(self): 105 with self.assertRaisesRegexp( 106 ValueError, 107 r"The debug tensor name in the to-be-evaluated expression is " 108 r"malformed:"): 109 evaluator._parse_debug_tensor_name( 110 "/job:ps/replica:0/task:2/cpu:0:foo:1:DebugNanCount:1337") 111 112 with self.assertRaisesRegexp( 113 ValueError, 114 r"The debug tensor name in the to-be-evaluated expression is " 115 r"malformed:"): 116 evaluator._parse_debug_tensor_name( 117 "/job:ps/replica:0/cpu:0:foo:1:DebugNanCount") 118 119 with self.assertRaises(ValueError): 120 evaluator._parse_debug_tensor_name( 121 "foo:1:DebugNanCount[]") 122 123 with self.assertRaises(ValueError): 124 evaluator._parse_debug_tensor_name( 125 "foo:1[DebugNanCount]") 126 127 def testParseNamesWithExecIndex(self): 128 device_name, node_name, output_slot, debug_op, exec_index = ( 129 evaluator._parse_debug_tensor_name("foo:1[20]")) 130 self.assertIsNone(device_name) 131 self.assertEqual("foo", node_name) 132 self.assertEqual(1, output_slot) 133 self.assertEqual("DebugIdentity", debug_op) 134 self.assertEqual(20, exec_index) 135 136 device_name, node_name, output_slot, debug_op, exec_index = ( 137 evaluator._parse_debug_tensor_name("hidden_0/Weights:0[3]")) 138 self.assertIsNone(device_name) 139 self.assertEqual("hidden_0/Weights", node_name) 140 self.assertEqual(0, output_slot) 141 self.assertEqual("DebugIdentity", debug_op) 142 self.assertEqual(3, exec_index) 143 144 145class EvaluatorTest(test_util.TensorFlowTestCase): 146 147 def testEvaluateSingleTensor(self): 148 dump = test.mock.MagicMock() 149 def fake_get_tensors(node_name, output_slot, debug_op, device_name=None): 150 del node_name, output_slot, debug_op, device_name # Unused. 151 return [np.array([[1.0, 2.0, 3.0]])] 152 153 with test.mock.patch.object( 154 dump, "get_tensors", side_effect=fake_get_tensors, autospec=True): 155 ev = evaluator.ExpressionEvaluator(dump) 156 self.assertEqual(3, ev.evaluate("np.size(`a:0`)")) 157 158 # Whitespace in backticks should be tolerated. 159 self.assertEqual(3, ev.evaluate("np.size(` a:0 `)")) 160 161 def testEvaluateTwoTensors(self): 162 dump = test.mock.MagicMock() 163 def fake_get_tensors(node_name, output_slot, debug_op, device_name=None): 164 del debug_op, device_name # Unused. 165 if node_name == "a" and output_slot == 0: 166 return [np.array([[1.0, -2.0], [0.0, 1.0]])] 167 elif node_name == "b" and output_slot == 0: 168 return [np.array([[-1.0], [1.0]])] 169 170 with test.mock.patch.object( 171 dump, "get_tensors", side_effect=fake_get_tensors, autospec=True): 172 ev = evaluator.ExpressionEvaluator(dump) 173 self.assertAllClose([[-3.0], [1.0]], 174 ev.evaluate("np.matmul(`a:0`, `b:0`)")) 175 self.assertAllClose( 176 [[-4.0], [2.0]], ev.evaluate("np.matmul(`a:0`, `b:0`) + `b:0`")) 177 178 def testEvaluateNoneExistentTensorGeneratesError(self): 179 dump = test.mock.MagicMock() 180 def fake_get_tensors(node_name, output_slot, debug_op, device_name=None): 181 del node_name, output_slot, debug_op, device_name # Unused. 182 raise debug_data.WatchKeyDoesNotExistInDebugDumpDirError() 183 184 with test.mock.patch.object( 185 dump, "get_tensors", side_effect=fake_get_tensors, autospec=True): 186 ev = evaluator.ExpressionEvaluator(dump) 187 with self.assertRaisesRegexp( 188 ValueError, "Eval failed due to the value of .* being unavailable"): 189 ev.evaluate("np.matmul(`a:0`, `b:0`)") 190 191 def testEvaluateWithMultipleDevicesContainingTheSameTensorName(self): 192 dump = test.mock.MagicMock() 193 def fake_get_tensors(node_name, output_slot, debug_op, device_name=None): 194 del output_slot, debug_op # Unused. 195 if node_name == "a" and device_name is None: 196 raise ValueError( 197 "There are multiple (2) devices with nodes named 'a' but " 198 "device_name is not specified") 199 elif (node_name == "a" and 200 device_name == "/job:worker/replica:0/task:0/cpu:0"): 201 return [np.array(10.0)] 202 elif (node_name == "a" and 203 device_name == "/job:worker/replica:0/task:1/cpu:0"): 204 return [np.array(20.0)] 205 206 with test.mock.patch.object( 207 dump, "get_tensors", side_effect=fake_get_tensors, autospec=True): 208 ev = evaluator.ExpressionEvaluator(dump) 209 with self.assertRaisesRegexp(ValueError, r"multiple \(2\) devices"): 210 ev.evaluate("`a:0` + `a:0`") 211 212 self.assertAllClose( 213 30.0, 214 ev.evaluate("`/job:worker/replica:0/task:0/cpu:0:a:0` + " 215 "`/job:worker/replica:0/task:1/cpu:0:a:0`")) 216 217 def testEvaluateWithNonDefaultDebugOp(self): 218 dump = test.mock.MagicMock() 219 def fake_get_tensors(node_name, output_slot, debug_op, device_name=None): 220 del device_name # Unused. 221 if node_name == "a" and output_slot == 0 and debug_op == "DebugIdentity": 222 return [np.array([[-1.0], [1.0]])] 223 elif node_name == "a" and output_slot == 0 and debug_op == "DebugFoo": 224 return [np.array([[-2.0, 2.0]])] 225 226 with test.mock.patch.object( 227 dump, "get_tensors", side_effect=fake_get_tensors, autospec=True): 228 ev = evaluator.ExpressionEvaluator(dump) 229 self.assertAllClose( 230 [[4.0]], 231 ev.evaluate("np.matmul(`a:0:DebugFoo`, `a:0:DebugIdentity`)")) 232 233 def testEvaluateWithMultipleExecIndexes(self): 234 dump = test.mock.MagicMock() 235 def fake_get_tensors(node_name, output_slot, debug_op, device_name=None): 236 del debug_op, device_name # Unused. 237 if node_name == "a" and output_slot == 0: 238 return [np.array([[-1.0], [1.0]]), np.array([[-2.0], [2.0]])] 239 240 with test.mock.patch.object( 241 dump, "get_tensors", side_effect=fake_get_tensors, autospec=True): 242 ev = evaluator.ExpressionEvaluator(dump) 243 self.assertAllClose( 244 [[4.0]], ev.evaluate("np.matmul(`a:0[1]`.T, `a:0[0]`)")) 245 246 def testEvaluateExpressionWithUnmatchedBacktick(self): 247 dump = test.mock.MagicMock() 248 ev = evaluator.ExpressionEvaluator(dump) 249 with self.assertRaises(SyntaxError): 250 ev.evaluate("np.matmul(`a:0`, `b:0`) + `b:0") 251 252 def testEvaluateExpressionWithInvalidDebugTensorName(self): 253 dump = test.mock.MagicMock() 254 ev = evaluator.ExpressionEvaluator(dump) 255 with self.assertRaisesRegexp( 256 ValueError, r".* tensor name .* expression .* malformed"): 257 ev.evaluate("np.matmul(`a`, `b`)") 258 259 with self.assertRaisesRegexp( 260 ValueError, r".* tensor name .* expression .* malformed"): 261 ev.evaluate("np.matmul(`a:0:DebugIdentity:0`, `b:1:DebugNanCount:2`)") 262 263 with self.assertRaises(ValueError): 264 ev.evaluate("np.matmul(`a:0[]`, `b:0[]`)") 265 266 267if __name__ == "__main__": 268 test.main() 269