1# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""Tests for arbitrary expression evaluator."""
16from __future__ import absolute_import
17from __future__ import division
18from __future__ import print_function
19
20import numpy as np
21
22from tensorflow.python.debug.cli import evaluator
23from tensorflow.python.debug.lib import debug_data
24from tensorflow.python.framework import test_util
25from tensorflow.python.platform import test
26
27
28class ParseDebugTensorNameTest(test_util.TensorFlowTestCase):
29
30  def testParseNamesWithoutPrefixOrSuffix(self):
31    device_name, node_name, output_slot, debug_op, exec_index = (
32        evaluator._parse_debug_tensor_name("foo:1"))
33    self.assertIsNone(device_name)
34    self.assertEqual("foo", node_name)
35    self.assertEqual(1, output_slot)
36    self.assertEqual("DebugIdentity", debug_op)
37    self.assertEqual(0, exec_index)
38
39    device_name, node_name, output_slot, debug_op, exec_index = (
40        evaluator._parse_debug_tensor_name("hidden_0/Weights:0"))
41    self.assertIsNone(device_name)
42    self.assertEqual("hidden_0/Weights", node_name)
43    self.assertEqual(0, output_slot)
44    self.assertEqual("DebugIdentity", debug_op)
45    self.assertEqual(0, exec_index)
46
47  def testParseNamesWithoutPrefixWithDebugOpSuffix(self):
48    device_name, node_name, output_slot, debug_op, exec_index = (
49        evaluator._parse_debug_tensor_name("foo:1:DebugNanCount"))
50    self.assertIsNone(device_name)
51    self.assertEqual("foo", node_name)
52    self.assertEqual(1, output_slot)
53    self.assertEqual("DebugNanCount", debug_op)
54    self.assertEqual(0, exec_index)
55
56    device_name, node_name, output_slot, debug_op, exec_index = (
57        evaluator._parse_debug_tensor_name(
58            "hidden_0/Weights:0:DebugNumericSummary"))
59    self.assertIsNone(device_name)
60    self.assertEqual("hidden_0/Weights", node_name)
61    self.assertEqual(0, output_slot)
62    self.assertEqual("DebugNumericSummary", debug_op)
63    self.assertEqual(0, exec_index)
64
65  def testParseNamesWithDeviceNamePrefixWithoutDebugOpSuffix(self):
66    device_name, node_name, output_slot, debug_op, exec_index = (
67        evaluator._parse_debug_tensor_name(
68            "/job:ps/replica:0/task:2/cpu:0:foo:1"))
69    self.assertEqual("/job:ps/replica:0/task:2/cpu:0", device_name)
70    self.assertEqual("foo", node_name)
71    self.assertEqual(1, output_slot)
72    self.assertEqual("DebugIdentity", debug_op)
73    self.assertEqual(0, exec_index)
74
75    device_name, node_name, output_slot, debug_op, exec_index = (
76        evaluator._parse_debug_tensor_name(
77            "/job:worker/replica:0/task:3/gpu:0:hidden_0/Weights:0"))
78    self.assertEqual("/job:worker/replica:0/task:3/gpu:0", device_name)
79    self.assertEqual("hidden_0/Weights", node_name)
80    self.assertEqual(0, output_slot)
81    self.assertEqual("DebugIdentity", debug_op)
82    self.assertEqual(0, exec_index)
83
84  def testParseNamesWithDeviceNamePrefixWithDebugOpSuffix(self):
85    device_name, node_name, output_slot, debug_op, exec_index = (
86        evaluator._parse_debug_tensor_name(
87            "/job:ps/replica:0/task:2/cpu:0:foo:1:DebugNanCount"))
88    self.assertEqual("/job:ps/replica:0/task:2/cpu:0", device_name)
89    self.assertEqual("foo", node_name)
90    self.assertEqual(1, output_slot)
91    self.assertEqual("DebugNanCount", debug_op)
92    self.assertEqual(0, exec_index)
93
94    device_name, node_name, output_slot, debug_op, exec_index = (
95        evaluator._parse_debug_tensor_name(
96            "/job:worker/replica:0/task:3/gpu:0:"
97            "hidden_0/Weights:0:DebugNumericSummary"))
98    self.assertEqual("/job:worker/replica:0/task:3/gpu:0", device_name)
99    self.assertEqual("hidden_0/Weights", node_name)
100    self.assertEqual(0, output_slot)
101    self.assertEqual("DebugNumericSummary", debug_op)
102    self.assertEqual(0, exec_index)
103
104  def testParseMalformedDebugTensorName(self):
105    with self.assertRaisesRegexp(
106        ValueError,
107        r"The debug tensor name in the to-be-evaluated expression is "
108        r"malformed:"):
109      evaluator._parse_debug_tensor_name(
110          "/job:ps/replica:0/task:2/cpu:0:foo:1:DebugNanCount:1337")
111
112    with self.assertRaisesRegexp(
113        ValueError,
114        r"The debug tensor name in the to-be-evaluated expression is "
115        r"malformed:"):
116      evaluator._parse_debug_tensor_name(
117          "/job:ps/replica:0/cpu:0:foo:1:DebugNanCount")
118
119    with self.assertRaises(ValueError):
120      evaluator._parse_debug_tensor_name(
121          "foo:1:DebugNanCount[]")
122
123    with self.assertRaises(ValueError):
124      evaluator._parse_debug_tensor_name(
125          "foo:1[DebugNanCount]")
126
127  def testParseNamesWithExecIndex(self):
128    device_name, node_name, output_slot, debug_op, exec_index = (
129        evaluator._parse_debug_tensor_name("foo:1[20]"))
130    self.assertIsNone(device_name)
131    self.assertEqual("foo", node_name)
132    self.assertEqual(1, output_slot)
133    self.assertEqual("DebugIdentity", debug_op)
134    self.assertEqual(20, exec_index)
135
136    device_name, node_name, output_slot, debug_op, exec_index = (
137        evaluator._parse_debug_tensor_name("hidden_0/Weights:0[3]"))
138    self.assertIsNone(device_name)
139    self.assertEqual("hidden_0/Weights", node_name)
140    self.assertEqual(0, output_slot)
141    self.assertEqual("DebugIdentity", debug_op)
142    self.assertEqual(3, exec_index)
143
144
145class EvaluatorTest(test_util.TensorFlowTestCase):
146
147  def testEvaluateSingleTensor(self):
148    dump = test.mock.MagicMock()
149    def fake_get_tensors(node_name, output_slot, debug_op, device_name=None):
150      del node_name, output_slot, debug_op, device_name  # Unused.
151      return [np.array([[1.0, 2.0, 3.0]])]
152
153    with test.mock.patch.object(
154        dump, "get_tensors", side_effect=fake_get_tensors, autospec=True):
155      ev = evaluator.ExpressionEvaluator(dump)
156      self.assertEqual(3, ev.evaluate("np.size(`a:0`)"))
157
158      # Whitespace in backticks should be tolerated.
159      self.assertEqual(3, ev.evaluate("np.size(` a:0 `)"))
160
161  def testEvaluateTwoTensors(self):
162    dump = test.mock.MagicMock()
163    def fake_get_tensors(node_name, output_slot, debug_op, device_name=None):
164      del debug_op, device_name  # Unused.
165      if node_name == "a" and output_slot == 0:
166        return [np.array([[1.0, -2.0], [0.0, 1.0]])]
167      elif node_name == "b" and output_slot == 0:
168        return [np.array([[-1.0], [1.0]])]
169
170    with test.mock.patch.object(
171        dump, "get_tensors", side_effect=fake_get_tensors, autospec=True):
172      ev = evaluator.ExpressionEvaluator(dump)
173      self.assertAllClose([[-3.0], [1.0]],
174                          ev.evaluate("np.matmul(`a:0`, `b:0`)"))
175      self.assertAllClose(
176          [[-4.0], [2.0]], ev.evaluate("np.matmul(`a:0`, `b:0`) + `b:0`"))
177
178  def testEvaluateNoneExistentTensorGeneratesError(self):
179    dump = test.mock.MagicMock()
180    def fake_get_tensors(node_name, output_slot, debug_op, device_name=None):
181      del node_name, output_slot, debug_op, device_name  # Unused.
182      raise debug_data.WatchKeyDoesNotExistInDebugDumpDirError()
183
184    with test.mock.patch.object(
185        dump, "get_tensors", side_effect=fake_get_tensors, autospec=True):
186      ev = evaluator.ExpressionEvaluator(dump)
187      with self.assertRaisesRegexp(
188          ValueError, "Eval failed due to the value of .* being unavailable"):
189        ev.evaluate("np.matmul(`a:0`, `b:0`)")
190
191  def testEvaluateWithMultipleDevicesContainingTheSameTensorName(self):
192    dump = test.mock.MagicMock()
193    def fake_get_tensors(node_name, output_slot, debug_op, device_name=None):
194      del output_slot, debug_op  # Unused.
195      if node_name == "a" and device_name is None:
196        raise ValueError(
197            "There are multiple (2) devices with nodes named 'a' but "
198            "device_name is not specified")
199      elif (node_name == "a" and
200            device_name == "/job:worker/replica:0/task:0/cpu:0"):
201        return [np.array(10.0)]
202      elif (node_name == "a" and
203            device_name == "/job:worker/replica:0/task:1/cpu:0"):
204        return [np.array(20.0)]
205
206    with test.mock.patch.object(
207        dump, "get_tensors", side_effect=fake_get_tensors, autospec=True):
208      ev = evaluator.ExpressionEvaluator(dump)
209      with self.assertRaisesRegexp(ValueError, r"multiple \(2\) devices"):
210        ev.evaluate("`a:0` + `a:0`")
211
212      self.assertAllClose(
213          30.0,
214          ev.evaluate("`/job:worker/replica:0/task:0/cpu:0:a:0` + "
215                      "`/job:worker/replica:0/task:1/cpu:0:a:0`"))
216
217  def testEvaluateWithNonDefaultDebugOp(self):
218    dump = test.mock.MagicMock()
219    def fake_get_tensors(node_name, output_slot, debug_op, device_name=None):
220      del device_name  # Unused.
221      if node_name == "a" and output_slot == 0 and debug_op == "DebugIdentity":
222        return [np.array([[-1.0], [1.0]])]
223      elif node_name == "a" and output_slot == 0 and debug_op == "DebugFoo":
224        return [np.array([[-2.0, 2.0]])]
225
226    with test.mock.patch.object(
227        dump, "get_tensors", side_effect=fake_get_tensors, autospec=True):
228      ev = evaluator.ExpressionEvaluator(dump)
229      self.assertAllClose(
230          [[4.0]],
231          ev.evaluate("np.matmul(`a:0:DebugFoo`, `a:0:DebugIdentity`)"))
232
233  def testEvaluateWithMultipleExecIndexes(self):
234    dump = test.mock.MagicMock()
235    def fake_get_tensors(node_name, output_slot, debug_op, device_name=None):
236      del debug_op, device_name  # Unused.
237      if node_name == "a" and output_slot == 0:
238        return [np.array([[-1.0], [1.0]]), np.array([[-2.0], [2.0]])]
239
240    with test.mock.patch.object(
241        dump, "get_tensors", side_effect=fake_get_tensors, autospec=True):
242      ev = evaluator.ExpressionEvaluator(dump)
243      self.assertAllClose(
244          [[4.0]], ev.evaluate("np.matmul(`a:0[1]`.T, `a:0[0]`)"))
245
246  def testEvaluateExpressionWithUnmatchedBacktick(self):
247    dump = test.mock.MagicMock()
248    ev = evaluator.ExpressionEvaluator(dump)
249    with self.assertRaises(SyntaxError):
250      ev.evaluate("np.matmul(`a:0`, `b:0`) + `b:0")
251
252  def testEvaluateExpressionWithInvalidDebugTensorName(self):
253    dump = test.mock.MagicMock()
254    ev = evaluator.ExpressionEvaluator(dump)
255    with self.assertRaisesRegexp(
256        ValueError, r".* tensor name .* expression .* malformed"):
257      ev.evaluate("np.matmul(`a`, `b`)")
258
259    with self.assertRaisesRegexp(
260        ValueError, r".* tensor name .* expression .* malformed"):
261      ev.evaluate("np.matmul(`a:0:DebugIdentity:0`, `b:1:DebugNanCount:2`)")
262
263    with self.assertRaises(ValueError):
264      ev.evaluate("np.matmul(`a:0[]`, `b:0[]`)")
265
266
267if __name__ == "__main__":
268  test.main()
269