Searched refs:can_fetch (Results 1 – 3 of 3) sorted by relevance
23 self.assertTrue(self.parser.can_fetch(agent, url))25 self.assertFalse(self.parser.can_fetch(agent, url))243 self.assertEqual(parser.can_fetch("*", url+"/robots.txt"), False)252 parser.can_fetch("*", "http://www.python.org/robots.txt"))
128 def can_fetch(self, useragent, url): member in RobotFileParser
483 return self.robots[root].can_fetch(AGENTNAME, url)