Mercurial > cpython-lru_cache-weakref
diff Lib/test/test_robotparser.py @ 48535:dece17edbed0
Bug 3347: robotparser failed because it didn't convert bytes to string.
The solution is to convert bytes to text via utf-8. I'm not entirely
sure if this is safe, but it looks like robots.txt is expected to be
ascii.
| author | Jeremy Hylton <jeremy@alum.mit.edu> |
|---|---|
| date | Fri, 18 Jul 2008 20:59:44 +0000 |
| parents | 5a416a6417d3 |
| children | b51cd5d3795c |
line wrap: on
line diff
--- a/Lib/test/test_robotparser.py Fri Jul 18 19:34:28 2008 +0000 +++ b/Lib/test/test_robotparser.py Fri Jul 18 20:59:44 2008 +0000 @@ -136,8 +136,9 @@ RobotTest(7, doc, good, bad) -class TestCase(unittest.TestCase): - def runTest(self): +class NetworkTestCase(unittest.TestCase): + + def testPasswordProtectedSite(self): support.requires('network') # whole site is password-protected. url = 'http://mueblesmoraleda.com' @@ -146,9 +147,17 @@ parser.read() self.assertEqual(parser.can_fetch("*", url+"/robots.txt"), False) + def testPythonOrg(self): + support.requires('network') + parser = urllib.robotparser.RobotFileParser( + "http://www.python.org/robots.txt") + parser.read() + self.assertTrue(parser.can_fetch("*", + "http://www.python.org/robots.txt")) + def test_main(): + support.run_unittest(NetworkTestCase) support.run_unittest(tests) - TestCase().run() if __name__=='__main__': support.Verbose = 1