Mercurial > cpython-lru_cache-weakref

annotate Lib/test/test_robotparser.py @ 87690:66c1c9f32567 default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add use_weakref option to functools.lru_cache The use_weakref option makes lru_cache create weak references to the function arguments when creating the cache key, which prevents them from staying in the memory forever even after all other references to them are gone. This is a proof of concept for http://bugs.python.org/issue19859
author Radomir Dopieralski <devel@sheep.art.pl>
date Tue, 03 Dec 2013 16:02:48 +0100
parents 30128355f53b
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
42501
f6069f9dd52d SF patch# 1770008 by Christian Heimes (plus some extras).
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
1 import io
48334
5a416a6417d3 Make a new urllib package .
Jeremy Hylton <jeremy@alum.mit.edu>
parents: 47201
diff changeset
2 import unittest
5a416a6417d3 Make a new urllib package .
Jeremy Hylton <jeremy@alum.mit.edu>
parents: 47201
diff changeset
3 import urllib.robotparser
71261
6e72490bbff6 Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68257
diff changeset
4 from urllib.error import URLError, HTTPError
6e72490bbff6 Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68257
diff changeset
5 from urllib.request import urlopen
47201
9c930a32cebb #2621 rename test.test_support to test.support
Benjamin Peterson <benjamin@python.org>
parents: 43095
diff changeset
6 from test import support
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
7
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
8 class RobotTestCase(unittest.TestCase):
82625
fa051c6276a0 #17066: test_robotparser now works with unittest test discovery. Patch by Zachary Ware.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 71261
diff changeset
9 def __init__(self, index=None, parser=None, url=None, good=None, agent=None):
fa051c6276a0 #17066: test_robotparser now works with unittest test discovery. Patch by Zachary Ware.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 71261
diff changeset
10 # workaround to make unittest discovery work (see #17066)
fa051c6276a0 #17066: test_robotparser now works with unittest test discovery. Patch by Zachary Ware.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 71261
diff changeset
11 if not isinstance(index, int):
fa051c6276a0 #17066: test_robotparser now works with unittest test discovery. Patch by Zachary Ware.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 71261
diff changeset
12 return
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
13 unittest.TestCase.__init__(self)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
14 if good:
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
15 self.str = "RobotTest(%d, good, %s)" % (index, url)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
16 else:
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
17 self.str = "RobotTest(%d, bad, %s)" % (index, url)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
18 self.parser = parser
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
19 self.url = url
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
20 self.good = good
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
21 self.agent = agent
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
22
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
23 def runTest(self):
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
24 if isinstance(self.url, tuple):
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
25 agent, url = self.url
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
26 else:
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
27 url = self.url
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
28 agent = self.agent
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
29 if self.good:
55593
0bb232078b8c Merged revisions 73715 via svnmerge from
Georg Brandl <georg@python.org>
parents: 48594
diff changeset
30 self.assertTrue(self.parser.can_fetch(agent, url))
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
31 else:
55593
0bb232078b8c Merged revisions 73715 via svnmerge from
Georg Brandl <georg@python.org>
parents: 48594
diff changeset
32 self.assertFalse(self.parser.can_fetch(agent, url))
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
33
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
34 def __str__(self):
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
35 return self.str
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
36
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
37 tests = unittest.TestSuite()
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
38
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
39 def RobotTest(index, robots_txt, good_urls, bad_urls,
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
40 agent="test_robotparser"):
23075
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
41
42501
f6069f9dd52d SF patch# 1770008 by Christian Heimes (plus some extras).
Guido van Rossum <guido@python.org>
parents: 41554
diff changeset
42 lines = io.StringIO(robots_txt).readlines()
48334
5a416a6417d3 Make a new urllib package .
Jeremy Hylton <jeremy@alum.mit.edu>
parents: 47201
diff changeset
43 parser = urllib.robotparser.RobotFileParser()
23075
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
44 parser.parse(lines)
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
45 for url in good_urls:
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
46 tests.addTest(RobotTestCase(index, parser, url, 1, agent))
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
47 for url in bad_urls:
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
48 tests.addTest(RobotTestCase(index, parser, url, 0, agent))
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
49
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
50 # Examples from http://www.robotstxt.org/wc/norobots.html (fetched 2002)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
51
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
52 # 1.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
53 doc = """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
54 User-agent: *
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
55 Disallow: /cyberworld/map/ # This is an infinite virtual URL space
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
56 Disallow: /tmp/ # these will soon disappear
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
57 Disallow: /foo.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
58 """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
59
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
60 good = ['/','/test.html']
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
61 bad = ['/cyberworld/map/index.html','/tmp/xxx','/foo.html']
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
62
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
63 RobotTest(1, doc, good, bad)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
64
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
65 # 2.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
66 doc = """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
67 # robots.txt for http://www.example.com/
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
68
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
69 User-agent: *
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
70 Disallow: /cyberworld/map/ # This is an infinite virtual URL space
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
71
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
72 # Cybermapper knows where to go.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
73 User-agent: cybermapper
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
74 Disallow:
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
75
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
76 """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
77
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
78 good = ['/','/test.html',('cybermapper','/cyberworld/map/index.html')]
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
79 bad = ['/cyberworld/map/index.html']
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
80
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
81 RobotTest(2, doc, good, bad)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
82
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
83 # 3.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
84 doc = """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
85 # go away
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
86 User-agent: *
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
87 Disallow: /
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
88 """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
89
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
90 good = []
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
91 bad = ['/cyberworld/map/index.html','/','/tmp/']
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
92
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
93 RobotTest(3, doc, good, bad)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
94
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
95 # Examples from http://www.robotstxt.org/wc/norobots-rfc.html (fetched 2002)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
96
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
97 # 4.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
98 doc = """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
99 User-agent: figtree
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
100 Disallow: /tmp
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
101 Disallow: /a%3cd.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
102 Disallow: /a%2fb.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
103 Disallow: /%7ejoe/index.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
104 """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
105
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
106 good = [] # XFAIL '/a/b.html'
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
107 bad = ['/tmp','/tmp.html','/tmp/a.html',
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
108 '/a%3cd.html','/a%3Cd.html','/a%2fb.html',
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
109 '/~joe/index.html'
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
110 ]
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
111
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
112 RobotTest(4, doc, good, bad, 'figtree')
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
113 RobotTest(5, doc, good, bad, 'FigTree Robot libwww-perl/5.04')
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
114
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
115 # 6.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
116 doc = """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
117 User-agent: *
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
118 Disallow: /tmp/
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
119 Disallow: /a%3Cd.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
120 Disallow: /a/b.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
121 Disallow: /%7ejoe/index.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
122 """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
123
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
124 good = ['/tmp',] # XFAIL: '/a%2fb.html'
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
125 bad = ['/tmp/','/tmp/a.html',
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
126 '/a%3cd.html','/a%3Cd.html',"/a/b.html",
23075
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
127 '/%7Ejoe/index.html']
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
128
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
129 RobotTest(6, doc, good, bad)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
130
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
131 # From bug report #523041
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
132
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
133 # 7.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
134 doc = """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
135 User-Agent: *
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
136 Disallow: /.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
137 """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
138
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
139 good = ['/foo.html']
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
140 bad = [] # Bug report says "/" should be denied, but that is not in the RFC
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
141
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
142 RobotTest(7, doc, good, bad)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
143
48594
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
144 # From Google: http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=40364
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
145
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
146 # 8.
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
147 doc = """
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
148 User-agent: Googlebot
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
149 Allow: /folder1/myfile.html
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
150 Disallow: /folder1/
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
151 """
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
152
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
153 good = ['/folder1/myfile.html']
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
154 bad = ['/folder1/anotherfile.html']
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
155
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
156 RobotTest(8, doc, good, bad, agent="Googlebot")
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
157
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
158 # 9. This file is incorrect because "Googlebot" is a substring of
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
159 # "Googlebot-Mobile", so test 10 works just like test 9.
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
160 doc = """
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
161 User-agent: Googlebot
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
162 Disallow: /
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
163
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
164 User-agent: Googlebot-Mobile
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
165 Allow: /
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
166 """
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
167
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
168 good = []
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
169 bad = ['/something.jpg']
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
170
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
171 RobotTest(9, doc, good, bad, agent="Googlebot")
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
172
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
173 good = []
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
174 bad = ['/something.jpg']
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
175
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
176 RobotTest(10, doc, good, bad, agent="Googlebot-Mobile")
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
177
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
178 # 11. Get the order correct.
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
179 doc = """
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
180 User-agent: Googlebot-Mobile
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
181 Allow: /
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
182
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
183 User-agent: Googlebot
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
184 Disallow: /
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
185 """
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
186
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
187 good = []
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
188 bad = ['/something.jpg']
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
189
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
190 RobotTest(11, doc, good, bad, agent="Googlebot")
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
191
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
192 good = ['/something.jpg']
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
193 bad = []
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
194
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
195 RobotTest(12, doc, good, bad, agent="Googlebot-Mobile")
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
196
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
197
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
198 # 13. Google also got the order wrong in #8. You need to specify the
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
199 # URLs from more specific to more general.
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
200 doc = """
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
201 User-agent: Googlebot
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
202 Allow: /folder1/myfile.html
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
203 Disallow: /folder1/
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
204 """
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
205
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
206 good = ['/folder1/myfile.html']
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
207 bad = ['/folder1/anotherfile.html']
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
208
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
209 RobotTest(13, doc, good, bad, agent="googlebot")
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
210
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
211
63271
3119c4d7276b Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60202
diff changeset
212 # 14. For issue #6325 (query string support)
3119c4d7276b Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60202
diff changeset
213 doc = """
3119c4d7276b Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60202
diff changeset
214 User-agent: *
3119c4d7276b Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60202
diff changeset
215 Disallow: /some/path?name=value
3119c4d7276b Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60202
diff changeset
216 """
3119c4d7276b Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60202
diff changeset
217
3119c4d7276b Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60202
diff changeset
218 good = ['/some/path']
3119c4d7276b Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60202
diff changeset
219 bad = ['/some/path?name=value']
3119c4d7276b Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60202
diff changeset
220
3119c4d7276b Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60202
diff changeset
221 RobotTest(14, doc, good, bad)
3119c4d7276b Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60202
diff changeset
222
63464
415ea2c3cfe8 Merged revisions 83238 via svnmerge from
Georg Brandl <georg@python.org>
parents: 63271
diff changeset
223 # 15. For issue #4108 (obey first * entry)
415ea2c3cfe8 Merged revisions 83238 via svnmerge from
Georg Brandl <georg@python.org>
parents: 63271
diff changeset
224 doc = """
415ea2c3cfe8 Merged revisions 83238 via svnmerge from
Georg Brandl <georg@python.org>
parents: 63271
diff changeset
225 User-agent: *
415ea2c3cfe8 Merged revisions 83238 via svnmerge from
Georg Brandl <georg@python.org>
parents: 63271
diff changeset
226 Disallow: /some/path
415ea2c3cfe8 Merged revisions 83238 via svnmerge from
Georg Brandl <georg@python.org>
parents: 63271
diff changeset
227
415ea2c3cfe8 Merged revisions 83238 via svnmerge from
Georg Brandl <georg@python.org>
parents: 63271
diff changeset
228 User-agent: *
415ea2c3cfe8 Merged revisions 83238 via svnmerge from
Georg Brandl <georg@python.org>
parents: 63271
diff changeset
229 Disallow: /another/path
415ea2c3cfe8 Merged revisions 83238 via svnmerge from
Georg Brandl <georg@python.org>
parents: 63271
diff changeset
230 """
415ea2c3cfe8 Merged revisions 83238 via svnmerge from
Georg Brandl <georg@python.org>
parents: 63271
diff changeset
231
415ea2c3cfe8 Merged revisions 83238 via svnmerge from
Georg Brandl <georg@python.org>
parents: 63271
diff changeset
232 good = ['/another/path']
415ea2c3cfe8 Merged revisions 83238 via svnmerge from
Georg Brandl <georg@python.org>
parents: 63271
diff changeset
233 bad = ['/some/path']
415ea2c3cfe8 Merged revisions 83238 via svnmerge from
Georg Brandl <georg@python.org>
parents: 63271
diff changeset
234
415ea2c3cfe8 Merged revisions 83238 via svnmerge from
Georg Brandl <georg@python.org>
parents: 63271
diff changeset
235 RobotTest(15, doc, good, bad)
415ea2c3cfe8 Merged revisions 83238 via svnmerge from
Georg Brandl <georg@python.org>
parents: 63271
diff changeset
236
83971
30128355f53b #17403: urllib.parse.robotparser normalizes the urls before adding to ruleline.
Senthil Kumaran <senthil@uthcode.com>
parents: 82625
diff changeset
237 # 16. Empty query (issue #17403). Normalizing the url first.
30128355f53b #17403: urllib.parse.robotparser normalizes the urls before adding to ruleline.
Senthil Kumaran <senthil@uthcode.com>
parents: 82625
diff changeset
238 doc = """
30128355f53b #17403: urllib.parse.robotparser normalizes the urls before adding to ruleline.
Senthil Kumaran <senthil@uthcode.com>
parents: 82625
diff changeset
239 User-agent: *
30128355f53b #17403: urllib.parse.robotparser normalizes the urls before adding to ruleline.
Senthil Kumaran <senthil@uthcode.com>
parents: 82625
diff changeset
240 Allow: /some/path?
30128355f53b #17403: urllib.parse.robotparser normalizes the urls before adding to ruleline.
Senthil Kumaran <senthil@uthcode.com>
parents: 82625
diff changeset
241 Disallow: /another/path?
30128355f53b #17403: urllib.parse.robotparser normalizes the urls before adding to ruleline.
Senthil Kumaran <senthil@uthcode.com>
parents: 82625
diff changeset
242 """
30128355f53b #17403: urllib.parse.robotparser normalizes the urls before adding to ruleline.
Senthil Kumaran <senthil@uthcode.com>
parents: 82625
diff changeset
243
30128355f53b #17403: urllib.parse.robotparser normalizes the urls before adding to ruleline.
Senthil Kumaran <senthil@uthcode.com>
parents: 82625
diff changeset
244 good = ['/some/path?']
30128355f53b #17403: urllib.parse.robotparser normalizes the urls before adding to ruleline.
Senthil Kumaran <senthil@uthcode.com>
parents: 82625
diff changeset
245 bad = ['/another/path?']
30128355f53b #17403: urllib.parse.robotparser normalizes the urls before adding to ruleline.
Senthil Kumaran <senthil@uthcode.com>
parents: 82625
diff changeset
246
30128355f53b #17403: urllib.parse.robotparser normalizes the urls before adding to ruleline.
Senthil Kumaran <senthil@uthcode.com>
parents: 82625
diff changeset
247 RobotTest(16, doc, good, bad)
30128355f53b #17403: urllib.parse.robotparser normalizes the urls before adding to ruleline.
Senthil Kumaran <senthil@uthcode.com>
parents: 82625
diff changeset
248
48594
35b1a20470db Merged revisions 65209-65216,65225-65226,65233,65239,65246-65247,65255-65256 via svnmerge from
Benjamin Peterson <benjamin@python.org>
parents: 48563
diff changeset
249
48535
dece17edbed0 Bug 3347: robotparser failed because it didn't convert bytes to string.
Jeremy Hylton <jeremy@alum.mit.edu>
parents: 48334
diff changeset
250 class NetworkTestCase(unittest.TestCase):
dece17edbed0 Bug 3347: robotparser failed because it didn't convert bytes to string.
Jeremy Hylton <jeremy@alum.mit.edu>
parents: 48334
diff changeset
251
dece17edbed0 Bug 3347: robotparser failed because it didn't convert bytes to string.
Jeremy Hylton <jeremy@alum.mit.edu>
parents: 48334
diff changeset
252 def testPasswordProtectedSite(self):
60202
5496503b8b1a Merged revision 79605 via svnmerge from
Florent Xicluna <florent.xicluna@gmail.com>
parents: 55593
diff changeset
253 support.requires('network')
64536
2400f55ea312 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63464
diff changeset
254 with support.transient_internet('mueblesmoraleda.com'):
2400f55ea312 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63464
diff changeset
255 url = 'http://mueblesmoraleda.com'
71261
6e72490bbff6 Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68257
diff changeset
256 robots_url = url + "/robots.txt"
6e72490bbff6 Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68257
diff changeset
257 # First check the URL is usable for our purposes, since the
6e72490bbff6 Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68257
diff changeset
258 # test site is a bit flaky.
6e72490bbff6 Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68257
diff changeset
259 try:
6e72490bbff6 Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68257
diff changeset
260 urlopen(robots_url)
6e72490bbff6 Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68257
diff changeset
261 except HTTPError as e:
6e72490bbff6 Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68257
diff changeset
262 if e.code not in {401, 403}:
6e72490bbff6 Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68257
diff changeset
263 self.skipTest(
6e72490bbff6 Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68257
diff changeset
264 "%r should return a 401 or 403 HTTP error, not %r"
6e72490bbff6 Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68257
diff changeset
265 % (robots_url, e.code))
6e72490bbff6 Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68257
diff changeset
266 else:
6e72490bbff6 Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68257
diff changeset
267 self.skipTest(
6e72490bbff6 Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68257
diff changeset
268 "%r should return a 401 or 403 HTTP error, not succeed"
6e72490bbff6 Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68257
diff changeset
269 % (robots_url))
64536
2400f55ea312 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63464
diff changeset
270 parser = urllib.robotparser.RobotFileParser()
2400f55ea312 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63464
diff changeset
271 parser.set_url(url)
2400f55ea312 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63464
diff changeset
272 try:
2400f55ea312 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63464
diff changeset
273 parser.read()
2400f55ea312 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63464
diff changeset
274 except URLError:
2400f55ea312 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63464
diff changeset
275 self.skipTest('%s is unavailable' % url)
71261
6e72490bbff6 Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68257
diff changeset
276 self.assertEqual(parser.can_fetch("*", robots_url), False)
43095
4a07a288f5c4 Merged revisions 57620-57771 via svnmerge from
Thomas Wouters <thomas@python.org>
parents: 42501
diff changeset
277
48535
dece17edbed0 Bug 3347: robotparser failed because it didn't convert bytes to string.
Jeremy Hylton <jeremy@alum.mit.edu>
parents: 48334
diff changeset
278 def testPythonOrg(self):
60202
5496503b8b1a Merged revision 79605 via svnmerge from
Florent Xicluna <florent.xicluna@gmail.com>
parents: 55593
diff changeset
279 support.requires('network')
64536
2400f55ea312 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63464
diff changeset
280 with support.transient_internet('www.python.org'):
2400f55ea312 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63464
diff changeset
281 parser = urllib.robotparser.RobotFileParser(
2400f55ea312 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63464
diff changeset
282 "http://www.python.org/robots.txt")
2400f55ea312 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63464
diff changeset
283 parser.read()
2400f55ea312 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63464
diff changeset
284 self.assertTrue(
2400f55ea312 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63464
diff changeset
285 parser.can_fetch("*", "http://www.python.org/robots.txt"))
48535
dece17edbed0 Bug 3347: robotparser failed because it didn't convert bytes to string.
Jeremy Hylton <jeremy@alum.mit.edu>
parents: 48334
diff changeset
286
82625
fa051c6276a0 #17066: test_robotparser now works with unittest test discovery. Patch by Zachary Ware.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 71261
diff changeset
287 def load_tests(loader, suite, pattern):
fa051c6276a0 #17066: test_robotparser now works with unittest test discovery. Patch by Zachary Ware.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 71261
diff changeset
288 suite = unittest.makeSuite(NetworkTestCase)
fa051c6276a0 #17066: test_robotparser now works with unittest test discovery. Patch by Zachary Ware.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 71261
diff changeset
289 suite.addTest(tests)
fa051c6276a0 #17066: test_robotparser now works with unittest test discovery. Patch by Zachary Ware.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 71261
diff changeset
290 return suite
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
291
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
292 if __name__=='__main__':
82625
fa051c6276a0 #17066: test_robotparser now works with unittest test discovery. Patch by Zachary Ware.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 71261
diff changeset
293 support.use_resources = ['network']
fa051c6276a0 #17066: test_robotparser now works with unittest test discovery. Patch by Zachary Ware.
Ezio Melotti <ezio.melotti@gmail.com>
parents: 71261
diff changeset
294 unittest.main()