Mercurial > cpython-lru_cache-weakref

annotate Lib/test/test_robotparser.py @ 80661:94de77bd0b4b 2.7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Issue #16194: document imp.load_dynamic problems when read several extension modules from the same binary object file.
author Andrew Svetlov <andrew.svetlov@gmail.com>
date Thu, 29 Nov 2012 19:51:28 +0200
parents e6e37f1e29ca
children bcbad715c2ce
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
24364
aa0d2d51fbde Get rid of relative imports in all unittests. Now anything that
Barry Warsaw <barry@python.org>
parents: 23075
diff changeset
1 import unittest, StringIO, robotparser
aa0d2d51fbde Get rid of relative imports in all unittests. Now anything that
Barry Warsaw <barry@python.org>
parents: 23075
diff changeset
2 from test import test_support
71260
e6e37f1e29ca Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68256
diff changeset
3 from urllib2 import urlopen, HTTPError
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
4
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
5 class RobotTestCase(unittest.TestCase):
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
6 def __init__(self, index, parser, url, good, agent):
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
7 unittest.TestCase.__init__(self)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
8 if good:
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
9 self.str = "RobotTest(%d, good, %s)" % (index, url)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
10 else:
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
11 self.str = "RobotTest(%d, bad, %s)" % (index, url)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
12 self.parser = parser
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
13 self.url = url
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
14 self.good = good
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
15 self.agent = agent
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
16
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
17 def runTest(self):
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
18 if isinstance(self.url, tuple):
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
19 agent, url = self.url
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
20 else:
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
21 url = self.url
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
22 agent = self.agent
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
23 if self.good:
55002
6768b2350c57 convert usage of fail* to assert*
Benjamin Peterson <benjamin@python.org>
parents: 48085
diff changeset
24 self.assertTrue(self.parser.can_fetch(agent, url))
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
25 else:
55002
6768b2350c57 convert usage of fail* to assert*
Benjamin Peterson <benjamin@python.org>
parents: 48085
diff changeset
26 self.assertFalse(self.parser.can_fetch(agent, url))
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
27
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
28 def __str__(self):
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
29 return self.str
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
30
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
31 tests = unittest.TestSuite()
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
32
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
33 def RobotTest(index, robots_txt, good_urls, bad_urls,
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
34 agent="test_robotparser"):
23075
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
35
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
36 lines = StringIO.StringIO(robots_txt).readlines()
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
37 parser = robotparser.RobotFileParser()
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
38 parser.parse(lines)
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
39 for url in good_urls:
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
40 tests.addTest(RobotTestCase(index, parser, url, 1, agent))
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
41 for url in bad_urls:
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
42 tests.addTest(RobotTestCase(index, parser, url, 0, agent))
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
43
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
44 # Examples from http://www.robotstxt.org/wc/norobots.html (fetched 2002)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
45
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
46 # 1.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
47 doc = """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
48 User-agent: *
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
49 Disallow: /cyberworld/map/ # This is an infinite virtual URL space
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
50 Disallow: /tmp/ # these will soon disappear
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
51 Disallow: /foo.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
52 """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
53
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
54 good = ['/','/test.html']
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
55 bad = ['/cyberworld/map/index.html','/tmp/xxx','/foo.html']
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
56
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
57 RobotTest(1, doc, good, bad)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
58
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
59 # 2.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
60 doc = """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
61 # robots.txt for http://www.example.com/
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
62
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
63 User-agent: *
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
64 Disallow: /cyberworld/map/ # This is an infinite virtual URL space
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
65
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
66 # Cybermapper knows where to go.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
67 User-agent: cybermapper
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
68 Disallow:
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
69
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
70 """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
71
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
72 good = ['/','/test.html',('cybermapper','/cyberworld/map/index.html')]
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
73 bad = ['/cyberworld/map/index.html']
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
74
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
75 RobotTest(2, doc, good, bad)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
76
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
77 # 3.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
78 doc = """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
79 # go away
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
80 User-agent: *
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
81 Disallow: /
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
82 """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
83
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
84 good = []
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
85 bad = ['/cyberworld/map/index.html','/','/tmp/']
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
86
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
87 RobotTest(3, doc, good, bad)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
88
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
89 # Examples from http://www.robotstxt.org/wc/norobots-rfc.html (fetched 2002)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
90
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
91 # 4.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
92 doc = """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
93 User-agent: figtree
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
94 Disallow: /tmp
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
95 Disallow: /a%3cd.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
96 Disallow: /a%2fb.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
97 Disallow: /%7ejoe/index.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
98 """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
99
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
100 good = [] # XFAIL '/a/b.html'
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
101 bad = ['/tmp','/tmp.html','/tmp/a.html',
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
102 '/a%3cd.html','/a%3Cd.html','/a%2fb.html',
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
103 '/~joe/index.html'
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
104 ]
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
105
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
106 RobotTest(4, doc, good, bad, 'figtree')
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
107 RobotTest(5, doc, good, bad, 'FigTree Robot libwww-perl/5.04')
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
108
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
109 # 6.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
110 doc = """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
111 User-agent: *
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
112 Disallow: /tmp/
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
113 Disallow: /a%3Cd.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
114 Disallow: /a/b.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
115 Disallow: /%7ejoe/index.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
116 """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
117
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
118 good = ['/tmp',] # XFAIL: '/a%2fb.html'
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
119 bad = ['/tmp/','/tmp/a.html',
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
120 '/a%3cd.html','/a%3Cd.html',"/a/b.html",
23075
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
121 '/%7Ejoe/index.html']
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
122
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
123 RobotTest(6, doc, good, bad)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
124
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
125 # From bug report #523041
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
126
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
127 # 7.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
128 doc = """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
129 User-Agent: *
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
130 Disallow: /.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
131 """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
132
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
133 good = ['/foo.html']
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
134 bad = [] # Bug report says "/" should be denied, but that is not in the RFC
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
135
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
136 RobotTest(7, doc, good, bad)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
137
48085
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
138 # From Google: http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=40364
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
139
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
140 # 8.
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
141 doc = """
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
142 User-agent: Googlebot
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
143 Allow: /folder1/myfile.html
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
144 Disallow: /folder1/
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
145 """
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
146
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
147 good = ['/folder1/myfile.html']
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
148 bad = ['/folder1/anotherfile.html']
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
149
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
150 RobotTest(8, doc, good, bad, agent="Googlebot")
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
151
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
152 # 9. This file is incorrect because "Googlebot" is a substring of
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
153 # "Googlebot-Mobile", so test 10 works just like test 9.
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
154 doc = """
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
155 User-agent: Googlebot
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
156 Disallow: /
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
157
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
158 User-agent: Googlebot-Mobile
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
159 Allow: /
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
160 """
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
161
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
162 good = []
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
163 bad = ['/something.jpg']
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
164
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
165 RobotTest(9, doc, good, bad, agent="Googlebot")
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
166
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
167 good = []
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
168 bad = ['/something.jpg']
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
169
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
170 RobotTest(10, doc, good, bad, agent="Googlebot-Mobile")
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
171
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
172 # 11. Get the order correct.
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
173 doc = """
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
174 User-agent: Googlebot-Mobile
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
175 Allow: /
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
176
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
177 User-agent: Googlebot
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
178 Disallow: /
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
179 """
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
180
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
181 good = []
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
182 bad = ['/something.jpg']
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
183
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
184 RobotTest(11, doc, good, bad, agent="Googlebot")
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
185
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
186 good = ['/something.jpg']
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
187 bad = []
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
188
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
189 RobotTest(12, doc, good, bad, agent="Googlebot-Mobile")
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
190
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
191
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
192 # 13. Google also got the order wrong in #8. You need to specify the
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
193 # URLs from more specific to more general.
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
194 doc = """
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
195 User-agent: Googlebot
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
196 Allow: /folder1/myfile.html
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
197 Disallow: /folder1/
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
198 """
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
199
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
200 good = ['/folder1/myfile.html']
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
201 bad = ['/folder1/anotherfile.html']
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
202
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
203 RobotTest(13, doc, good, bad, agent="googlebot")
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
204
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
205
63272
0bb47c51e933 Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60170
diff changeset
206 # 14. For issue #6325 (query string support)
0bb47c51e933 Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60170
diff changeset
207 doc = """
0bb47c51e933 Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60170
diff changeset
208 User-agent: *
0bb47c51e933 Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60170
diff changeset
209 Disallow: /some/path?name=value
0bb47c51e933 Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60170
diff changeset
210 """
0bb47c51e933 Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60170
diff changeset
211
0bb47c51e933 Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60170
diff changeset
212 good = ['/some/path']
0bb47c51e933 Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60170
diff changeset
213 bad = ['/some/path?name=value']
0bb47c51e933 Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60170
diff changeset
214
0bb47c51e933 Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60170
diff changeset
215 RobotTest(14, doc, good, bad)
0bb47c51e933 Merged revisions 83209 via svnmerge from
Senthil Kumaran <orsenthil@gmail.com>
parents: 60170
diff changeset
216
63465
4a002da7fbfb Merged revisions 83238 via svnmerge from
Georg Brandl <georg@python.org>
parents: 63272
diff changeset
217 # 15. For issue #4108 (obey first * entry)
63506
2c253f6e02e7 Merged revisions 83449 via svnmerge from
Georg Brandl <georg@python.org>
parents: 48085
diff changeset
218 doc = """
2c253f6e02e7 Merged revisions 83449 via svnmerge from
Georg Brandl <georg@python.org>
parents: 48085
diff changeset
219 User-agent: *
2c253f6e02e7 Merged revisions 83449 via svnmerge from
Georg Brandl <georg@python.org>
parents: 48085
diff changeset
220 Disallow: /some/path
2c253f6e02e7 Merged revisions 83449 via svnmerge from
Georg Brandl <georg@python.org>
parents: 48085
diff changeset
221
2c253f6e02e7 Merged revisions 83449 via svnmerge from
Georg Brandl <georg@python.org>
parents: 48085
diff changeset
222 User-agent: *
2c253f6e02e7 Merged revisions 83449 via svnmerge from
Georg Brandl <georg@python.org>
parents: 48085
diff changeset
223 Disallow: /another/path
2c253f6e02e7 Merged revisions 83449 via svnmerge from
Georg Brandl <georg@python.org>
parents: 48085
diff changeset
224 """
2c253f6e02e7 Merged revisions 83449 via svnmerge from
Georg Brandl <georg@python.org>
parents: 48085
diff changeset
225
2c253f6e02e7 Merged revisions 83449 via svnmerge from
Georg Brandl <georg@python.org>
parents: 48085
diff changeset
226 good = ['/another/path']
2c253f6e02e7 Merged revisions 83449 via svnmerge from
Georg Brandl <georg@python.org>
parents: 48085
diff changeset
227 bad = ['/some/path']
2c253f6e02e7 Merged revisions 83449 via svnmerge from
Georg Brandl <georg@python.org>
parents: 48085
diff changeset
228
63465
4a002da7fbfb Merged revisions 83238 via svnmerge from
Georg Brandl <georg@python.org>
parents: 63272
diff changeset
229 RobotTest(15, doc, good, bad)
63506
2c253f6e02e7 Merged revisions 83449 via svnmerge from
Georg Brandl <georg@python.org>
parents: 48085
diff changeset
230
48085
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
231
60170
43a837d2728a Backport some robotparser test and skip the test if the external resource is not available.
Florent Xicluna <florent.xicluna@gmail.com>
parents: 55002
diff changeset
232 class NetworkTestCase(unittest.TestCase):
43a837d2728a Backport some robotparser test and skip the test if the external resource is not available.
Florent Xicluna <florent.xicluna@gmail.com>
parents: 55002
diff changeset
233
43a837d2728a Backport some robotparser test and skip the test if the external resource is not available.
Florent Xicluna <florent.xicluna@gmail.com>
parents: 55002
diff changeset
234 def testPasswordProtectedSite(self):
42957
533fb5321d6b backport r57626
Skip Montanaro <skip@pobox.com>
parents: 24364
diff changeset
235 test_support.requires('network')
64539
af414b29d467 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63465
diff changeset
236 with test_support.transient_internet('mueblesmoraleda.com'):
af414b29d467 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63465
diff changeset
237 url = 'http://mueblesmoraleda.com'
71260
e6e37f1e29ca Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68256
diff changeset
238 robots_url = url + "/robots.txt"
e6e37f1e29ca Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68256
diff changeset
239 # First check the URL is usable for our purposes, since the
e6e37f1e29ca Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68256
diff changeset
240 # test site is a bit flaky.
e6e37f1e29ca Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68256
diff changeset
241 try:
e6e37f1e29ca Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68256
diff changeset
242 urlopen(robots_url)
e6e37f1e29ca Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68256
diff changeset
243 except HTTPError as e:
e6e37f1e29ca Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68256
diff changeset
244 if e.code not in {401, 403}:
e6e37f1e29ca Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68256
diff changeset
245 self.skipTest(
e6e37f1e29ca Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68256
diff changeset
246 "%r should return a 401 or 403 HTTP error, not %r"
e6e37f1e29ca Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68256
diff changeset
247 % (robots_url, e.code))
e6e37f1e29ca Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68256
diff changeset
248 else:
e6e37f1e29ca Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68256
diff changeset
249 self.skipTest(
e6e37f1e29ca Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68256
diff changeset
250 "%r should return a 401 or 403 HTTP error, not succeed"
e6e37f1e29ca Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68256
diff changeset
251 % (robots_url))
64539
af414b29d467 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63465
diff changeset
252 parser = robotparser.RobotFileParser()
af414b29d467 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63465
diff changeset
253 parser.set_url(url)
af414b29d467 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63465
diff changeset
254 try:
af414b29d467 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63465
diff changeset
255 parser.read()
af414b29d467 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63465
diff changeset
256 except IOError:
af414b29d467 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63465
diff changeset
257 self.skipTest('%s is unavailable' % url)
71260
e6e37f1e29ca Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and
Antoine Pitrou <solipsis@pitrou.net>
parents: 68256
diff changeset
258 self.assertEqual(parser.can_fetch("*", robots_url), False)
60170
43a837d2728a Backport some robotparser test and skip the test if the external resource is not available.
Florent Xicluna <florent.xicluna@gmail.com>
parents: 55002
diff changeset
259
43a837d2728a Backport some robotparser test and skip the test if the external resource is not available.
Florent Xicluna <florent.xicluna@gmail.com>
parents: 55002
diff changeset
260 def testPythonOrg(self):
43a837d2728a Backport some robotparser test and skip the test if the external resource is not available.
Florent Xicluna <florent.xicluna@gmail.com>
parents: 55002
diff changeset
261 test_support.requires('network')
64539
af414b29d467 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63465
diff changeset
262 with test_support.transient_internet('www.python.org'):
af414b29d467 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63465
diff changeset
263 parser = robotparser.RobotFileParser(
af414b29d467 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63465
diff changeset
264 "http://www.python.org/robots.txt")
af414b29d467 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63465
diff changeset
265 parser.read()
af414b29d467 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63465
diff changeset
266 self.assertTrue(
af414b29d467 Merged revisions 84597-84599 via svnmerge from
Antoine Pitrou <solipsis@pitrou.net>
parents: 63465
diff changeset
267 parser.can_fetch("*", "http://www.python.org/robots.txt"))
60170
43a837d2728a Backport some robotparser test and skip the test if the external resource is not available.
Florent Xicluna <florent.xicluna@gmail.com>
parents: 55002
diff changeset
268
42957
533fb5321d6b backport r57626
Skip Montanaro <skip@pobox.com>
parents: 24364
diff changeset
269
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
270 def test_main():
41554
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 24364
diff changeset
271 test_support.run_unittest(tests)
60170
43a837d2728a Backport some robotparser test and skip the test if the external resource is not available.
Florent Xicluna <florent.xicluna@gmail.com>
parents: 55002
diff changeset
272 test_support.run_unittest(NetworkTestCase)
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
273
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
274 if __name__=='__main__':
48014
e85d5690c296 Correct attribute name.
Georg Brandl <georg@python.org>
parents: 42956
diff changeset
275 test_support.verbose = 1
41554
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 24364
diff changeset
276 test_main()