Mercurial > cpython-lru_cache-weakref

annotate Lib/test/test_robotparser.py @ 58765:2ba7677bfa95 legacy-trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
testCopysign was defined twice in test_math; combine the definitions
author Mark Dickinson <dickinsm@gmail.com>
date Sat, 06 Feb 2010 23:11:25 +0000
parents 6768b2350c57
children 43a837d2728a
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
24364
aa0d2d51fbde Get rid of relative imports in all unittests. Now anything that
Barry Warsaw <barry@python.org>
parents: 23075
diff changeset
1 import unittest, StringIO, robotparser
aa0d2d51fbde Get rid of relative imports in all unittests. Now anything that
Barry Warsaw <barry@python.org>
parents: 23075
diff changeset
2 from test import test_support
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
3
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
4 class RobotTestCase(unittest.TestCase):
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
5 def __init__(self, index, parser, url, good, agent):
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
6 unittest.TestCase.__init__(self)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
7 if good:
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
8 self.str = "RobotTest(%d, good, %s)" % (index, url)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
9 else:
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
10 self.str = "RobotTest(%d, bad, %s)" % (index, url)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
11 self.parser = parser
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
12 self.url = url
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
13 self.good = good
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
14 self.agent = agent
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
15
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
16 def runTest(self):
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
17 if isinstance(self.url, tuple):
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
18 agent, url = self.url
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
19 else:
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
20 url = self.url
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
21 agent = self.agent
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
22 if self.good:
55002
6768b2350c57 convert usage of fail* to assert*
Benjamin Peterson <benjamin@python.org>
parents: 48085
diff changeset
23 self.assertTrue(self.parser.can_fetch(agent, url))
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
24 else:
55002
6768b2350c57 convert usage of fail* to assert*
Benjamin Peterson <benjamin@python.org>
parents: 48085
diff changeset
25 self.assertFalse(self.parser.can_fetch(agent, url))
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
26
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
27 def __str__(self):
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
28 return self.str
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
29
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
30 tests = unittest.TestSuite()
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
31
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
32 def RobotTest(index, robots_txt, good_urls, bad_urls,
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
33 agent="test_robotparser"):
23075
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
34
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
35 lines = StringIO.StringIO(robots_txt).readlines()
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
36 parser = robotparser.RobotFileParser()
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
37 parser.parse(lines)
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
38 for url in good_urls:
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
39 tests.addTest(RobotTestCase(index, parser, url, 1, agent))
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
40 for url in bad_urls:
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
41 tests.addTest(RobotTestCase(index, parser, url, 0, agent))
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
42
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
43 # Examples from http://www.robotstxt.org/wc/norobots.html (fetched 2002)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
44
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
45 # 1.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
46 doc = """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
47 User-agent: *
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
48 Disallow: /cyberworld/map/ # This is an infinite virtual URL space
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
49 Disallow: /tmp/ # these will soon disappear
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
50 Disallow: /foo.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
51 """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
52
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
53 good = ['/','/test.html']
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
54 bad = ['/cyberworld/map/index.html','/tmp/xxx','/foo.html']
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
55
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
56 RobotTest(1, doc, good, bad)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
57
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
58 # 2.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
59 doc = """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
60 # robots.txt for http://www.example.com/
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
61
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
62 User-agent: *
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
63 Disallow: /cyberworld/map/ # This is an infinite virtual URL space
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
64
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
65 # Cybermapper knows where to go.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
66 User-agent: cybermapper
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
67 Disallow:
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
68
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
69 """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
70
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
71 good = ['/','/test.html',('cybermapper','/cyberworld/map/index.html')]
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
72 bad = ['/cyberworld/map/index.html']
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
73
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
74 RobotTest(2, doc, good, bad)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
75
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
76 # 3.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
77 doc = """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
78 # go away
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
79 User-agent: *
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
80 Disallow: /
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
81 """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
82
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
83 good = []
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
84 bad = ['/cyberworld/map/index.html','/','/tmp/']
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
85
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
86 RobotTest(3, doc, good, bad)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
87
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
88 # Examples from http://www.robotstxt.org/wc/norobots-rfc.html (fetched 2002)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
89
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
90 # 4.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
91 doc = """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
92 User-agent: figtree
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
93 Disallow: /tmp
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
94 Disallow: /a%3cd.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
95 Disallow: /a%2fb.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
96 Disallow: /%7ejoe/index.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
97 """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
98
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
99 good = [] # XFAIL '/a/b.html'
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
100 bad = ['/tmp','/tmp.html','/tmp/a.html',
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
101 '/a%3cd.html','/a%3Cd.html','/a%2fb.html',
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
102 '/~joe/index.html'
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
103 ]
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
104
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
105 RobotTest(4, doc, good, bad, 'figtree')
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
106 RobotTest(5, doc, good, bad, 'FigTree Robot libwww-perl/5.04')
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
107
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
108 # 6.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
109 doc = """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
110 User-agent: *
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
111 Disallow: /tmp/
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
112 Disallow: /a%3Cd.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
113 Disallow: /a/b.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
114 Disallow: /%7ejoe/index.html
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
115 """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
116
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
117 good = ['/tmp',] # XFAIL: '/a%2fb.html'
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
118 bad = ['/tmp/','/tmp/a.html',
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
119 '/a%3cd.html','/a%3Cd.html',"/a/b.html",
23075
b7e87732f6ab Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 22088
diff changeset
120 '/%7Ejoe/index.html']
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
121
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
122 RobotTest(6, doc, good, bad)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
123
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
124 # From bug report #523041
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
125
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
126 # 7.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
127 doc = """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
128 User-Agent: *
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
129 Disallow: /.
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
130 """
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
131
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
132 good = ['/foo.html']
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
133 bad = [] # Bug report says "/" should be denied, but that is not in the RFC
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
134
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
135 RobotTest(7, doc, good, bad)
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
136
48085
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
137 # From Google: http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=40364
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
138
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
139 # 8.
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
140 doc = """
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
141 User-agent: Googlebot
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
142 Allow: /folder1/myfile.html
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
143 Disallow: /folder1/
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
144 """
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
145
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
146 good = ['/folder1/myfile.html']
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
147 bad = ['/folder1/anotherfile.html']
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
148
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
149 RobotTest(8, doc, good, bad, agent="Googlebot")
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
150
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
151 # 9. This file is incorrect because "Googlebot" is a substring of
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
152 # "Googlebot-Mobile", so test 10 works just like test 9.
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
153 doc = """
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
154 User-agent: Googlebot
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
155 Disallow: /
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
156
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
157 User-agent: Googlebot-Mobile
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
158 Allow: /
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
159 """
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
160
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
161 good = []
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
162 bad = ['/something.jpg']
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
163
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
164 RobotTest(9, doc, good, bad, agent="Googlebot")
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
165
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
166 good = []
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
167 bad = ['/something.jpg']
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
168
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
169 RobotTest(10, doc, good, bad, agent="Googlebot-Mobile")
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
170
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
171 # 11. Get the order correct.
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
172 doc = """
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
173 User-agent: Googlebot-Mobile
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
174 Allow: /
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
175
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
176 User-agent: Googlebot
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
177 Disallow: /
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
178 """
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
179
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
180 good = []
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
181 bad = ['/something.jpg']
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
182
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
183 RobotTest(11, doc, good, bad, agent="Googlebot")
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
184
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
185 good = ['/something.jpg']
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
186 bad = []
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
187
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
188 RobotTest(12, doc, good, bad, agent="Googlebot-Mobile")
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
189
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
190
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
191 # 13. Google also got the order wrong in #8. You need to specify the
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
192 # URLs from more specific to more general.
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
193 doc = """
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
194 User-agent: Googlebot
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
195 Allow: /folder1/myfile.html
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
196 Disallow: /folder1/
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
197 """
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
198
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
199 good = ['/folder1/myfile.html']
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
200 bad = ['/folder1/anotherfile.html']
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
201
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
202 RobotTest(13, doc, good, bad, agent="googlebot")
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
203
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
204
9f9e5fe63ec2 Close issue 3437 - missing state change when Allow lines are processed.
Skip Montanaro <skip@pobox.com>
parents: 48014
diff changeset
205
42956
e6a9fcd1b44e fixes 813986
Skip Montanaro <skip@pobox.com>
parents: 41554
diff changeset
206 class TestCase(unittest.TestCase):
e6a9fcd1b44e fixes 813986
Skip Montanaro <skip@pobox.com>
parents: 41554
diff changeset
207 def runTest(self):
e6a9fcd1b44e fixes 813986
Skip Montanaro <skip@pobox.com>
parents: 41554
diff changeset
208 test_support.requires('network')
e6a9fcd1b44e fixes 813986
Skip Montanaro <skip@pobox.com>
parents: 41554
diff changeset
209 # whole site is password-protected.
e6a9fcd1b44e fixes 813986
Skip Montanaro <skip@pobox.com>
parents: 41554
diff changeset
210 url = 'http://mueblesmoraleda.com'
e6a9fcd1b44e fixes 813986
Skip Montanaro <skip@pobox.com>
parents: 41554
diff changeset
211 parser = robotparser.RobotFileParser()
e6a9fcd1b44e fixes 813986
Skip Montanaro <skip@pobox.com>
parents: 41554
diff changeset
212 parser.set_url(url)
e6a9fcd1b44e fixes 813986
Skip Montanaro <skip@pobox.com>
parents: 41554
diff changeset
213 parser.read()
e6a9fcd1b44e fixes 813986
Skip Montanaro <skip@pobox.com>
parents: 41554
diff changeset
214 self.assertEqual(parser.can_fetch("*", url+"/robots.txt"), False)
e6a9fcd1b44e fixes 813986
Skip Montanaro <skip@pobox.com>
parents: 41554
diff changeset
215
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
216 def test_main():
41554
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 24364
diff changeset
217 test_support.run_unittest(tests)
42956
e6a9fcd1b44e fixes 813986
Skip Montanaro <skip@pobox.com>
parents: 41554
diff changeset
218 TestCase().run()
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
219
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents:
diff changeset
220 if __name__=='__main__':
48014
e85d5690c296 Correct attribute name.
Georg Brandl <georg@python.org>
parents: 42956
diff changeset
221 test_support.verbose = 1
41554
06145fbc7ab9 Merged revisions 53952-54987 via svnmerge from
Guido van Rossum <guido@python.org>
parents: 24364
diff changeset
222 test_main()