Mercurial > cpython-lru_cache-weakref

annotate Lib/robotparser.py @ 22194:fc6dc2ea25cf legacy-trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Fix leak of NotImplemented in previous checkin to PyNumber_Add(). If result == Py_NotImplemented, always DECREF it before assigning a new value to result.
author Jeremy Hylton <jeremy@alum.mit.edu>
date Fri, 08 Mar 2002 21:28:54 +0000
parents 736fb5aec9b3
children c210562e1558
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
1 """ robotparser.py
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
2
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
3 Copyright (C) 2000 Bastian Kleineidam
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
4
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
5 You can choose between two licenses when using this package:
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
6 1) GNU GPLv2
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
7 2) PYTHON 2.0 OPEN SOURCE LICENSE
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
8
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
9 The robots.txt Exclusion Protocol is implemented as specified in
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
10 http://info.webcrawler.com/mak/projects/robots/norobots-rfc.html
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
11 """
16494
f4d15fdd15dd String method conversion.
Eric S. Raymond <esr@thyrsus.com>
parents: 16074
diff changeset
12 import re,urlparse,urllib
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
13
16070
b5345e99596b added __all__ lists to a number of Python modules
Skip Montanaro <skip@pobox.com>
parents: 16067
diff changeset
14 __all__ = ["RobotFileParser"]
b5345e99596b added __all__ lists to a number of Python modules
Skip Montanaro <skip@pobox.com>
parents: 16067
diff changeset
15
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
16 debug = 0
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
17
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
18 def _debug(msg):
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
19 if debug: print msg
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
20
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
21
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
22 class RobotFileParser:
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
23 def __init__(self, url=''):
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
24 self.entries = []
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents: 19213
diff changeset
25 self.default_entry = None
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
26 self.disallow_all = 0
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
27 self.allow_all = 0
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
28 self.set_url(url)
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
29 self.last_checked = 0
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
30
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
31 def mtime(self):
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
32 return self.last_checked
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
33
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
34 def modified(self):
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
35 import time
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
36 self.last_checked = time.time()
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
37
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
38 def set_url(self, url):
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
39 self.url = url
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
40 self.host, self.path = urlparse.urlparse(url)[1:3]
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
41
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
42 def read(self):
16579
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
43 opener = URLopener()
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
44 f = opener.open(self.url)
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
45 lines = f.readlines()
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
46 self.errcode = opener.errcode
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
47 if self.errcode == 401 or self.errcode == 403:
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
48 self.disallow_all = 1
16579
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
49 _debug("disallow all")
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
50 elif self.errcode >= 400:
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
51 self.allow_all = 1
16579
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
52 _debug("allow all")
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
53 elif self.errcode == 200 and lines:
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
54 _debug("parse lines")
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
55 self.parse(lines)
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
56
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
57 def parse(self, lines):
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
58 """parse the input lines from a robot.txt file.
16074
7e55fa93f4db Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 16070
diff changeset
59 We allow that a user-agent: line is not preceded by
7e55fa93f4db Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 16070
diff changeset
60 one or more blank lines."""
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
61 state = 0
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
62 linenumber = 0
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
63 entry = Entry()
16074
7e55fa93f4db Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 16070
diff changeset
64
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
65 for line in lines:
16494
f4d15fdd15dd String method conversion.
Eric S. Raymond <esr@thyrsus.com>
parents: 16074
diff changeset
66 line = line.strip()
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
67 linenumber = linenumber + 1
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
68 if not line:
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
69 if state==1:
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
70 _debug("line %d: warning: you should insert"
16074
7e55fa93f4db Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 16070
diff changeset
71 " allow: or disallow: directives below any"
7e55fa93f4db Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 16070
diff changeset
72 " user-agent: line" % linenumber)
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
73 entry = Entry()
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
74 state = 0
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
75 elif state==2:
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents: 19213
diff changeset
76 if "*" in entry.useragents:
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents: 19213
diff changeset
77 # the default entry is considered last
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents: 19213
diff changeset
78 self.default_entry = entry
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents: 19213
diff changeset
79 else:
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents: 19213
diff changeset
80 self.entries.append(entry)
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
81 entry = Entry()
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
82 state = 0
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
83 # remove optional comment and strip line
16494
f4d15fdd15dd String method conversion.
Eric S. Raymond <esr@thyrsus.com>
parents: 16074
diff changeset
84 i = line.find('#')
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
85 if i>=0:
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
86 line = line[:i]
16494
f4d15fdd15dd String method conversion.
Eric S. Raymond <esr@thyrsus.com>
parents: 16074
diff changeset
87 line = line.strip()
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
88 if not line:
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
89 continue
16494
f4d15fdd15dd String method conversion.
Eric S. Raymond <esr@thyrsus.com>
parents: 16074
diff changeset
90 line = line.split(':', 1)
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
91 if len(line) == 2:
16494
f4d15fdd15dd String method conversion.
Eric S. Raymond <esr@thyrsus.com>
parents: 16074
diff changeset
92 line[0] = line[0].strip().lower()
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents: 19213
diff changeset
93 line[1] = urllib.unquote(line[1].strip())
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
94 if line[0] == "user-agent":
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
95 if state==2:
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
96 _debug("line %d: warning: you should insert a blank"
16074
7e55fa93f4db Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 16070
diff changeset
97 " line before any user-agent"
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
98 " directive" % linenumber)
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
99 self.entries.append(entry)
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
100 entry = Entry()
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
101 entry.useragents.append(line[1])
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
102 state = 1
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
103 elif line[0] == "disallow":
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
104 if state==0:
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
105 _debug("line %d: error: you must insert a user-agent:"
16074
7e55fa93f4db Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 16070
diff changeset
106 " directive before this line" % linenumber)
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
107 else:
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
108 entry.rulelines.append(RuleLine(line[1], 0))
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
109 state = 2
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
110 elif line[0] == "allow":
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
111 if state==0:
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
112 _debug("line %d: error: you must insert a user-agent:"
16074
7e55fa93f4db Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 16070
diff changeset
113 " directive before this line" % linenumber)
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
114 else:
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
115 entry.rulelines.append(RuleLine(line[1], 1))
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
116 else:
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
117 _debug("line %d: warning: unknown key %s" % (linenumber,
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
118 line[0]))
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
119 else:
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
120 _debug("line %d: error: malformed line %s"%(linenumber, line))
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
121 if state==2:
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
122 self.entries.append(entry)
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
123 _debug("Parsed rules:\n%s" % str(self))
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
124
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
125
12062
c43b38079228 Skip Montanaro:
Guido van Rossum <guido@python.org>
parents: 7887
diff changeset
126 def can_fetch(self, useragent, url):
c43b38079228 Skip Montanaro:
Guido van Rossum <guido@python.org>
parents: 7887
diff changeset
127 """using the parsed robots.txt decide if useragent can fetch url"""
16579
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
128 _debug("Checking robot.txt allowance for:\n user agent: %s\n url: %s" %
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
129 (useragent, url))
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
130 if self.disallow_all:
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
131 return 0
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
132 if self.allow_all:
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
133 return 1
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
134 # search for given user agent matches
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
135 # the first match counts
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents: 19213
diff changeset
136 url = urllib.quote(urlparse.urlparse(urllib.unquote(url))[2]) or "/"
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
137 for entry in self.entries:
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
138 if entry.applies_to(useragent):
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
139 return entry.allowance(url)
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents: 19213
diff changeset
140 # try the default entry last
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents: 19213
diff changeset
141 if self.default_entry:
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents: 19213
diff changeset
142 return self.default_entry.allowance(url)
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
143 # agent not found ==> access granted
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
144 return 1
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
145
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
146
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
147 def __str__(self):
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
148 ret = ""
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
149 for entry in self.entries:
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
150 ret = ret + str(entry) + "\n"
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
151 return ret
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
152
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
153
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
154 class RuleLine:
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
155 """A rule line is a single "Allow:" (allowance==1) or "Disallow:"
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
156 (allowance==0) followed by a path."""
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
157 def __init__(self, path, allowance):
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents: 19213
diff changeset
158 if path == '' and not allowance:
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents: 19213
diff changeset
159 # an empty value means allow all
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents: 19213
diff changeset
160 allowance = 1
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
161 self.path = urllib.quote(path)
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
162 self.allowance = allowance
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
163
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
164 def applies_to(self, filename):
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents: 19213
diff changeset
165 return self.path=="*" or filename.startswith(self.path)
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
166
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
167 def __str__(self):
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
168 return (self.allowance and "Allow" or "Disallow")+": "+self.path
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
169
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
170
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
171 class Entry:
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
172 """An entry has one or more user-agents and zero or more rulelines"""
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
173 def __init__(self):
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
174 self.useragents = []
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
175 self.rulelines = []
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
176
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
177 def __str__(self):
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
178 ret = ""
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
179 for agent in self.useragents:
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
180 ret = ret + "User-agent: "+agent+"\n"
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
181 for line in self.rulelines:
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
182 ret = ret + str(line) + "\n"
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
183 return ret
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
184
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
185 def applies_to(self, useragent):
16579
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
186 """check if this entry applies to the specified agent"""
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
187 # split the name token and make it lower case
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
188 useragent = useragent.split("/")[0].lower()
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
189 for agent in self.useragents:
16579
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
190 if agent=='*':
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
191 # we have the catch-all agent
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
192 return 1
16579
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
193 agent = agent.lower()
22088
736fb5aec9b3 Correct various errors:
Martin v. Löwis <martin@v.loewis.de>
parents: 19213
diff changeset
194 if useragent.find(agent) != -1:
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
195 return 1
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
196 return 0
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
197
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
198 def allowance(self, filename):
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
199 """Preconditions:
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
200 - our agent applies to this entry
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
201 - filename is URL decoded"""
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
202 for line in self.rulelines:
16579
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
203 _debug((filename, str(line), line.allowance))
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
204 if line.applies_to(filename):
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
205 return line.allowance
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
206 return 1
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
207
16579
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
208 class URLopener(urllib.FancyURLopener):
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
209 def __init__(self, *args):
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
210 apply(urllib.FancyURLopener.__init__, (self,) + args)
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
211 self.errcode = 200
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
212 self.tries = 0
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
213 self.maxtries = 10
16616
e2a63584566f Whitespace normalization.
Tim Peters <tim.peters@gmail.com>
parents: 16579
diff changeset
214
16579
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
215 def http_error_default(self, url, fp, errcode, errmsg, headers):
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
216 self.errcode = errcode
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
217 return urllib.FancyURLopener.http_error_default(self, url, fp, errcode,
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
218 errmsg, headers)
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
219
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
220 def http_error_302(self, url, fp, errcode, errmsg, headers, data=None):
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
221 self.tries += 1
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
222 if self.tries >= self.maxtries:
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
223 return self.http_error_default(url, fp, 500,
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
224 "Internal Server Error: Redirect Recursion",
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
225 headers)
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
226 result = urllib.FancyURLopener.http_error_302(self, url, fp, errcode,
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
227 errmsg, headers, data)
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
228 self.tries = 0
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
229 return result
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
230
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
231 def _check(a,b):
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
232 if not b:
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
233 ac = "access denied"
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
234 else:
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
235 ac = "access allowed"
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
236 if a!=b:
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
237 print "failed"
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
238 else:
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
239 print "ok (%s)" % ac
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
240 print
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
241
12062
c43b38079228 Skip Montanaro:
Guido van Rossum <guido@python.org>
parents: 7887
diff changeset
242 def _test():
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
243 global debug
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
244 rp = RobotFileParser()
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
245 debug = 1
16579
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
246
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
247 # robots.txt that exists, gotten to by redirection
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
248 rp.set_url('http://www.musi-cal.com/robots.txt')
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
249 rp.read()
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
250
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
251 # test for re.escape
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
252 _check(rp.can_fetch('*', 'http://www.musi-cal.com/'), 1)
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
253 # this should match the first rule, which is a disallow
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
254 _check(rp.can_fetch('', 'http://www.musi-cal.com/'), 0)
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
255 # various cherry pickers
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
256 _check(rp.can_fetch('CherryPickerSE',
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
257 'http://www.musi-cal.com/cgi-bin/event-search'
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
258 '?city=San+Francisco'), 0)
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
259 _check(rp.can_fetch('CherryPickerSE/1.0',
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
260 'http://www.musi-cal.com/cgi-bin/event-search'
16579
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
261 '?city=San+Francisco'), 0)
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
262 _check(rp.can_fetch('CherryPickerSE/1.5',
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
263 'http://www.musi-cal.com/cgi-bin/event-search'
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
264 '?city=San+Francisco'), 0)
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
265 # case sensitivity
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
266 _check(rp.can_fetch('ExtractorPro', 'http://www.musi-cal.com/blubba'), 0)
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
267 _check(rp.can_fetch('extractorpro', 'http://www.musi-cal.com/blubba'), 0)
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
268 # substring test
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
269 _check(rp.can_fetch('toolpak/1.1', 'http://www.musi-cal.com/blubba'), 0)
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
270 # tests for catch-all * agent
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
271 _check(rp.can_fetch('spam', 'http://www.musi-cal.com/search'), 0)
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
272 _check(rp.can_fetch('spam', 'http://www.musi-cal.com/Musician/me'), 1)
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
273 _check(rp.can_fetch('spam', 'http://www.musi-cal.com/'), 1)
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
274 _check(rp.can_fetch('spam', 'http://www.musi-cal.com/'), 1)
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
275
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
276 # robots.txt that does not exist
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
277 rp.set_url('http://www.lycos.com/robots.txt')
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
278 rp.read()
a63dc59be0f9 The bulk of the credit for these changes goes to Bastian Kleineidam
Skip Montanaro <skip@pobox.com>
parents: 16494
diff changeset
279 _check(rp.can_fetch('Mozilla', 'http://www.lycos.com/search'), 1)
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
280
16067
fe36ab4b3e97 rewrite of robotparser.py by Bastian Kleineidam. Closes patch 102229.
Skip Montanaro <skip@pobox.com>
parents: 12062
diff changeset
281 if __name__ == '__main__':
12062
c43b38079228 Skip Montanaro:
Guido van Rossum <guido@python.org>
parents: 7887
diff changeset
282 _test()