Mercurial > cpython-lru_cache-weakref
annotate Lib/robotparser.py @ 15164:8930ec3bc00a legacy-trunk
Added additional test cases for pulldom modifications.
| author | Lars Gustäbel <lars@gustaebel.de> |
|---|---|
| date | Wed, 11 Oct 2000 22:36:00 +0000 |
| parents | c43b38079228 |
| children | fe36ab4b3e97 |
| rev | line source |
|---|---|
|
5063
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
1 """ |
|
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
2 |
|
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
3 Robots.txt file parser class. Accepts a list of lines or robots.txt URL as |
|
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
4 input, builds a set of rules from that list, then answers questions about |
|
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
5 fetchability of other URLs. |
|
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
6 |
|
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
7 """ |
|
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
8 |
|
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
9 class RobotFileParser: |
|
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
10 |
|
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
11 def __init__(self): |
| 7887 | 12 self.rules = {} |
| 13 self.debug = 0 | |
| 14 self.url = '' | |
| 15 self.last_checked = 0 | |
|
5063
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
16 |
|
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
17 def mtime(self): |
| 7887 | 18 return self.last_checked |
|
5063
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
19 |
|
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
20 def modified(self): |
| 7887 | 21 import time |
| 22 self.last_checked = time.time() | |
|
5063
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
23 |
|
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
24 def set_url(self, url): |
| 7887 | 25 self.url = url |
|
5063
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
26 |
|
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
27 def read(self): |
| 7887 | 28 import urllib |
| 29 self.parse(urllib.urlopen(self.url).readlines()) | |
|
5063
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
30 |
|
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
31 def parse(self, lines): |
| 12062 | 32 """parse the input lines from a robot.txt file""" |
| 33 import string, re | |
| 7887 | 34 active = [] |
| 35 for line in lines: | |
| 36 if self.debug: print '>', line, | |
| 37 # blank line terminates current record | |
| 38 if not line[:-1]: | |
| 39 active = [] | |
| 40 continue | |
| 41 # remove optional comment and strip line | |
| 42 line = string.strip(line[:string.find(line, '#')]) | |
| 43 if not line: | |
| 44 continue | |
| 12062 | 45 line = re.split(' *: *', line) |
| 7887 | 46 if len(line) == 2: |
| 47 line[0] = string.lower(line[0]) | |
| 48 if line[0] == 'user-agent': | |
| 49 # this record applies to this user agent | |
| 50 if self.debug: print '>> user-agent:', line[1] | |
| 51 active.append(line[1]) | |
| 52 if not self.rules.has_key(line[1]): | |
| 53 self.rules[line[1]] = [] | |
| 54 elif line[0] == 'disallow': | |
| 55 if line[1]: | |
| 56 if self.debug: print '>> disallow:', line[1] | |
| 57 for agent in active: | |
| 12062 | 58 self.rules[agent].append(re.compile(line[1])) |
| 7887 | 59 else: |
| 60 pass | |
| 61 for agent in active: | |
| 62 if self.debug: print '>> allow', agent | |
| 63 self.rules[agent] = [] | |
| 64 else: | |
| 65 if self.debug: print '>> unknown:', line | |
|
5063
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
66 |
| 7887 | 67 self.modified() |
|
5063
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
68 |
|
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
69 # returns true if agent is allowed to fetch url |
| 12062 | 70 def can_fetch(self, useragent, url): |
| 71 """using the parsed robots.txt decide if useragent can fetch url""" | |
| 7887 | 72 import urlparse |
| 12062 | 73 ag = useragent |
| 7887 | 74 if not self.rules.has_key(ag): ag = '*' |
| 75 if not self.rules.has_key(ag): | |
| 12062 | 76 if self.debug: print '>> allowing', url, 'fetch by', useragent |
| 7887 | 77 return 1 |
| 78 path = urlparse.urlparse(url)[2] | |
| 79 for rule in self.rules[ag]: | |
| 12062 | 80 if rule.match(path) is not None: |
| 81 if self.debug: print '>> disallowing', url, 'fetch by', useragent | |
| 7887 | 82 return 0 |
| 12062 | 83 if self.debug: print '>> allowing', url, 'fetch by', useragent |
| 7887 | 84 return 1 |
|
5063
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
85 |
| 12062 | 86 def _test(): |
|
5063
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
87 rp = RobotFileParser() |
|
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
88 rp.debug = 1 |
| 12062 | 89 rp.set_url('http://www.musi-cal.com/robots.txt') |
|
5063
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
90 rp.read() |
|
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
91 print rp.rules |
| 12062 | 92 print rp.can_fetch('*', 'http://www.musi-cal.com.com/') |
|
5063
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
93 print rp.can_fetch('Musi-Cal-Robot', |
| 12062 | 94 'http://www.musi-cal.com/cgi-bin/event-search?city=San+Francisco') |
|
5063
f0731fa4447f
Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff
changeset
|
95 |
| 12062 | 96 if __name__ == "__main__": |
| 97 _test() |