Mercurial > cpython-lru_cache-weakref

annotate Lib/robotparser.py @ 6719:faebc1940432 legacy-trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Understand -Xlinker option.
author Guido van Rossum <guido@python.org>
date Tue, 02 Dec 1997 20:38:11 +0000
parents f0731fa4447f
children efbb7e57e6f8
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
1 """
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
2
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
3 Robots.txt file parser class. Accepts a list of lines or robots.txt URL as
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
4 input, builds a set of rules from that list, then answers questions about
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
5 fetchability of other URLs.
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
6
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
7 """
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
8
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
9 class RobotFileParser:
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
10
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
11 def __init__(self):
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
12 self.rules = {}
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
13 self.debug = 0
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
14 self.url = ''
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
15 self.last_checked = 0
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
16
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
17 def mtime(self):
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
18 return self.last_checked
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
19
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
20 def modified(self):
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
21 import time
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
22 self.last_checked = time.time()
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
23
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
24 def set_url(self, url):
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
25 self.url = url
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
26 ## import urlmisc
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
27 ## self.url = urlmisc.canonical_url(url)
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
28
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
29 def read(self):
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
30 import urllib
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
31 self.parse(urllib.urlopen(self.url).readlines())
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
32
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
33 def parse(self, lines):
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
34 import regsub, string, regex
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
35 active = []
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
36 for line in lines:
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
37 if self.debug: print '>', line,
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
38 # blank line terminates current record
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
39 if not line[:-1]:
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
40 active = []
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
41 continue
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
42 # remove optional comment and strip line
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
43 line = string.strip(line[:string.find(line, '#')])
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
44 if not line:
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
45 continue
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
46 line = regsub.split(line, ' *: *')
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
47 if len(line) == 2:
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
48 line[0] = string.lower(line[0])
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
49 if line[0] == 'user-agent':
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
50 # this record applies to this user agent
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
51 if self.debug: print '>> user-agent:', line[1]
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
52 active.append(line[1])
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
53 if not self.rules.has_key(line[1]):
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
54 self.rules[line[1]] = []
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
55 elif line[0] == 'disallow':
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
56 if line[1]:
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
57 if self.debug: print '>> disallow:', line[1]
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
58 for agent in active:
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
59 self.rules[agent].append(regex.compile(line[1]))
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
60 else:
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
61 pass
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
62 for agent in active:
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
63 if self.debug: print '>> allow', agent
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
64 self.rules[agent] = []
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
65 else:
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
66 if self.debug: print '>> unknown:', line
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
67
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
68 self.modified()
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
69
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
70 # returns true if agent is allowed to fetch url
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
71 def can_fetch(self, agent, url):
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
72 import urlparse
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
73 ag = agent
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
74 if not self.rules.has_key(ag): ag = '*'
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
75 if not self.rules.has_key(ag):
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
76 if self.debug: print '>> allowing', url, 'fetch by', agent
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
77 return 1
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
78 path = urlparse.urlparse(url)[2]
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
79 for rule in self.rules[ag]:
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
80 if rule.match(path) != -1:
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
81 if self.debug: print '>> disallowing', url, 'fetch by', agent
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
82 return 0
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
83 if self.debug: print '>> allowing', url, 'fetch by', agent
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
84 return 1
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
85
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
86 def test():
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
87 rp = RobotFileParser()
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
88 rp.debug = 1
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
89 rp.set_url('http://www.automatrix.com/robots.txt')
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
90 rp.read()
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
91 print rp.rules
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
92 print rp.can_fetch('*', 'http://www.calendar.com/concerts/')
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
93 print rp.can_fetch('Musi-Cal-Robot',
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
94 'http://dolphin:80/cgi-bin/music-search?performer=Rolling+Stones')
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
95
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
96 print rp.can_fetch('Lycos', 'http://www/~skip/volkswagen/')
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
97 print rp.can_fetch('Lycos', 'http://www/~skip/volkswagen/vanagon-list-001')