Mercurial > cpython-lru_cache-weakref

annotate Lib/robotparser.py @ 9383:2b34461ba4a3 legacy-trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
PyMapping_SetItemString() returns int, not an object.
author Guido van Rossum <guido@python.org>
date Fri, 16 Oct 1998 17:43:53 +0000
parents efbb7e57e6f8
children c43b38079228
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
1 """
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
2
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
3 Robots.txt file parser class. Accepts a list of lines or robots.txt URL as
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
4 input, builds a set of rules from that list, then answers questions about
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
5 fetchability of other URLs.
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
6
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
7 """
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
8
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
9 class RobotFileParser:
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
10
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
11 def __init__(self):
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
12 self.rules = {}
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
13 self.debug = 0
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
14 self.url = ''
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
15 self.last_checked = 0
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
16
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
17 def mtime(self):
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
18 return self.last_checked
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
19
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
20 def modified(self):
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
21 import time
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
22 self.last_checked = time.time()
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
23
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
24 def set_url(self, url):
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
25 self.url = url
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
26 ## import urlmisc
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
27 ## self.url = urlmisc.canonical_url(url)
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
28
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
29 def read(self):
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
30 import urllib
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
31 self.parse(urllib.urlopen(self.url).readlines())
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
32
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
33 def parse(self, lines):
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
34 import regsub, string, regex
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
35 active = []
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
36 for line in lines:
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
37 if self.debug: print '>', line,
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
38 # blank line terminates current record
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
39 if not line[:-1]:
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
40 active = []
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
41 continue
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
42 # remove optional comment and strip line
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
43 line = string.strip(line[:string.find(line, '#')])
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
44 if not line:
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
45 continue
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
46 line = regsub.split(line, ' *: *')
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
47 if len(line) == 2:
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
48 line[0] = string.lower(line[0])
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
49 if line[0] == 'user-agent':
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
50 # this record applies to this user agent
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
51 if self.debug: print '>> user-agent:', line[1]
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
52 active.append(line[1])
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
53 if not self.rules.has_key(line[1]):
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
54 self.rules[line[1]] = []
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
55 elif line[0] == 'disallow':
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
56 if line[1]:
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
57 if self.debug: print '>> disallow:', line[1]
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
58 for agent in active:
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
59 self.rules[agent].append(regex.compile(line[1]))
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
60 else:
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
61 pass
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
62 for agent in active:
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
63 if self.debug: print '>> allow', agent
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
64 self.rules[agent] = []
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
65 else:
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
66 if self.debug: print '>> unknown:', line
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
67
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
68 self.modified()
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
69
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
70 # returns true if agent is allowed to fetch url
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
71 def can_fetch(self, agent, url):
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
72 import urlparse
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
73 ag = agent
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
74 if not self.rules.has_key(ag): ag = '*'
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
75 if not self.rules.has_key(ag):
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
76 if self.debug: print '>> allowing', url, 'fetch by', agent
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
77 return 1
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
78 path = urlparse.urlparse(url)[2]
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
79 for rule in self.rules[ag]:
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
80 if rule.match(path) != -1:
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
81 if self.debug: print '>> disallowing', url, 'fetch by', agent
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
82 return 0
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
83 if self.debug: print '>> allowing', url, 'fetch by', agent
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
84 return 1
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
85
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
86 def test():
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
87 rp = RobotFileParser()
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
88 rp.debug = 1
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
89 rp.set_url('http://www.automatrix.com/robots.txt')
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
90 rp.read()
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
91 print rp.rules
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
92 print rp.can_fetch('*', 'http://www.calendar.com/concerts/')
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
93 print rp.can_fetch('Musi-Cal-Robot',
7887
efbb7e57e6f8 Give in to tabnanny
Guido van Rossum <guido@python.org>
parents: 5063
diff changeset
94 'http://dolphin:80/cgi-bin/music-search?performer=Rolling+Stones')
5063
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
95
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
96 print rp.can_fetch('Lycos', 'http://www/~skip/volkswagen/')
f0731fa4447f Skip Montanaro's robots.txt parser.
Guido van Rossum <guido@python.org>
parents:
diff changeset
97 print rp.can_fetch('Lycos', 'http://www/~skip/volkswagen/vanagon-list-001')