Mercurial > cpython-lru_cache-weakref

annotate Lib/urlopen.py @ 4566:6403c94de50d legacy-trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added safeguard against failure in __del__.
author Guido van Rossum <guido@python.org>
date Wed, 27 Nov 1996 19:39:58 +0000
parents 91f846f6d6a6
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
1 # Open an arbitrary URL
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
2 #
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
3 # See the following document for a tentative description of URLs:
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
4 # Uniform Resource Locators Tim Berners-Lee
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
5 # INTERNET DRAFT CERN
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
6 # IETF URL Working Group 14 July 1993
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
7 # draft-ietf-uri-url-01.txt
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
8 #
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
9 # The object returned by URLopener().open(file) will differ per
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
10 # protocol. All you know is that is has methods read(), readline(),
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
11 # readlines(), fileno(), close() and info(). The read*(), fileno()
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
12 # and close() methods work like those of open files.
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
13 # The info() method returns an rfc822.Message object which can be
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
14 # used to query various info about the object, if available.
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
15 # (rfc822.Message objects are queried with the getheader() method.)
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
16
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
17 import socket
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
18 import regex
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
19
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
20
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
21 # This really consists of two pieces:
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
22 # (1) a class which handles opening of all sorts of URLs
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
23 # (plus assorted utilities etc.)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
24 # (2) a set of functions for parsing URLs
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
25 # XXX Should these be separated out into different modules?
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
26
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
27
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
28 # Shortcut for basic usage
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
29 _urlopener = None
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
30 def urlopen(url):
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
31 global _urlopener
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
32 if not _urlopener:
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
33 _urlopener = URLopener()
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
34 return _urlopener.open(url)
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
35 def urlretrieve(url):
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
36 global _urlopener
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
37 if not _urlopener:
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
38 _urlopener = URLopener()
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
39 return _urlopener.retrieve(url)
1568
e44499a44cb8 use caching of temp files; added cleanup
Guido van Rossum <guido@python.org>
parents: 1563
diff changeset
40 def urlcleanup():
e44499a44cb8 use caching of temp files; added cleanup
Guido van Rossum <guido@python.org>
parents: 1563
diff changeset
41 if _urlopener:
e44499a44cb8 use caching of temp files; added cleanup
Guido van Rossum <guido@python.org>
parents: 1563
diff changeset
42 _urlopener.cleanup()
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
43
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
44
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
45 # Class to open URLs.
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
46 # This is a class rather than just a subroutine because we may need
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
47 # more than one set of global protocol-specific options.
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
48 ftpcache = {}
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
49 class URLopener:
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
50
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
51 # Constructor
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
52 def __init__(self):
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
53 self.addheaders = []
1568
e44499a44cb8 use caching of temp files; added cleanup
Guido van Rossum <guido@python.org>
parents: 1563
diff changeset
54 self.tempcache = {}
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
55 self.ftpcache = ftpcache
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
56 # Undocumented feature: you can use a different
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
57 # ftp cache by assigning to the .ftpcache member;
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
58 # in case you want logically independent URL openers
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
59
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
60 def __del__(self):
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
61 self.close()
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
62
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
63 def close(self):
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
64 self.cleanup()
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
65
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
66 def cleanup(self):
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
67 import os
1568
e44499a44cb8 use caching of temp files; added cleanup
Guido van Rossum <guido@python.org>
parents: 1563
diff changeset
68 for url in self.tempcache.keys():
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
69 try:
1568
e44499a44cb8 use caching of temp files; added cleanup
Guido van Rossum <guido@python.org>
parents: 1563
diff changeset
70 os.unlink(self.tempcache[url][0])
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
71 except os.error:
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
72 pass
1568
e44499a44cb8 use caching of temp files; added cleanup
Guido van Rossum <guido@python.org>
parents: 1563
diff changeset
73 del self.tempcache[url]
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
74
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
75 # Add a header to be used by the HTTP interface only
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
76 # e.g. u.addheader('Accept', 'sound/basic')
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
77 def addheader(self, *args):
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
78 self.addheaders.append(args)
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
79
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
80 # External interface
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
81 # Use URLopener().open(file) instead of open(file, 'r')
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
82 def open(self, url):
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
83 type, url = splittype(unwrap(url))
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
84 if not type: type = 'file'
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
85 name = 'open_' + type
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
86 if '-' in name:
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
87 import regsub
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
88 name = regsub.gsub('-', '_', name)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
89 if not hasattr(self, name):
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
90 raise IOError, ('url error', 'unknown url type', type)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
91 try:
1563
bd1a4d717e9d Added __repr__ to addbase class; delete more objects
Guido van Rossum <guido@python.org>
parents: 1562
diff changeset
92 return getattr(self, name)(url)
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
93 except socket.error, msg:
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
94 raise IOError, ('socket error', msg)
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
95
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
96 # External interface
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
97 # retrieve(url) returns (filename, None) for a local object
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
98 # or (tempfilename, headers) for a remote object
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
99 def retrieve(self, url):
1568
e44499a44cb8 use caching of temp files; added cleanup
Guido van Rossum <guido@python.org>
parents: 1563
diff changeset
100 if self.tempcache.has_key(url):
e44499a44cb8 use caching of temp files; added cleanup
Guido van Rossum <guido@python.org>
parents: 1563
diff changeset
101 return self.tempcache[url]
e44499a44cb8 use caching of temp files; added cleanup
Guido van Rossum <guido@python.org>
parents: 1563
diff changeset
102 url1 = unwrap(url)
e44499a44cb8 use caching of temp files; added cleanup
Guido van Rossum <guido@python.org>
parents: 1563
diff changeset
103 if self.tempcache.has_key(url1):
e44499a44cb8 use caching of temp files; added cleanup
Guido van Rossum <guido@python.org>
parents: 1563
diff changeset
104 self.tempcache[url] = self.tempcache[url1]
e44499a44cb8 use caching of temp files; added cleanup
Guido van Rossum <guido@python.org>
parents: 1563
diff changeset
105 return self.tempcache[url1]
e44499a44cb8 use caching of temp files; added cleanup
Guido van Rossum <guido@python.org>
parents: 1563
diff changeset
106 type, url1 = splittype(url1)
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
107 if not type or type == 'file':
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
108 try:
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
109 fp = self.open_local_file(url1)
1563
bd1a4d717e9d Added __repr__ to addbase class; delete more objects
Guido van Rossum <guido@python.org>
parents: 1562
diff changeset
110 del fp
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
111 return splithost(url1)[1], None
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
112 except IOError, msg:
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
113 pass
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
114 fp = self.open(url)
1568
e44499a44cb8 use caching of temp files; added cleanup
Guido van Rossum <guido@python.org>
parents: 1563
diff changeset
115 headers = fp.info()
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
116 import tempfile
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
117 tfn = tempfile.mktemp()
1568
e44499a44cb8 use caching of temp files; added cleanup
Guido van Rossum <guido@python.org>
parents: 1563
diff changeset
118 self.tempcache[url] = result = tfn, headers
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
119 tfp = open(tfn, 'w')
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
120 bs = 1024*8
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
121 block = fp.read(bs)
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
122 while block:
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
123 tfp.write(block)
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
124 block = fp.read(bs)
1563
bd1a4d717e9d Added __repr__ to addbase class; delete more objects
Guido van Rossum <guido@python.org>
parents: 1562
diff changeset
125 del fp
bd1a4d717e9d Added __repr__ to addbase class; delete more objects
Guido van Rossum <guido@python.org>
parents: 1562
diff changeset
126 del tfp
1568
e44499a44cb8 use caching of temp files; added cleanup
Guido van Rossum <guido@python.org>
parents: 1563
diff changeset
127 return result
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
128
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
129 # Each method named open_<type> knows how to open that type of URL
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
130
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
131 # Use HTTP protocol
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
132 def open_http(self, url):
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
133 import httplib
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
134 host, selector = splithost(url)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
135 h = httplib.HTTP(host)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
136 h.putrequest('GET', selector)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
137 for args in self.addheaders: apply(h.putheader, args)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
138 errcode, errmsg, headers = h.getreply()
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
139 if errcode == 200: return addinfo(h.getfile(), headers)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
140 else: raise IOError, ('http error', errcode, errmsg, headers)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
141
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
142 # Use Gopher protocol
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
143 def open_gopher(self, url):
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
144 import gopherlib
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
145 host, selector = splithost(url)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
146 type, selector = splitgophertype(selector)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
147 selector, query = splitquery(selector)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
148 if query: fp = gopherlib.send_query(selector, query, host)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
149 else: fp = gopherlib.send_selector(selector, host)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
150 return addinfo(fp, noheaders())
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
151
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
152 # Use local file or FTP depending on form of URL
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
153 def open_file(self, url):
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
154 try:
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
155 return self.open_local_file(url)
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
156 except IOError:
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
157 return self.open_ftp(url)
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
158
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
159 # Use local file
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
160 def open_local_file(self, url):
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
161 host, file = splithost(url)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
162 if not host: return addinfo(open(file, 'r'), noheaders())
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
163 host, port = splitport(host)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
164 if not port and socket.gethostbyname(host) in (
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
165 localhost(), thishost()):
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
166 return addinfo(open(file, 'r'), noheaders())
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
167 raise IOError, ('local file error', 'not on local host')
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
168
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
169 # Use FTP protocol
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
170 def open_ftp(self, url):
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
171 host, file = splithost(url)
1571
06aa608c589e avoid crash in open_ftp when no host in url
Guido van Rossum <guido@python.org>
parents: 1568
diff changeset
172 if not host: raise IOError, ('ftp error', 'no host given')
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
173 host, port = splitport(host)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
174 host = socket.gethostbyname(host)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
175 if not port:
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
176 import ftplib
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
177 port = ftplib.FTP_PORT
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
178 key = (host, port)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
179 try:
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
180 if not self.ftpcache.has_key(key):
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
181 self.ftpcache[key] = ftpwrapper(host, port)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
182 return addinfo(self.ftpcache[key].retrfile(file),
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
183 noheaders())
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
184 except ftperrors(), msg:
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
185 raise IOError, ('ftp error', msg)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
186
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
187
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
188 # Utility functions
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
189
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
190 # Return the IP address of the magic hostname 'localhost'
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
191 _localhost = None
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
192 def localhost():
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
193 global _localhost
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
194 if not _localhost:
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
195 _localhost = socket.gethostbyname('localhost')
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
196 return _localhost
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
197
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
198 # Return the IP address of the current host
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
199 _thishost = None
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
200 def thishost():
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
201 global _thishost
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
202 if not _thishost:
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
203 _thishost = socket.gethostbyname(socket.gethostname())
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
204 return _thishost
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
205
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
206 # Return the set of errors raised by the FTP class
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
207 _ftperrors = None
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
208 def ftperrors():
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
209 global _ftperrors
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
210 if not _ftperrors:
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
211 import ftplib
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
212 _ftperrors = (ftplib.error_reply,
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
213 ftplib.error_temp,
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
214 ftplib.error_perm,
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
215 ftplib.error_proto)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
216 return _ftperrors
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
217
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
218 # Return an empty rfc822.Message object
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
219 _noheaders = None
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
220 def noheaders():
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
221 global _noheaders
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
222 if not _noheaders:
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
223 import rfc822
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
224 _noheaders = rfc822.Message(open('/dev/null', 'r'))
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
225 _noheaders.fp.close() # Recycle file descriptor
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
226 return _noheaders
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
227
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
228
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
229 # Utility classes
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
230
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
231 # Class used by open_ftp() for cache of open FTP connections
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
232 class ftpwrapper:
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
233 def __init__(self, host, port):
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
234 self.host = host
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
235 self.port = port
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
236 self.init()
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
237 def init(self):
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
238 import ftplib
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
239 self.ftp = ftplib.FTP()
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
240 self.ftp.connect(self.host, self.port)
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
241 self.ftp.login()
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
242 def retrfile(self, file):
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
243 import ftplib
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
244 try:
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
245 self.ftp.voidcmd('TYPE I')
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
246 except ftplib.all_errors:
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
247 self.init()
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
248 self.ftp.voidcmd('TYPE I')
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
249 conn = None
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
250 if file:
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
251 try:
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
252 cmd = 'RETR ' + file
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
253 conn = self.ftp.transfercmd(cmd)
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
254 except ftplib.error_perm, reason:
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
255 if reason[:3] != '550':
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
256 raise IOError, ('ftp error', reason)
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
257 if not conn:
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
258 # Try a directory listing
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
259 if file: cmd = 'LIST ' + file
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
260 else: cmd = 'LIST'
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
261 conn = self.ftp.transfercmd(cmd)
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
262 return addclosehook(conn.makefile('r'), self.ftp.voidresp)
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
263
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
264 # Base class for addinfo and addclosehook
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
265 class addbase:
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
266 def __init__(self, fp):
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
267 self.fp = fp
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
268 self.read = self.fp.read
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
269 self.readline = self.fp.readline
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
270 self.readlines = self.fp.readlines
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
271 self.fileno = self.fp.fileno
1563
bd1a4d717e9d Added __repr__ to addbase class; delete more objects
Guido van Rossum <guido@python.org>
parents: 1562
diff changeset
272 def __repr__(self):
bd1a4d717e9d Added __repr__ to addbase class; delete more objects
Guido van Rossum <guido@python.org>
parents: 1562
diff changeset
273 return '<%s at %s whose fp = %s>' % (
bd1a4d717e9d Added __repr__ to addbase class; delete more objects
Guido van Rossum <guido@python.org>
parents: 1562
diff changeset
274 self.__class__.__name__, `id(self)`, `self.fp`)
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
275 def __del__(self):
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
276 self.close()
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
277 def close(self):
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
278 self.read = None
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
279 self.readline = None
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
280 self.readlines = None
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
281 self.fileno = None
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
282 self.fp = None
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
283
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
284 # Class to add a close hook to an open file
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
285 class addclosehook(addbase):
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
286 def __init__(self, fp, closehook, *hookargs):
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
287 addbase.__init__(self, fp)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
288 self.closehook = closehook
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
289 self.hookargs = hookargs
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
290 def close(self):
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
291 if self.closehook:
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
292 apply(self.closehook, self.hookargs)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
293 self.closehook = None
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
294 self.hookargs = None
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
295 addbase.close(self)
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
296
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
297 # class to add an info() method to an open file
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
298 class addinfo(addbase):
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
299 def __init__(self, fp, headers):
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
300 addbase.__init__(self, fp)
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
301 self.headers = headers
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
302 def info(self):
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
303 return self.headers
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
304
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
305
1572
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
306 # Utility to combine a URL with a base URL to form a new URL
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
307
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
308 def basejoin(base, url):
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
309 type, path = splittype(url)
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
310 if type: return url
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
311 host, path = splithost(path)
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
312 basetype, basepath = splittype(base)
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
313 basehost, basepath = splithost(basepath)
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
314 basepath, basetag = splittag(basepath)
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
315 basepath, basequery = splitquery(basepath)
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
316 type = basetype or 'file'
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
317 if path[:1] != '/':
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
318 import string
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
319 i = string.rfind(basepath, '/')
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
320 if i < 0: basepath = '/'
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
321 else: basepath = basepath[:i+1]
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
322 path = basepath + path
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
323 if not host: host = basehost
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
324 if host: return type + '://' + host + path
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
325 else: return type + ':' + path
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
326
91f846f6d6a6 urlopen: add basejoin() function.
Guido van Rossum <guido@python.org>
parents: 1571
diff changeset
327
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
328 # Utilities to parse URLs:
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
329 # unwrap('<URL:type//host/path>') --> 'type//host/path'
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
330 # splittype('type:opaquestring') --> 'type', 'opaquestring'
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
331 # splithost('//host[:port]/path') --> 'host[:port]', '/path'
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
332 # splitport('host:port') --> 'host', 'port'
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
333 # splitquery('/path?query') --> '/path', 'query'
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
334 # splittag('/path#tag') --> '/path', 'tag'
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
335 # splitgophertype('/Xselector') --> 'X', 'selector'
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
336
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
337 def unwrap(url):
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
338 import string
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
339 url = string.strip(url)
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
340 if url[:1] == '<' and url[-1:] == '>':
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
341 url = string.strip(url[1:-1])
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
342 if url[:4] == 'URL:': url = string.strip(url[4:])
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
343 return url
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
344
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
345 _typeprog = regex.compile('^\([^/:]+\):\(.*\)$')
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
346 def splittype(url):
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
347 if _typeprog.match(url) >= 0: return _typeprog.group(1, 2)
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
348 return None, url
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
349
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
350 _hostprog = regex.compile('^//\([^/]+\)\(.*\)$')
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
351 def splithost(url):
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
352 if _hostprog.match(url) >= 0: return _hostprog.group(1, 2)
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
353 return None, url
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
354
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
355 _portprog = regex.compile('^\(.*\):\([0-9]+\)$')
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
356 def splitport(host):
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
357 if _portprog.match(host) >= 0: return _portprog.group(1, 2)
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
358 return host, None
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
359
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
360 _queryprog = regex.compile('^\(.*\)\?\([^?]*\)$')
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
361 def splitquery(url):
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
362 if _queryprog.match(url) >= 0: return _queryprog.group(1, 2)
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
363 return url, None
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
364
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
365 _tagprog = regex.compile('^\(.*\)#\([^#]*\)$')
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
366 def splittag(url):
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
367 if _tagprog.match(url) >= 0: return _tagprog.group(1, 2)
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
368 return url, None
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
369
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
370 def splitgophertype(selector):
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
371 if selector[:1] == '/' and selector[1:2]:
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
372 return selector[1], selector[2:]
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
373 return None, selector
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
374
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
375
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
376 # Test program
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
377 def test():
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
378 import sys
1558
274066bde241 Redesigned as a class
Guido van Rossum <guido@python.org>
parents: 1555
diff changeset
379 import regsub
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
380 args = sys.argv[1:]
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
381 if not args:
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
382 args = [
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
383 '/etc/passwd',
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
384 'file:/etc/passwd',
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
385 'file://localhost/etc/passwd',
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
386 'ftp://ftp.cwi.nl/etc/passwd',
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
387 'gopher://gopher.cwi.nl/11/',
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
388 'http://www.cwi.nl/index.html',
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
389 ]
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
390 try:
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
391 for url in args:
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
392 print '-'*10, url, '-'*10
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
393 fn, h = urlretrieve(url)
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
394 print fn, h
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
395 if h:
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
396 print '======'
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
397 for k in h.keys(): print k + ':', h[k]
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
398 print '======'
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
399 fp = open(fn, 'r')
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
400 data = fp.read()
1563
bd1a4d717e9d Added __repr__ to addbase class; delete more objects
Guido van Rossum <guido@python.org>
parents: 1562
diff changeset
401 del fp
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
402 print regsub.gsub('\r', '', data)
1563
bd1a4d717e9d Added __repr__ to addbase class; delete more objects
Guido van Rossum <guido@python.org>
parents: 1562
diff changeset
403 fn, h = None, None
1562
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
404 print '-'*40
020d8c2d9d3c Added __del__(), close(), cleanup(), retrieve() and open_local_file()
Guido van Rossum <guido@python.org>
parents: 1558
diff changeset
405 finally:
1568
e44499a44cb8 use caching of temp files; added cleanup
Guido van Rossum <guido@python.org>
parents: 1563
diff changeset
406 urlcleanup()
1555
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
407
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
408 # Run test program when run as a script
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
409 if __name__ == '__main__':
6d35308261f4 Towards a standard access mechanism for URLs.
Guido van Rossum <guido@python.org>
parents:
diff changeset
410 test()