Mercurial > cpython-lru_cache-weakref
annotate Tools/scripts/ftpmirror.py @ 5817:48b94de85bf3 legacy-trunk
Extensive changes by AMK.
| author | Guido van Rossum <guido@python.org> |
|---|---|
| date | Thu, 17 Jul 1997 16:29:42 +0000 |
| parents | 6c07d44a7081 |
| children | e17fe79305c7 |
| rev | line source |
|---|---|
|
4573
73758db2f464
/usr/local/bin/python -> /usr/bin/env python
Guido van Rossum <guido@python.org>
parents:
4544
diff
changeset
|
1 #! /usr/bin/env python |
| 2761 | 2 |
| 3 # Mirror a remote ftp subtree into a local directory tree. | |
| 4 # Basic usage: ftpmirror [options] host remotedir localdir | |
| 5 # | |
| 6 # XXX To do: | |
| 7 # - handle symbolic links | |
| 8 # - back up .mirrorinfo before overwriting | |
| 9 # - use pickles for .mirrorinfo? | |
| 10 | |
| 11 import os | |
| 12 import sys | |
| 13 import time | |
| 14 import getopt | |
| 15 import string | |
| 16 import ftplib | |
| 17 from fnmatch import fnmatch | |
| 18 | |
| 19 usage_msg = """ | |
| 20 usage: ftpmirror [-v] [-q] [-i] [-m] [-n] [-r] [-s pat] | |
| 21 [-l username [-p passwd [-a account]]] | |
| 22 hostname [remotedir [localdir]] | |
| 23 -v: verbose | |
| 24 -q: quiet | |
| 25 -i: interactive mode | |
| 26 -m: macintosh server (NCSA telnet 2.4) (implies -n -s '*.o') | |
| 27 -n: don't log in | |
| 28 -r: remove files no longer pertinent | |
| 29 -l username [-p passwd [-a account]]: login info (default anonymous ftp) | |
| 30 -s pat: skip files matching pattern | |
| 31 hostname: remote host | |
| 32 remotedir: remote directory (default initial) | |
| 33 localdir: local directory (default current) | |
| 34 """ | |
| 35 def usage(*args): | |
| 36 sys.stdout = sys.stderr | |
| 37 for msg in args: print msg | |
| 38 print usage_msg | |
| 39 sys.exit(2) | |
| 40 | |
| 41 verbose = 1 # 0 for -q, 2 for -v | |
| 42 interactive = 0 | |
| 43 mac = 0 | |
| 44 rmok = 0 | |
| 45 nologin = 0 | |
|
3374
0c8821bf6db5
don't die on unreadable files etc.
Guido van Rossum <guido@python.org>
parents:
2761
diff
changeset
|
46 skippats = ['.', '..', '.mirrorinfo'] |
| 2761 | 47 |
| 48 def main(): | |
| 49 global verbose, interactive, mac, rmok, nologin | |
| 50 try: | |
| 51 opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v') | |
| 52 except getopt.error, msg: | |
| 53 usage(msg) | |
| 54 login = '' | |
| 55 passwd = '' | |
| 56 account = '' | |
| 57 for o, a in opts: | |
| 58 if o == '-l': login = a | |
| 59 if o == '-p': passwd = a | |
| 60 if o == '-a': account = a | |
| 61 if o == '-v': verbose = verbose + 1 | |
| 62 if o == '-q': verbose = 0 | |
| 63 if o == '-i': interactive = 1 | |
| 64 if o == '-m': mac = 1; nologin = 1; skippats.append('*.o') | |
| 65 if o == '-n': nologin = 1 | |
| 66 if o == '-r': rmok = 1 | |
| 67 if o == '-s': skippats.append(a) | |
| 68 if not args: usage('hostname missing') | |
| 69 host = args[0] | |
| 70 remotedir = '' | |
| 71 localdir = '' | |
| 72 if args[1:]: | |
| 73 remotedir = args[1] | |
| 74 if args[2:]: | |
| 75 localdir = args[2] | |
| 76 if args[3:]: usage('too many arguments') | |
| 77 # | |
| 78 f = ftplib.FTP() | |
| 79 if verbose: print 'Connecting to %s...' % host | |
| 80 f.connect(host) | |
| 81 if not nologin: | |
| 82 if verbose: | |
| 83 print 'Logging in as %s...' % (login or 'anonymous') | |
| 84 f.login(login, passwd, account) | |
| 85 if verbose: print 'OK.' | |
| 86 pwd = f.pwd() | |
| 87 if verbose > 1: print 'PWD =', `pwd` | |
| 88 if remotedir: | |
| 89 if verbose > 1: print 'cwd(%s)' % `remotedir` | |
| 90 f.cwd(remotedir) | |
| 91 if verbose > 1: print 'OK.' | |
| 92 pwd = f.pwd() | |
| 93 if verbose > 1: print 'PWD =', `pwd` | |
| 94 # | |
| 95 mirrorsubdir(f, localdir) | |
| 96 | |
| 97 def mirrorsubdir(f, localdir): | |
| 98 pwd = f.pwd() | |
| 99 if localdir and not os.path.isdir(localdir): | |
| 100 if verbose: print 'Creating local directory', localdir | |
|
3495
f70dbe608f03
Create files as temp file and move to real location only when complete.
Guido van Rossum <guido@python.org>
parents:
3374
diff
changeset
|
101 try: |
|
f70dbe608f03
Create files as temp file and move to real location only when complete.
Guido van Rossum <guido@python.org>
parents:
3374
diff
changeset
|
102 makedir(localdir) |
|
f70dbe608f03
Create files as temp file and move to real location only when complete.
Guido van Rossum <guido@python.org>
parents:
3374
diff
changeset
|
103 except os.error, msg: |
|
f70dbe608f03
Create files as temp file and move to real location only when complete.
Guido van Rossum <guido@python.org>
parents:
3374
diff
changeset
|
104 print "Failed to establish local directory", localdir |
|
f70dbe608f03
Create files as temp file and move to real location only when complete.
Guido van Rossum <guido@python.org>
parents:
3374
diff
changeset
|
105 return |
| 2761 | 106 infofilename = os.path.join(localdir, '.mirrorinfo') |
| 107 try: | |
| 108 text = open(infofilename, 'r').read() | |
| 109 except IOError, msg: | |
| 110 text = '{}' | |
| 111 try: | |
| 112 info = eval(text) | |
| 113 except (SyntaxError, NameError): | |
| 114 print 'Bad mirror info in %s' % infofilename | |
| 115 info = {} | |
| 116 subdirs = [] | |
| 117 listing = [] | |
| 118 if verbose: print 'Listing remote directory %s...' % pwd | |
| 119 f.retrlines('LIST', listing.append) | |
|
4544
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
120 filesfound = [] |
| 2761 | 121 for line in listing: |
| 122 if verbose > 1: print '-->', `line` | |
| 123 if mac: | |
| 124 # Mac listing has just filenames; | |
| 125 # trailing / means subdirectory | |
| 126 filename = string.strip(line) | |
| 127 mode = '-' | |
| 128 if filename[-1:] == '/': | |
| 129 filename = filename[:-1] | |
| 130 mode = 'd' | |
| 131 infostuff = '' | |
| 132 else: | |
| 133 # Parse, assuming a UNIX listing | |
| 134 words = string.split(line) | |
| 135 if len(words) < 6: | |
| 136 if verbose > 1: print 'Skipping short line' | |
| 137 continue | |
| 138 if words[-2] == '->': | |
| 139 if verbose > 1: | |
| 140 print 'Skipping symbolic link %s -> %s' % \ | |
| 141 (words[-3], words[-1]) | |
| 142 continue | |
| 143 filename = words[-1] | |
| 144 infostuff = words[-5:-1] | |
| 145 mode = words[0] | |
| 146 skip = 0 | |
| 147 for pat in skippats: | |
| 148 if fnmatch(filename, pat): | |
| 149 if verbose > 1: | |
| 150 print 'Skip pattern', pat, | |
| 151 print 'matches', filename | |
| 152 skip = 1 | |
| 153 break | |
| 154 if skip: | |
| 155 continue | |
| 156 if mode[0] == 'd': | |
| 157 if verbose > 1: | |
| 158 print 'Remembering subdirectory', filename | |
| 159 subdirs.append(filename) | |
| 160 continue | |
|
4544
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
161 filesfound.append(filename) |
| 2761 | 162 if info.has_key(filename) and info[filename] == infostuff: |
| 163 if verbose > 1: | |
| 164 print 'Already have this version of', filename | |
| 165 continue | |
| 166 fullname = os.path.join(localdir, filename) | |
|
3495
f70dbe608f03
Create files as temp file and move to real location only when complete.
Guido van Rossum <guido@python.org>
parents:
3374
diff
changeset
|
167 tempname = os.path.join(localdir, '@'+filename) |
| 2761 | 168 if interactive: |
| 169 doit = askabout('file', filename, pwd) | |
| 170 if not doit: | |
| 171 if not info.has_key(filename): | |
| 172 info[filename] = 'Not retrieved' | |
| 173 continue | |
| 174 try: | |
|
3495
f70dbe608f03
Create files as temp file and move to real location only when complete.
Guido van Rossum <guido@python.org>
parents:
3374
diff
changeset
|
175 os.unlink(tempname) |
|
3374
0c8821bf6db5
don't die on unreadable files etc.
Guido van Rossum <guido@python.org>
parents:
2761
diff
changeset
|
176 except os.error: |
|
0c8821bf6db5
don't die on unreadable files etc.
Guido van Rossum <guido@python.org>
parents:
2761
diff
changeset
|
177 pass |
|
0c8821bf6db5
don't die on unreadable files etc.
Guido van Rossum <guido@python.org>
parents:
2761
diff
changeset
|
178 try: |
|
5532
30d559827047
Open the output files with 'wb', not 'w'.
Guido van Rossum <guido@python.org>
parents:
4573
diff
changeset
|
179 fp = open(tempname, 'wb') |
| 2761 | 180 except IOError, msg: |
|
3495
f70dbe608f03
Create files as temp file and move to real location only when complete.
Guido van Rossum <guido@python.org>
parents:
3374
diff
changeset
|
181 print "Can't create %s: %s" % (tempname, str(msg)) |
| 2761 | 182 continue |
| 183 if verbose: | |
| 184 print 'Retrieving %s from %s as %s...' % \ | |
| 185 (filename, pwd, fullname) | |
| 186 if verbose: | |
| 187 fp1 = LoggingFile(fp, 1024, sys.stdout) | |
| 188 else: | |
| 189 fp1 = fp | |
| 190 t0 = time.time() | |
|
3374
0c8821bf6db5
don't die on unreadable files etc.
Guido van Rossum <guido@python.org>
parents:
2761
diff
changeset
|
191 try: |
|
0c8821bf6db5
don't die on unreadable files etc.
Guido van Rossum <guido@python.org>
parents:
2761
diff
changeset
|
192 f.retrbinary('RETR ' + filename, fp1.write, 8*1024) |
|
0c8821bf6db5
don't die on unreadable files etc.
Guido van Rossum <guido@python.org>
parents:
2761
diff
changeset
|
193 except ftplib.error_perm, msg: |
|
0c8821bf6db5
don't die on unreadable files etc.
Guido van Rossum <guido@python.org>
parents:
2761
diff
changeset
|
194 print msg |
| 2761 | 195 t1 = time.time() |
| 196 bytes = fp.tell() | |
| 197 fp.close() | |
| 198 if fp1 != fp: | |
| 199 fp1.close() | |
|
3495
f70dbe608f03
Create files as temp file and move to real location only when complete.
Guido van Rossum <guido@python.org>
parents:
3374
diff
changeset
|
200 try: |
|
5546
6c07d44a7081
Unlink before rename (for NT).
Guido van Rossum <guido@python.org>
parents:
5532
diff
changeset
|
201 os.unlink(fullname) |
|
6c07d44a7081
Unlink before rename (for NT).
Guido van Rossum <guido@python.org>
parents:
5532
diff
changeset
|
202 except os.error: |
|
6c07d44a7081
Unlink before rename (for NT).
Guido van Rossum <guido@python.org>
parents:
5532
diff
changeset
|
203 pass # Ignore the error |
|
6c07d44a7081
Unlink before rename (for NT).
Guido van Rossum <guido@python.org>
parents:
5532
diff
changeset
|
204 try: |
|
6c07d44a7081
Unlink before rename (for NT).
Guido van Rossum <guido@python.org>
parents:
5532
diff
changeset
|
205 os.rename(tempname, fullname) |
|
3495
f70dbe608f03
Create files as temp file and move to real location only when complete.
Guido van Rossum <guido@python.org>
parents:
3374
diff
changeset
|
206 except os.error, msg: |
|
f70dbe608f03
Create files as temp file and move to real location only when complete.
Guido van Rossum <guido@python.org>
parents:
3374
diff
changeset
|
207 print "Can't rename %s to %s: %s" % (tempname, |
|
f70dbe608f03
Create files as temp file and move to real location only when complete.
Guido van Rossum <guido@python.org>
parents:
3374
diff
changeset
|
208 fullname, |
|
f70dbe608f03
Create files as temp file and move to real location only when complete.
Guido van Rossum <guido@python.org>
parents:
3374
diff
changeset
|
209 str(msg)) |
|
f70dbe608f03
Create files as temp file and move to real location only when complete.
Guido van Rossum <guido@python.org>
parents:
3374
diff
changeset
|
210 continue |
| 2761 | 211 info[filename] = infostuff |
| 212 writedict(info, infofilename) | |
| 213 if verbose: | |
| 214 dt = t1 - t0 | |
| 215 kbytes = bytes / 1024.0 | |
| 216 print int(round(kbytes)), | |
| 217 print 'Kbytes in', | |
| 218 print int(round(dt)), | |
| 219 print 'seconds', | |
| 220 if t1 > t0: | |
| 221 print '(~%d Kbytes/sec)' % \ | |
| 222 int(round(kbytes/dt),) | |
| 223 print | |
| 224 # | |
|
4544
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
225 # Remove files from info that are no longer remote |
|
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
226 deletions = 0 |
|
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
227 for filename in info.keys(): |
|
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
228 if filename not in filesfound: |
|
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
229 if verbose: |
|
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
230 print "Removing obsolete info entry for", |
|
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
231 print filename, "in", localdir or "." |
|
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
232 del info[filename] |
|
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
233 deletions = deletions + 1 |
|
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
234 if deletions: |
|
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
235 writedict(info, infofilename) |
|
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
236 # |
| 2761 | 237 # Remove local files that are no longer in the remote directory |
|
3495
f70dbe608f03
Create files as temp file and move to real location only when complete.
Guido van Rossum <guido@python.org>
parents:
3374
diff
changeset
|
238 try: |
|
f70dbe608f03
Create files as temp file and move to real location only when complete.
Guido van Rossum <guido@python.org>
parents:
3374
diff
changeset
|
239 if not localdir: names = os.listdir(os.curdir) |
|
f70dbe608f03
Create files as temp file and move to real location only when complete.
Guido van Rossum <guido@python.org>
parents:
3374
diff
changeset
|
240 else: names = os.listdir(localdir) |
|
f70dbe608f03
Create files as temp file and move to real location only when complete.
Guido van Rossum <guido@python.org>
parents:
3374
diff
changeset
|
241 except os.error: |
|
f70dbe608f03
Create files as temp file and move to real location only when complete.
Guido van Rossum <guido@python.org>
parents:
3374
diff
changeset
|
242 names = [] |
| 2761 | 243 for name in names: |
| 244 if name[0] == '.' or info.has_key(name) or name in subdirs: | |
| 245 continue | |
|
4544
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
246 skip = 0 |
|
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
247 for pat in skippats: |
|
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
248 if fnmatch(name, pat): |
|
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
249 if verbose > 1: |
|
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
250 print 'Skip pattern', pat, |
|
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
251 print 'matches', name |
|
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
252 skip = 1 |
|
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
253 break |
|
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
254 if skip: |
|
8d0b00ed8c2b
Fixed a bug: files that no longer exist remotely would never be
Guido van Rossum <guido@python.org>
parents:
3495
diff
changeset
|
255 continue |
| 2761 | 256 fullname = os.path.join(localdir, name) |
| 257 if not rmok: | |
| 258 if verbose: | |
| 259 print 'Local file', fullname, | |
| 260 print 'is no longer pertinent' | |
| 261 continue | |
| 262 if verbose: print 'Removing local file', fullname | |
| 263 try: | |
| 264 os.unlink(fullname) | |
| 265 except os.error, msg: | |
| 266 print "Can't remove local file %s: %s" % \ | |
| 267 (fullname, str(msg)) | |
| 268 # | |
| 269 # Recursively mirror subdirectories | |
| 270 for subdir in subdirs: | |
| 271 if interactive: | |
| 272 doit = askabout('subdirectory', subdir, pwd) | |
| 273 if not doit: continue | |
| 274 if verbose: print 'Processing subdirectory', subdir | |
| 275 localsubdir = os.path.join(localdir, subdir) | |
| 276 pwd = f.pwd() | |
| 277 if verbose > 1: | |
| 278 print 'Remote directory now:', pwd | |
| 279 print 'Remote cwd', subdir | |
| 280 try: | |
| 281 f.cwd(subdir) | |
| 282 except ftplib.error_perm, msg: | |
| 283 print "Can't chdir to", subdir, ":", msg | |
| 284 else: | |
| 285 if verbose: print 'Mirroring as', localsubdir | |
| 286 mirrorsubdir(f, localsubdir) | |
| 287 if verbose > 1: print 'Remote cwd ..' | |
| 288 f.cwd('..') | |
| 289 newpwd = f.pwd() | |
| 290 if newpwd != pwd: | |
| 291 print 'Ended up in wrong directory after cd + cd ..' | |
| 292 print 'Giving up now.' | |
| 293 break | |
| 294 else: | |
| 295 if verbose > 1: print 'OK.' | |
| 296 | |
| 297 # Wrapper around a file for writing to write a hash sign every block. | |
| 298 class LoggingFile: | |
| 299 def __init__(self, fp, blocksize, outfp): | |
| 300 self.fp = fp | |
| 301 self.bytes = 0 | |
| 302 self.hashes = 0 | |
| 303 self.blocksize = blocksize | |
| 304 self.outfp = outfp | |
| 305 def write(self, data): | |
| 306 self.bytes = self.bytes + len(data) | |
| 307 hashes = int(self.bytes) / self.blocksize | |
| 308 while hashes > self.hashes: | |
| 309 self.outfp.write('#') | |
| 310 self.outfp.flush() | |
| 311 self.hashes = self.hashes + 1 | |
| 312 self.fp.write(data) | |
| 313 def close(self): | |
| 314 self.outfp.write('\n') | |
| 315 | |
| 316 # Ask permission to download a file. | |
| 317 def askabout(filetype, filename, pwd): | |
| 318 prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd) | |
| 319 while 1: | |
| 320 reply = string.lower(string.strip(raw_input(prompt))) | |
| 321 if reply in ['y', 'ye', 'yes']: | |
| 322 return 1 | |
| 323 if reply in ['', 'n', 'no', 'nop', 'nope']: | |
| 324 return 0 | |
| 325 print 'Please answer yes or no.' | |
| 326 | |
| 327 # Create a directory if it doesn't exist. Recursively create the | |
| 328 # parent directory as well if needed. | |
| 329 def makedir(pathname): | |
| 330 if os.path.isdir(pathname): | |
| 331 return | |
| 332 dirname = os.path.dirname(pathname) | |
| 333 if dirname: makedir(dirname) | |
| 334 os.mkdir(pathname, 0777) | |
| 335 | |
| 336 # Write a dictionary to a file in a way that can be read back using | |
| 337 # rval() but is still somewhat readable (i.e. not a single long line). | |
| 338 def writedict(dict, filename): | |
| 339 fp = open(filename, 'w') | |
| 340 fp.write('{\n') | |
| 341 for key, value in dict.items(): | |
| 342 fp.write('%s: %s,\n' % (`key`, `value`)) | |
| 343 fp.write('}\n') | |
| 344 fp.close() | |
| 345 | |
| 346 main() |