Changeset 1603 in products for quintagroup.distrpoxy
- Timestamp:
- Jan 28, 2010 8:59:39 PM (14 years ago)
- Location:
- quintagroup.distrpoxy/trunk/quintagroup/distproxy
- Files:
-
- 2 edited
Legend:
- Unmodified
- Added
- Removed
-
quintagroup.distrpoxy/trunk/quintagroup/distproxy/spider.py
r1065 r1603 3 3 ## Copyright (c) 1999 - 2003 L. C. Rees. All rights reserved. 4 4 ## See COPYRIGHT file for license terms. 5 6 __name__ = 'spider'7 __version__ = '0.5'8 __author__ = 'L.C. Rees (xanimal@users.sf.net)'9 __all__ = ['ftpurls', 'ftppaths', 'weburls', 'ftpmirror', 'ftpspider',10 'webpaths', 'webreport', 'webmirror', 'webspider', 'urlreport',11 'badurlreport', 'badhtmreport', 'redireport', 'outreport', 'othereport']12 13 '''Multithreaded crawling, reporting, and mirroring for Web and FTP.'''14 5 15 6 from __future__ import generators -
quintagroup.distrpoxy/trunk/quintagroup/distproxy/wsgi.py
r1587 r1603 8 8 from paste import request 9 9 from paste.httpexceptions import HTTPNotFound 10 from spider import webmirror 10 import urllib2 11 11 12 12 13 class PackageProxyApp(object): 13 14 14 def __init__(self, index_url=None, pack_dir=None ):15 def __init__(self, index_url=None, pack_dir=None, username=None,password=None,realm=None): 15 16 if not index_url: 16 17 print "No repository index provided" … … 22 23 print 'You must create the %r directory' % pack_dir 23 24 sys.exit() 25 if username and password: 26 # authenticate 27 password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() 28 top_level_url = index_url 29 password_mgr.add_password(realm, top_level_url, username, password) 30 handler = urllib2.HTTPBasicAuthHandler(password_mgr) 31 opener = urllib2.build_opener(handler) 32 urllib2.install_opener(opener) 24 33 self.index_url = index_url 25 34 self.pack_dir = pack_dir 26 35 27 36 def __call__(self, environ, start_response): 37 """ serve the static files """ 28 38 path = environ.get('PATH_INFO', '').strip() 29 39 path_parts = path.split('/') 30 40 if len(path_parts) > 1 and path_parts[1] == "favicon.ico": 31 41 return HTTPNotFound()(environ, start_response) 32 filename = self.checkCache(path )42 filename = self.checkCache(path[1:]) 33 43 if filename is None: 34 44 return HTTPNotFound()(environ, start_response) 35 45 return FileApp(filename)(environ, start_response) 36 46 37 def checkCache(self, path): 47 def checkCache(self, path): 48 """check if we already have the file and download it if not""" 38 49 pth = self.pack_dir + path 39 pth1 = pth 40 if not (path[-3:] in ['tgz','.gz','egg','zip','exe','cfg']): 41 pth1 = pth + 'index.html' 50 index =0 51 if not (path[-3:] in ['ico','txt','tgz','.gz','egg','zip','exe','cfg']): 52 if not os.path.exists(pth): 53 os.makedirs(pth) #create dir if it is not there 54 # add index.html for supposedly folders 55 pth = pth + 'index.html' 56 index = 1 42 57 else: 43 pth = '/'.join(pth.split('/')[:-1]) 44 if not os.path.exists(pth1): 45 webmirror(pth,1,self.index_url+path,0,1) 46 elif int(time()) - os.path.getmtime(pth1) > 3600: 47 webmirror(pth,1,self.index_url+path,0,1) 48 if pth1: 49 return pth1 58 pth1 = '/'.join(pth.split('/')[:-1]) 59 if not os.path.exists(pth): 60 os.makedirs(pth1)#create parent dir if it is not there 61 url = self.index_url+path 62 #if we dont have download it 63 if not os.path.exists(pth): 64 f = urllib2.urlopen(url) 65 lf = open(pth,'wb') 66 lf.write(f.read()) 67 lf.close() 68 #if we have the index.html file if it is older the 1 hour update 69 elif index and int(time()) - os.path.getmtime(pth) > 3600: 70 f = urllib2.urlopen(url) 71 lf = open(pth,'wb') 72 lf.write(f.read()) 73 lf.close() 50 74 return pth 51 75 … … 55 79 pack_dir = local_conf.get('pack_directory', None) 56 80 index_url = local_conf.get('index', None) 57 return PackageProxyApp(index_url, pack_dir) 81 username = local_conf.get('username', None) 82 password = local_conf.get('password', None) 83 realm = local_conf.get('realm', None) 84 return PackageProxyApp(index_url, pack_dir, username, password, realm) 58 85 59 86 class StaticURLParser(urlparser.StaticURLParser):
Note: See TracChangeset
for help on using the changeset viewer.