Index: quintagroup.distrpoxy/tags/1.0/README.txt
===================================================================
--- quintagroup.distrpoxy/tags/1.0/README.txt	(revision 1073)
+++ quintagroup.distrpoxy/tags/1.0/README.txt	(revision 1073)
@@ -0,0 +1,4 @@
+Introduction
+============
+
+
Index: quintagroup.distrpoxy/tags/1.0/docs/HISTORY.txt
===================================================================
--- quintagroup.distrpoxy/tags/1.0/docs/HISTORY.txt	(revision 1073)
+++ quintagroup.distrpoxy/tags/1.0/docs/HISTORY.txt	(revision 1073)
@@ -0,0 +1,8 @@
+Changelog
+=========
+
+1.0 (2009-07-08)
+----------------
+
+* Initial release
+
Index: quintagroup.distrpoxy/tags/1.0/quintagroup/__init__.py
===================================================================
--- quintagroup.distrpoxy/tags/1.0/quintagroup/__init__.py	(revision 1073)
+++ quintagroup.distrpoxy/tags/1.0/quintagroup/__init__.py	(revision 1073)
@@ -0,0 +1,6 @@
+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)
Index: quintagroup.distrpoxy/tags/1.0/quintagroup/distproxy/README.txt
===================================================================
--- quintagroup.distrpoxy/tags/1.0/quintagroup/distproxy/README.txt	(revision 1073)
+++ quintagroup.distrpoxy/tags/1.0/quintagroup/distproxy/README.txt	(revision 1073)
@@ -0,0 +1,9 @@
+The quintagroup.distproxy is a smart mirror for package directories like dist.plone.org
+
+For mirroring the packaes index website spider.py used.
+
+Check wsgi.ini and update it with your directory.
+
+Use PasteDeploy to run the application
+
+    paster serve wsgi.ini
Index: quintagroup.distrpoxy/tags/1.0/quintagroup/distproxy/spider.py
===================================================================
--- quintagroup.distrpoxy/tags/1.0/quintagroup/distproxy/spider.py	(revision 1073)
+++ quintagroup.distrpoxy/tags/1.0/quintagroup/distproxy/spider.py	(revision 1073)
@@ -0,0 +1,1005 @@
+#! /usr/bin/env python
+
+## Copyright (c) 1999 - 2003 L. C. Rees.  All rights reserved.
+## See COPYRIGHT file for license terms.
+
+__name__ = 'spider'
+__version__ = '0.5'
+__author__ = 'L.C. Rees (xanimal@users.sf.net)'
+__all__ = ['ftpurls', 'ftppaths', 'weburls', 'ftpmirror', 'ftpspider',
+    'webpaths', 'webreport', 'webmirror', 'webspider', 'urlreport',
+    'badurlreport', 'badhtmreport', 'redireport', 'outreport', 'othereport']
+
+'''Multithreaded crawling, reporting, and mirroring for Web and FTP.'''
+
+from __future__ import generators
+
+
+class Spider:
+
+    '''HTTP and FTP crawling, reporting, and checking'''
+    
+    import os as _os
+    import urllib as _ulib
+    import urlparse as _uparse
+    from os import path as _path 
+    from ftplib import FTP as _ftp
+    from time import strftime as _formtime
+    from time import localtime as _localtime
+    from ftplib import error_perm as _ftperr        
+    from sgmllib import SGMLParseError as _sperror
+    from robotparser import RobotFileParser as _rparser
+    # Use threads if available 
+    try: from threading import Thread as _thread
+    except ImportError: pass
+    _bdsig, _bfsig, _session, _newparser = None, None, None, None
+    # HTML tags with URLs
+    _urltags = {'a':1, 'img':1, 'link':1, 'script':1, 'iframe':1, 'object':1,
+        'embed':1, 'area':1, 'frame':1, 'applet':1, 'input':1, 'base':1,
+        'div':1, 'layer':1, 'ilayer':1, 'bgsound':1}
+    # Supported protocols
+    _supported = {'HTTP':1, 'http':1, 'HTTPS':1, 'https':1, 'FTP':1, 'ftp':1}
+    # HTML attributes with URLs
+    _urlattrs = {'href':1, 'src':1, 'data':1}
+
+    def __init__(self, base=None, width=None, depth=None):
+        '''Initializes a Spider instance and its base attributes
+
+        Arguments:
+        base -- URL to crawl (default: None)
+        width -- maximum resources to crawl (default: None)
+        depth -- how deep in a hierarchy to crawl (default: None)'''                             
+        if base: self.base = base
+        else: self.base = None
+        if width: self.width = width
+        else: self.width = None
+        if depth: self.depth = depth
+        else: self.depth = None
+        
+    def _ftpopen(self, base, name='anonymous', password=None, attempts=3):
+        '''Returns FTP client session
+
+        Arguments:
+        base -- FTP server URL
+        name -- login name (default: 'anonymous')
+        password -- login password (default: None)
+        attempts -- number of login attempts to try (default: 3)'''
+
+        def ftpprompt(tries=0):        
+            '''Prompts for FTP username and password
+
+            Arguments:
+            tries -- number of login attempts'''
+            tries += tries
+            try:
+                self._name = raw_input('Enter login name: ')
+                self._password = raw_input('Enter password: ')
+                session = ftp(base, self._name, self._password)
+                return session
+            # If login attempt fails, retry login
+            except ftperr:               
+                if attempts >= tries:
+                    session = ftpprompt(tries)
+                    return session
+                # Too many login attempts? End program
+                elif attempts <= tries:
+                    raise IOError, 'Permission denied.'
+                    import sys
+                    sys.exit(0)           
+
+        # Assignments
+        self._name, self._password, ftperr = name, password, self._ftperr
+        su, ftp = self._uparse.urlsplit(base), self._ftp
+        # Set URL, path, and strip 'ftp://' off
+        base, path = su[1], '/'.join([su[2], ''])
+        try: session = ftp(base, name, password)
+        # Prompt for username, password if initial arguments are incorrect
+        except ftperr: session = ftpprompt()
+        # Change to remote path if it exits
+        if path: session.cwd(path)
+        return session
+
+    def ftpmirror(self, l, t=None, b=None, w=200, d=6, n='anonymous', p=None):
+        '''Mirrors an FTP site on a local filesystem
+
+        Arguments:
+        l -- local filesystem path (default: None)
+        b -- FTP server URL (default: None)
+        t -- number of download threads (default: None)
+        w -- maximum amount of resources to crawl (default: 200)   
+        d -- depth in hierarchy to crawl (default: 6)             
+        n -- login username (default: 'anonymous')
+        p -- login password (default: None)'''
+        if b: self.ftpspider(b, w, d, n, p)
+        return self._mirror((self.paths, self.urls), l, t)
+
+    def ftppaths(self, b=None, w=200, d=6, n='anonymous', p=None):
+        '''Returns a list of FTP paths.
+        
+        Arguments:
+        b -- FTP server URL (default: None)
+        w -- maximum amount of resources to crawl (default: 200) 
+        d -- depth in hierarchy to crawl (default: 6)               
+        n -- login username (default: 'anonymous')
+        p -- login password (default: None)'''
+        
+        def sortftp(rdir):
+            '''Returns a list of entries marked as files or directories
+
+            Arguments:
+            rdir -- remote directory list'''
+            rlist = []
+            rappend = rlist.append
+            for rl in rdir:
+                # Split remote file based on whitespace
+                ri = rl.split()[-1]
+                # Add tuple of remote item type, permissions & name to rlist
+                if ri not in ('.', '..'): rappend((rl[0], rl[7], ri))
+            return rlist
+       
+        def visitftp():
+            '''Extracts contents of an FTP directory'''
+            wd = pwd()
+            if wd[-1] != '/': wd = '/'.join([wd, ''])
+            # Add present working directory to visited directories
+            dirs[wd], rlist = None, []
+            # Get list of current directory's contents
+            retr('LIST -a', rlist.append)
+            for url in sortftp(rlist):
+                # Test if remote item is a file (indicated by '-')
+                if url[0] == '-':
+                    # Resolve path of file
+                    purl = ''.join([wd, url[2]])
+                    # Ensure file list don't exceed max number of resources
+                    if len(files) >= width: return None
+                    # Add files to file dictionary
+                    elif purl not in files: files[purl] = None
+                # Test if it's a directory ('d') and allows scanning ('-')
+                elif url[0] == 'd':
+                    if url[1] != '-':
+                        # Resolve path of directory
+                        purl = ''.join([wd, url[2], '/'])
+                        # Ensure no recursion beyond depth allowed
+                        if len(purl.split('/')) >= depth: dirs[purl] = None
+                        # Visit directory if it hasn't been visited yet 
+                        elif purl not in dirs:
+                            # Change to new directory
+                            cwd(purl)
+                            # Run 'visitftp' on new directory
+                            visitftp()
+                            
+        # Use classwide attributes if set
+        if b: self.base = b
+        else: b = self.base
+        # Use classwide width if different from method default
+        if self.width and w == 200: width = self.width
+        else: width = w
+        # Use classwide depth if different from method default
+        if self.depth and d == 6: depth = self.depth + 1
+        else: depth = d + 1
+        # File and directory dicts
+        files, dirs = {}, {}
+        # Use existing FTP client session if present
+        if self._session: ftp = self._session
+        # Create new FTP client session if necessary
+        else:
+            ftp = self._ftpopen(b, n, p)
+            self._session = ftp
+        # Avoid outside namespace lookups
+        cwd, pwd, retr = ftp.cwd, ftp.pwd, ftp.retrlines
+        # Walk FTP site
+        visitftp()
+        # Make path list out of files' keys and return it
+        self.paths = files.keys()
+        self.paths.sort()
+        return self.paths
+
+    def ftpspider(self, b=None, w=200, d=6, n='anonymous', p=None):
+        '''Returns lists of URLs and paths plus a live FTP client session
+        
+        Arguments:
+        b -- FTP server URL (default: None)
+        w -- maximum amount of resources to crawl (default: 200) 
+        d -- depth in hierarchy to crawl (default: 6)               
+        n -- login username (default: 'anonymous')
+        p -- login password (default: None)'''
+        if b: ftppaths(b, w, d, n, p)
+        return self.paths, ftpurls(), self._session
+
+    def ftpurls(self, b=None, w=200, d=6, n='anonymous', p=None):
+        '''Returns a list of FTP URLs
+        
+        Arguments:
+        b -- FTP server URL (default: None)
+        w -- maximum amount of resources to crawl (default: 200) 
+        d -- depth in hierarchy to crawl (default: 6)               
+        n -- login username (default: 'anonymous')
+        p -- login password (default: None)'''
+        if b:
+            ftppaths(b, w, d, n, p)
+            # Get rid of trailing '/' in base if present before joining
+            if b[-1] == '/': base = b[:-1]       
+        else:           
+            base = self.base
+            # Get rid of trailing '/' in base if present before joining
+            if base[-1] == '/': base = self.base[:-1]
+        paths = self.paths
+        # Add FTP URL
+        self.urls = [''.join([base, i]) for i in paths]
+        return self.urls
+
+    def _parserpick(self, old=None):
+        '''Returns a class using the sgmllib parser or the sgmlop parser
+
+        Arguments:
+        old -- use classic sgmllib SGMLParser'''
+        # Assignments
+        urltags, urlattrs = self._urltags, self._urlattrs
+        # Lists for bad file and bad directory signatures
+        self._bfsig, self._bdsig = [], []
+        bfsig, bdsig = self._bfsig, self._bdsig        
+        # Use faster SGMLParser if available
+        try:
+            from sgmlop import SGMLParser as newparser
+            self._newparser = newparser
+        # If unavailable, use classic SGML parser
+        except ImportError:
+            from sgmllib import SGMLParser as oldparser
+            old = 1
+        # Classes using classic sgmllib SGML Parser
+        if old:
+            from sgmllib import SGMLParser as oldparser
+            # Remove sgmlop parser if present
+            self._newparser = None
+            # UrlExtract class using classic parser
+            class UrlExtract(oldparser):
+                '''Extracts URLs from a SGMLish document'''
+                def reset(self):
+                    '''Resets SGML parser and clears lists'''
+                    oldparser.reset(self)
+                    self.urls, self.text, self.badurl = [], [], None
+                def handle_data(self, data):
+                    '''Handles non-markup data'''            
+                    # Get first 5 lines of non-markup data
+                    if len(self.text) <= 5: self.text.append(data)
+                    # Compare signature of known bad URL to a new web page
+                    if self.text == bfsig: self.badurl = 1
+                    elif self.text == bdsig: self.badurl = 1
+                def finish_starttag(self, tag, attrs):
+                    '''Extracts URL bearing tags'''
+                    if tag in urltags:
+                        # Get key, vale in attributes if they match
+                        url = [v for k, v in attrs if k in urlattrs]
+                        if url: self.urls.extend(url)
+            # BadUrl class using classic parser
+            class BadUrl(oldparser):            
+                '''Collects results of intentionally incorrect URLs'''
+                def reset(self):
+                    '''Resets SGML parser and clears lists'''
+                    oldparser.reset(self)
+                    self.text = []
+                def handle_data(self, data):
+                    '''Collects lines to profile bad URLs'''
+                    # Adds first 5 lines of non-markup data to text
+                    if len(self.text) <= 5: self.text.append(data)
+        # If no old flag, use SGMLParser from sgmlop and related classes
+        else:
+            # UrlExtract class using sgmlop parser
+            class UrlExtract:
+                '''Extracts URLs from a SGMLish document'''            
+                def __init__(self):
+                    '''Resets SGML parser and clears lists'''
+                    self.urls, self.text, self.badurl = [], [], None
+                def handle_data(self, data):
+                    '''Handles non-markup data'''            
+                    # Get first 5 lines of non-markup data
+                    if len(self.text) <= 5: self.text.append(data)
+                    # Compare signature of known bad URL to a new web page
+                    if self.text == bfsig: self.badurl = 1
+                    elif self.text == bdsig: self.badurl = 1
+                def finish_starttag(self, tag, attrs):
+                    '''Extracts URL bearing tags'''
+                    if tag in urltags:
+                        # Get key, vale in attributes if they match
+                        url = [v for k, v in attrs if k in urlattrs]
+                        if url: self.urls.extend(url)
+            # BadUrl class using sgmlop parser
+            class BadUrl:            
+                '''Collects results of intentionally incorrect URLs'''
+                def __init__(self):
+                    '''Resets SGML parser and clears lists'''
+                    self.text = []            
+                def handle_data(self, data):
+                    '''Collects lines to profile not found responses'''
+                    # Adds first 5 lines of non-markup data to list 'text'
+                    if len(self.text) <= 5: self.text.append(data)
+        # Make resulting classes available class wide
+        self._UrlExtract, self._BadUrl = UrlExtract, BadUrl
+
+    def _webtest(self):
+        '''Generates signatures for identifying bad URLs'''
+
+        def badurl(url):
+            '''Returns first 5 lines of a bad URL
+
+            Arguments:                
+            url -- Bad URL to open and parse'''
+            # Use different classes if faster SGML Parser is available
+            if self._newparser:
+                # sgmlop parser must have a handler passed to it
+                parser, urlget = self._newparser(), BadUrl()
+                # Pass handler (sgmlop cannot be subclassed)
+                parser.register(urlget)
+                parser.feed(urlopen(url).read())
+                parser.close()
+            # Use classic parser
+            else:
+                urlget = BadUrl()
+                urlget.feed(urlopen(url).read())
+                urlget.close()
+            # Return singature of bad URL
+            return urlget.text
+
+        # Make globals local
+        base, urljoin = self.base, self._uparse.urljoin
+        urlopen, BadUrl = self._ulib.urlopen, self._BadUrl
+        # Generate random string of jibber
+        from string import letters, digits
+        from random import choice, randint
+        jibber = ''.join([letters, digits])
+        ru = ''.join([choice(jibber) for x in range(randint(1, 30))])
+        # Builds signature of a bad URL for a file
+        self._bfsig.extend(badurl(urljoin(base, '%s.html' % ru)))
+        # Builds signature of a bad URL for a directory
+        self._bdsig.extend(badurl(urljoin(base,'%s/' % ru)))
+
+    def _webparser(self, html):
+        '''Parses HTML and returns bad URL indicator and extracted URLs
+
+        Arguments:
+        html -- HTML data'''
+        # Use different classes if faster SGML Parser is available
+        if self._newparser:
+            # Make instances of SGML parser and URL extracting handler
+            parser, urlget = self._newparser(), self._UrlExtract()
+            # Pass handler to parser
+            parser.register(urlget)
+            # Feed data to parser
+            parser.feed(html)
+            parser.close()
+            # Return bad URL indicator and extracted URLs
+        else:
+            urlget = self._UrlExtract()
+            urlget.feed(html)
+            urlget.close()
+        # Return badurl marker and list of child URLS
+        return urlget.badurl, urlget.urls
+
+    def _webopen(self, base):
+        '''Verifies URL and returns actual URL and extracted child URLs
+
+        Arguments:
+        base -- tuple containing a URL and its referring URL'''
+        # Assignments
+        good, cbase = self._good, base[0]        
+        try:
+            # If webspiders can access URL, open it            
+            if self._robot.can_fetch('*', cbase):
+                url = self._ulib.urlopen(cbase)
+            # Otherwise, mark as visited and abort
+            else:
+                self._visited[cbase] = 1
+                return False
+        # If HTTP error, log bad URL and abort
+        except IOError:
+            self._visited[cbase] = 1
+            self.badurls.append((base[1], cbase))
+            return False
+        # Get real URL
+        newbase = url.geturl()
+        # Change URL if different from old URL
+        if newbase != cbase: cbase, base = newbase, (newbase, base[1])    
+        # URLs with mimetype 'text/html" scanned for URLs
+        if url.headers.type == 'text/html':
+            # Feed parser
+            contents = url.read()
+            try: badurl, urls = self._webparser(contents)
+            # Log URL if SGML parser can't parse it 
+            except self._sperror:
+                self._visited[cbase], self.badhtm[cbase] = 1, 1
+                return False
+            url.close()
+            # Return URL and extracted urls if it's good
+            if not badurl: return cbase, urls
+            # If the URL is bad (after BadUrl), stop processing and log URL
+            else:
+                self._visited[cbase] = 1
+                self.badurls.append((base[1], cbase))
+                return False
+        # Return URL of non-HTML resources and empty list
+        else:
+            url.close()
+            return cbase, []
+
+    def _genverify(self, urls, base):
+        '''Verifies a list of full URL relative to a base URL
+
+        Arguments:
+        urls -- list of raw URLs
+        base -- referring URL'''
+        # Assignments
+        cache, visit, urlverify = self._cache, self._visited, self._urlverify
+        # Strip file off base URL for joining
+        newbase = base.replace(base.split('/')[-1], '') 
+        for url in urls:
+            # Get resolved url and raw child URLs
+            url, rawurls = urlverify(url, base, newbase)
+            # Handle any child URLs
+            if rawurls:
+                newurls = {}
+                # Eliminate duplicate URLs
+                for rawurl in rawurls:
+                    # Eliminate known visited URLs
+                    if rawurl not in visit: newurls[rawurl] = 1
+                # Put new URLs in cache if present
+                if newurls: cache[url] = newurls
+            # Yield new URL
+            if url: yield url
+
+    def _multiverify(self, url, base):
+        '''Verifies a full URL relative to a base URL
+
+        Arguments:
+        url -- a raw URLs
+        base -- referring URL'''
+        # Assignments
+        cache, visited = self._cache, self._visited
+        # Strip file off base URL for joining
+        newbase = base.replace(base.split('/')[-1], '')
+        # Get resolved url and raw child URLs
+        url, rawurls = self._urlverify(url, base, newbase)
+        # Handle any child URLs
+        if rawurls:
+            # Eliminate known visited URLs and duplicates
+            for rawurl in rawurls:
+                # Put new URLs in cache if present
+                if rawurl not in visited: cache[rawurl] = url
+        # Put URL in list of good URLs
+        if url: self._good[url] = 1
+
+    def _urlverify(self, url, base, newbase):
+        '''Returns a full URL relative to a base URL
+
+        Arguments:
+        urls -- list of raw URLs
+        base -- referring URL
+        newbase -- temporary version of referring URL for joining'''
+        # Assignments
+        visited, webopen, other = self._visited, self._webopen, self.other
+        sb, depth, urljoin = self._sb[2], self.depth, self._uparse.urljoin
+        urlsplit, urldefrag = self._uparse.urlsplit, self._uparse.urldefrag
+        outside, redirs, supported = self.outside, self.redirs, self._supported
+        if url not in visited:
+            # Remove whitespace from URL
+            if url.find(' ') != -1:
+                visited[url], url = 1, url.replace(' ', '')
+                if url in visited: return 0, 0
+            # Remove fragments i.e. 'http:foo/bar#frag'
+            if url.find('#') != -1:
+                visited[url], url = 1, urldefrag(url)[0]
+                if url in visited: return 0, 0
+            # Process full URLs i.e. 'http://foo/bar
+            if url.find(':') != -1:
+                urlseg = urlsplit(url)
+                # Block non-FTP, HTTP URLs
+                if urlseg[0] not in supported:
+                    # Log as non-FTP/HTTP URL
+                    other[url], visited[url] = 1, 1
+                    return 0, 0
+                # If URL is not in root domain, block it
+                if urlseg[1] not in sb:
+                    visited[url], outside[url] = 1, 1                        
+                    return 0, 0
+                # Block duplicate root URLs
+                elif not urlseg[2] and urlseg[1] == sb:
+                    visited[url] = 1
+                    return 0, 0
+            # Handle relative URLs i.e. ../foo/bar
+            elif url.find(':') == -1:
+                # Join root domain and relative URL
+                visited[url], url = 1, urljoin(newbase, url)
+                if url in visited: return 0, 0
+            # Test URL by attempting to open it
+            rurl = webopen((url, base))
+            if rurl and rurl[0] not in visited:
+                # Get URL
+                turl, rawurls = rurl
+                visited[url], visited[turl] = 1, 1
+                # If URL resolved to a different URL, process it
+                if turl != url:
+                    urlseg = urlsplit(turl)
+                    # If URL is not in root domain, block it
+                    if urlseg[1] not in sb:
+                        # Log as a redirected internal URL
+                        redirs[(url, turl)] = 1
+                        return 0, 0
+                    # Block duplicate root URLs
+                    elif not urlseg[2] and urlseg[1] == sb: return 0, 0
+                # If URL exceeds depth, don't process 
+                if len(turl.split('/')) >= depth: return 0, 0
+                # Otherwise return URL
+                else:
+                    if rawurls: return turl, rawurls
+                    else: return turl, []
+            else: return 0,0
+        else: return 0, 0
+
+    def _onewalk(self):
+        '''Yields good URLs from under a base URL'''
+        # Assignments
+        cache, genverify = self._cache, self._genverify
+        # End processing if cache is empty
+        while cache:
+            # Fetch item from cache
+            base, urls = cache.popitem()
+            # If item has child URLs, process them and yield good URLs
+            if urls:
+                for url in genverify(urls, base): yield url
+
+    def _multiwalk(self, threads):
+        '''Extracts good URLs from under a base URL
+        
+        Arguments:
+        threads -- number of threads to run'''
+
+        def urlthread(url, base):
+            '''Spawns a thread containing a multiverify function
+
+            Arguments:
+
+            url -- URL to verify
+            base -- referring URL'''
+            # Create instance of Thread
+            dthread = Thread(target=multiverify, args=(url, base))
+            # Put in pool
+            pool.append(dthread)
+
+        # Assignments        
+        pool, cache, multiverify = [], self._cache, self._multiverify
+        Thread, width, good = self._thread, self.width, self._good
+        # End processing if cache is empty
+        while cache:
+            # Process URLs as long as width not exceeded
+            if len(good) <= width:
+                # Fetch item from cache
+                url, base = cache.popitem()
+                # Make thread
+                if url: urlthread(url, base)
+                # Run threads once pool size is reached
+                if len(pool) == threads or threads >= len(cache):
+                    # Start threads
+                    for thread in pool: thread.start()
+                    # Empty thread pool as threads complete
+                    while pool:
+                        for thread in pool:
+                            if not thread.isAlive(): pool.remove(thread)
+            # End if width reached
+            elif len(good) >= width: break
+
+    def weburls(self, base=None, width=200, depth=5, thread=None):
+        '''Returns a list of web paths.
+        
+        Arguments:
+        base -- base web URL (default: None)
+        width -- amount of resources to crawl (default: 200)
+        depth -- depth in hierarchy to crawl (default: 5)
+        thread -- number of threads to run (default: None)'''
+        # Assignments
+        self._visited, self._good, self._cache, self.badurls = {}, {}, {}, []
+        self.redirs, self.outside, self.badhtm, self.other = {}, {}, {}, {}
+        onewalk, good, self._robot = self._onewalk, self._good, self._rparser()
+        uparse, robot, multiwalk = self._uparse, self._robot, self._multiwalk
+        cache = self._cache
+        # Assign width
+        if self.width and width == 200: width = self.width
+        else: self.width = width
+        # sgmlop crashes Python after too many iterations
+        if width > 5000: self._parserpick(1)
+        else: self._parserpick() 
+        # Use global base if present
+        if not base: base = self.base
+        # Verify URL and get child URLs
+        newbase, rawurls = self._webopen((base, ''))
+        if newbase:
+            # Change base URL if different
+            if newbase != base: base = newbase            
+            # Ensure there's a trailing '/' in base URL
+            if base[-1] != '/':
+                url = list(uparse.urlsplit(base))
+                url[1] = ''.join([url[1], '/'])
+                base = uparse.urlunsplit(url)            
+            # Eliminate duplicates and put raw URLs in cache
+            newurls = {}
+            for rawurl in rawurls: newurls[rawurl] = 1
+            if newurls:
+                # Cache URLs individually if threads are desired
+                if thread:
+                    for newurl in newurls: cache[newurl] = base
+                # Cache in group if no threads
+                else: cache[base] = newurls
+            # Make base URL, get split, and put in verified URL list
+            self.base, self._sb = base, base.split('/')
+            self._visited[base], good[base] = 1, 1
+        # If URL is bad, abort and raise error
+        else: raise IOError, "URL is invalid"
+        # Adjust dept to length of base URL
+        if self.depth and depth == 6: self.depth += len(self._sb)
+        else: self.depth = depth + len(self._sb)
+        # Get robot limits
+        robot.set_url(''.join([base, 'robots.txt']))
+        robot.read()
+        # Get signature of bad URL
+        self._webtest()
+        # Get good URLs as long as total width isn't exceeded
+        try:
+            # Multiwalk if threaded
+            if thread: self._multiwalk(thread)
+            # Otherwise, use single thread
+            else:
+                for item in onewalk():
+                    # Don't exceed maximum width
+                    if len(good) <= width: good[item] = 1
+                    elif len(good) >= width: break
+        # If user interrupts crawl, return what's done
+        except KeyboardInterrupt: pass
+        # Get URLs, sort them, and return list
+        self.urls = good.keys()
+        self.urls.sort()
+        return self.urls                        
+
+    def webpaths(self, b=None, w=200, d=5, t=None):
+        '''Returns a list of web paths.
+        
+        Arguments:
+        b -- base web URL (default: None)
+        w -- amount of resources to crawl (default: 200)
+        d -- depth in hierarchy to crawl (default: 5)
+        t -- number of threads (default: None)'''
+
+        def pathize():            
+            '''Strips base URL from full URLs to produce paths'''            
+            for url in urls:
+                # Remove base URL from path list
+                url = url.replace(self.base, '')
+                # Add default name 'index.html' to root URLs and directories
+                if not url: url = 'index.html'
+                elif url[-1] == '/': url = ''.join([url, 'index.html'])
+                # Verify removal of base URL and remove it if found
+                if url.find(':') != -1: url = urlsplit(url)[2:][0]                
+                yield url
+
+        # Assignments
+        urlsplit = self._uparse.urlsplit
+        # Run weburls if base passed as an argument
+        if b: self.weburls(b, w, d, t)
+        # Strip off trailing resource or query from base URL
+        if self.base[-1] != '/': self.base = '/'.join(self._sb[:-1])
+        urls = self.urls
+        # Return path list after stripping base URL
+        self.paths = list(pathize())
+        return self.paths
+        
+    def webmirror(self, root=None, t=None, base=None, width=200, depth=5):
+        '''Mirrors a website on a local filesystem
+
+        Arguments:
+        root -- local filesystem path (default: None)
+        t -- number of threads (default: None)
+        base -- base web URL (default: None)
+        width -- amount of resources to crawl (default: 200)
+        depth -- depth in hierarchy to crawl (default: 5)'''
+        if base: self.webspider(base, width, depth, t)
+        return self._mirror((self.paths, self.urls), root, t)
+    
+    def webspider(self, b=None, w=200, d=5, t=None):
+        '''Returns two lists of child URLs and paths
+        
+        Arguments:
+        b -- base web URL (default: None)
+        w -- amount of resources to crawl (default: 200)
+        d -- depth in hierarchy to crawl (default: 5)
+        t -- number of threads (default: None)'''
+        if b: self.weburls(b, w, d, t)
+        return self.webpaths(), self.urls
+
+    def badurlreport(self, f=None, b=None, w=200, d=5, t=None):
+        '''Pretties up a list of bad URLs
+        
+        Arguments:
+        f -- output file for report (default: None)
+        b -- base web URL (default: None)
+        w -- amount of resources to crawl (default: 200)
+        d -- depth in hierarchy to crawl (default: 5)
+        t -- number of threads (default: None)'''
+        if b: self.weburls(b, w, d, t)
+        # Format report if information is available
+        if self.badurls:
+            # Number of bad URLs
+            amount = str(len(self.badurls))
+            header = '%s broken URLs under %s on %s:\n'
+            # Print referring URL pointing to bad URL
+            body = '\n'.join([' -> '.join([i[0], i[1]]) for i in self.badurls])
+            report = self._formatreport(amount, header, body, f)
+            # Return if just getting string
+            if report: return report
+
+    def badhtmreport(self, f=None, b=None, w=200, d=5, t=None):
+        '''Pretties up a list of unparsed HTML URLs
+        
+        Arguments:
+        f -- output file for report (default: None)
+        b -- base web URL (default: None)
+        w -- amount of resources to crawl (default: 200)
+        d -- depth in hierarchy to crawl (default: 5)
+        t -- number of threads (default: None)'''
+        if b: self.weburls(b, w, d, t)
+        # Format report if information is available
+        if self.badhtm:
+            amount = str(len(self.badhtm))
+            header = '%s unparsable HTML URLs under %s on %s:\n'
+            body = '\n'.join(self.badhtm)
+            report = self._formatreport(amount, header, body, f)
+            # Return if just getting string
+            if report: return report
+
+    def redireport(self, f=None, b=None, w=200, d=5, t=None):
+        '''Pretties up a list of URLs redirected to an external URL
+        
+        Arguments:
+        f -- output file for report (default: None)
+        b -- base web URL (default: None)
+        w -- amount of resources to crawl (default: 200)
+        d -- depth in hierarchy to crawl (default: 5)
+        t -- number of threads (default: None)'''
+        if b: self.weburls(b, w, d, t)
+        # Format report if information is available
+        if self.redirs:
+            amount = str(len(self.redirs))
+            header = '%s redirects to external URLs under %s on %s:\n'
+            # Print referring URL pointing to new URL
+            body = '\n'.join([' -> '.join([i[0], i[1]]) for i in self.redirs])
+            report = self._formatreport(amount, header, body, f)
+            # Return if just getting string
+            if report: return report
+
+    def outreport(self, f=None, b=None, w=200, d=5, t=None):
+        '''Pretties up a list of outside URLs referenced under the base URL
+        
+        Arguments:
+        f -- output file for report (default: None)
+        b -- base web URL (default: None)
+        w -- amount of resources to crawl (default: 200)
+        d -- depth in hierarchy to crawl (default: 5)
+        t -- number of threads (default: None)'''
+        if b: self.weburls(b, w, d, t)
+        # Format report if information is available
+        if self.outside:
+            amount = str(len(self.outside))
+            header = '%s links to external URLs under %s on %s:\n'
+            body = '\n'.join(self.outside)
+            report = self._formatreport(amount, header, body, f)
+            # Return if just getting string
+            if report: return report            
+
+    def othereport(self, f=None, b=None, w=200, d=5, t=None):
+        '''Pretties up a list of non-HTTP/FTP URLs
+        
+        Arguments:
+        f -- output file for report (default: None)
+        b -- base web URL (default: None)
+        w -- amount of resources to crawl (default: 200)
+        d -- depth in hierarchy to crawl (default: 5)
+        t -- number of threads (default: None)'''
+        if b: self.weburls(b, w, d, t)
+        # Format report if information is available
+        if self.other:
+            amount = str(len(self.other))
+            header = '%s non-FTP/non-HTTP URLs under %s on %s:\n'
+            body = '\n'.join(self.other)
+            report = self._formatreport(amount, header, body, f)
+            # Return if just getting string
+            if report: return report
+
+    def urlreport(self, f=None, b=None, w=200, d=5, t=None):
+        '''Pretties up a list of all URLs under a URL
+        
+        Arguments:
+        f -- output file for report (default: None)
+        b -- base web URL (default: None)
+        w -- amount of resources to crawl (default: 200)
+        d -- depth in hierarchy to crawl (default: 5)
+        t -- number of threads (default: None)'''
+        if b: self.weburls(b, w, d, t)
+        # Format report if information is available
+        if self.urls:
+            amount = str(len(self.urls))
+            header = '%s verified URLs under %s on %s:\n'
+            body = '\n'.join(self.urls)
+            report = self._formatreport(amount, header, body, f)
+            # Return if just getting string
+            if report: return report
+
+    def webreport(self, f=None, b=None, w=200, d=5, t=None, *vargs):
+        '''Pretties up a list of logged information under a URL
+        
+        Arguments:
+        f -- output file for report (default: None)
+        b -- base web URL (default: None)
+        w -- amount of resources to crawl (default: 200)
+        d -- depth in hierarchy to crawl (default: 5)
+        t -- number of threads (default: None)
+        vargs -- report sections to include or exclude
+        To override defaults:
+        To include a section add 'badhtm', 'redirs', 'outside', or 'other'
+        To exclude a section add 'badurls' or "urls"'''
+        if b: self.weburls(b, w, d, t)
+        # Defaults for report
+        badurls, badhtm, redirs, urls, outside, other = 1, 0, 0, 1, 0, 0
+        # Create compilation list
+        compile = []
+        # Override default report settings if argument is passed to vargs
+        for arg in vargs:
+            if arg == 'badurls': badurls = 0
+            elif arg == 'badhtm': badhtm = 1
+            elif arg == 'redirs': redirs = 1
+            elif arg == 'urls': urls = 0
+            elif arg == 'outside': outside = 1
+            elif arg == 'other': other = 1
+        # Compile report
+        if badurls:
+            badurls = self.badurlreport()
+            if badurls: compile.append(badurls)
+        if urls:
+            urls = self.urlreport()
+            if urls: compile.append(urls)
+        if outside:
+            outside = self.outreport()
+            if outside: compile.append(outside)
+        if redirs:
+            redirs = self.redireport()
+            if redirs: compile.append(redirs)
+        if badhtm:
+            badhtm = self.badhtmreport()
+            if badhtm: compile.append(badhtm)        
+        if other:
+            other = self.othereport()
+            if other: compile.append(other)
+        # Make report
+        report = '\n\n'.join(compile)
+        # Write to file if argument present
+        if file: open(f, 'w').write(report)
+        # Or return string
+        else: return report
+        
+    def _formatreport(self, amount, header, body, file=None):
+        '''Generic prettifier with date/time stamper
+        
+        Arguments:
+        header -- title of report
+        body -- body of report
+        file -- output file for report (default: None)'''
+        # Get current time
+        localtime, strftime = self._localtime, self._formtime        
+        curtime = strftime('%A, %B %d, %Y at %I:%M %p', localtime())
+        # Make section header
+        header = header % (amount, self.base, curtime)
+        # Add header to body
+        report = '\n'.join([header, body])
+        # Write to file if argument present
+        if file: open(file, 'w').write(report)
+        # Or return string
+        else: return report
+
+    def _mirror(self, lists, root=None, threads=None):
+        '''Mirrors a site on a local filesystem based on lists passed to it
+
+        Argument:
+        lists -- lists of URLs and paths
+        root -- local filesystem path (default: None)
+        threads -- number of threads (default: None)'''
+
+        def download(url, np, op):
+            '''Downloads files that need to be mirrored.'''
+            # If ftp...
+            if url[:3] == 'ftp':
+                # Open local file
+                local = open(np, 'wb')
+                # Download using FTP session
+                ftp = ftpopen(base, name, password)
+                ftp.retrbinary('RETR %s' % op, local.write)
+                ftp.close()
+                # Close local file
+                local.close()
+            # Use normal urlretrieve if no FTP required
+            else: ulib.urlretrieve(url, np)
+            
+        def dlthread(url, np, op):
+            '''Spawns a thread containing the download function'''
+            # Create thread
+            dthread = Thread(target=download, args=(url, np, op))
+            # Add to thread pool
+            pool.append(dthread)
+                
+        # Extract path and URL lists
+        paths, urls = lists
+        # Avoid outside namespace lookups
+        ulib, makedirs, sep = self._ulib, self._os.makedirs, self._os.sep
+        normcase, split = self._path.normcase, self._path.split
+        exists, isdir = self._path.exists, self._path.isdir
+        ftpopen = self._ftpopen
+        # Create local names for thread class and thread pool
+        if threads: Thread, pool = self._thread, []
+        # Localize name and password if exists
+        try: base, name, password = self.base, self._name, self._password
+        except AttributeError: pass
+        # Change to directory if given...
+        if root:
+            if exists(root):
+                if isdir(root): self._os.chdir(root)
+            # Create root if it doesn't exist
+            else:
+                makedirs(root)
+                self._os.chdir(root)
+        # Otherwise use current directory
+        else: root = self._os.getcwd()
+        # Iterate over paths and download files
+        for oldpath in paths:
+            # Sync with the URL for oldpath
+            url = urls[paths.index(oldpath)]
+            # Create name of local copy
+            newpath = normcase(oldpath).lstrip(sep)            
+            # Get directory name
+            dirname = split(newpath)[0]
+            # If the directory exists, download the file directly        
+            if exists(dirname):
+                if isdir(dirname):
+                    if threads: dlthread(url, newpath, oldpath)
+                    else: download(url, newpath, oldpath)
+            # Don't create local directory if path in root of remote URL
+            elif not dirname:
+                if threads: dlthread(url, newpath, oldpath)
+                else: download(url, newpath, oldpath)
+            # Make local directory if it doesn't exist, then dowload file
+            else:
+                makedirs(dirname)
+                if threads: dlthread(url, newpath, oldpath)
+                else: download(url, newpath, oldpath)
+            # Run threads if they've hit the max number of threads allowed
+            if threads:
+                # Run if max threads or final thread reached
+                if len(pool) == threads or paths[-1] == oldpath:
+                    # Start all threads
+                    for thread in pool: thread.start()
+                    # Clear the thread pool as they finish
+                    while pool:
+                        for thread in pool:
+                            if not thread.isAlive(): pool.remove(thread)
+
+
+# Instance of Spider enables exporting Spider's methods as standalone functions
+_inst = Spider()
+ftpurls = _inst.ftpurls
+weburls = _inst.weburls
+ftppaths = _inst.ftppaths
+webpaths = _inst.webpaths
+ftpmirror = _inst.ftpmirror
+ftpspider = _inst.ftpspider
+webmirror = _inst.webmirror
+webspider = _inst.webspider
+webreport = _inst.webreport
+urlreport = _inst.urlreport
+outreport = _inst.outreport
+redireport = _inst.redireport
+othereport = _inst.othereport
+badurlreport = _inst.badurlreport
+badhtmreport = _inst.badhtmreport
Index: quintagroup.distrpoxy/tags/1.0/quintagroup/distproxy/wsgi.ini
===================================================================
--- quintagroup.distrpoxy/tags/1.0/quintagroup/distproxy/wsgi.ini	(revision 1073)
+++ quintagroup.distrpoxy/tags/1.0/quintagroup/distproxy/wsgi.ini	(revision 1073)
@@ -0,0 +1,8 @@
+[server:main]
+use = egg:Paste#http
+port = 8898
+
+[app:main]
+use = egg:quintagroup.distproxy
+pack_directory = /var/pack_directory
+index = http://dist.plone.org
Index: quintagroup.distrpoxy/tags/1.0/quintagroup/distproxy/wsgi.py
===================================================================
--- quintagroup.distrpoxy/tags/1.0/quintagroup/distproxy/wsgi.py	(revision 1073)
+++ quintagroup.distrpoxy/tags/1.0/quintagroup/distproxy/wsgi.py	(revision 1073)
@@ -0,0 +1,65 @@
+import os
+import socket
+import sys
+from paste.script.appinstall import Installer as BaseInstaller
+from paste.fileapp import FileApp
+from paste.httpexceptions import HTTPNotFound
+from spider import webmirror
+
+class PackageProxyApp(object):
+
+    def __init__(self, index_url=None, pack_dir=None):
+        if not index_url: 
+            print "No repository index provided"
+            sys.exit()
+        if not pack_dir:
+            print "No packages cache directory provided"
+            sys.exit()
+        if not os.path.isdir(pack_dir):
+            print 'You must create the %r directory' % pack_dir
+            sys.exit()
+        self.index_url = index_url
+        self.pack_dir = pack_dir
+
+    def __call__(self, environ, start_response):
+        path = environ.get('PATH_INFO', '').strip()
+        filename = self.checkCache(path)        
+        if path.split('/')[0] == "favicon.ico":
+                return HTTPNotFound()(environ, start_response)
+        if filename is None:
+            return HTTPNotFound()(environ, start_response)
+        return FileApp(filename)(environ, start_response)
+
+    def checkCache(self, path):   
+        pth = self.pack_dir + path
+        pth1 = pth
+        if not (path[-3:] in ['tgz','.gz','egg','zip','exe']): 
+            pth1 = pth + 'index.html'
+        else:
+            pth = '/'.join(pth.split('/')[:-1])
+        if not os.path.exists(pth1):
+            webmirror(pth,1,self.index_url+path,0,1)            
+        if pth1:
+            return pth1
+        return pth
+
+def app_factory(global_config, **local_conf):
+    # Grab config from wsgi .ini file. If not specified, config.py's values
+    # take over.
+    pack_dir = local_conf.get('pack_directory', None)
+    index_url = local_conf.get('index', None)
+    return PackageProxyApp(index_url, pack_dir)
+"""
+class Installer(BaseInstaller):
+    use_cheetah = False
+    config_file = 'deployment.ini_tmpl'
+
+    def config_content(self, command, vars):
+        import pkg_resources
+        module = 'collective.eggproxy'
+        if pkg_resources.resource_exists(module, self.config_file):
+            return self.template_renderer(
+                pkg_resources.resource_string(module, self.config_file),
+                vars,
+                filename=self.config_file)
+"""
Index: quintagroup.distrpoxy/tags/1.0/setup.cfg
===================================================================
--- quintagroup.distrpoxy/tags/1.0/setup.cfg	(revision 1073)
+++ quintagroup.distrpoxy/tags/1.0/setup.cfg	(revision 1073)
@@ -0,0 +1,3 @@
+[egg_info]
+tag_build = dev
+tag_svn_revision = true
Index: quintagroup.distrpoxy/tags/1.0/setup.py
===================================================================
--- quintagroup.distrpoxy/tags/1.0/setup.py	(revision 1073)
+++ quintagroup.distrpoxy/tags/1.0/setup.py	(revision 1073)
@@ -0,0 +1,36 @@
+from setuptools import setup, find_packages
+import os
+
+version = '1.0'
+
+setup(name='quintagroup.distproxy',
+      version=version,
+      description="quintgroup.distproxy is a smart mirror for dist.plone.org",
+      long_description=open("README.txt").read() + "\n" +
+                       open(os.path.join("docs", "HISTORY.txt")).read(),
+      # Get more strings from http://www.python.org/pypi?%3Aaction=list_classifiers
+      classifiers=[
+        "Programming Language :: Python",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+        ],
+      keywords='',
+      author='Volodymyr Cherepanyak, Quintagroup',
+      author_email='<support at quintagroup.com>',
+      url='http://svn.quintagroup.com/products/quintagroup.distrpoxy',
+      license='GPL',
+      packages=find_packages(exclude=['ez_setup']),
+      namespace_packages=['quintagroup'],
+      include_package_data=True,
+      zip_safe=False,
+      install_requires=[
+          'setuptools',
+          'PasteScript',
+          'PasteDeploy',
+          # -*- Extra requirements: -*-
+      ],
+      entry_points={
+          'paste.app_factory': [
+              'main = quintagroup.distproxy.wsgi:app_factory',
+              ],
+          },
+      )
