Changeset 2895 in products

Timestamp:

Oct 19, 2010 5:52:01 PM (14 years ago)

Author:

mylan

Message:

#233: Updated keywords calculation - replace lynx usage with portal_transforms, added optional external page retrieval

File:

• quintagroup.seoptimizer/trunk/quintagroup/seoptimizer/browser/keywords.py (modified) (4 diffs)

quintagroup.seoptimizer/trunk/quintagroup/seoptimizer/browser/keywords.py

@@ -1 @@
-import re, commands
+import re, commands, urllib2
 from xml.dom import Node
 
 from zope.interface import implements
+from zope.component import queryAdapter
+from zope.component import queryMultiAdapter
 from Products.Five.browser import BrowserView
 
@@ -10 +12 @@
 from interfaces import IValidateSEOKeywordsView
 from quintagroup.seoptimizer import SeoptimizerMessageFactory as _
-
-#from pyquery import PyQuery as pq
+from quintagroup.seoptimizer.browser.seo_configlet import ISEOConfigletSchema
 
 class ValidateSEOKeywordsView(BrowserView):
@@ -21 +22 @@
         text = self.request.get('text')
         ts = getToolByName(self.context, 'translation_service')
+        transforms = getToolByName(self.context, 'portal_transforms')
+        portal = getToolByName(self.context, 'portal_url').getPortalObject()
+        isExternal = queryAdapter(portal, ISEOConfigletSchema).external_keywords_test
         # extract keywords from text
         enc = getSiteEncoding(self.context)
@@ -31 +35 @@
                                  context=self.context)
         # request html page of context object
-        url = '%s?without_metatag_keywords=1' % self.context.absolute_url()
-
-        # extract words from url page using lynx browser (test page by 'url'
-        # randered without metatag keywords)
-        #import pdb;pdb.set_trace()
-        page_text = commands.getoutput('lynx --dump --nolist %s' % url).lower()
-        if page_text and page_text != 'sh: lynx: command not found':
-            page_text = safe_unicode(page_text, 'utf-8')
+        if isExternal:
+            # Not pass timeout option because:
+            # 1. its value get from the global default timeout settings by default.
+            # 2. timeout option added in python 2.6 (so acceptable only in plone4+)
+            try:
+                html = urllib2.urlopen(self.context.absolute_url())
+            except urllib2.URLError:
+                # In case of exceed timeout period
+                # or other URL connection errors.
+                html = unicode(self.context()).encode(enc)
         else:
-            return ts.utranslate(domain='quintagroup.seoptimizer',
-                                 msgid=_(u'Could not find lynx browser!'),
-                                 context=self.context)
-
-        # html = self.context()
-        # page_text = pq("body", html).text()
-
+            html = unicode(self.context()).encode(enc)
+        page_text = transforms.convert("html_to_text", html).getData()
+                                 
         # check every keyword on appearing in body of html page
         result = []