[IMP] clean sitemap, enumerate pages + fixes
authorFabien Pinckaers <fp@tinyerp.com>
Sun, 11 May 2014 11:52:31 +0000 (13:52 +0200)
committerFabien Pinckaers <fp@tinyerp.com>
Sun, 11 May 2014 11:52:31 +0000 (13:52 +0200)
bzr revid: fp@tinyerp.com-20140511115231-g8ke14r9iepyypv4

14 files changed:
addons/website/controllers/main.py
addons/website/data/data.xml
addons/website/models/ir_http.py
addons/website/models/website.py
addons/website/tests/test_requests.py
addons/website/views/website_templates.xml
addons/website_blog/controllers/main.py
addons/website_blog/models/website_blog.py
addons/website_crm_partner_assign/controllers/main.py
addons/website_forum/controllers/main.py
addons/website_forum_doc/controllers/main.py
addons/website_mail_group/controllers/main.py
addons/website_quote/controllers/main.py
addons/website_sale/controllers/main.py

index eade32e..7880603 100644 (file)
@@ -26,7 +26,7 @@ logger = logging.getLogger(__name__)
 
 # Completely arbitrary limits
 MAX_IMAGE_WIDTH, MAX_IMAGE_HEIGHT = IMAGE_LIMITS = (1024, 768)
-
+LOC_PER_SITEMAP = 45000
 
 class Website(openerp.addons.web.controllers.main.Home):
     #------------------------------------------------------
@@ -69,20 +69,34 @@ class Website(openerp.addons.web.controllers.main.Home):
 
         return request.render(page, values)
 
-    @http.route(['/robots.txt'], type='http', auth="public", website=True)
+    @http.route(['/robots.txt'], type='http', auth="public")
     def robots(self):
         return request.render('website.robots', {'url_root': request.httprequest.url_root}, mimetype='text/plain')
 
-    @http.route('/sitemap', type='http', auth='public', website=True, multilang=True)
-    def sitemap(self):
-        return request.render('website.sitemap', {
-            'pages': request.website.enumerate_pages()
-        })
-
     @http.route('/sitemap.xml', type='http', auth="public", website=True)
-    def sitemap_xml(self):
+    def sitemap_xml_index(self):
+        pages = list(request.website.enumerate_pages())
+        if len(pages)<=LOC_PER_SITEMAP:
+            return self.__sitemap_xml(pages, 0)
+        # Sitemaps must be split in several smaller files with a sitemap index
+        values = {
+            'pages': range(len(pages)/LOC_PER_SITEMAP+1),
+            'url_root': request.httprequest.url_root
+        }
+        headers = {
+            'Content-Type': 'application/xml;charset=utf-8',
+        }
+        return request.render('website.sitemap_index_xml', values, headers=headers)
+
+    @http.route('/sitemap-<int:page>.xml', type='http', auth="public", website=True)
+    def sitemap_xml(self, page):
+        pages = list(request.website.enumerate_pages())
+        return self.__sitemap_xml(pages, page)
+
+    def __sitemap_xml(self, pages, index=0):
         values = {
-            'pages': request.website.enumerate_pages()
+            'pages': pages[index*LOC_PER_SITEMAP:(index+1)*LOC_PER_SITEMAP],
+            'url_root': request.httprequest.url_root.rstrip('/')
         }
         headers = {
             'Content-Type': 'application/xml;charset=utf-8',
@@ -428,4 +442,3 @@ class Website(openerp.addons.web.controllers.main.Home):
             return res
         return request.redirect('/')
 
-# vim:et:
index 4eff1b8..c5254d6 100644 (file)
@@ -15,7 +15,7 @@
 
         <record id="menu_homepage" model="website.menu">
             <field name="name">Home</field>
-            <field name="url">/</field>
+            <field name="url">/page/homepage</field>
             <field name="parent_id" ref="website.main_menu"/>
             <field name="sequence" type="int">10</field>
         </record>
index 2a77493..d90b40b 100644 (file)
@@ -178,8 +178,9 @@ class ir_http(orm.AbstractModel):
         return super(ir_http, self)._handle_exception(exception)
 
 class ModelConverter(ir.ir_http.ModelConverter):
-    def __init__(self, url_map, model=False):
+    def __init__(self, url_map, model=False, domain='[]'):
         super(ModelConverter, self).__init__(url_map, model)
+        self.domain = domain
         self.regex = r'(?:[A-Za-z0-9-_]+?-)?(\d+)(?=$|/)'
 
     def to_url(self, value):
@@ -191,24 +192,28 @@ class ModelConverter(ir.ir_http.ModelConverter):
         return request.registry[self.model].browse(
             request.cr, _uid, int(m.group(1)), context=request.context)
 
-    def generate(self, cr, uid, query=None, context=None):
-        return request.registry[self.model].name_search(
-            cr, uid, name=query or '', context=context)
+    def generate(self, cr, uid, query=None, args=None, context=None):
+        for record in request.registry[self.model].name_search(
+            cr, uid, name=query or '', args=eval( self.domain, (args or {}).copy()),
+            context=context):
+            yield {'loc': record}
 
 class PageConverter(werkzeug.routing.PathConverter):
-    """ Only point of this converter is to bundle pages enumeration logic
-
-    Sads got: no way to get the view's human-readable name even if one exists
-    """
-    def generate(self, cr, uid, query=None, context=None):
+    """ Only point of this converter is to bundle pages enumeration logic """
+    def generate(self, cr, uid, query=None, args={}, context=None):
         View = request.registry['ir.ui.view']
-        views = View.search_read(
-            cr, uid, [['page', '=', True]],
-            fields=[], order='name', context=context)
-        xids = View.get_external_id(
-            cr, uid, [view['id'] for view in views], context=context)
-
+        views = View.search_read(cr, uid, [['page', '=', True]],
+            fields=['xml_id','priority','write_date'], order='name', context=context)
         for view in views:
-            xid = xids[view['id']]
-            if xid and (not query or query.lower() in xid.lower()):
-                yield xid
+            xid = view['xml_id'].startswith('website.') and view['xml_id'][8:] or view['xml_id']
+            # the 'page/homepage' url is indexed as '/', avoid aving the same page referenced twice
+            # when we will have an url mapping mechanism, replace this by a rule: page/homepage --> /
+            if xid=='homepage': continue
+            if query and query.lower() not in xid.lower():
+                continue
+            record = {'loc': xid}
+            if view['priority'] <> 16:
+                record['__priority'] = min(round(view['priority'] / 32.0,1), 1)
+            if view['write_date']:
+                record['__lastmod'] = view['write_date'][:10]
+            yield record
index fcaa184..40a296a 100644 (file)
@@ -283,44 +283,23 @@ class website(osv.osv):
         endpoint = rule.endpoint
         methods = rule.methods or ['GET']
         converters = rule._converters.values()
-
-        return (
-            'GET' in methods
+        if not ('GET' in methods
             and endpoint.routing['type'] == 'http'
             and endpoint.routing['auth'] in ('none', 'public')
             and endpoint.routing.get('website', False)
-            # preclude combinatorial explosion by only allowing a single converter
-            and len(converters) <= 1
-            # ensure all converters on the rule are able to generate values for
-            # themselves
             and all(hasattr(converter, 'generate') for converter in converters)
-        ) and self.endpoint_is_enumerable(rule)
-
-    def endpoint_is_enumerable(self, rule):
-        """ Verifies that it's possible to generate a valid url for the rule's
-        endpoint
-
-        :type rule: werkzeug.routing.Rule
-        :rtype: bool
-        """
-        spec = inspect.getargspec(rule.endpoint.method)
-
-        # if *args bail the fuck out, only dragons can live there
-        if spec.varargs:
+            and endpoint.routing.get('website')):
             return False
 
-        # remove all arguments with a default value from the list
-        defaults_count = len(spec.defaults or []) # spec.defaults can be None
-        # a[:-0] ~ a[:0] ~ [] -> replace defaults_count == 0 by None to get
-        # a[:None] ~ a
-        args = spec.args[:(-defaults_count or None)]
+        # dont't list routes without argument having no default value or converter
+        spec = inspect.getargspec(endpoint.method.original_func)
+
+        # remove self and arguments having a default value
+        defaults_count = len(spec.defaults or [])
+        args = spec.args[1:(-defaults_count or None)]
 
-        # params with defaults were removed, leftover allowed are:
-        # * self (technically should be first-parameter-of-instance-method but whatever)
-        # * any parameter mapping to a converter
-        return all(
-            (arg == 'self' or arg in rule._converters)
-            for arg in args)
+        # check that all args have a converter
+        return all( (arg in rule._converters) for arg in args)
 
     def enumerate_pages(self, cr, uid, ids, query_string=None, context=None):
         """ Available pages in the website/CMS. This is mostly used for links
@@ -344,27 +323,30 @@ class website(osv.osv):
             if not self.rule_is_enumerable(rule):
                 continue
 
-            converters = rule._converters
-            filtered = bool(converters)
-            if converters:
-                # allow single converter as decided by fp, checked by
-                # rule_is_enumerable
-                [(name, converter)] = converters.items()
-                converter_values = converter.generate(
-                    request.cr, uid, query=query_string, context=context)
-                generated = ({k: v} for k, v in itertools.izip(
-                    itertools.repeat(name), converter_values))
-            else:
-                # force single iteration for literal urls
-                generated = [{}]
-
-            for values in generated:
-                domain_part, url = rule.build(values, append_unknown=False)
-                page = {'name': url, 'url': url}
+            converters = rule._converters or {}
+            values = [{}]
+            for (name, converter) in converters.items():
+                newval = []
+                for val in values:
+                    for v in converter.generate(request.cr, uid, query=query_string, args=val, context=context):
+                        newval.append( val.copy() )
+                        v[name] = v['loc']
+                        del v['loc']
+                        newval[-1].update(v)
+                values = newval
+
+            for value in values:
+                domain_part, url = rule.build(value, append_unknown=False)
+                page = {'loc': url}
+                for key,val in value.items():
+                    if key.startswith('__'):
+                        page[key[2:]] = val
+                if url in ('/sitemap.xml',):
+                    continue
                 if url in url_list:
                     continue
                 url_list.append(url)
-                if not filtered and query_string and not self.page_matches(cr, uid, page, query_string, context=context):
+                if query_string and not self.page_matches(cr, uid, page, query_string, context=context):
                     continue
                 yield page
 
index 127aea2..2d14cbc 100644 (file)
@@ -93,7 +93,7 @@ class CrawlSuite(unittest2.TestSuite):
             # switch registry to test mode, so that requests can be made
             registry.enter_test_mode()
 
-            paths = [URL('/'), URL('/sitemap')]
+            paths = [URL('/')]
             seen = set(paths)
 
             while paths:
index 0b99d7a..890d608 100644 (file)
@@ -39,7 +39,7 @@
         ((submenu.url != '/' and request.httprequest.path.startswith(submenu.url)) or
          request.httprequest.path == submenu.url) and 'active'
         ">
-        <a t-att-href="submenu.url" t-ignore="true" t-att-target="'blank' if submenu.new_window else None">
+        <a t-att-href="(website.menu_id.child_id[0] == submenu) and '/' or submenu.url" t-ignore="true" t-att-target="'blank' if submenu.new_window else None">
             <span t-field="submenu.name"/>
         </a>
     </li>
             <div class="well mt32">
                 <p>This page does not exists, but you can create it as you are administrator of this site.</p>
                 <a class="btn btn-primary" t-attf-href="/website/add/#{ path }">Create Page</a>
-                <span class="text-muted">or</span> <a href="/sitemap">Search a Page</a>
             </div>
             <div class="text-center text-muted">Edit the content below this line to adapt the default "page not found" page.</div>
         </div>
 </template>
 
 <template id="robots">
-# robotstxt.org/
 User-agent: *
 Sitemap: <t t-esc="url_root"/>sitemap.xml
 </template>
 
-<template id="sitemap" name="Site Map">
-    <t t-call="website.layout">
-        <ul>
-            <li t-foreach="pages" t-as="page">
-                <a t-att-href="page['url']"><t t-esc="page['name']"/></a>
-            </li>
-        </ul>
-    </t>
+<template id="sitemap_xml">&lt;?xml version="1.0" encoding="UTF-8"?&gt;
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url t-foreach="pages" t-as="page">
+    <loc><t t-esc="url_root"/><t t-esc="page['loc']"/></loc><t t-if="page.get('lastmod', False)">
+    <lastmod t-esc="page['lastmod']"/></t><t t-if="page.get('priority', False)">
+    <priority t-esc="page['priority']"/></t><t t-if="page.get('changefreq', False)">
+    <changefreq t-esc="page['changefreq']"/></t>
+  </url>
+</urlset>
 </template>
 
-<template id="sitemap_xml">
-    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-        <t t-foreach="pages" t-as="page">
-            <url>
-                <loc><t t-esc="page['url']"/></loc>
-            </url>
-        </t>
-    </urlset>
+<template id="sitemap_index_xml">&lt;?xml version="1.0" encoding="UTF-8"?&gt;
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <sitemap t-foreach="pages" t-as="page">
+    <loc><t t-esc="url_root"/>sitemap-<t t-esc="page"/>.xml</loc>
+  </sitemap>
+</sitemapindex>
 </template>
 
+
 <!-- Actual pages -->
 
-<template id="homepage" name="Homepage" page="True">
+<template id="homepage" name="Homepage" page="True" priority="29">
     <t t-call="website.layout">
       <div id="wrap" class="oe_structure oe_empty"></div>
     </t>
@@ -741,7 +739,7 @@ Sitemap: <t t-esc="url_root"/>sitemap.xml
 
 <template id="company_description" name="Company Description">
     <address itemscope="itemscope" itemtype="http://schema.org/Organization">
-       <!-- TODO widget contact must add itemprop attributes -->
+        <!-- TODO widget contact must add itemprop attributes -->
         <div t-field="res_company.partner_id" t-field-options='{
                 "widget": "contact",
                 "fields": ["name", "address", "phone", "mobile", "fax", "email"]}'/>
index e2eac59..133a95d 100644 (file)
@@ -155,7 +155,7 @@ class WebsiteBlog(http.Controller):
         return response
 
     @http.route([
-        '/blog/<model("blog.blog"):blog>/post/<model("blog.post"):blog_post>',
+            '''/blog/<model("blog.blog"):blog>/post/<model("blog.post", "[('blog_id','=',blog[0])]"):blog_post>''',
     ], type='http', auth="public", website=True, multilang=True)
     def blog_post(self, blog, blog_post, tag_id=None, page=1, enable_editor=None, **post):
         """ Prepare all values to display the blog.
@@ -294,7 +294,7 @@ class WebsiteBlog(http.Controller):
         return values
 
     @http.route(['/blogpost/post_discussion'], type='json', auth="public", website=True)
-    def post_discussion(self, blog_post_id=0, **post):
+    def post_discussion(self, blog_post_id, **post):
         cr, uid, context = request.cr, request.uid, request.context
         publish = request.registry['res.users'].has_group(cr, uid, 'base.group_website_publisher')
         user = request.registry['res.users'].browse(cr, uid, uid, context=context)
index ab96cc5..37678e9 100644 (file)
@@ -16,7 +16,6 @@ class Blog(osv.Model):
     _description = 'Blogs'
     _inherit = ['mail.thread', 'website.seo.metadata']
     _order = 'name'
-
     _columns = {
         'name': fields.char('Blog Name', required=True),
         'subtitle': fields.char('Blog Subtitle'),
@@ -29,7 +28,6 @@ class BlogTag(osv.Model):
     _description = 'Blog Tag'
     _inherit = ['website.seo.metadata']
     _order = 'name'
-
     _columns = {
         'name': fields.char('Name', required=True),
     }
index 6d6dda5..4f56392 100644 (file)
@@ -110,6 +110,7 @@ class WebsiteCrmPartnerAssign(http.Controller):
         }
         return request.website.render("website_crm_partner_assign.index", values)
 
+    # Do not use semantic controller due to SUPERUSER_ID
     @http.route(['/partners/<int:partner_id>', '/partners/<partner_name>-<int:partner_id>'], type='http', auth="public", website=True, multilang=True)
     def partners_ref(self, partner_id, **post):
         partner = request.registry['res.partner'].browse(request.cr, SUPERUSER_ID, partner_id, context=request.context)
index f02a6f9..1464019 100644 (file)
@@ -87,7 +87,7 @@ class WebsiteForum(http.Controller):
 
     @http.route(['/forum/<model("forum.forum"):forum>',
                  '/forum/<model("forum.forum"):forum>/page/<int:page>',
-                 '/forum/<model("forum.forum"):forum>/tag/<model("forum.tag"):tag>/questions'
+                 '''/forum/<model("forum.forum"):forum>/tag/<model("forum.tag", "[('forum_id','=',forum[0])]"):tag>/questions'''
                  ], type='http', auth="public", website=True, multilang=True)
     def questions(self, forum, tag=None, page=1, filters='all', sorting='date', search='', **post):
         cr, uid, context = request.cr, request.uid, request.context
@@ -190,7 +190,7 @@ class WebsiteForum(http.Controller):
             }, context=context)
         return werkzeug.utils.redirect("/forum/%s/question/%s" % (slug(forum), new_question_id))
 
-    @http.route(['/forum/<model("forum.forum"):forum>/question/<model("forum.post"):question>'], type='http', auth="public", website=True, multilang=True)
+    @http.route(['''/forum/<model("forum.forum"):forum>/question/<model("forum.post", "[('forum_id','=',forum[0])]"):question>'''], type='http', auth="public", website=True, multilang=True)
     def question(self, forum, question, **post):
         cr, uid, context = request.cr, request.uid, request.context
         # increment view counter
@@ -576,7 +576,7 @@ class WebsiteForum(http.Controller):
         })
         return request.website.render("website_forum.badge", values)
 
-    @http.route(['/forum/<model("forum.forum"):forum>/badge/<model("gamification.badge"):badge>'], type='http', auth="public", website=True, multilang=True)
+    @http.route(['''/forum/<model("forum.forum"):forum>/badge/<model("gamification.badge", "[('challenge_ids.category', '=', 'forum')]"):badge>'''], type='http', auth="public", website=True, multilang=True)
     def badge_users(self, forum, badge, **kwargs):
         user_ids = [badge_user.user_id.id for badge_user in badge.owner_ids]
         users = request.registry['res.users'].browse(request.cr, SUPERUSER_ID, user_ids, context=request.context)
index 4075479..25fdf9a 100644 (file)
@@ -26,7 +26,7 @@ class WebsiteDoc(http.Controller):
         }
         return request.website.render("website_forum_doc.documentation", value)
 
-    @http.route(['/forum/how-to/<model("forum.documentation.toc"):toc>/<model("forum.post"):post>'], type='http', auth="public", website=True, multilang=True)
+    @http.route(['''/forum/how-to/<model("forum.documentation.toc"):toc>/<model("forum.post", "[('documentation_toc_id','=',toc)]"):post>'''], type='http', auth="public", website=True, multilang=True)
     def post(self, toc, post, **kwargs):
         # TODO: implement a redirect instead of crash
         assert post.documentation_toc_id.id == toc.id, "Wrong post!"
@@ -42,7 +42,7 @@ class WebsiteDoc(http.Controller):
     def post_toc(self, forum, post, **kwargs):
         cr, uid, context, toc_id = request.cr, request.uid, request.context, False
         user = request.registry['res.users'].browse(cr, uid, uid, context=context)
-        assert user.karma >= 200, 'Not enough karma'
+        assert user.karma >= 200, 'You need 200 karma to promote a post to the documentation'
         toc_obj = request.registry['forum.documentation.toc']
         obj_ids = toc_obj.search(cr, uid, [], context=context)
         tocs = toc_obj.browse(cr, uid, obj_ids, context=context)
@@ -57,7 +57,7 @@ class WebsiteDoc(http.Controller):
     def post_toc_ok(self, forum, post_id, toc_id, **kwargs):
         cr, uid, context = request.cr, request.uid, request.context
         user = request.registry['res.users'].browse(cr, uid, uid, context=context)
-        assert user.karma >= 200, 'Not enough karma'
+        assert user.karma >= 200, 'Not enough karma, you need 200 to promote a documentation.'
 
         toc_obj = request.registry['forum.documentation.toc']
         stage_ids = toc_obj.search(cr, uid, [], limit=1, context=context)
index 628aa87..88eb1a7 100644 (file)
@@ -77,7 +77,7 @@ class MailGroup(http.Controller):
         return request.website.render('website_mail_group.group_messages', values)
 
     @http.route([
-        "/groups/<model('mail.group'):group>/<model('mail.message'):message>",
+        '''/groups/<model('mail.group'):group>/<model('mail.message', "[('model','=','mail.group'), ('res_id','=',group[0])]"):message>''',
     ], type='http', auth="public", website=True)
     def thread_discussion(self, group, message, mode='thread', date_begin=None, date_end=None, **post):
         cr, uid, context = request.cr, request.uid, request.context
index 03ba273..0cc56f2 100644 (file)
@@ -59,7 +59,7 @@ class sale_quote(http.Controller):
         return request.website.render('website_quote.so_quotation', values)
 
     @http.route(['/quote/accept'], type='json', auth="public", website=True)
-    def accept(self, order_id=None, token=None, signer=None, sign=None, **post):
+    def accept(self, order_id, token=None, signer=None, sign=None, **post):
         order_obj = request.registry.get('sale.order')
         order = order_obj.browse(request.cr, SUPERUSER_ID, order_id)
         if token != order.access_token:
@@ -111,7 +111,7 @@ class sale_quote(http.Controller):
         return True
 
     @http.route(['/quote/update_line'], type='json', auth="public", website=True)
-    def update(self, line_id=None, remove=False, unlink=False, order_id=None, token=None, **post):
+    def update(self, line_id, remove=False, unlink=False, order_id=None, token=None, **post):
         order = request.registry.get('sale.order').browse(request.cr, SUPERUSER_ID, int(order_id))
         if token != order.access_token:
             return request.website.render('website.404')
index dd09479..1b96f8e 100644 (file)
@@ -427,6 +427,8 @@ class website_sale(http.Controller):
         cr, uid, context, registry = request.cr, request.uid, request.context, request.registry
 
         order = request.website.sale_get_order(context=context)
+        if not order:
+            return request.redirect("/shop")
 
         redirection = self.checkout_redirection(order)
         if redirection: