#!/usr/bin/env python # DT WSGI Robots # # Douglas Thrift # # $Id$ # Copyright 2010 Douglas Thrift # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from cStringIO import StringIO import urlparse class Robots(object): def __init__(self, disallows = None, sitemaps = None): self.disallows = disallows self.sitemaps = sitemaps def __iter__(self): yield '# $Id$\n' if self.disallows: if isinstance(self.disallows, str): yield self._agent('*', [self.disallows]) elif isinstance(self.disallows, (list, tuple)) and isinstance(self.disallows[0], str): yield self._agent('*', self.disallows) else: for agent, disallows in self.disallows.iteritems() if isinstance(self.disallows, dict) else self.disallows: yield self._agent(agent, [disallows] if isinstance(disallows, str) else disallows) if self.sitemaps: robots = StringIO() robots.write('\n') for sitemap in [self.sitemaps] if isinstance(self.sitemaps, str) else self.sitemaps: robots.write('Sitemap: %s\n' % sitemap) yield robots.getvalue() def __str__(self): return ''.join(self) @staticmethod def _agent(agent, disallows): robots = StringIO() robots.write('\nUser-agent: %s\n' % agent) for disallow in disallows: robots.write('Disallow: %s\n' % disallow) return robots.getvalue() def robots(disallows = None, sitemaps = None, string = False): robots = Robots(disallows, sitemaps) return str(robots) if string else robots def wsgi(function = None, string = False): class WSGI(object): def __init__(self, function): self.function = function self.__name__ = function.__name__ self.__module__ = function.__module__ self.__doc__ = function.__doc__ def __call__(self, *args, **kwargs): value = self.function(*args, **kwargs) if isinstance(value, dict): return robots(value.get('disallows'), value.get('sitemaps'), string = string) return value return WSGI(function) if function is not None else WSGI def bottle(function): @wsgi def _bottle(*args, **kwargs): import bottle value = function(*args, **kwargs) if isinstance(value, dict): if 'sitemaps' in value: url = urlparse.urlsplit(bottle.request.url) def _url(value): value = urlparse.urlsplit(value) if value.scheme and value.netloc: return value.geturl() return urlparse.urlunsplit(url[:2] + value[2:]) sitemaps = value['sitemaps'] value['sitemaps'] = map(_url, [sitemaps] if isinstance(sitemaps, str) else sitemaps) bottle.response.content_type = 'text/plain' return value return _bottle