1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/OpenSecurity/install/web.py-0.37/build/lib/web/utils.py Mon Dec 02 14:02:05 2013 +0100
1.3 @@ -0,0 +1,1526 @@
1.4 +#!/usr/bin/env python
1.5 +"""
1.6 +General Utilities
1.7 +(part of web.py)
1.8 +"""
1.9 +
1.10 +__all__ = [
1.11 + "Storage", "storage", "storify",
1.12 + "Counter", "counter",
1.13 + "iters",
1.14 + "rstrips", "lstrips", "strips",
1.15 + "safeunicode", "safestr", "utf8",
1.16 + "TimeoutError", "timelimit",
1.17 + "Memoize", "memoize",
1.18 + "re_compile", "re_subm",
1.19 + "group", "uniq", "iterview",
1.20 + "IterBetter", "iterbetter",
1.21 + "safeiter", "safewrite",
1.22 + "dictreverse", "dictfind", "dictfindall", "dictincr", "dictadd",
1.23 + "requeue", "restack",
1.24 + "listget", "intget", "datestr",
1.25 + "numify", "denumify", "commify", "dateify",
1.26 + "nthstr", "cond",
1.27 + "CaptureStdout", "capturestdout", "Profile", "profile",
1.28 + "tryall",
1.29 + "ThreadedDict", "threadeddict",
1.30 + "autoassign",
1.31 + "to36",
1.32 + "safemarkdown",
1.33 + "sendmail"
1.34 +]
1.35 +
1.36 +import re, sys, time, threading, itertools, traceback, os
1.37 +
1.38 +try:
1.39 + import subprocess
1.40 +except ImportError:
1.41 + subprocess = None
1.42 +
1.43 +try: import datetime
1.44 +except ImportError: pass
1.45 +
1.46 +try: set
1.47 +except NameError:
1.48 + from sets import Set as set
1.49 +
1.50 +try:
1.51 + from threading import local as threadlocal
1.52 +except ImportError:
1.53 + from python23 import threadlocal
1.54 +
1.55 +class Storage(dict):
1.56 + """
1.57 + A Storage object is like a dictionary except `obj.foo` can be used
1.58 + in addition to `obj['foo']`.
1.59 +
1.60 + >>> o = storage(a=1)
1.61 + >>> o.a
1.62 + 1
1.63 + >>> o['a']
1.64 + 1
1.65 + >>> o.a = 2
1.66 + >>> o['a']
1.67 + 2
1.68 + >>> del o.a
1.69 + >>> o.a
1.70 + Traceback (most recent call last):
1.71 + ...
1.72 + AttributeError: 'a'
1.73 +
1.74 + """
1.75 + def __getattr__(self, key):
1.76 + try:
1.77 + return self[key]
1.78 + except KeyError, k:
1.79 + raise AttributeError, k
1.80 +
1.81 + def __setattr__(self, key, value):
1.82 + self[key] = value
1.83 +
1.84 + def __delattr__(self, key):
1.85 + try:
1.86 + del self[key]
1.87 + except KeyError, k:
1.88 + raise AttributeError, k
1.89 +
1.90 + def __repr__(self):
1.91 + return '<Storage ' + dict.__repr__(self) + '>'
1.92 +
1.93 +storage = Storage
1.94 +
1.95 +def storify(mapping, *requireds, **defaults):
1.96 + """
1.97 + Creates a `storage` object from dictionary `mapping`, raising `KeyError` if
1.98 + d doesn't have all of the keys in `requireds` and using the default
1.99 + values for keys found in `defaults`.
1.100 +
1.101 + For example, `storify({'a':1, 'c':3}, b=2, c=0)` will return the equivalent of
1.102 + `storage({'a':1, 'b':2, 'c':3})`.
1.103 +
1.104 + If a `storify` value is a list (e.g. multiple values in a form submission),
1.105 + `storify` returns the last element of the list, unless the key appears in
1.106 + `defaults` as a list. Thus:
1.107 +
1.108 + >>> storify({'a':[1, 2]}).a
1.109 + 2
1.110 + >>> storify({'a':[1, 2]}, a=[]).a
1.111 + [1, 2]
1.112 + >>> storify({'a':1}, a=[]).a
1.113 + [1]
1.114 + >>> storify({}, a=[]).a
1.115 + []
1.116 +
1.117 + Similarly, if the value has a `value` attribute, `storify will return _its_
1.118 + value, unless the key appears in `defaults` as a dictionary.
1.119 +
1.120 + >>> storify({'a':storage(value=1)}).a
1.121 + 1
1.122 + >>> storify({'a':storage(value=1)}, a={}).a
1.123 + <Storage {'value': 1}>
1.124 + >>> storify({}, a={}).a
1.125 + {}
1.126 +
1.127 + Optionally, keyword parameter `_unicode` can be passed to convert all values to unicode.
1.128 +
1.129 + >>> storify({'x': 'a'}, _unicode=True)
1.130 + <Storage {'x': u'a'}>
1.131 + >>> storify({'x': storage(value='a')}, x={}, _unicode=True)
1.132 + <Storage {'x': <Storage {'value': 'a'}>}>
1.133 + >>> storify({'x': storage(value='a')}, _unicode=True)
1.134 + <Storage {'x': u'a'}>
1.135 + """
1.136 + _unicode = defaults.pop('_unicode', False)
1.137 +
1.138 + # if _unicode is callable object, use it convert a string to unicode.
1.139 + to_unicode = safeunicode
1.140 + if _unicode is not False and hasattr(_unicode, "__call__"):
1.141 + to_unicode = _unicode
1.142 +
1.143 + def unicodify(s):
1.144 + if _unicode and isinstance(s, str): return to_unicode(s)
1.145 + else: return s
1.146 +
1.147 + def getvalue(x):
1.148 + if hasattr(x, 'file') and hasattr(x, 'value'):
1.149 + return x.value
1.150 + elif hasattr(x, 'value'):
1.151 + return unicodify(x.value)
1.152 + else:
1.153 + return unicodify(x)
1.154 +
1.155 + stor = Storage()
1.156 + for key in requireds + tuple(mapping.keys()):
1.157 + value = mapping[key]
1.158 + if isinstance(value, list):
1.159 + if isinstance(defaults.get(key), list):
1.160 + value = [getvalue(x) for x in value]
1.161 + else:
1.162 + value = value[-1]
1.163 + if not isinstance(defaults.get(key), dict):
1.164 + value = getvalue(value)
1.165 + if isinstance(defaults.get(key), list) and not isinstance(value, list):
1.166 + value = [value]
1.167 + setattr(stor, key, value)
1.168 +
1.169 + for (key, value) in defaults.iteritems():
1.170 + result = value
1.171 + if hasattr(stor, key):
1.172 + result = stor[key]
1.173 + if value == () and not isinstance(result, tuple):
1.174 + result = (result,)
1.175 + setattr(stor, key, result)
1.176 +
1.177 + return stor
1.178 +
1.179 +class Counter(storage):
1.180 + """Keeps count of how many times something is added.
1.181 +
1.182 + >>> c = counter()
1.183 + >>> c.add('x')
1.184 + >>> c.add('x')
1.185 + >>> c.add('x')
1.186 + >>> c.add('x')
1.187 + >>> c.add('x')
1.188 + >>> c.add('y')
1.189 + >>> c
1.190 + <Counter {'y': 1, 'x': 5}>
1.191 + >>> c.most()
1.192 + ['x']
1.193 + """
1.194 + def add(self, n):
1.195 + self.setdefault(n, 0)
1.196 + self[n] += 1
1.197 +
1.198 + def most(self):
1.199 + """Returns the keys with maximum count."""
1.200 + m = max(self.itervalues())
1.201 + return [k for k, v in self.iteritems() if v == m]
1.202 +
1.203 + def least(self):
1.204 + """Returns the keys with mininum count."""
1.205 + m = min(self.itervalues())
1.206 + return [k for k, v in self.iteritems() if v == m]
1.207 +
1.208 + def percent(self, key):
1.209 + """Returns what percentage a certain key is of all entries.
1.210 +
1.211 + >>> c = counter()
1.212 + >>> c.add('x')
1.213 + >>> c.add('x')
1.214 + >>> c.add('x')
1.215 + >>> c.add('y')
1.216 + >>> c.percent('x')
1.217 + 0.75
1.218 + >>> c.percent('y')
1.219 + 0.25
1.220 + """
1.221 + return float(self[key])/sum(self.values())
1.222 +
1.223 + def sorted_keys(self):
1.224 + """Returns keys sorted by value.
1.225 +
1.226 + >>> c = counter()
1.227 + >>> c.add('x')
1.228 + >>> c.add('x')
1.229 + >>> c.add('y')
1.230 + >>> c.sorted_keys()
1.231 + ['x', 'y']
1.232 + """
1.233 + return sorted(self.keys(), key=lambda k: self[k], reverse=True)
1.234 +
1.235 + def sorted_values(self):
1.236 + """Returns values sorted by value.
1.237 +
1.238 + >>> c = counter()
1.239 + >>> c.add('x')
1.240 + >>> c.add('x')
1.241 + >>> c.add('y')
1.242 + >>> c.sorted_values()
1.243 + [2, 1]
1.244 + """
1.245 + return [self[k] for k in self.sorted_keys()]
1.246 +
1.247 + def sorted_items(self):
1.248 + """Returns items sorted by value.
1.249 +
1.250 + >>> c = counter()
1.251 + >>> c.add('x')
1.252 + >>> c.add('x')
1.253 + >>> c.add('y')
1.254 + >>> c.sorted_items()
1.255 + [('x', 2), ('y', 1)]
1.256 + """
1.257 + return [(k, self[k]) for k in self.sorted_keys()]
1.258 +
1.259 + def __repr__(self):
1.260 + return '<Counter ' + dict.__repr__(self) + '>'
1.261 +
1.262 +counter = Counter
1.263 +
1.264 +iters = [list, tuple]
1.265 +import __builtin__
1.266 +if hasattr(__builtin__, 'set'):
1.267 + iters.append(set)
1.268 +if hasattr(__builtin__, 'frozenset'):
1.269 + iters.append(set)
1.270 +if sys.version_info < (2,6): # sets module deprecated in 2.6
1.271 + try:
1.272 + from sets import Set
1.273 + iters.append(Set)
1.274 + except ImportError:
1.275 + pass
1.276 +
1.277 +class _hack(tuple): pass
1.278 +iters = _hack(iters)
1.279 +iters.__doc__ = """
1.280 +A list of iterable items (like lists, but not strings). Includes whichever
1.281 +of lists, tuples, sets, and Sets are available in this version of Python.
1.282 +"""
1.283 +
1.284 +def _strips(direction, text, remove):
1.285 + if isinstance(remove, iters):
1.286 + for subr in remove:
1.287 + text = _strips(direction, text, subr)
1.288 + return text
1.289 +
1.290 + if direction == 'l':
1.291 + if text.startswith(remove):
1.292 + return text[len(remove):]
1.293 + elif direction == 'r':
1.294 + if text.endswith(remove):
1.295 + return text[:-len(remove)]
1.296 + else:
1.297 + raise ValueError, "Direction needs to be r or l."
1.298 + return text
1.299 +
1.300 +def rstrips(text, remove):
1.301 + """
1.302 + removes the string `remove` from the right of `text`
1.303 +
1.304 + >>> rstrips("foobar", "bar")
1.305 + 'foo'
1.306 +
1.307 + """
1.308 + return _strips('r', text, remove)
1.309 +
1.310 +def lstrips(text, remove):
1.311 + """
1.312 + removes the string `remove` from the left of `text`
1.313 +
1.314 + >>> lstrips("foobar", "foo")
1.315 + 'bar'
1.316 + >>> lstrips('http://foo.org/', ['http://', 'https://'])
1.317 + 'foo.org/'
1.318 + >>> lstrips('FOOBARBAZ', ['FOO', 'BAR'])
1.319 + 'BAZ'
1.320 + >>> lstrips('FOOBARBAZ', ['BAR', 'FOO'])
1.321 + 'BARBAZ'
1.322 +
1.323 + """
1.324 + return _strips('l', text, remove)
1.325 +
1.326 +def strips(text, remove):
1.327 + """
1.328 + removes the string `remove` from the both sides of `text`
1.329 +
1.330 + >>> strips("foobarfoo", "foo")
1.331 + 'bar'
1.332 +
1.333 + """
1.334 + return rstrips(lstrips(text, remove), remove)
1.335 +
1.336 +def safeunicode(obj, encoding='utf-8'):
1.337 + r"""
1.338 + Converts any given object to unicode string.
1.339 +
1.340 + >>> safeunicode('hello')
1.341 + u'hello'
1.342 + >>> safeunicode(2)
1.343 + u'2'
1.344 + >>> safeunicode('\xe1\x88\xb4')
1.345 + u'\u1234'
1.346 + """
1.347 + t = type(obj)
1.348 + if t is unicode:
1.349 + return obj
1.350 + elif t is str:
1.351 + return obj.decode(encoding)
1.352 + elif t in [int, float, bool]:
1.353 + return unicode(obj)
1.354 + elif hasattr(obj, '__unicode__') or isinstance(obj, unicode):
1.355 + return unicode(obj)
1.356 + else:
1.357 + return str(obj).decode(encoding)
1.358 +
1.359 +def safestr(obj, encoding='utf-8'):
1.360 + r"""
1.361 + Converts any given object to utf-8 encoded string.
1.362 +
1.363 + >>> safestr('hello')
1.364 + 'hello'
1.365 + >>> safestr(u'\u1234')
1.366 + '\xe1\x88\xb4'
1.367 + >>> safestr(2)
1.368 + '2'
1.369 + """
1.370 + if isinstance(obj, unicode):
1.371 + return obj.encode(encoding)
1.372 + elif isinstance(obj, str):
1.373 + return obj
1.374 + elif hasattr(obj, 'next'): # iterator
1.375 + return itertools.imap(safestr, obj)
1.376 + else:
1.377 + return str(obj)
1.378 +
1.379 +# for backward-compatibility
1.380 +utf8 = safestr
1.381 +
1.382 +class TimeoutError(Exception): pass
1.383 +def timelimit(timeout):
1.384 + """
1.385 + A decorator to limit a function to `timeout` seconds, raising `TimeoutError`
1.386 + if it takes longer.
1.387 +
1.388 + >>> import time
1.389 + >>> def meaningoflife():
1.390 + ... time.sleep(.2)
1.391 + ... return 42
1.392 + >>>
1.393 + >>> timelimit(.1)(meaningoflife)()
1.394 + Traceback (most recent call last):
1.395 + ...
1.396 + TimeoutError: took too long
1.397 + >>> timelimit(1)(meaningoflife)()
1.398 + 42
1.399 +
1.400 + _Caveat:_ The function isn't stopped after `timeout` seconds but continues
1.401 + executing in a separate thread. (There seems to be no way to kill a thread.)
1.402 +
1.403 + inspired by <http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/473878>
1.404 + """
1.405 + def _1(function):
1.406 + def _2(*args, **kw):
1.407 + class Dispatch(threading.Thread):
1.408 + def __init__(self):
1.409 + threading.Thread.__init__(self)
1.410 + self.result = None
1.411 + self.error = None
1.412 +
1.413 + self.setDaemon(True)
1.414 + self.start()
1.415 +
1.416 + def run(self):
1.417 + try:
1.418 + self.result = function(*args, **kw)
1.419 + except:
1.420 + self.error = sys.exc_info()
1.421 +
1.422 + c = Dispatch()
1.423 + c.join(timeout)
1.424 + if c.isAlive():
1.425 + raise TimeoutError, 'took too long'
1.426 + if c.error:
1.427 + raise c.error[0], c.error[1]
1.428 + return c.result
1.429 + return _2
1.430 + return _1
1.431 +
1.432 +class Memoize:
1.433 + """
1.434 + 'Memoizes' a function, caching its return values for each input.
1.435 + If `expires` is specified, values are recalculated after `expires` seconds.
1.436 + If `background` is specified, values are recalculated in a separate thread.
1.437 +
1.438 + >>> calls = 0
1.439 + >>> def howmanytimeshaveibeencalled():
1.440 + ... global calls
1.441 + ... calls += 1
1.442 + ... return calls
1.443 + >>> fastcalls = memoize(howmanytimeshaveibeencalled)
1.444 + >>> howmanytimeshaveibeencalled()
1.445 + 1
1.446 + >>> howmanytimeshaveibeencalled()
1.447 + 2
1.448 + >>> fastcalls()
1.449 + 3
1.450 + >>> fastcalls()
1.451 + 3
1.452 + >>> import time
1.453 + >>> fastcalls = memoize(howmanytimeshaveibeencalled, .1, background=False)
1.454 + >>> fastcalls()
1.455 + 4
1.456 + >>> fastcalls()
1.457 + 4
1.458 + >>> time.sleep(.2)
1.459 + >>> fastcalls()
1.460 + 5
1.461 + >>> def slowfunc():
1.462 + ... time.sleep(.1)
1.463 + ... return howmanytimeshaveibeencalled()
1.464 + >>> fastcalls = memoize(slowfunc, .2, background=True)
1.465 + >>> fastcalls()
1.466 + 6
1.467 + >>> timelimit(.05)(fastcalls)()
1.468 + 6
1.469 + >>> time.sleep(.2)
1.470 + >>> timelimit(.05)(fastcalls)()
1.471 + 6
1.472 + >>> timelimit(.05)(fastcalls)()
1.473 + 6
1.474 + >>> time.sleep(.2)
1.475 + >>> timelimit(.05)(fastcalls)()
1.476 + 7
1.477 + >>> fastcalls = memoize(slowfunc, None, background=True)
1.478 + >>> threading.Thread(target=fastcalls).start()
1.479 + >>> time.sleep(.01)
1.480 + >>> fastcalls()
1.481 + 9
1.482 + """
1.483 + def __init__(self, func, expires=None, background=True):
1.484 + self.func = func
1.485 + self.cache = {}
1.486 + self.expires = expires
1.487 + self.background = background
1.488 + self.running = {}
1.489 +
1.490 + def __call__(self, *args, **keywords):
1.491 + key = (args, tuple(keywords.items()))
1.492 + if not self.running.get(key):
1.493 + self.running[key] = threading.Lock()
1.494 + def update(block=False):
1.495 + if self.running[key].acquire(block):
1.496 + try:
1.497 + self.cache[key] = (self.func(*args, **keywords), time.time())
1.498 + finally:
1.499 + self.running[key].release()
1.500 +
1.501 + if key not in self.cache:
1.502 + update(block=True)
1.503 + elif self.expires and (time.time() - self.cache[key][1]) > self.expires:
1.504 + if self.background:
1.505 + threading.Thread(target=update).start()
1.506 + else:
1.507 + update()
1.508 + return self.cache[key][0]
1.509 +
1.510 +memoize = Memoize
1.511 +
1.512 +re_compile = memoize(re.compile) #@@ threadsafe?
1.513 +re_compile.__doc__ = """
1.514 +A memoized version of re.compile.
1.515 +"""
1.516 +
1.517 +class _re_subm_proxy:
1.518 + def __init__(self):
1.519 + self.match = None
1.520 + def __call__(self, match):
1.521 + self.match = match
1.522 + return ''
1.523 +
1.524 +def re_subm(pat, repl, string):
1.525 + """
1.526 + Like re.sub, but returns the replacement _and_ the match object.
1.527 +
1.528 + >>> t, m = re_subm('g(oo+)fball', r'f\\1lish', 'goooooofball')
1.529 + >>> t
1.530 + 'foooooolish'
1.531 + >>> m.groups()
1.532 + ('oooooo',)
1.533 + """
1.534 + compiled_pat = re_compile(pat)
1.535 + proxy = _re_subm_proxy()
1.536 + compiled_pat.sub(proxy.__call__, string)
1.537 + return compiled_pat.sub(repl, string), proxy.match
1.538 +
1.539 +def group(seq, size):
1.540 + """
1.541 + Returns an iterator over a series of lists of length size from iterable.
1.542 +
1.543 + >>> list(group([1,2,3,4], 2))
1.544 + [[1, 2], [3, 4]]
1.545 + >>> list(group([1,2,3,4,5], 2))
1.546 + [[1, 2], [3, 4], [5]]
1.547 + """
1.548 + def take(seq, n):
1.549 + for i in xrange(n):
1.550 + yield seq.next()
1.551 +
1.552 + if not hasattr(seq, 'next'):
1.553 + seq = iter(seq)
1.554 + while True:
1.555 + x = list(take(seq, size))
1.556 + if x:
1.557 + yield x
1.558 + else:
1.559 + break
1.560 +
1.561 +def uniq(seq, key=None):
1.562 + """
1.563 + Removes duplicate elements from a list while preserving the order of the rest.
1.564 +
1.565 + >>> uniq([9,0,2,1,0])
1.566 + [9, 0, 2, 1]
1.567 +
1.568 + The value of the optional `key` parameter should be a function that
1.569 + takes a single argument and returns a key to test the uniqueness.
1.570 +
1.571 + >>> uniq(["Foo", "foo", "bar"], key=lambda s: s.lower())
1.572 + ['Foo', 'bar']
1.573 + """
1.574 + key = key or (lambda x: x)
1.575 + seen = set()
1.576 + result = []
1.577 + for v in seq:
1.578 + k = key(v)
1.579 + if k in seen:
1.580 + continue
1.581 + seen.add(k)
1.582 + result.append(v)
1.583 + return result
1.584 +
1.585 +def iterview(x):
1.586 + """
1.587 + Takes an iterable `x` and returns an iterator over it
1.588 + which prints its progress to stderr as it iterates through.
1.589 + """
1.590 + WIDTH = 70
1.591 +
1.592 + def plainformat(n, lenx):
1.593 + return '%5.1f%% (%*d/%d)' % ((float(n)/lenx)*100, len(str(lenx)), n, lenx)
1.594 +
1.595 + def bars(size, n, lenx):
1.596 + val = int((float(n)*size)/lenx + 0.5)
1.597 + if size - val:
1.598 + spacing = ">" + (" "*(size-val))[1:]
1.599 + else:
1.600 + spacing = ""
1.601 + return "[%s%s]" % ("="*val, spacing)
1.602 +
1.603 + def eta(elapsed, n, lenx):
1.604 + if n == 0:
1.605 + return '--:--:--'
1.606 + if n == lenx:
1.607 + secs = int(elapsed)
1.608 + else:
1.609 + secs = int((elapsed/n) * (lenx-n))
1.610 + mins, secs = divmod(secs, 60)
1.611 + hrs, mins = divmod(mins, 60)
1.612 +
1.613 + return '%02d:%02d:%02d' % (hrs, mins, secs)
1.614 +
1.615 + def format(starttime, n, lenx):
1.616 + out = plainformat(n, lenx) + ' '
1.617 + if n == lenx:
1.618 + end = ' '
1.619 + else:
1.620 + end = ' ETA '
1.621 + end += eta(time.time() - starttime, n, lenx)
1.622 + out += bars(WIDTH - len(out) - len(end), n, lenx)
1.623 + out += end
1.624 + return out
1.625 +
1.626 + starttime = time.time()
1.627 + lenx = len(x)
1.628 + for n, y in enumerate(x):
1.629 + sys.stderr.write('\r' + format(starttime, n, lenx))
1.630 + yield y
1.631 + sys.stderr.write('\r' + format(starttime, n+1, lenx) + '\n')
1.632 +
1.633 +class IterBetter:
1.634 + """
1.635 + Returns an object that can be used as an iterator
1.636 + but can also be used via __getitem__ (although it
1.637 + cannot go backwards -- that is, you cannot request
1.638 + `iterbetter[0]` after requesting `iterbetter[1]`).
1.639 +
1.640 + >>> import itertools
1.641 + >>> c = iterbetter(itertools.count())
1.642 + >>> c[1]
1.643 + 1
1.644 + >>> c[5]
1.645 + 5
1.646 + >>> c[3]
1.647 + Traceback (most recent call last):
1.648 + ...
1.649 + IndexError: already passed 3
1.650 +
1.651 + For boolean test, IterBetter peeps at first value in the itertor without effecting the iteration.
1.652 +
1.653 + >>> c = iterbetter(iter(range(5)))
1.654 + >>> bool(c)
1.655 + True
1.656 + >>> list(c)
1.657 + [0, 1, 2, 3, 4]
1.658 + >>> c = iterbetter(iter([]))
1.659 + >>> bool(c)
1.660 + False
1.661 + >>> list(c)
1.662 + []
1.663 + """
1.664 + def __init__(self, iterator):
1.665 + self.i, self.c = iterator, 0
1.666 +
1.667 + def __iter__(self):
1.668 + if hasattr(self, "_head"):
1.669 + yield self._head
1.670 +
1.671 + while 1:
1.672 + yield self.i.next()
1.673 + self.c += 1
1.674 +
1.675 + def __getitem__(self, i):
1.676 + #todo: slices
1.677 + if i < self.c:
1.678 + raise IndexError, "already passed "+str(i)
1.679 + try:
1.680 + while i > self.c:
1.681 + self.i.next()
1.682 + self.c += 1
1.683 + # now self.c == i
1.684 + self.c += 1
1.685 + return self.i.next()
1.686 + except StopIteration:
1.687 + raise IndexError, str(i)
1.688 +
1.689 + def __nonzero__(self):
1.690 + if hasattr(self, "__len__"):
1.691 + return len(self) != 0
1.692 + elif hasattr(self, "_head"):
1.693 + return True
1.694 + else:
1.695 + try:
1.696 + self._head = self.i.next()
1.697 + except StopIteration:
1.698 + return False
1.699 + else:
1.700 + return True
1.701 +
1.702 +iterbetter = IterBetter
1.703 +
1.704 +def safeiter(it, cleanup=None, ignore_errors=True):
1.705 + """Makes an iterator safe by ignoring the exceptions occured during the iteration.
1.706 + """
1.707 + def next():
1.708 + while True:
1.709 + try:
1.710 + return it.next()
1.711 + except StopIteration:
1.712 + raise
1.713 + except:
1.714 + traceback.print_exc()
1.715 +
1.716 + it = iter(it)
1.717 + while True:
1.718 + yield next()
1.719 +
1.720 +def safewrite(filename, content):
1.721 + """Writes the content to a temp file and then moves the temp file to
1.722 + given filename to avoid overwriting the existing file in case of errors.
1.723 + """
1.724 + f = file(filename + '.tmp', 'w')
1.725 + f.write(content)
1.726 + f.close()
1.727 + os.rename(f.name, filename)
1.728 +
1.729 +def dictreverse(mapping):
1.730 + """
1.731 + Returns a new dictionary with keys and values swapped.
1.732 +
1.733 + >>> dictreverse({1: 2, 3: 4})
1.734 + {2: 1, 4: 3}
1.735 + """
1.736 + return dict([(value, key) for (key, value) in mapping.iteritems()])
1.737 +
1.738 +def dictfind(dictionary, element):
1.739 + """
1.740 + Returns a key whose value in `dictionary` is `element`
1.741 + or, if none exists, None.
1.742 +
1.743 + >>> d = {1:2, 3:4}
1.744 + >>> dictfind(d, 4)
1.745 + 3
1.746 + >>> dictfind(d, 5)
1.747 + """
1.748 + for (key, value) in dictionary.iteritems():
1.749 + if element is value:
1.750 + return key
1.751 +
1.752 +def dictfindall(dictionary, element):
1.753 + """
1.754 + Returns the keys whose values in `dictionary` are `element`
1.755 + or, if none exists, [].
1.756 +
1.757 + >>> d = {1:4, 3:4}
1.758 + >>> dictfindall(d, 4)
1.759 + [1, 3]
1.760 + >>> dictfindall(d, 5)
1.761 + []
1.762 + """
1.763 + res = []
1.764 + for (key, value) in dictionary.iteritems():
1.765 + if element is value:
1.766 + res.append(key)
1.767 + return res
1.768 +
1.769 +def dictincr(dictionary, element):
1.770 + """
1.771 + Increments `element` in `dictionary`,
1.772 + setting it to one if it doesn't exist.
1.773 +
1.774 + >>> d = {1:2, 3:4}
1.775 + >>> dictincr(d, 1)
1.776 + 3
1.777 + >>> d[1]
1.778 + 3
1.779 + >>> dictincr(d, 5)
1.780 + 1
1.781 + >>> d[5]
1.782 + 1
1.783 + """
1.784 + dictionary.setdefault(element, 0)
1.785 + dictionary[element] += 1
1.786 + return dictionary[element]
1.787 +
1.788 +def dictadd(*dicts):
1.789 + """
1.790 + Returns a dictionary consisting of the keys in the argument dictionaries.
1.791 + If they share a key, the value from the last argument is used.
1.792 +
1.793 + >>> dictadd({1: 0, 2: 0}, {2: 1, 3: 1})
1.794 + {1: 0, 2: 1, 3: 1}
1.795 + """
1.796 + result = {}
1.797 + for dct in dicts:
1.798 + result.update(dct)
1.799 + return result
1.800 +
1.801 +def requeue(queue, index=-1):
1.802 + """Returns the element at index after moving it to the beginning of the queue.
1.803 +
1.804 + >>> x = [1, 2, 3, 4]
1.805 + >>> requeue(x)
1.806 + 4
1.807 + >>> x
1.808 + [4, 1, 2, 3]
1.809 + """
1.810 + x = queue.pop(index)
1.811 + queue.insert(0, x)
1.812 + return x
1.813 +
1.814 +def restack(stack, index=0):
1.815 + """Returns the element at index after moving it to the top of stack.
1.816 +
1.817 + >>> x = [1, 2, 3, 4]
1.818 + >>> restack(x)
1.819 + 1
1.820 + >>> x
1.821 + [2, 3, 4, 1]
1.822 + """
1.823 + x = stack.pop(index)
1.824 + stack.append(x)
1.825 + return x
1.826 +
1.827 +def listget(lst, ind, default=None):
1.828 + """
1.829 + Returns `lst[ind]` if it exists, `default` otherwise.
1.830 +
1.831 + >>> listget(['a'], 0)
1.832 + 'a'
1.833 + >>> listget(['a'], 1)
1.834 + >>> listget(['a'], 1, 'b')
1.835 + 'b'
1.836 + """
1.837 + if len(lst)-1 < ind:
1.838 + return default
1.839 + return lst[ind]
1.840 +
1.841 +def intget(integer, default=None):
1.842 + """
1.843 + Returns `integer` as an int or `default` if it can't.
1.844 +
1.845 + >>> intget('3')
1.846 + 3
1.847 + >>> intget('3a')
1.848 + >>> intget('3a', 0)
1.849 + 0
1.850 + """
1.851 + try:
1.852 + return int(integer)
1.853 + except (TypeError, ValueError):
1.854 + return default
1.855 +
1.856 +def datestr(then, now=None):
1.857 + """
1.858 + Converts a (UTC) datetime object to a nice string representation.
1.859 +
1.860 + >>> from datetime import datetime, timedelta
1.861 + >>> d = datetime(1970, 5, 1)
1.862 + >>> datestr(d, now=d)
1.863 + '0 microseconds ago'
1.864 + >>> for t, v in {
1.865 + ... timedelta(microseconds=1): '1 microsecond ago',
1.866 + ... timedelta(microseconds=2): '2 microseconds ago',
1.867 + ... -timedelta(microseconds=1): '1 microsecond from now',
1.868 + ... -timedelta(microseconds=2): '2 microseconds from now',
1.869 + ... timedelta(microseconds=2000): '2 milliseconds ago',
1.870 + ... timedelta(seconds=2): '2 seconds ago',
1.871 + ... timedelta(seconds=2*60): '2 minutes ago',
1.872 + ... timedelta(seconds=2*60*60): '2 hours ago',
1.873 + ... timedelta(days=2): '2 days ago',
1.874 + ... }.iteritems():
1.875 + ... assert datestr(d, now=d+t) == v
1.876 + >>> datestr(datetime(1970, 1, 1), now=d)
1.877 + 'January 1'
1.878 + >>> datestr(datetime(1969, 1, 1), now=d)
1.879 + 'January 1, 1969'
1.880 + >>> datestr(datetime(1970, 6, 1), now=d)
1.881 + 'June 1, 1970'
1.882 + >>> datestr(None)
1.883 + ''
1.884 + """
1.885 + def agohence(n, what, divisor=None):
1.886 + if divisor: n = n // divisor
1.887 +
1.888 + out = str(abs(n)) + ' ' + what # '2 day'
1.889 + if abs(n) != 1: out += 's' # '2 days'
1.890 + out += ' ' # '2 days '
1.891 + if n < 0:
1.892 + out += 'from now'
1.893 + else:
1.894 + out += 'ago'
1.895 + return out # '2 days ago'
1.896 +
1.897 + oneday = 24 * 60 * 60
1.898 +
1.899 + if not then: return ""
1.900 + if not now: now = datetime.datetime.utcnow()
1.901 + if type(now).__name__ == "DateTime":
1.902 + now = datetime.datetime.fromtimestamp(now)
1.903 + if type(then).__name__ == "DateTime":
1.904 + then = datetime.datetime.fromtimestamp(then)
1.905 + elif type(then).__name__ == "date":
1.906 + then = datetime.datetime(then.year, then.month, then.day)
1.907 +
1.908 + delta = now - then
1.909 + deltaseconds = int(delta.days * oneday + delta.seconds + delta.microseconds * 1e-06)
1.910 + deltadays = abs(deltaseconds) // oneday
1.911 + if deltaseconds < 0: deltadays *= -1 # fix for oddity of floor
1.912 +
1.913 + if deltadays:
1.914 + if abs(deltadays) < 4:
1.915 + return agohence(deltadays, 'day')
1.916 +
1.917 + try:
1.918 + out = then.strftime('%B %e') # e.g. 'June 3'
1.919 + except ValueError:
1.920 + # %e doesn't work on Windows.
1.921 + out = then.strftime('%B %d') # e.g. 'June 03'
1.922 +
1.923 + if then.year != now.year or deltadays < 0:
1.924 + out += ', %s' % then.year
1.925 + return out
1.926 +
1.927 + if int(deltaseconds):
1.928 + if abs(deltaseconds) > (60 * 60):
1.929 + return agohence(deltaseconds, 'hour', 60 * 60)
1.930 + elif abs(deltaseconds) > 60:
1.931 + return agohence(deltaseconds, 'minute', 60)
1.932 + else:
1.933 + return agohence(deltaseconds, 'second')
1.934 +
1.935 + deltamicroseconds = delta.microseconds
1.936 + if delta.days: deltamicroseconds = int(delta.microseconds - 1e6) # datetime oddity
1.937 + if abs(deltamicroseconds) > 1000:
1.938 + return agohence(deltamicroseconds, 'millisecond', 1000)
1.939 +
1.940 + return agohence(deltamicroseconds, 'microsecond')
1.941 +
1.942 +def numify(string):
1.943 + """
1.944 + Removes all non-digit characters from `string`.
1.945 +
1.946 + >>> numify('800-555-1212')
1.947 + '8005551212'
1.948 + >>> numify('800.555.1212')
1.949 + '8005551212'
1.950 +
1.951 + """
1.952 + return ''.join([c for c in str(string) if c.isdigit()])
1.953 +
1.954 +def denumify(string, pattern):
1.955 + """
1.956 + Formats `string` according to `pattern`, where the letter X gets replaced
1.957 + by characters from `string`.
1.958 +
1.959 + >>> denumify("8005551212", "(XXX) XXX-XXXX")
1.960 + '(800) 555-1212'
1.961 +
1.962 + """
1.963 + out = []
1.964 + for c in pattern:
1.965 + if c == "X":
1.966 + out.append(string[0])
1.967 + string = string[1:]
1.968 + else:
1.969 + out.append(c)
1.970 + return ''.join(out)
1.971 +
1.972 +def commify(n):
1.973 + """
1.974 + Add commas to an integer `n`.
1.975 +
1.976 + >>> commify(1)
1.977 + '1'
1.978 + >>> commify(123)
1.979 + '123'
1.980 + >>> commify(1234)
1.981 + '1,234'
1.982 + >>> commify(1234567890)
1.983 + '1,234,567,890'
1.984 + >>> commify(123.0)
1.985 + '123.0'
1.986 + >>> commify(1234.5)
1.987 + '1,234.5'
1.988 + >>> commify(1234.56789)
1.989 + '1,234.56789'
1.990 + >>> commify('%.2f' % 1234.5)
1.991 + '1,234.50'
1.992 + >>> commify(None)
1.993 + >>>
1.994 +
1.995 + """
1.996 + if n is None: return None
1.997 + n = str(n)
1.998 + if '.' in n:
1.999 + dollars, cents = n.split('.')
1.1000 + else:
1.1001 + dollars, cents = n, None
1.1002 +
1.1003 + r = []
1.1004 + for i, c in enumerate(str(dollars)[::-1]):
1.1005 + if i and (not (i % 3)):
1.1006 + r.insert(0, ',')
1.1007 + r.insert(0, c)
1.1008 + out = ''.join(r)
1.1009 + if cents:
1.1010 + out += '.' + cents
1.1011 + return out
1.1012 +
1.1013 +def dateify(datestring):
1.1014 + """
1.1015 + Formats a numified `datestring` properly.
1.1016 + """
1.1017 + return denumify(datestring, "XXXX-XX-XX XX:XX:XX")
1.1018 +
1.1019 +
1.1020 +def nthstr(n):
1.1021 + """
1.1022 + Formats an ordinal.
1.1023 + Doesn't handle negative numbers.
1.1024 +
1.1025 + >>> nthstr(1)
1.1026 + '1st'
1.1027 + >>> nthstr(0)
1.1028 + '0th'
1.1029 + >>> [nthstr(x) for x in [2, 3, 4, 5, 10, 11, 12, 13, 14, 15]]
1.1030 + ['2nd', '3rd', '4th', '5th', '10th', '11th', '12th', '13th', '14th', '15th']
1.1031 + >>> [nthstr(x) for x in [91, 92, 93, 94, 99, 100, 101, 102]]
1.1032 + ['91st', '92nd', '93rd', '94th', '99th', '100th', '101st', '102nd']
1.1033 + >>> [nthstr(x) for x in [111, 112, 113, 114, 115]]
1.1034 + ['111th', '112th', '113th', '114th', '115th']
1.1035 +
1.1036 + """
1.1037 +
1.1038 + assert n >= 0
1.1039 + if n % 100 in [11, 12, 13]: return '%sth' % n
1.1040 + return {1: '%sst', 2: '%snd', 3: '%srd'}.get(n % 10, '%sth') % n
1.1041 +
1.1042 +def cond(predicate, consequence, alternative=None):
1.1043 + """
1.1044 + Function replacement for if-else to use in expressions.
1.1045 +
1.1046 + >>> x = 2
1.1047 + >>> cond(x % 2 == 0, "even", "odd")
1.1048 + 'even'
1.1049 + >>> cond(x % 2 == 0, "even", "odd") + '_row'
1.1050 + 'even_row'
1.1051 + """
1.1052 + if predicate:
1.1053 + return consequence
1.1054 + else:
1.1055 + return alternative
1.1056 +
1.1057 +class CaptureStdout:
1.1058 + """
1.1059 + Captures everything `func` prints to stdout and returns it instead.
1.1060 +
1.1061 + >>> def idiot():
1.1062 + ... print "foo"
1.1063 + >>> capturestdout(idiot)()
1.1064 + 'foo\\n'
1.1065 +
1.1066 + **WARNING:** Not threadsafe!
1.1067 + """
1.1068 + def __init__(self, func):
1.1069 + self.func = func
1.1070 + def __call__(self, *args, **keywords):
1.1071 + from cStringIO import StringIO
1.1072 + # Not threadsafe!
1.1073 + out = StringIO()
1.1074 + oldstdout = sys.stdout
1.1075 + sys.stdout = out
1.1076 + try:
1.1077 + self.func(*args, **keywords)
1.1078 + finally:
1.1079 + sys.stdout = oldstdout
1.1080 + return out.getvalue()
1.1081 +
1.1082 +capturestdout = CaptureStdout
1.1083 +
1.1084 +class Profile:
1.1085 + """
1.1086 + Profiles `func` and returns a tuple containing its output
1.1087 + and a string with human-readable profiling information.
1.1088 +
1.1089 + >>> import time
1.1090 + >>> out, inf = profile(time.sleep)(.001)
1.1091 + >>> out
1.1092 + >>> inf[:10].strip()
1.1093 + 'took 0.0'
1.1094 + """
1.1095 + def __init__(self, func):
1.1096 + self.func = func
1.1097 + def __call__(self, *args): ##, **kw): kw unused
1.1098 + import hotshot, hotshot.stats, os, tempfile ##, time already imported
1.1099 + f, filename = tempfile.mkstemp()
1.1100 + os.close(f)
1.1101 +
1.1102 + prof = hotshot.Profile(filename)
1.1103 +
1.1104 + stime = time.time()
1.1105 + result = prof.runcall(self.func, *args)
1.1106 + stime = time.time() - stime
1.1107 + prof.close()
1.1108 +
1.1109 + import cStringIO
1.1110 + out = cStringIO.StringIO()
1.1111 + stats = hotshot.stats.load(filename)
1.1112 + stats.stream = out
1.1113 + stats.strip_dirs()
1.1114 + stats.sort_stats('time', 'calls')
1.1115 + stats.print_stats(40)
1.1116 + stats.print_callers()
1.1117 +
1.1118 + x = '\n\ntook '+ str(stime) + ' seconds\n'
1.1119 + x += out.getvalue()
1.1120 +
1.1121 + # remove the tempfile
1.1122 + try:
1.1123 + os.remove(filename)
1.1124 + except IOError:
1.1125 + pass
1.1126 +
1.1127 + return result, x
1.1128 +
1.1129 +profile = Profile
1.1130 +
1.1131 +
1.1132 +import traceback
1.1133 +# hack for compatibility with Python 2.3:
1.1134 +if not hasattr(traceback, 'format_exc'):
1.1135 + from cStringIO import StringIO
1.1136 + def format_exc(limit=None):
1.1137 + strbuf = StringIO()
1.1138 + traceback.print_exc(limit, strbuf)
1.1139 + return strbuf.getvalue()
1.1140 + traceback.format_exc = format_exc
1.1141 +
1.1142 +def tryall(context, prefix=None):
1.1143 + """
1.1144 + Tries a series of functions and prints their results.
1.1145 + `context` is a dictionary mapping names to values;
1.1146 + the value will only be tried if it's callable.
1.1147 +
1.1148 + >>> tryall(dict(j=lambda: True))
1.1149 + j: True
1.1150 + ----------------------------------------
1.1151 + results:
1.1152 + True: 1
1.1153 +
1.1154 + For example, you might have a file `test/stuff.py`
1.1155 + with a series of functions testing various things in it.
1.1156 + At the bottom, have a line:
1.1157 +
1.1158 + if __name__ == "__main__": tryall(globals())
1.1159 +
1.1160 + Then you can run `python test/stuff.py` and get the results of
1.1161 + all the tests.
1.1162 + """
1.1163 + context = context.copy() # vars() would update
1.1164 + results = {}
1.1165 + for (key, value) in context.iteritems():
1.1166 + if not hasattr(value, '__call__'):
1.1167 + continue
1.1168 + if prefix and not key.startswith(prefix):
1.1169 + continue
1.1170 + print key + ':',
1.1171 + try:
1.1172 + r = value()
1.1173 + dictincr(results, r)
1.1174 + print r
1.1175 + except:
1.1176 + print 'ERROR'
1.1177 + dictincr(results, 'ERROR')
1.1178 + print ' ' + '\n '.join(traceback.format_exc().split('\n'))
1.1179 +
1.1180 + print '-'*40
1.1181 + print 'results:'
1.1182 + for (key, value) in results.iteritems():
1.1183 + print ' '*2, str(key)+':', value
1.1184 +
1.1185 +class ThreadedDict(threadlocal):
1.1186 + """
1.1187 + Thread local storage.
1.1188 +
1.1189 + >>> d = ThreadedDict()
1.1190 + >>> d.x = 1
1.1191 + >>> d.x
1.1192 + 1
1.1193 + >>> import threading
1.1194 + >>> def f(): d.x = 2
1.1195 + ...
1.1196 + >>> t = threading.Thread(target=f)
1.1197 + >>> t.start()
1.1198 + >>> t.join()
1.1199 + >>> d.x
1.1200 + 1
1.1201 + """
1.1202 + _instances = set()
1.1203 +
1.1204 + def __init__(self):
1.1205 + ThreadedDict._instances.add(self)
1.1206 +
1.1207 + def __del__(self):
1.1208 + ThreadedDict._instances.remove(self)
1.1209 +
1.1210 + def __hash__(self):
1.1211 + return id(self)
1.1212 +
1.1213 + def clear_all():
1.1214 + """Clears all ThreadedDict instances.
1.1215 + """
1.1216 + for t in list(ThreadedDict._instances):
1.1217 + t.clear()
1.1218 + clear_all = staticmethod(clear_all)
1.1219 +
1.1220 + # Define all these methods to more or less fully emulate dict -- attribute access
1.1221 + # is built into threading.local.
1.1222 +
1.1223 + def __getitem__(self, key):
1.1224 + return self.__dict__[key]
1.1225 +
1.1226 + def __setitem__(self, key, value):
1.1227 + self.__dict__[key] = value
1.1228 +
1.1229 + def __delitem__(self, key):
1.1230 + del self.__dict__[key]
1.1231 +
1.1232 + def __contains__(self, key):
1.1233 + return key in self.__dict__
1.1234 +
1.1235 + has_key = __contains__
1.1236 +
1.1237 + def clear(self):
1.1238 + self.__dict__.clear()
1.1239 +
1.1240 + def copy(self):
1.1241 + return self.__dict__.copy()
1.1242 +
1.1243 + def get(self, key, default=None):
1.1244 + return self.__dict__.get(key, default)
1.1245 +
1.1246 + def items(self):
1.1247 + return self.__dict__.items()
1.1248 +
1.1249 + def iteritems(self):
1.1250 + return self.__dict__.iteritems()
1.1251 +
1.1252 + def keys(self):
1.1253 + return self.__dict__.keys()
1.1254 +
1.1255 + def iterkeys(self):
1.1256 + return self.__dict__.iterkeys()
1.1257 +
1.1258 + iter = iterkeys
1.1259 +
1.1260 + def values(self):
1.1261 + return self.__dict__.values()
1.1262 +
1.1263 + def itervalues(self):
1.1264 + return self.__dict__.itervalues()
1.1265 +
1.1266 + def pop(self, key, *args):
1.1267 + return self.__dict__.pop(key, *args)
1.1268 +
1.1269 + def popitem(self):
1.1270 + return self.__dict__.popitem()
1.1271 +
1.1272 + def setdefault(self, key, default=None):
1.1273 + return self.__dict__.setdefault(key, default)
1.1274 +
1.1275 + def update(self, *args, **kwargs):
1.1276 + self.__dict__.update(*args, **kwargs)
1.1277 +
1.1278 + def __repr__(self):
1.1279 + return '<ThreadedDict %r>' % self.__dict__
1.1280 +
1.1281 + __str__ = __repr__
1.1282 +
1.1283 +threadeddict = ThreadedDict
1.1284 +
1.1285 +def autoassign(self, locals):
1.1286 + """
1.1287 + Automatically assigns local variables to `self`.
1.1288 +
1.1289 + >>> self = storage()
1.1290 + >>> autoassign(self, dict(a=1, b=2))
1.1291 + >>> self
1.1292 + <Storage {'a': 1, 'b': 2}>
1.1293 +
1.1294 + Generally used in `__init__` methods, as in:
1.1295 +
1.1296 + def __init__(self, foo, bar, baz=1): autoassign(self, locals())
1.1297 + """
1.1298 + for (key, value) in locals.iteritems():
1.1299 + if key == 'self':
1.1300 + continue
1.1301 + setattr(self, key, value)
1.1302 +
1.1303 +def to36(q):
1.1304 + """
1.1305 + Converts an integer to base 36 (a useful scheme for human-sayable IDs).
1.1306 +
1.1307 + >>> to36(35)
1.1308 + 'z'
1.1309 + >>> to36(119292)
1.1310 + '2k1o'
1.1311 + >>> int(to36(939387374), 36)
1.1312 + 939387374
1.1313 + >>> to36(0)
1.1314 + '0'
1.1315 + >>> to36(-393)
1.1316 + Traceback (most recent call last):
1.1317 + ...
1.1318 + ValueError: must supply a positive integer
1.1319 +
1.1320 + """
1.1321 + if q < 0: raise ValueError, "must supply a positive integer"
1.1322 + letters = "0123456789abcdefghijklmnopqrstuvwxyz"
1.1323 + converted = []
1.1324 + while q != 0:
1.1325 + q, r = divmod(q, 36)
1.1326 + converted.insert(0, letters[r])
1.1327 + return "".join(converted) or '0'
1.1328 +
1.1329 +
1.1330 +r_url = re_compile('(?<!\()(http://(\S+))')
1.1331 +def safemarkdown(text):
1.1332 + """
1.1333 + Converts text to HTML following the rules of Markdown, but blocking any
1.1334 + outside HTML input, so that only the things supported by Markdown
1.1335 + can be used. Also converts raw URLs to links.
1.1336 +
1.1337 + (requires [markdown.py](http://webpy.org/markdown.py))
1.1338 + """
1.1339 + from markdown import markdown
1.1340 + if text:
1.1341 + text = text.replace('<', '<')
1.1342 + # TODO: automatically get page title?
1.1343 + text = r_url.sub(r'<\1>', text)
1.1344 + text = markdown(text)
1.1345 + return text
1.1346 +
1.1347 +def sendmail(from_address, to_address, subject, message, headers=None, **kw):
1.1348 + """
1.1349 + Sends the email message `message` with mail and envelope headers
1.1350 + for from `from_address_` to `to_address` with `subject`.
1.1351 + Additional email headers can be specified with the dictionary
1.1352 + `headers.
1.1353 +
1.1354 + Optionally cc, bcc and attachments can be specified as keyword arguments.
1.1355 + Attachments must be an iterable and each attachment can be either a
1.1356 + filename or a file object or a dictionary with filename, content and
1.1357 + optionally content_type keys.
1.1358 +
1.1359 + If `web.config.smtp_server` is set, it will send the message
1.1360 + to that SMTP server. Otherwise it will look for
1.1361 + `/usr/sbin/sendmail`, the typical location for the sendmail-style
1.1362 + binary. To use sendmail from a different path, set `web.config.sendmail_path`.
1.1363 + """
1.1364 + attachments = kw.pop("attachments", [])
1.1365 + mail = _EmailMessage(from_address, to_address, subject, message, headers, **kw)
1.1366 +
1.1367 + for a in attachments:
1.1368 + if isinstance(a, dict):
1.1369 + mail.attach(a['filename'], a['content'], a.get('content_type'))
1.1370 + elif hasattr(a, 'read'): # file
1.1371 + filename = os.path.basename(getattr(a, "name", ""))
1.1372 + content_type = getattr(a, 'content_type', None)
1.1373 + mail.attach(filename, a.read(), content_type)
1.1374 + elif isinstance(a, basestring):
1.1375 + f = open(a, 'rb')
1.1376 + content = f.read()
1.1377 + f.close()
1.1378 + filename = os.path.basename(a)
1.1379 + mail.attach(filename, content, None)
1.1380 + else:
1.1381 + raise ValueError, "Invalid attachment: %s" % repr(a)
1.1382 +
1.1383 + mail.send()
1.1384 +
1.1385 +class _EmailMessage:
1.1386 + def __init__(self, from_address, to_address, subject, message, headers=None, **kw):
1.1387 + def listify(x):
1.1388 + if not isinstance(x, list):
1.1389 + return [safestr(x)]
1.1390 + else:
1.1391 + return [safestr(a) for a in x]
1.1392 +
1.1393 + subject = safestr(subject)
1.1394 + message = safestr(message)
1.1395 +
1.1396 + from_address = safestr(from_address)
1.1397 + to_address = listify(to_address)
1.1398 + cc = listify(kw.get('cc', []))
1.1399 + bcc = listify(kw.get('bcc', []))
1.1400 + recipients = to_address + cc + bcc
1.1401 +
1.1402 + import email.Utils
1.1403 + self.from_address = email.Utils.parseaddr(from_address)[1]
1.1404 + self.recipients = [email.Utils.parseaddr(r)[1] for r in recipients]
1.1405 +
1.1406 + self.headers = dictadd({
1.1407 + 'From': from_address,
1.1408 + 'To': ", ".join(to_address),
1.1409 + 'Subject': subject
1.1410 + }, headers or {})
1.1411 +
1.1412 + if cc:
1.1413 + self.headers['Cc'] = ", ".join(cc)
1.1414 +
1.1415 + self.message = self.new_message()
1.1416 + self.message.add_header("Content-Transfer-Encoding", "7bit")
1.1417 + self.message.add_header("Content-Disposition", "inline")
1.1418 + self.message.add_header("MIME-Version", "1.0")
1.1419 + self.message.set_payload(message, 'utf-8')
1.1420 + self.multipart = False
1.1421 +
1.1422 + def new_message(self):
1.1423 + from email.Message import Message
1.1424 + return Message()
1.1425 +
1.1426 + def attach(self, filename, content, content_type=None):
1.1427 + if not self.multipart:
1.1428 + msg = self.new_message()
1.1429 + msg.add_header("Content-Type", "multipart/mixed")
1.1430 + msg.attach(self.message)
1.1431 + self.message = msg
1.1432 + self.multipart = True
1.1433 +
1.1434 + import mimetypes
1.1435 + try:
1.1436 + from email import encoders
1.1437 + except:
1.1438 + from email import Encoders as encoders
1.1439 +
1.1440 + content_type = content_type or mimetypes.guess_type(filename)[0] or "applcation/octet-stream"
1.1441 +
1.1442 + msg = self.new_message()
1.1443 + msg.set_payload(content)
1.1444 + msg.add_header('Content-Type', content_type)
1.1445 + msg.add_header('Content-Disposition', 'attachment', filename=filename)
1.1446 +
1.1447 + if not content_type.startswith("text/"):
1.1448 + encoders.encode_base64(msg)
1.1449 +
1.1450 + self.message.attach(msg)
1.1451 +
1.1452 + def prepare_message(self):
1.1453 + for k, v in self.headers.iteritems():
1.1454 + if k.lower() == "content-type":
1.1455 + self.message.set_type(v)
1.1456 + else:
1.1457 + self.message.add_header(k, v)
1.1458 +
1.1459 + self.headers = {}
1.1460 +
1.1461 + def send(self):
1.1462 + try:
1.1463 + import webapi
1.1464 + except ImportError:
1.1465 + webapi = Storage(config=Storage())
1.1466 +
1.1467 + self.prepare_message()
1.1468 + message_text = self.message.as_string()
1.1469 +
1.1470 + if webapi.config.get('smtp_server'):
1.1471 + server = webapi.config.get('smtp_server')
1.1472 + port = webapi.config.get('smtp_port', 0)
1.1473 + username = webapi.config.get('smtp_username')
1.1474 + password = webapi.config.get('smtp_password')
1.1475 + debug_level = webapi.config.get('smtp_debuglevel', None)
1.1476 + starttls = webapi.config.get('smtp_starttls', False)
1.1477 +
1.1478 + import smtplib
1.1479 + smtpserver = smtplib.SMTP(server, port)
1.1480 +
1.1481 + if debug_level:
1.1482 + smtpserver.set_debuglevel(debug_level)
1.1483 +
1.1484 + if starttls:
1.1485 + smtpserver.ehlo()
1.1486 + smtpserver.starttls()
1.1487 + smtpserver.ehlo()
1.1488 +
1.1489 + if username and password:
1.1490 + smtpserver.login(username, password)
1.1491 +
1.1492 + smtpserver.sendmail(self.from_address, self.recipients, message_text)
1.1493 + smtpserver.quit()
1.1494 + elif webapi.config.get('email_engine') == 'aws':
1.1495 + import boto.ses
1.1496 + c = boto.ses.SESConnection(
1.1497 + aws_access_key_id=webapi.config.get('aws_access_key_id'),
1.1498 + aws_secret_access_key=web.api.config.get('aws_secret_access_key'))
1.1499 + c.send_raw_email(self.from_address, message_text, self.from_recipients)
1.1500 + else:
1.1501 + sendmail = webapi.config.get('sendmail_path', '/usr/sbin/sendmail')
1.1502 +
1.1503 + assert not self.from_address.startswith('-'), 'security'
1.1504 + for r in self.recipients:
1.1505 + assert not r.startswith('-'), 'security'
1.1506 +
1.1507 + cmd = [sendmail, '-f', self.from_address] + self.recipients
1.1508 +
1.1509 + if subprocess:
1.1510 + p = subprocess.Popen(cmd, stdin=subprocess.PIPE)
1.1511 + p.stdin.write(message_text)
1.1512 + p.stdin.close()
1.1513 + p.wait()
1.1514 + else:
1.1515 + i, o = os.popen2(cmd)
1.1516 + i.write(message)
1.1517 + i.close()
1.1518 + o.close()
1.1519 + del i, o
1.1520 +
1.1521 + def __repr__(self):
1.1522 + return "<EmailMessage>"
1.1523 +
1.1524 + def __str__(self):
1.1525 + return self.message.as_string()
1.1526 +
1.1527 +if __name__ == "__main__":
1.1528 + import doctest
1.1529 + doctest.testmod()