8 "Storage", "storage", "storify",
11 "rstrips", "lstrips", "strips",
12 "safeunicode", "safestr", "utf8",
13 "TimeoutError", "timelimit",
15 "re_compile", "re_subm",
16 "group", "uniq", "iterview",
17 "IterBetter", "iterbetter",
18 "safeiter", "safewrite",
19 "dictreverse", "dictfind", "dictfindall", "dictincr", "dictadd",
21 "listget", "intget", "datestr",
22 "numify", "denumify", "commify", "dateify",
24 "CaptureStdout", "capturestdout", "Profile", "profile",
26 "ThreadedDict", "threadeddict",
33 import re, sys, time, threading, itertools, traceback, os
41 except ImportError: pass
45 from sets import Set as set
48 from threading import local as threadlocal
50 from python23 import threadlocal
54 A Storage object is like a dictionary except `obj.foo` can be used
55 in addition to `obj['foo']`.
67 Traceback (most recent call last):
72 def __getattr__(self, key):
76 raise AttributeError, k
78 def __setattr__(self, key, value):
81 def __delattr__(self, key):
85 raise AttributeError, k
88 return '<Storage ' + dict.__repr__(self) + '>'
92 def storify(mapping, *requireds, **defaults):
94 Creates a `storage` object from dictionary `mapping`, raising `KeyError` if
95 d doesn't have all of the keys in `requireds` and using the default
96 values for keys found in `defaults`.
98 For example, `storify({'a':1, 'c':3}, b=2, c=0)` will return the equivalent of
99 `storage({'a':1, 'b':2, 'c':3})`.
101 If a `storify` value is a list (e.g. multiple values in a form submission),
102 `storify` returns the last element of the list, unless the key appears in
103 `defaults` as a list. Thus:
105 >>> storify({'a':[1, 2]}).a
107 >>> storify({'a':[1, 2]}, a=[]).a
109 >>> storify({'a':1}, a=[]).a
111 >>> storify({}, a=[]).a
114 Similarly, if the value has a `value` attribute, `storify will return _its_
115 value, unless the key appears in `defaults` as a dictionary.
117 >>> storify({'a':storage(value=1)}).a
119 >>> storify({'a':storage(value=1)}, a={}).a
120 <Storage {'value': 1}>
121 >>> storify({}, a={}).a
124 Optionally, keyword parameter `_unicode` can be passed to convert all values to unicode.
126 >>> storify({'x': 'a'}, _unicode=True)
127 <Storage {'x': u'a'}>
128 >>> storify({'x': storage(value='a')}, x={}, _unicode=True)
129 <Storage {'x': <Storage {'value': 'a'}>}>
130 >>> storify({'x': storage(value='a')}, _unicode=True)
131 <Storage {'x': u'a'}>
133 _unicode = defaults.pop('_unicode', False)
135 # if _unicode is callable object, use it convert a string to unicode.
136 to_unicode = safeunicode
137 if _unicode is not False and hasattr(_unicode, "__call__"):
138 to_unicode = _unicode
141 if _unicode and isinstance(s, str): return to_unicode(s)
145 if hasattr(x, 'file') and hasattr(x, 'value'):
147 elif hasattr(x, 'value'):
148 return unicodify(x.value)
153 for key in requireds + tuple(mapping.keys()):
155 if isinstance(value, list):
156 if isinstance(defaults.get(key), list):
157 value = [getvalue(x) for x in value]
160 if not isinstance(defaults.get(key), dict):
161 value = getvalue(value)
162 if isinstance(defaults.get(key), list) and not isinstance(value, list):
164 setattr(stor, key, value)
166 for (key, value) in defaults.iteritems():
168 if hasattr(stor, key):
170 if value == () and not isinstance(result, tuple):
172 setattr(stor, key, result)
176 class Counter(storage):
177 """Keeps count of how many times something is added.
187 <Counter {'y': 1, 'x': 5}>
192 self.setdefault(n, 0)
196 """Returns the keys with maximum count."""
197 m = max(self.itervalues())
198 return [k for k, v in self.iteritems() if v == m]
201 """Returns the keys with mininum count."""
202 m = min(self.itervalues())
203 return [k for k, v in self.iteritems() if v == m]
205 def percent(self, key):
206 """Returns what percentage a certain key is of all entries.
218 return float(self[key])/sum(self.values())
220 def sorted_keys(self):
221 """Returns keys sorted by value.
230 return sorted(self.keys(), key=lambda k: self[k], reverse=True)
232 def sorted_values(self):
233 """Returns values sorted by value.
239 >>> c.sorted_values()
242 return [self[k] for k in self.sorted_keys()]
244 def sorted_items(self):
245 """Returns items sorted by value.
254 return [(k, self[k]) for k in self.sorted_keys()]
257 return '<Counter ' + dict.__repr__(self) + '>'
261 iters = [list, tuple]
263 if hasattr(__builtin__, 'set'):
265 if hasattr(__builtin__, 'frozenset'):
267 if sys.version_info < (2,6): # sets module deprecated in 2.6
274 class _hack(tuple): pass
277 A list of iterable items (like lists, but not strings). Includes whichever
278 of lists, tuples, sets, and Sets are available in this version of Python.
281 def _strips(direction, text, remove):
282 if isinstance(remove, iters):
284 text = _strips(direction, text, subr)
288 if text.startswith(remove):
289 return text[len(remove):]
290 elif direction == 'r':
291 if text.endswith(remove):
292 return text[:-len(remove)]
294 raise ValueError, "Direction needs to be r or l."
297 def rstrips(text, remove):
299 removes the string `remove` from the right of `text`
301 >>> rstrips("foobar", "bar")
305 return _strips('r', text, remove)
307 def lstrips(text, remove):
309 removes the string `remove` from the left of `text`
311 >>> lstrips("foobar", "foo")
313 >>> lstrips('http://foo.org/', ['http://', 'https://'])
315 >>> lstrips('FOOBARBAZ', ['FOO', 'BAR'])
317 >>> lstrips('FOOBARBAZ', ['BAR', 'FOO'])
321 return _strips('l', text, remove)
323 def strips(text, remove):
325 removes the string `remove` from the both sides of `text`
327 >>> strips("foobarfoo", "foo")
331 return rstrips(lstrips(text, remove), remove)
333 def safeunicode(obj, encoding='utf-8'):
335 Converts any given object to unicode string.
337 >>> safeunicode('hello')
341 >>> safeunicode('\xe1\x88\xb4')
348 return obj.decode(encoding)
349 elif t in [int, float, bool]:
351 elif hasattr(obj, '__unicode__') or isinstance(obj, unicode):
354 return str(obj).decode(encoding)
356 def safestr(obj, encoding='utf-8'):
358 Converts any given object to utf-8 encoded string.
362 >>> safestr(u'\u1234')
367 if isinstance(obj, unicode):
368 return obj.encode(encoding)
369 elif isinstance(obj, str):
371 elif hasattr(obj, 'next'): # iterator
372 return itertools.imap(safestr, obj)
376 # for backward-compatibility
379 class TimeoutError(Exception): pass
380 def timelimit(timeout):
382 A decorator to limit a function to `timeout` seconds, raising `TimeoutError`
386 >>> def meaningoflife():
390 >>> timelimit(.1)(meaningoflife)()
391 Traceback (most recent call last):
393 TimeoutError: took too long
394 >>> timelimit(1)(meaningoflife)()
397 _Caveat:_ The function isn't stopped after `timeout` seconds but continues
398 executing in a separate thread. (There seems to be no way to kill a thread.)
400 inspired by <http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/473878>
404 class Dispatch(threading.Thread):
406 threading.Thread.__init__(self)
415 self.result = function(*args, **kw)
417 self.error = sys.exc_info()
422 raise TimeoutError, 'took too long'
424 raise c.error[0], c.error[1]
431 'Memoizes' a function, caching its return values for each input.
432 If `expires` is specified, values are recalculated after `expires` seconds.
433 If `background` is specified, values are recalculated in a separate thread.
436 >>> def howmanytimeshaveibeencalled():
440 >>> fastcalls = memoize(howmanytimeshaveibeencalled)
441 >>> howmanytimeshaveibeencalled()
443 >>> howmanytimeshaveibeencalled()
450 >>> fastcalls = memoize(howmanytimeshaveibeencalled, .1, background=False)
460 ... return howmanytimeshaveibeencalled()
461 >>> fastcalls = memoize(slowfunc, .2, background=True)
464 >>> timelimit(.05)(fastcalls)()
467 >>> timelimit(.05)(fastcalls)()
469 >>> timelimit(.05)(fastcalls)()
472 >>> timelimit(.05)(fastcalls)()
474 >>> fastcalls = memoize(slowfunc, None, background=True)
475 >>> threading.Thread(target=fastcalls).start()
480 def __init__(self, func, expires=None, background=True):
483 self.expires = expires
484 self.background = background
487 def __call__(self, *args, **keywords):
488 key = (args, tuple(keywords.items()))
489 if not self.running.get(key):
490 self.running[key] = threading.Lock()
491 def update(block=False):
492 if self.running[key].acquire(block):
494 self.cache[key] = (self.func(*args, **keywords), time.time())
496 self.running[key].release()
498 if key not in self.cache:
500 elif self.expires and (time.time() - self.cache[key][1]) > self.expires:
502 threading.Thread(target=update).start()
505 return self.cache[key][0]
509 re_compile = memoize(re.compile) #@@ threadsafe?
510 re_compile.__doc__ = """
511 A memoized version of re.compile.
514 class _re_subm_proxy:
517 def __call__(self, match):
521 def re_subm(pat, repl, string):
523 Like re.sub, but returns the replacement _and_ the match object.
525 >>> t, m = re_subm('g(oo+)fball', r'f\\1lish', 'goooooofball')
531 compiled_pat = re_compile(pat)
532 proxy = _re_subm_proxy()
533 compiled_pat.sub(proxy.__call__, string)
534 return compiled_pat.sub(repl, string), proxy.match
536 def group(seq, size):
538 Returns an iterator over a series of lists of length size from iterable.
540 >>> list(group([1,2,3,4], 2))
542 >>> list(group([1,2,3,4,5], 2))
543 [[1, 2], [3, 4], [5]]
549 if not hasattr(seq, 'next'):
552 x = list(take(seq, size))
558 def uniq(seq, key=None):
560 Removes duplicate elements from a list while preserving the order of the rest.
562 >>> uniq([9,0,2,1,0])
565 The value of the optional `key` parameter should be a function that
566 takes a single argument and returns a key to test the uniqueness.
568 >>> uniq(["Foo", "foo", "bar"], key=lambda s: s.lower())
571 key = key or (lambda x: x)
584 Takes an iterable `x` and returns an iterator over it
585 which prints its progress to stderr as it iterates through.
589 def plainformat(n, lenx):
590 return '%5.1f%% (%*d/%d)' % ((float(n)/lenx)*100, len(str(lenx)), n, lenx)
592 def bars(size, n, lenx):
593 val = int((float(n)*size)/lenx + 0.5)
595 spacing = ">" + (" "*(size-val))[1:]
598 return "[%s%s]" % ("="*val, spacing)
600 def eta(elapsed, n, lenx):
606 secs = int((elapsed/n) * (lenx-n))
607 mins, secs = divmod(secs, 60)
608 hrs, mins = divmod(mins, 60)
610 return '%02d:%02d:%02d' % (hrs, mins, secs)
612 def format(starttime, n, lenx):
613 out = plainformat(n, lenx) + ' '
618 end += eta(time.time() - starttime, n, lenx)
619 out += bars(WIDTH - len(out) - len(end), n, lenx)
623 starttime = time.time()
625 for n, y in enumerate(x):
626 sys.stderr.write('\r' + format(starttime, n, lenx))
628 sys.stderr.write('\r' + format(starttime, n+1, lenx) + '\n')
632 Returns an object that can be used as an iterator
633 but can also be used via __getitem__ (although it
634 cannot go backwards -- that is, you cannot request
635 `iterbetter[0]` after requesting `iterbetter[1]`).
638 >>> c = iterbetter(itertools.count())
644 Traceback (most recent call last):
646 IndexError: already passed 3
648 For boolean test, IterBetter peeps at first value in the itertor without effecting the iteration.
650 >>> c = iterbetter(iter(range(5)))
655 >>> c = iterbetter(iter([]))
661 def __init__(self, iterator):
662 self.i, self.c = iterator, 0
665 if hasattr(self, "_head"):
672 def __getitem__(self, i):
675 raise IndexError, "already passed "+str(i)
683 except StopIteration:
684 raise IndexError, str(i)
686 def __nonzero__(self):
687 if hasattr(self, "__len__"):
688 return len(self) != 0
689 elif hasattr(self, "_head"):
693 self._head = self.i.next()
694 except StopIteration:
699 iterbetter = IterBetter
701 def safeiter(it, cleanup=None, ignore_errors=True):
702 """Makes an iterator safe by ignoring the exceptions occured during the iteration.
708 except StopIteration:
711 traceback.print_exc()
717 def safewrite(filename, content):
718 """Writes the content to a temp file and then moves the temp file to
719 given filename to avoid overwriting the existing file in case of errors.
721 f = file(filename + '.tmp', 'w')
724 os.rename(f.name, filename)
726 def dictreverse(mapping):
728 Returns a new dictionary with keys and values swapped.
730 >>> dictreverse({1: 2, 3: 4})
733 return dict([(value, key) for (key, value) in mapping.iteritems()])
735 def dictfind(dictionary, element):
737 Returns a key whose value in `dictionary` is `element`
738 or, if none exists, None.
745 for (key, value) in dictionary.iteritems():
749 def dictfindall(dictionary, element):
751 Returns the keys whose values in `dictionary` are `element`
752 or, if none exists, [].
755 >>> dictfindall(d, 4)
757 >>> dictfindall(d, 5)
761 for (key, value) in dictionary.iteritems():
766 def dictincr(dictionary, element):
768 Increments `element` in `dictionary`,
769 setting it to one if it doesn't exist.
781 dictionary.setdefault(element, 0)
782 dictionary[element] += 1
783 return dictionary[element]
787 Returns a dictionary consisting of the keys in the argument dictionaries.
788 If they share a key, the value from the last argument is used.
790 >>> dictadd({1: 0, 2: 0}, {2: 1, 3: 1})
798 def requeue(queue, index=-1):
799 """Returns the element at index after moving it to the beginning of the queue.
811 def restack(stack, index=0):
812 """Returns the element at index after moving it to the top of stack.
824 def listget(lst, ind, default=None):
826 Returns `lst[ind]` if it exists, `default` otherwise.
828 >>> listget(['a'], 0)
830 >>> listget(['a'], 1)
831 >>> listget(['a'], 1, 'b')
838 def intget(integer, default=None):
840 Returns `integer` as an int or `default` if it can't.
850 except (TypeError, ValueError):
853 def datestr(then, now=None):
855 Converts a (UTC) datetime object to a nice string representation.
857 >>> from datetime import datetime, timedelta
858 >>> d = datetime(1970, 5, 1)
859 >>> datestr(d, now=d)
862 ... timedelta(microseconds=1): '1 microsecond ago',
863 ... timedelta(microseconds=2): '2 microseconds ago',
864 ... -timedelta(microseconds=1): '1 microsecond from now',
865 ... -timedelta(microseconds=2): '2 microseconds from now',
866 ... timedelta(microseconds=2000): '2 milliseconds ago',
867 ... timedelta(seconds=2): '2 seconds ago',
868 ... timedelta(seconds=2*60): '2 minutes ago',
869 ... timedelta(seconds=2*60*60): '2 hours ago',
870 ... timedelta(days=2): '2 days ago',
872 ... assert datestr(d, now=d+t) == v
873 >>> datestr(datetime(1970, 1, 1), now=d)
875 >>> datestr(datetime(1969, 1, 1), now=d)
877 >>> datestr(datetime(1970, 6, 1), now=d)
882 def agohence(n, what, divisor=None):
883 if divisor: n = n // divisor
885 out = str(abs(n)) + ' ' + what # '2 day'
886 if abs(n) != 1: out += 's' # '2 days'
887 out += ' ' # '2 days '
892 return out # '2 days ago'
894 oneday = 24 * 60 * 60
896 if not then: return ""
897 if not now: now = datetime.datetime.utcnow()
898 if type(now).__name__ == "DateTime":
899 now = datetime.datetime.fromtimestamp(now)
900 if type(then).__name__ == "DateTime":
901 then = datetime.datetime.fromtimestamp(then)
902 elif type(then).__name__ == "date":
903 then = datetime.datetime(then.year, then.month, then.day)
906 deltaseconds = int(delta.days * oneday + delta.seconds + delta.microseconds * 1e-06)
907 deltadays = abs(deltaseconds) // oneday
908 if deltaseconds < 0: deltadays *= -1 # fix for oddity of floor
911 if abs(deltadays) < 4:
912 return agohence(deltadays, 'day')
915 out = then.strftime('%B %e') # e.g. 'June 3'
917 # %e doesn't work on Windows.
918 out = then.strftime('%B %d') # e.g. 'June 03'
920 if then.year != now.year or deltadays < 0:
921 out += ', %s' % then.year
924 if int(deltaseconds):
925 if abs(deltaseconds) > (60 * 60):
926 return agohence(deltaseconds, 'hour', 60 * 60)
927 elif abs(deltaseconds) > 60:
928 return agohence(deltaseconds, 'minute', 60)
930 return agohence(deltaseconds, 'second')
932 deltamicroseconds = delta.microseconds
933 if delta.days: deltamicroseconds = int(delta.microseconds - 1e6) # datetime oddity
934 if abs(deltamicroseconds) > 1000:
935 return agohence(deltamicroseconds, 'millisecond', 1000)
937 return agohence(deltamicroseconds, 'microsecond')
941 Removes all non-digit characters from `string`.
943 >>> numify('800-555-1212')
945 >>> numify('800.555.1212')
949 return ''.join([c for c in str(string) if c.isdigit()])
951 def denumify(string, pattern):
953 Formats `string` according to `pattern`, where the letter X gets replaced
954 by characters from `string`.
956 >>> denumify("8005551212", "(XXX) XXX-XXXX")
963 out.append(string[0])
971 Add commas to an integer `n`.
979 >>> commify(1234567890)
985 >>> commify(1234.56789)
987 >>> commify('%.2f' % 1234.5)
993 if n is None: return None
996 dollars, cents = n.split('.')
998 dollars, cents = n, None
1001 for i, c in enumerate(str(dollars)[::-1]):
1002 if i and (not (i % 3)):
1010 def dateify(datestring):
1012 Formats a numified `datestring` properly.
1014 return denumify(datestring, "XXXX-XX-XX XX:XX:XX")
1020 Doesn't handle negative numbers.
1026 >>> [nthstr(x) for x in [2, 3, 4, 5, 10, 11, 12, 13, 14, 15]]
1027 ['2nd', '3rd', '4th', '5th', '10th', '11th', '12th', '13th', '14th', '15th']
1028 >>> [nthstr(x) for x in [91, 92, 93, 94, 99, 100, 101, 102]]
1029 ['91st', '92nd', '93rd', '94th', '99th', '100th', '101st', '102nd']
1030 >>> [nthstr(x) for x in [111, 112, 113, 114, 115]]
1031 ['111th', '112th', '113th', '114th', '115th']
1036 if n % 100 in [11, 12, 13]: return '%sth' % n
1037 return {1: '%sst', 2: '%snd', 3: '%srd'}.get(n % 10, '%sth') % n
1039 def cond(predicate, consequence, alternative=None):
1041 Function replacement for if-else to use in expressions.
1044 >>> cond(x % 2 == 0, "even", "odd")
1046 >>> cond(x % 2 == 0, "even", "odd") + '_row'
1054 class CaptureStdout:
1056 Captures everything `func` prints to stdout and returns it instead.
1060 >>> capturestdout(idiot)()
1063 **WARNING:** Not threadsafe!
1065 def __init__(self, func):
1067 def __call__(self, *args, **keywords):
1068 from cStringIO import StringIO
1071 oldstdout = sys.stdout
1074 self.func(*args, **keywords)
1076 sys.stdout = oldstdout
1077 return out.getvalue()
1079 capturestdout = CaptureStdout
1083 Profiles `func` and returns a tuple containing its output
1084 and a string with human-readable profiling information.
1087 >>> out, inf = profile(time.sleep)(.001)
1089 >>> inf[:10].strip()
1092 def __init__(self, func):
1094 def __call__(self, *args): ##, **kw): kw unused
1095 import hotshot, hotshot.stats, os, tempfile ##, time already imported
1096 f, filename = tempfile.mkstemp()
1099 prof = hotshot.Profile(filename)
1102 result = prof.runcall(self.func, *args)
1103 stime = time.time() - stime
1107 out = cStringIO.StringIO()
1108 stats = hotshot.stats.load(filename)
1111 stats.sort_stats('time', 'calls')
1112 stats.print_stats(40)
1113 stats.print_callers()
1115 x = '\n\ntook '+ str(stime) + ' seconds\n'
1118 # remove the tempfile
1130 # hack for compatibility with Python 2.3:
1131 if not hasattr(traceback, 'format_exc'):
1132 from cStringIO import StringIO
1133 def format_exc(limit=None):
1135 traceback.print_exc(limit, strbuf)
1136 return strbuf.getvalue()
1137 traceback.format_exc = format_exc
1139 def tryall(context, prefix=None):
1141 Tries a series of functions and prints their results.
1142 `context` is a dictionary mapping names to values;
1143 the value will only be tried if it's callable.
1145 >>> tryall(dict(j=lambda: True))
1147 ----------------------------------------
1151 For example, you might have a file `test/stuff.py`
1152 with a series of functions testing various things in it.
1153 At the bottom, have a line:
1155 if __name__ == "__main__": tryall(globals())
1157 Then you can run `python test/stuff.py` and get the results of
1160 context = context.copy() # vars() would update
1162 for (key, value) in context.iteritems():
1163 if not hasattr(value, '__call__'):
1165 if prefix and not key.startswith(prefix):
1170 dictincr(results, r)
1174 dictincr(results, 'ERROR')
1175 print ' ' + '\n '.join(traceback.format_exc().split('\n'))
1179 for (key, value) in results.iteritems():
1180 print ' '*2, str(key)+':', value
1182 class ThreadedDict(threadlocal):
1184 Thread local storage.
1186 >>> d = ThreadedDict()
1190 >>> import threading
1191 >>> def f(): d.x = 2
1193 >>> t = threading.Thread(target=f)
1202 ThreadedDict._instances.add(self)
1205 ThreadedDict._instances.remove(self)
1211 """Clears all ThreadedDict instances.
1213 for t in list(ThreadedDict._instances):
1215 clear_all = staticmethod(clear_all)
1217 # Define all these methods to more or less fully emulate dict -- attribute access
1218 # is built into threading.local.
1220 def __getitem__(self, key):
1221 return self.__dict__[key]
1223 def __setitem__(self, key, value):
1224 self.__dict__[key] = value
1226 def __delitem__(self, key):
1227 del self.__dict__[key]
1229 def __contains__(self, key):
1230 return key in self.__dict__
1232 has_key = __contains__
1235 self.__dict__.clear()
1238 return self.__dict__.copy()
1240 def get(self, key, default=None):
1241 return self.__dict__.get(key, default)
1244 return self.__dict__.items()
1246 def iteritems(self):
1247 return self.__dict__.iteritems()
1250 return self.__dict__.keys()
1253 return self.__dict__.iterkeys()
1258 return self.__dict__.values()
1260 def itervalues(self):
1261 return self.__dict__.itervalues()
1263 def pop(self, key, *args):
1264 return self.__dict__.pop(key, *args)
1267 return self.__dict__.popitem()
1269 def setdefault(self, key, default=None):
1270 return self.__dict__.setdefault(key, default)
1272 def update(self, *args, **kwargs):
1273 self.__dict__.update(*args, **kwargs)
1276 return '<ThreadedDict %r>' % self.__dict__
1280 threadeddict = ThreadedDict
1282 def autoassign(self, locals):
1284 Automatically assigns local variables to `self`.
1286 >>> self = storage()
1287 >>> autoassign(self, dict(a=1, b=2))
1289 <Storage {'a': 1, 'b': 2}>
1291 Generally used in `__init__` methods, as in:
1293 def __init__(self, foo, bar, baz=1): autoassign(self, locals())
1295 for (key, value) in locals.iteritems():
1298 setattr(self, key, value)
1302 Converts an integer to base 36 (a useful scheme for human-sayable IDs).
1308 >>> int(to36(939387374), 36)
1313 Traceback (most recent call last):
1315 ValueError: must supply a positive integer
1318 if q < 0: raise ValueError, "must supply a positive integer"
1319 letters = "0123456789abcdefghijklmnopqrstuvwxyz"
1322 q, r = divmod(q, 36)
1323 converted.insert(0, letters[r])
1324 return "".join(converted) or '0'
1327 r_url = re_compile('(?<!\()(http://(\S+))')
1328 def safemarkdown(text):
1330 Converts text to HTML following the rules of Markdown, but blocking any
1331 outside HTML input, so that only the things supported by Markdown
1332 can be used. Also converts raw URLs to links.
1334 (requires [markdown.py](http://webpy.org/markdown.py))
1336 from markdown import markdown
1338 text = text.replace('<', '<')
1339 # TODO: automatically get page title?
1340 text = r_url.sub(r'<\1>', text)
1341 text = markdown(text)
1344 def sendmail(from_address, to_address, subject, message, headers=None, **kw):
1346 Sends the email message `message` with mail and envelope headers
1347 for from `from_address_` to `to_address` with `subject`.
1348 Additional email headers can be specified with the dictionary
1351 Optionally cc, bcc and attachments can be specified as keyword arguments.
1352 Attachments must be an iterable and each attachment can be either a
1353 filename or a file object or a dictionary with filename, content and
1354 optionally content_type keys.
1356 If `web.config.smtp_server` is set, it will send the message
1357 to that SMTP server. Otherwise it will look for
1358 `/usr/sbin/sendmail`, the typical location for the sendmail-style
1359 binary. To use sendmail from a different path, set `web.config.sendmail_path`.
1361 attachments = kw.pop("attachments", [])
1362 mail = _EmailMessage(from_address, to_address, subject, message, headers, **kw)
1364 for a in attachments:
1365 if isinstance(a, dict):
1366 mail.attach(a['filename'], a['content'], a.get('content_type'))
1367 elif hasattr(a, 'read'): # file
1368 filename = os.path.basename(getattr(a, "name", ""))
1369 content_type = getattr(a, 'content_type', None)
1370 mail.attach(filename, a.read(), content_type)
1371 elif isinstance(a, basestring):
1375 filename = os.path.basename(a)
1376 mail.attach(filename, content, None)
1378 raise ValueError, "Invalid attachment: %s" % repr(a)
1382 class _EmailMessage:
1383 def __init__(self, from_address, to_address, subject, message, headers=None, **kw):
1385 if not isinstance(x, list):
1388 return [safestr(a) for a in x]
1390 subject = safestr(subject)
1391 message = safestr(message)
1393 from_address = safestr(from_address)
1394 to_address = listify(to_address)
1395 cc = listify(kw.get('cc', []))
1396 bcc = listify(kw.get('bcc', []))
1397 recipients = to_address + cc + bcc
1400 self.from_address = email.Utils.parseaddr(from_address)[1]
1401 self.recipients = [email.Utils.parseaddr(r)[1] for r in recipients]
1403 self.headers = dictadd({
1404 'From': from_address,
1405 'To': ", ".join(to_address),
1410 self.headers['Cc'] = ", ".join(cc)
1412 self.message = self.new_message()
1413 self.message.add_header("Content-Transfer-Encoding", "7bit")
1414 self.message.add_header("Content-Disposition", "inline")
1415 self.message.add_header("MIME-Version", "1.0")
1416 self.message.set_payload(message, 'utf-8')
1417 self.multipart = False
1419 def new_message(self):
1420 from email.Message import Message
1423 def attach(self, filename, content, content_type=None):
1424 if not self.multipart:
1425 msg = self.new_message()
1426 msg.add_header("Content-Type", "multipart/mixed")
1427 msg.attach(self.message)
1429 self.multipart = True
1433 from email import encoders
1435 from email import Encoders as encoders
1437 content_type = content_type or mimetypes.guess_type(filename)[0] or "applcation/octet-stream"
1439 msg = self.new_message()
1440 msg.set_payload(content)
1441 msg.add_header('Content-Type', content_type)
1442 msg.add_header('Content-Disposition', 'attachment', filename=filename)
1444 if not content_type.startswith("text/"):
1445 encoders.encode_base64(msg)
1447 self.message.attach(msg)
1449 def prepare_message(self):
1450 for k, v in self.headers.iteritems():
1451 if k.lower() == "content-type":
1452 self.message.set_type(v)
1454 self.message.add_header(k, v)
1462 webapi = Storage(config=Storage())
1464 self.prepare_message()
1465 message_text = self.message.as_string()
1467 if webapi.config.get('smtp_server'):
1468 server = webapi.config.get('smtp_server')
1469 port = webapi.config.get('smtp_port', 0)
1470 username = webapi.config.get('smtp_username')
1471 password = webapi.config.get('smtp_password')
1472 debug_level = webapi.config.get('smtp_debuglevel', None)
1473 starttls = webapi.config.get('smtp_starttls', False)
1476 smtpserver = smtplib.SMTP(server, port)
1479 smtpserver.set_debuglevel(debug_level)
1483 smtpserver.starttls()
1486 if username and password:
1487 smtpserver.login(username, password)
1489 smtpserver.sendmail(self.from_address, self.recipients, message_text)
1491 elif webapi.config.get('email_engine') == 'aws':
1493 c = boto.ses.SESConnection(
1494 aws_access_key_id=webapi.config.get('aws_access_key_id'),
1495 aws_secret_access_key=web.api.config.get('aws_secret_access_key'))
1496 c.send_raw_email(self.from_address, message_text, self.from_recipients)
1498 sendmail = webapi.config.get('sendmail_path', '/usr/sbin/sendmail')
1500 assert not self.from_address.startswith('-'), 'security'
1501 for r in self.recipients:
1502 assert not r.startswith('-'), 'security'
1504 cmd = [sendmail, '-f', self.from_address] + self.recipients
1507 p = subprocess.Popen(cmd, stdin=subprocess.PIPE)
1508 p.stdin.write(message_text)
1512 i, o = os.popen2(cmd)
1519 return "<EmailMessage>"
1522 return self.message.as_string()
1524 if __name__ == "__main__":