I needed to link some page explaining what inlineCallbacks and
deferredGenerator are, and ... failed to find one. So here is a
short explanation and the example code (working scripts).
Code size comparison: raw deferreds - 29 lines,
deferredGenerator - 26 lines, inlineCallbacks - 17 lines (non-empty lines, lookup function measured).
inlineCallbacks and deferredGenerator are syntax glues
which make Twisted programs far easier to read and write.
Thanks to both, one can avoid creating the callback functions
explicite and handle exceptions in natural manner.
Below I wrote 3 versions of the simple (but working) script. Only the
lookup function body is different (and this function is expected to
make some HTTP queries and return deferred fired with the info
extracted from results).
Raw deferreds
from twisted.internet import reactor, defer
from twisted.web.client import getPage
import re
def lookup(country, search_term):
main_d = defer.Deferred()
def first_step():
query = "http://www.google.%s/search?q=%s" % (country,search_term)
d = getPage(query)
d.addCallback(second_step, country)
d.addErrback(failure, country)
def second_step(content, country):
m = re.search('<div id="?res.*?href="(?P<url>http://[^"]+)"',
content, re.DOTALL)
if not m:
main_d.callback(None)
return
url = m.group('url')
d = getPage(url)
d.addCallback(third_step, country, url)
d.addErrback(failure, country)
def third_step(content, country, url):
m = re.search("<title>(.*?)</title>", content)
if m:
title = m.group(1)
main_d.callback(dict(url = url, title = title))
else:
main_d.callback(dict(url=url, title="{not-specified}"))
def failure(e, country):
print ".%s FAILED: %s" % (country, str(e))
main_d.callback(None)
first_step()
return main_d
def printResult(result, country):
if result:
print ".%s result: %s (%s)" % (country, result['url'], result['title'])
else:
print ".%s result: nothing found" % country
def runme():
all = []
for country in ["com", "pl", "nonexistant"]:
d = lookup(country, "Twisted")
d.addCallback(printResult, country)
all.append(d)
defer.DeferredList(all).addCallback(lambda _: reactor.stop())
reactor.callLater(0, runme)
reactor.run()
Note: it is of course not necessary to pass country as callback
parameter, I left it as an example of how does this work.
Deferred generators
Note: Python 2.4 required
General idea:
@defer.deferredGenerator
def someFunction():
a = 1
x = defer.waitForDeferred(
deferredReturningFunction(a))
yield x
b = x.getResult()
x = defer.waitForDeferred(
anotherDeferredReturningFunction(a, b))
yield x
c = x.getResult()
yield c
Our script:
from twisted.internet import reactor, defer
from twisted.web.client import getPage
import re
@defer.deferredGenerator
def lookup(country, search_term):
try:
d = defer.waitForDeferred(
getPage("http://www.google.%s/search?q=%s" % (country,search_term)))
yield d
content = d.getResult()
m = re.search('<div id="?res.*?href="(?P<url>http://[^"]+)"',
content, re.DOTALL)
if not m:
yield None
return
url = m.group('url')
d = defer.waitForDeferred(
getPage(url))
yield d
content = d.getResult()
m = re.search("<title>(.*?)</title>", content)
if m:
title = m.group(1)
yield dict(url=url, title=title)
return
else:
yield dict(url=url, title="{not-specified}")
return
except Exception, e:
print ".%s FAILED: %s" % (country, str(e))
def printResult(result, country):
if result:
print ".%s result: %s (%s)" % (country, result['url'], result['title'])
else:
print ".%s result: nothing found" % country
def runme():
all = []
for country in ["com", "pl", "nonexistant"]:
d = lookup(country, "Twisted")
d.addCallback(printResult, country)
all.append(d)
defer.DeferredList(all).addCallback(lambda _: reactor.stop())
reactor.callLater(0, runme)
reactor.run()
Inline callbacks
Note: Python 2.5 required
General idea:
@defer.inlineCallbacks
def someFunction():
a = 1
b = yield deferredReturningFunction(a)
c = yield anotherDeferredReturningFunction(a, b)
defer.returnValue(c)
Our script:
from twisted.internet import reactor, defer
from twisted.web.client import getPage
import re
@defer.inlineCallbacks
def lookup(country, search_term):
try:
query = "http://www.google.%s/search?q=%s" % (country,search_term)
content = yield getPage(query)
m = re.search('<div id="?res.*?href="(?P<url>http://[^"]+)"',
content, re.DOTALL)
if not m:
defer.returnValue(None)
url = m.group('url')
content = yield getPage(url)
m = re.search("<title>(.*?)</title>", content)
if m:
defer.returnValue(dict(url=url, title=m.group(1)))
else:
defer.returnValue(dict(url=url, title="{not-specified}"))
except Exception, e:
print ".%s FAILED: %s" % (country, str(e))
def printResult(result, country):
if result:
print ".%s result: %s (%s)" % (country, result['url'], result['title'])
else:
print ".%s result: nothing found" % country
def runme():
all = []
for country in ["com", "pl", "nonexistant"]:
d = lookup(country, "Twisted")
d.addCallback(printResult, country)
all.append(d)
defer.DeferredList(all).addCallback(lambda _: reactor.stop())
reactor.callLater(0, runme)
reactor.run()
Additional remarks
After I posted this article, Jan-Paul Calderone (one of the Twisted core developers) offered his, more idiomatic version of the raw deferred example. Here it is (I fixed two small typos):
def lookup(country, search_term):
query = "http://www.google.%s/search?q=%s" % (country,search_term)
d = getPage(query)
def cbFirstPage(content, country):
m = re.search('<div id="?res.*?href="(?P<url>http://[^"]+)"',
content, re.DOTALL)
if not m:
return None
url = m.group('url')
d = getPage(url)
def cbSecondPage(content, country, url):
m = re.search("<title>(.*?)</title>", content)
if m:
title = m.group(1)
return dict(url = url, title = title)
else:
return dict(url=url, title="{not-specified}")
d.addCallback(cbSecondPage, country, url)
return d
d.addCallback(cbFirstPage, country)
def ebGeneric(err, country):
print ".%s FAILED: %s" % (country, str(err))
d.addErrback(ebGeneric, country)
return d