Twisted inlineCallbacks and deferredGenerator

inlineCallbacks and deferredGenerator are syntax glues which make Twisted programs far easier to read and write. Thanks to both, one can avoid creating the callback functions explicite and handle exceptions in natural manner.

Below I wrote 3 versions of the simple (but working) script. Only the lookup function body is different (and this function is expected to make some HTTP queries and return deferred fired with the info extracted from results).

Raw deferreds

from twisted.internet import reactor, defer
from twisted.web.client import getPage
import re

def lookup(country, search_term):
    main_d = defer.Deferred()

    def first_step():
        query = "http://www.google.%s/search?q=%s" % (country,search_term)
        d = getPage(query)
        d.addCallback(second_step, country)
        d.addErrback(failure, country)

    def second_step(content, country):
        m = re.search('<div id="?res.*?href="(?P<url>http://[^"]+)"', 
                      content, re.DOTALL)
        if not m:
            main_d.callback(None)
            return
        url = m.group('url')
        d = getPage(url)
        d.addCallback(third_step, country, url)
        d.addErrback(failure, country)

    def third_step(content, country, url):
        m = re.search("", content)
        if m:
            title = m.group(1)
            main_d.callback(dict(url = url, title = title))
        else:
            main_d.callback(dict(url=url, title="{not-specified}"))

    def failure(e, country):
        print ".%s FAILED: %s" % (country, str(e))
        main_d.callback(None)

    first_step()
    return main_d

def printResult(result, country):
    if result:
        print ".%s result: %s (%s)" % (country, result['url'], result['title'])
    else:
        print ".%s result: nothing found" % country

def runme():
    all = []
    for country in ["com", "pl", "nonexistant"]:
        d = lookup(country, "Twisted")
        d.addCallback(printResult, country)
        all.append(d)
    defer.DeferredList(all).addCallback(lambda _: reactor.stop())

reactor.callLater(0, runme)
reactor.run()

Note: it is of course not necessary to pass country as callback parameter, I left it as an example of how does this work.

Deferred generators

Note: Python 2.4 required

General idea:

@defer.deferredGenerator
def someFunction():
    a = 1
    x = defer.waitForDeferred(
           deferredReturningFunction(a))
    yield x
    b = x.getResult()
    x = defer.waitForDeferred(
           anotherDeferredReturningFunction(a, b))
    yield x
    c = x.getResult()
    yield c

Our script:

from twisted.internet import reactor, defer
from twisted.web.client import getPage
import re

@defer.deferredGenerator
def lookup(country, search_term):
    try:
        d = defer.waitForDeferred(
            getPage("http://www.google.%s/search?q=%s" % (country,search_term)))
        yield d
        content = d.getResult()

        m = re.search('<div id="?res.*?href="(?P<url>http://[^"]+)"', 
                      content, re.DOTALL)
        if not m:
            yield None
            return

        url = m.group('url')
        d = defer.waitForDeferred(
            getPage(url))
        yield d
        content = d.getResult()

        m = re.search("", content)
        if m:
            title = m.group(1)
            yield dict(url=url, title=title)
            return
        else:
            yield dict(url=url, title="{not-specified}")
            return
    except Exception, e:
        print ".%s FAILED: %s" % (country, str(e))

def printResult(result, country):
    if result:
        print ".%s result: %s (%s)" % (country, result['url'], result['title'])
    else:
        print ".%s result: nothing found" % country

def runme():
    all = []
    for country in ["com", "pl", "nonexistant"]:
        d = lookup(country, "Twisted")
        d.addCallback(printResult, country)
        all.append(d)
    defer.DeferredList(all).addCallback(lambda _: reactor.stop())

reactor.callLater(0, runme)
reactor.run()

Inline callbacks

Note: Python 2.5 required

General idea:

@defer.inlineCallbacks
def someFunction():
    a = 1
    b = yield deferredReturningFunction(a)
    c = yield anotherDeferredReturningFunction(a, b)
    defer.returnValue(c)

Our script:

from twisted.internet import reactor, defer
from twisted.web.client import getPage
import re

@defer.inlineCallbacks
def lookup(country, search_term):
    try:
        query = "http://www.google.%s/search?q=%s" % (country,search_term)
        content = yield getPage(query)

        m = re.search('<div id="?res.*?href="(?P<url>http://[^"]+)"', 
                      content, re.DOTALL)
        if not m:
            defer.returnValue(None)

        url = m.group('url')
        content = yield getPage(url)

        m = re.search("", content)
        if m:
            defer.returnValue(dict(url=url, title=m.group(1)))
        else:
            defer.returnValue(dict(url=url, title="{not-specified}"))

    except Exception, e:
        print ".%s FAILED: %s" % (country, str(e))

def printResult(result, country):
    if result:
        print ".%s result: %s (%s)" % (country, result['url'], result['title'])
    else:
        print ".%s result: nothing found" % country

def runme():
    all = []
    for country in ["com", "pl", "nonexistant"]:
        d = lookup(country, "Twisted")
        d.addCallback(printResult, country)
        all.append(d)
    defer.DeferredList(all).addCallback(lambda _: reactor.stop())

reactor.callLater(0, runme)
reactor.run()

Additional remarks

After I posted this article, Jan-Paul Calderone (one of the Twisted core developers) offered his, more idiomatic version of the raw deferred example. Here it is (I fixed two small typos):

def lookup(country, search_term):
    query = "http://www.google.%s/search?q=%s" % (country,search_term)
    d = getPage(query)
    def cbFirstPage(content, country):
        m = re.search('<div id="?res.*?href="(?P<url>http://[^"]+)"',
                      content, re.DOTALL)
        if not m:
            return None
        url = m.group('url')
        d = getPage(url)
        def cbSecondPage(content, country, url):
            m = re.search("", content)
            if m:
                title = m.group(1)
                return dict(url = url, title = title)
            else:
                return dict(url=url, title="{not-specified}")
        d.addCallback(cbSecondPage, country, url)
        return d
    d.addCallback(cbFirstPage, country)
    def ebGeneric(err, country):
        print ".%s FAILED: %s" % (country, str(err))
    d.addErrback(ebGeneric, country)
    return d