lsb_master.cfg

# -*- python -*-
# ex: set syntax=python:

import os

from buildbot import scheduler, locks, revlinks
from buildbot.buildslave import BuildSlave
from buildbot.process import factory
from buildbot.steps import trigger
from buildbot.steps.shell import ShellCommand, Configure, Compile
from buildbot.steps.master import MasterShellCommand
from buildbot.steps.source.bzr import Bzr
from buildbot.steps.transfer import FileUpload
from buildbot.process.properties import WithProperties
from buildbot.status import html
from buildbot.status import words
from buildbot.status.web.auth import HTPasswdAuth
from buildbot.changes import pb

import lfbuildbot

# ForceSchedulers are new for 0.8.6; don't fail if it's not present.
try:
    from buildbot.schedulers.forcesched import ForceScheduler
except ImportError:
    ForceScheduler = None

# This buildbot config file defines the LSB builds that are needed.
# 
# Each project needs one build per architecture; this includes the
# "sdk" meta-project and all the other pieces we build.  So, to
# prevent cut-n-pasting, we use conventions.  All architectures are
# defined in an architectures list, and builds are named
# "project-arch".  Schedulers for these are named "sch-project-arch".
# For the special "sdk" project, the job's name is "build-sdk-arch".
# Its scheduler is prefixed with "sch-", like the others.

# Path where buildbot lives on the slaves.

buildbot_slave_path = "/opt/buildbot"

# Toplevel configuration, used by the builder/scheduler setup.

web_htpasswd_path = os.path.join(buildbot_slave_path, "htpasswd")
bzr_toplevel = "http://bzr.linuxfoundation.org/"

# Timers and timeouts for builds.  We define them as hours and minutes
# here, and translate them into seconds below for buildbot's benefit.
stable_timer_hours = 0
stable_timer_minutes = 1

slow_stable_timer_hours = 6
slow_stable_timer_minutes = 0

bzr_timeout_hours = 4
bzr_timeout_minutes = 0

build_timeout_hours = 4
build_timeout_minutes = 0

# Supported architectures.
lsb_archs = ["x86_64", "x86", "ia64", "ppc32", "ppc64", "s390", "s390x"]

# This funny-sounding setting controls which architecture we hold
# responsible for building architecture-independent components.
lsb_buildslave_arch_indep_arch = "x86_64"

# Some architectures, for whatever reason, need to use as few resources
# as possible while still allowing it to be supported.  We don't build
# these archs on every version control change, and we may limit builds
# in other ways as well.  Note that the arch independent arch, defined
# immediately above, CANNOT be part of this list.
low_resource_archs = []

# Some architectures are served by multiple build slaves; these are
# listed here.
multi_slave_archs = {}

# Extra projects to build as part of the SDK project.
lsb_bzr_sdk_projects = ["lsbdev-c++", "lsbdev-qt", "lsbdev-qt3"]

# "Package subdir" projects.  These have a "package" directory off the
# source root, which has a Makefile which acts a certain way.  See any
# of the current projects' package/Makefile for an example.  We declare
# each project here as the key to the dict, and any extra repositories
# needed during the build in the array value.
lsb_pkg_subdir_projects = { "app-checker": [],
                            "azov-qt4-tests": ["t2c-harness"],
                            "azov-xml2-tests": ["t2c-harness"],
                            "desktop-test": ["xml-test", "misc-test"],
                            "distribution-checker": [],
                            "libstdcpp-test": [],
                            "lsb-setup": [],
                            "lsb-xvfb": [],
                            "lsb-sigchk": [],
                            "lsbappchk-perl": [],
                            "lsbappchk-python": ["misc-test"],
                            "lsbappchk-sh": [],
                            "makelsbpkg": [],
                            "misc-test": [],
                            "olver-core-tests": [],
                            "perl-test": [],
                            "printing-test": ["runtime-test"],
                            "python-test": ["misc-test"],
                            "qmtest-harness": [],
                            "runtime-test": [],
                            "t2c-alsa-tests": ["t2c-harness"],
                            "t2c-cpp-tests": ["t2c-harness"],
                            "t2c-desktop-tests": ["t2c-harness"],
                            "t2c-runtime-tests": ["t2c-harness"],
                            "task-pkgs": [],
                            "tet-harness": [],
                            "xts5-test": ["vsw4-test"] }

# "Packaging" projects.  These are built out of a tree called "packaging".
# The name of the project in packaging is the first value of each tuple,
# while the repository used to build it is the second.
lsb_packaging_projects = []

lsb_arch_indep_projects = ["lsb-setup", "lsbappchk-perl", "lsbappchk-python", 
                           "lsbappchk-sh", "perl-test", "task-pkgs"]

# Dependencies.  The key is the package being built, and the values are
# the list of packages which it needs to build.  For example, if you need
# "foo" to build "bar", you'd do "bar": ["foo"].  Dependencies cannot be
# recursive, as this could cause builder deadlocks.  Each dependency can
# be just the name of a project, or a 2-tuple with the name of the project
# and the packages to actually install in a list; each package name is
# used in a "<name>*.rpm" search to find the packages.
lsb_dependencies = { "appbat": ["tet-harness", "libbat",
                                ("misc-test", ["lsb-appchk"])],
                     "libstdcpp-test": ["qmtest-harness"],
                     "azov-qt4-tests": ["tet-harness"],
                     "azov-xml2-tests": ["tet-harness"],
                     "desktop-test": ["tet-harness"],
                     "printing-test": ["tet-harness"],
                     "runtime-test": ["tet-harness"],
                     "t2c-alsa-tests": ["tet-harness"],
                     "t2c-cpp-tests": ["tet-harness"],
                     "t2c-desktop-tests": ["tet-harness"],
                     "t2c-runtime-tests": ["tet-harness"],
                     "xts5-test": ["tet-harness"] }

# Job locking.  We don't limit the number of concurrent jobs per
# slave in the slave config; instead, we only allow one build at a time
# for builds not triggered via dependencies.
nondep_job_lock = locks.SlaveLock("nondep_builds", maxCount = 1)
dep_job_lock = locks.SlaveLock("dep_builds", maxCount = 1)

# Lock for all package installs.
pkg_lock = locks.SlaveLock("package_manager", maxCount = 1)

# Per-repo locks.  This handles when multiple projects are built
# out of the same repository.  If any of them are dependencies,
# we risk concurrent builds clobbering each other.  Right now,
# only misc-test works this way.
repo_locks = { "misc-test": locks.SlaveLock("misc-test", maxCount = 1) }

# Branches we watch, besides devel.
branch_watch_list = { "4.0": [ "appbat", "azov-qt4-tests",
                               "desktop-test", "libstdcpp-test", "misc-test",
                               "olver-core-tests", "perl-test", 
                               "printing-test", "python-test", "runtime-test",
                               "t2c-cpp-tests", "t2c-desktop-tests", 
                               "xml-test", "xts5-test" ],
                      "4.1": [ "appbat", "desktop-test",
                               "libstdcpp-test", "misc-test",
                               "olver-core-tests", "perl-test",
                               "printing-test", "python-test", "runtime-test", 
                               "t2c-alsa-tests", "t2c-desktop-tests"] }

# Values calculated from configured variables.
stable_timer_seconds = \
    (stable_timer_hours * 3600) + (stable_timer_minutes * 60)
slow_stable_timer_seconds = \
    (slow_stable_timer_hours * 3600) + (slow_stable_timer_minutes * 60)
bzr_timeout_seconds = \
    (bzr_timeout_hours * 3600) + (bzr_timeout_minutes * 60)
build_timeout_seconds = \
    (build_timeout_hours * 3600) + (build_timeout_minutes * 60)

# "devchk build slaves".  These are build slaves that only build devchk.
# We do this because devchk requires a lot of packages that can interfere
# with clean builds of everything else.  It's also useful to have devchk
# run against a number of different native build environments.  Because
# we use the list of build slaves in a number of places, we keep the
# master list in a separate file.
devchk_build_slaves = \
    [x.strip() 
     for x in open(os.path.join(buildbot_slave_path,
                                "buildbot-config/devchk_build_slave_list"))
     if x]

# Helper functions

def add_pre_dependency_triggers(project, arch, builder):
    pkg_install_command = \
        "ls results/%s*.rpm | egrep -v src.rpm | " + \
        "xargs sudo rpm -Uvh --force"
    remove_old_libbat_command = \
        "rpm -qa --queryformat '%{NAME}\\n' | grep libbat | " + \
        "xargs sudo rpm -e"
    if project in lsb_dependencies:
        for dep_entry in lsb_dependencies[project]:
            # Figure out whether we only need individual pkgs.
            if isinstance(dep_entry, tuple):
                (dep, dep_pkg_list) = dep_entry
            else:
                dep = dep_entry
                dep_pkg_list = [""]

            # First, see if we need this from devel always.
            always_devel = True
            for branch in branch_watch_list.keys():
                if dep in branch_watch_list[branch]:
                    always_devel = False
                    break

            # If this is libbat, add a step to remove all libbat-like
            # packages and force not-devel.
            if dep == "libbat":
                builder.addStep(ShellCommand(command=remove_old_libbat_command,
                                             name="remove-old-libbat",
                                             flunkOnFailure=False,
                                             haltOnFailure=False,
                                             workdir="deps"))
                always_devel = False

            if always_devel:
                unpack_command="tar xzvf ../../saved/devel/%s-%s.tar.gz" % (dep, arch)
            else:
                unpack_command=WithProperties("tar xzvf ../../saved/%%(branch_name:-devel)s/%s-%s.tar.gz" % (dep, arch))
            builder.addStep(ShellCommand(command=unpack_command, 
                                         name="unpack-dep-%s-%s" % (dep, arch),
                                         workdir="deps"))

            for dep_pkg in dep_pkg_list:
                if dep_pkg:
                    dep_pkg_name = dep_pkg
                else:
                    dep_pkg_name = "all"
                builder.addStep(ShellCommand(
                    command=pkg_install_command % dep_pkg,
                    name="install-dep-%s-%s-%s" % (dep, dep_pkg_name, arch),
                    locks=[pkg_lock], workdir="deps", haltOnFailure=True))

            builder.addStep(ShellCommand(
                command=["rm", "-rf", "../deps"],
                name="remove-dep-%s-%s" % (dep, arch), alwaysRun=True))

def add_post_dependency_triggers(project, arch, builder):
    # Figure out if the project is a dependency of something.
    dep_found = False
    for dep_prj in lsb_dependencies:
        dep_prj_deps = [x for x in lsb_dependencies[dep_prj]
                        if not isinstance(x, tuple)] + \
                       [x[0] for x in lsb_dependencies[dep_prj]
                        if isinstance(x, tuple)]
        if project in dep_prj_deps:
            dep_found = True
            break

    # If it is, save its tarball.
    if dep_found:
        prj_name = "%s-%s" % (project, arch)

        builder.addStep(ShellCommand(
            command=["mkdir", "-p",
                     WithProperties("../../saved/%(branch_name:-devel)s")],
            name="create-saved-dep-dir"))
        builder.addStep(ShellCommand(
            command=["rm", "-f", 
                     WithProperties("../../saved/%%(branch_name:-devel)s/%s.tar.gz" % prj_name)],
            name="remove-old-saved-dep-tarball"))
        builder.addStep(ShellCommand(
            command=["cp", "../results.tar.gz",
                     WithProperties("../../saved/%%(branch_name:-devel)s/%s.tar.gz" % prj_name)],
            name="save-dep-tarball"))

def always_pull_from_devel(project):
    # Figure out if the project needs to be built from devel
    # every time.  For now, just use "branches not specific
    # to 4.0" as the rule for this.
    return project not in branch_watch_list["4.0"]


# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

# Priority order.  In general, we don't care what order things build in.
# Builds related to the SDK and dependencies should be built first,
# however, just in case they are missing (on a new build slave, for
# example).

def prioritize(buildmaster, builders):
    dependencies = [isinstance(x, tuple) and x[0] or x
                    for x in reduce(lambda x, y: x + y[1],
                                    lsb_dependencies.items(), [])]

    priorities = ["build-sdk"] + dependencies

    prioritized_builders = []
    for jobname in priorities:
        for builder in builders[:]:
            if builder.name[:len(jobname)] == jobname:
                builders.remove(builder)
                prioritized_builders.append(builder)

    return prioritized_builders + builders

c['prioritizeBuilders'] = prioritize

####### BUILDSLAVES

# the 'slaves' list defines the set of allowable buildslaves. Each element is
# a tuple of bot-name and bot-password. These correspond to values given to
# the buildslave's mktap invocation.

c['slaves'] = []
slave_password_file = open(os.path.join(buildbot_slave_path, "slave_pwds"))
for line in slave_password_file:
    (user, pw) = line.strip().split(":", 1)
    c['slaves'].append(BuildSlave(user, pw, max_builds=1))
slave_password_file.close()

# to limit to two concurrent builds on a slave, use
#  c['slaves'] = [BuildSlave("bot1name", "bot1passwd", max_builds=2)]


# 'slavePortnum' defines the TCP port to listen on. This must match the value
# configured into the buildslaves (with their --master option)

c['slavePortnum'] = 9989


####### CHANGESOURCES

# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes. Any class which implements IChangeSource can be
# put here: there are several in buildbot/changes/*.py to choose from.

# For the LSB builder, we let bzr push change notifications via email.
# Thus, the only change source we need is the email parser, as well as a
# branch map.

branch_map = {}
lsb_change_sources = lsb_pkg_subdir_projects.keys() + \
    [x[1] for x in lsb_packaging_projects if x[1] is not None] + \
    ["build_env", "appbat", "devchk"]
added = []
for project in lsb_change_sources:
    if project in added:
        continue
    devel_path = "lsb/devel/" + project
    branch_map["/var/www/bzr/" + devel_path] = devel_path
    for branch in branch_watch_list:
        if project in branch_watch_list[branch]:
            branch_path = "lsb/%s/%s" % (branch, project)
            branch_map["/var/www/bzr/" + branch_path] = branch_path
    added.append(project)

c['change_source'] = lfbuildbot.BzrLsbMaildirSource(
    os.path.join(buildbot_slave_path, "Maildir"), branchMap=branch_map)


####### SCHEDULERS

## configure the Schedulers

# LSB schedulers

# Build the SDK daily.  Since it's not complicated, we can schedule
# all the builds with a single scheduler.
# XXX: disable for now; build the SDK like everything else.

sch_sdk_names = ["build-sdk-" + x for x in lsb_archs]
#sch_build = scheduler.Nightly(name="build-sdk", branch=None,
#                              builderNames=sch_sdk_names, hour=19, minute=0)

# Since we have to report a list of all builders to several schedulers,
# create a list of all projects to pass in.

all_builder_list = sch_sdk_names[:]

arch_prj_list = lsb_pkg_subdir_projects.keys() + \
    ["appbat", "libbat"] + [x[0] for x in lsb_packaging_projects]

for prj in arch_prj_list:
    if prj in lsb_arch_indep_projects:
        all_builder_list.append("%s-%s" % (prj, 
                                           lsb_buildslave_arch_indep_arch))
    else:
        for arch in lsb_archs:
            all_builder_list.append("%s-%s" % (prj, arch))

for devchk_job in ["devchk", "build-sdk"]:
    all_builder_list.extend([devchk_job + "-" + x 
                             for x in devchk_build_slaves])

# ForceScheduler, for forcing builds via one of the status targets.

if ForceScheduler:
    sch_force = ForceScheduler(name="force", builderNames=all_builder_list)

# Accept multiple jobs via a job directory.

jobdir_repos = dict(lsb_packaging_projects)
jobdir_repos["build-sdk"] = "build_env"
jobdir_repos["libbat"] = "appbat"

sch_jobdir = lfbuildbot.MultiScheduler(
    name="sch_jobdir", builderNames=all_builder_list, 
    jobdir=os.path.join(buildbot_slave_path, "jobdir"),
    repos=jobdir_repos, archs=lsb_archs, 
    indep_prj=lsb_arch_indep_projects,
    indep_arch=lsb_buildslave_arch_indep_arch,
    devchk_builders=devchk_build_slaves)

# Schedulers for devchk.  The builders and build slaves are entirely
# separate from the rest of the builders and slaves, and so need their
# own scheduler.

devchk_builder_list = ["devchk-" + x for x in devchk_build_slaves]
sch_devchk = scheduler.Scheduler(name="sch-devchk",
                                 branch="lsb/devel/devchk",
                                 builderNames=devchk_builder_list,
                                 treeStableTimer=stable_timer_seconds)

# SDK schedulers.  We use a different timeout for the low-resource
# architectures, and thus a different scheduler.

sdk_builder_list = ["build-sdk-" + x
                    for x in lsb_archs + devchk_build_slaves
                    if x not in low_resource_archs]
sch_sdk = scheduler.Scheduler(name="sch-build-sdk",
                              branch="lsb/devel/build_env",
                              builderNames=sdk_builder_list,
                              treeStableTimer=stable_timer_seconds)

if low_resource_archs:
    low_sdk_builder_list = ["build-sdk-" + x for x in low_resource_archs]
    sch_sdk_low = scheduler.Scheduler(name="sch-build-sdk-low",
                                      branch="lsb/devel/build_env",
                                      builderNames=low_sdk_builder_list,
                                      treeStableTimer=slow_stable_timer_seconds)

# Set up first static set of schedulers.

c['schedulers'] = [sch_jobdir, sch_devchk, sch_sdk]
if low_resource_archs:
    c['schedulers'].append(sch_sdk_low)
if ForceScheduler:
    c['schedulers'].append(sch_force)

# The rest of the schedulers (except the weekly one) react to events, 
# such as a previous build succeeding or a change in version control,
# and so have to be defined per project and per architecture.  First,
# "strange" builders, such as appbat and libbat:

for arch in lsb_archs:
    if arch not in low_resource_archs:
        sch_appbat = scheduler.Scheduler(
            name="sch-appbat-" + arch, branch="lsb/devel/appbat",
            builderNames=["libbat-" + arch, "appbat-" + arch], 
            treeStableTimer=slow_stable_timer_seconds)

        c['schedulers'].append(sch_appbat)

# Package subdir schedulers.

for prj in lsb_pkg_subdir_projects:
    if prj in lsb_arch_indep_projects:
        archlist = [lsb_buildslave_arch_indep_arch]
    else:
        archlist = lsb_archs
    for arch in archlist:
        prj_name = "%s-%s" % (prj, arch)

        if arch not in low_resource_archs:
            sch = scheduler.Scheduler(name="sch-" + prj_name,
                                      branch=os.path.join("lsb/devel", prj),
                                      builderNames=[prj_name],
                                      treeStableTimer=stable_timer_seconds)
            c['schedulers'].append(sch)

            for branch in branch_watch_list:
                if prj in branch_watch_list[branch]:
                    sch = scheduler.Scheduler(
                        name="sch-%s-%s" % (prj_name, branch),
                        branch=os.path.join("lsb", branch, prj),
                        builderNames=[prj_name],
                        treeStableTimer=stable_timer_seconds)
                    c['schedulers'].append(sch)

# Packaging project schedulers.

for (prj, repo) in lsb_packaging_projects:
    if prj in lsb_arch_indep_projects:
        archlist = [lsb_buildslave_arch_indep_arch]
    else:
        archlist = [x for x in lsb_archs if x not in low_resource_archs]
    branches = ["lsb/devel/packaging"]
    if repo is not None:
        branches.append(os.path.join("lsb/devel", repo))
    for arch in archlist:
        prj_name = "%s-%s" % (prj, arch)
        sch = \
            scheduler.AnyBranchScheduler(name="sch-" + prj_name,
                                         branches=branches,
                                         builderNames=[prj_name],
                                         treeStableTimer=stable_timer_seconds)
        c['schedulers'].append(sch)


####### BUILDERS

# the 'builders' list defines the Builders. Each one is configured with a
# dictionary, using the following keys:
#  name (required): the name used to describe this bilder
#  slavename (required): which slave to use, must appear in c['bots']
#  builddir (required): which subdirectory to run the builder in
#  factory (required): a BuildFactory to define how the build is run
#  periodicBuildTime (optional): if set, force a build every N seconds

# buildbot/process/factory.py provides several BuildFactory classes you can
# start with, which implement build processes for common targets (GNU
# autoconf projects, CPAN perl modules, etc). The factory.BuildFactory is the
# base class, and is configured with a series of BuildSteps. When the build
# is run, the appropriate buildslave is told to execute each Step in turn.

# the first BuildStep is typically responsible for obtaining a copy of the
# sources. There are source-obtaining Steps in buildbot/process/step.py for
# CVS, SVN, and others.

#cvsroot = ":pserver:anonymous@cvs.sourceforge.net:/cvsroot/buildbot"
#cvsmodule = "buildbot"

#from buildbot.process import factory
#from buildbot.steps.source import CVS
#from buildbot.steps.shell import Compile
#from buildbot.steps.python_twisted import Trial
#f1 = factory.BuildFactory()
#f1.addStep(CVS(cvsroot=cvsroot, cvsmodule=cvsmodule, login="", mode="copy"))
#f1.addStep(Compile(command=["python", "./setup.py", "build"]))
#f1.addStep(Trial(testpath="."))

#b1 = {'name': "buildbot-full",
#      'slavename': "bot1name",
#      'builddir': "full",
#      'factory': f1,
#      }
#c['builders'] = [b1]

# LSB builds

lsb_builds = []
indep_added = []

# First, let's set up the SDK builds.  We do this separately because
# every build slave (regular and devchk) needs a SDK build job.

for slave_id in lsb_archs + devchk_build_slaves:
    if slave_id in multi_slave_archs:
        slave_list = multi_slave_archs[slave_id]
    elif slave_id in lsb_archs:
        slave_list = ["lfbuild-" + slave_id]
    else:
        slave_list = ["devchk-" + slave_id]

    # SDK.  This is unique because it is used to build everything
    # else.  So we have to start with a known good SDK, build the base
    # packages with it, install those base packages, and then build
    # everything else with them.  This results in a "hybrid SDK", so
    # we also have to force buildbot to reinstall the SDK for the next
    # build.

    build_sdk = factory.BuildFactory()
    build_sdk.addStep(ShellCommand(command=["df", "-h"],
                                   name="report-space"))
    build_sdk.addStep(Bzr(baseURL=bzr_toplevel, 
                          defaultBranch="lsb/devel/build_env", 
                          mode="full", method="clobber",
                          workdir="build_env", timeout=bzr_timeout_seconds))
    build_sdk.addStep(ShellCommand(command=["rm", "-rf", "../packaging"],
                                   name="clear-old-packaging",
                                   workdir="build_env"))
    build_sdk.addStep(ShellCommand(
            command=["bzr", "checkout", 
                     os.path.join(bzr_toplevel, "lsb/devel/packaging"),
                     "../packaging"],
            name="checkout-packaging", workdir="build_env", 
            timeout=bzr_timeout_seconds))
    build_sdk.addStep(ShellCommand(command=['reset-sdk'], name="reset-sdk",
                                   workdir="build_env", locks=[pkg_lock]))
    build_sdk.addStep(ShellCommand(command=["rm", "-rf", 
                                            "../sdk-results"],
                                   name="rm-old-results", workdir="build_env"))
    build_sdk.addStep(
        ShellCommand(command=["mkdir", "-p", "../sdk-results/build_env"],
                     name="mkdir_base", workdir="build_env"))
    build_sdk.addStep(
        lfbuildbot.LSBBuildCommand(command="cd package && make clean " + 
                                   "&& make BZRTREES=../..", 
                                   name="build_base", workdir="build_env", 
                                   haltOnFailure=True, makeargs=True,
                                   timeout=build_timeout_seconds))
    build_sdk.addStep(
        ShellCommand(command="cp package/*.rpm ../sdk-results/build_env", 
                     name="copy_base", workdir="build_env"))
    build_sdk.addStep(
        ShellCommand(command="ls package/*.rpm | grep -v src.rpm | xargs sudo rpm -Uvh --force; rm -f /tmp/last_installed_sdk",
                     name="update_base", workdir="build_env"))
    build_sdk.addStep(
        ShellCommand(command="cd package && make clean",
                     name="clean_base", workdir="build_env", alwaysRun=True))
    for sdk_project in lsb_bzr_sdk_projects:
        build_sdk.addStep(ShellCommand(command=['mkdir', '-p', 
                                                '../sdk-results/' + sdk_project],
                                       name="mkdir_" + sdk_project,
                                       workdir="packaging"))
        sdk_command = "cd " + sdk_project + \
            " && make rpm_package BZR_ROOT=../.."
        build_sdk.addStep(
            lfbuildbot.LSBBuildCommand(command=sdk_command,
                                       name="build_" + sdk_project,
                                       workdir="packaging",
                                       haltOnFailure=True, makeargs=True,
                                       timeout=build_timeout_seconds))
        build_sdk.addStep(
            ShellCommand(command="cp %s/*.rpm ../sdk-results/%s" 
                                 % (sdk_project, sdk_project),
                         name="copy_" + sdk_project, workdir="packaging"))
    build_sdk.addStep(ShellCommand(command=['tar', 'czvf', '../sdk-results.tar.gz',
                                            '../sdk-results'],
                                   name="pack-sdk", workdir="build_env"))
    build_sdk.addStep(ShellCommand(
        command=["mkdir", "-p",
                 WithProperties("../../saved/%(result_type:-devel)s")],
        name="create-saved-sdk-dir", workdir="build_env"))
    build_sdk.addStep(ShellCommand(
        command=["rm", "-f", 
                 WithProperties("../../saved/%(result_type:-devel)s/sdk.tar.gz")],
        name="remove-old-saved-sdk-tarball", workdir="build_env"))
    build_sdk.addStep(ShellCommand(
        command=["cp", "../sdk-results.tar.gz",
                 WithProperties("../../saved/%(result_type:-devel)s/sdk.tar.gz")],
        name="save-sdk-tarball", workdir="build_env"))

    if slave_id in lsb_archs:
        build_sdk.addStep(
            FileUpload(slavesrc="../sdk-results.tar.gz", 
                       masterdest=WithProperties("lsb-sdk-%%(result_type:-results)s-%s.tar.gz" % slave_id),
                       workdir="build_env"))

    # A note on locks: we grab both dependent and non-dependent locks
    # for the SDK.  This is in an attempt to avoid the possibility of
    # dependent packages being triggered by version control at the same
    # time as SDK builds, which can be particularly bad as the SDK build
    # always replaces the SDK.

    lsb_builds.append(
        { 'name': 'build-sdk-' + slave_id,
          'slavenames': slave_list,
          'builddir': 'build-sdk-' + slave_id, 'factory': build_sdk, 
          'category': 'lsb', 
          'locks': [nondep_job_lock.access('counting'), 
                    dep_job_lock.access('counting')] })

# Now set up the builders for the traditional build slaves.

for arch in lsb_archs:
    # "Package subdir" projects.  These have a "package" directory in
    # them; to build their packages, you go to the package directory
    # and type "make".  (Well, more than that; see the custom builder
    # class we use.)

    for prj in lsb_pkg_subdir_projects:
        if prj in lsb_arch_indep_projects:
            if prj in indep_added:
                continue
            else:
                buildslaves = ["lfbuild-" + lsb_buildslave_arch_indep_arch]
                build_arch = lsb_buildslave_arch_indep_arch
                indep_added.append(prj)
        elif arch in multi_slave_archs:
            buildslaves = multi_slave_archs[arch]
            build_arch = arch
        else:
            buildslaves = ["lfbuild-" + arch]
            build_arch = arch

        b = factory.BuildFactory()
        b.addStep(ShellCommand(command=["df", "-h"],
                               name="report-space"))
        b.addStep(lfbuildbot.LSBReloadSDK(name="reload-sdk", locks=[pkg_lock]))
        add_pre_dependency_triggers(prj, build_arch, b)
        b.addStep(Bzr(baseURL=bzr_toplevel,
                      defaultBranch="lsb/devel/%s" % prj,
                      mode="full", method="clobber",
                      timeout=bzr_timeout_seconds, workdir=prj))
        for other_repo in lsb_pkg_subdir_projects[prj]:
            if always_pull_from_devel(other_repo):
                checkout_url = "%slsb/devel/%s" % (bzr_toplevel, other_repo)
            else:
                checkout_url = "%slsb/%%(branch_name)s/%s" \
                    % (bzr_toplevel, other_repo)

            b.addStep(ShellCommand(command=["rm", "-rf", "../" + other_repo],
                                   name="clear-old-" + other_repo,
                                   workdir=prj))
            b.addStep(ShellCommand(
                    command=["bzr", "checkout", 
                             WithProperties(checkout_url),
                             "../" + other_repo],
                    name="checkout-" + other_repo, timeout=bzr_timeout_seconds,
                    workdir=prj))
        b.addStep(ShellCommand(command=["rm", "-rf", "../results"], 
                               name="clear-old-results", workdir=prj))
        b.addStep(ShellCommand(command=["mkdir", "-p", "../results"],
                               name="create-results-dir", workdir=prj))
        b.addStep(lfbuildbot.LSBBuildPackage(name="build", haltOnFailure=True,
                                             timeout=build_timeout_seconds,
                                             workdir=prj))
        b.addStep(ShellCommand(
                command="find -L package -name '*.rpm' " +
                        "-exec cp '{}' ../results ';'",
                name="copy-results", workdir=prj))
        b.addStep(ShellCommand(command=["tar", "czvf", "../results.tar.gz", 
                                        "../results"],
                               name="pack-results", workdir=prj))
        b.addStep(
            FileUpload(slavesrc="../results.tar.gz", 
                       masterdest=WithProperties("%s-%%(result_type:-results)s-%s.tar.gz" % (prj, build_arch)),
                       name="upload-results", workdir=prj))
        add_post_dependency_triggers(prj, build_arch, b)
        b.addStep(ShellCommand(command=["rm", "-rf", "../results"],
                               name="clear-results-dir", workdir=prj))

        locks = [nondep_job_lock.access('counting')]

        lsb_builds.append(
            { 'name': '%s-%s' % (prj, build_arch), 'slavenames': buildslaves,
              'builddir': "%s-%s" % (prj, build_arch), 'factory': b, 
              'category': 'lsb', 'locks': locks })

    # "Packaging" projects.  These are actually built from a separate
    # bzr tree, called "packaging", which contains the instructions for
    # building those packages.

    for (prj, repo) in lsb_packaging_projects:
        if prj in lsb_arch_indep_projects:
            if prj in indep_added:
                continue
            else:
                buildslaves = ["lfbuild-" + lsb_buildslave_arch_indep_arch]
                build_arch = lsb_buildslave_arch_indep_arch
                indep_added.append(prj)
        elif arch in multi_slave_archs:
            buildslaves = multi_slave_archs[arch]
            build_arch = arch
        else:
            buildslaves = ["lfbuild-" + arch]
            build_arch = arch

        b = factory.BuildFactory()
        b.addStep(ShellCommand(command=["df", "-h"],
                               name="report-space"))
        b.addStep(lfbuildbot.LSBReloadSDK(name="reload-sdk", locks=[pkg_lock]))
        add_pre_dependency_triggers(prj, build_arch, b)
        if repo is not None:
            b.addStep(Bzr(baseURL=bzr_toplevel,
                          defaultBranch="lsb/devel/%s" % repo,
                          workdir=repo, timeout=bzr_timeout_seconds,
                          mode="full", method="clobber"))
        b.addStep(ShellCommand(command=["rm", "-rf", "../packaging"],
                               name="clear-old-packaging"))
        b.addStep(ShellCommand(
                command=["bzr", "checkout", 
                         WithProperties("%slsb/%%(branch_name)s/packaging"
                                        % bzr_toplevel),
                         "../packaging"],
                name="checkout-packaging", timeout=bzr_timeout_seconds))
        b.addStep(ShellCommand(command=["rm", "-rf", "../results"],
                               name="clear-old-results"))
        b.addStep(ShellCommand(command=["mkdir", "-p", "../results"],
                               name="make-results-dir"))
        b.addStep(
            lfbuildbot.LSBBuildFromPackaging(subdir=prj,
                                             env={"BZRTREES": "..",
                                                  "BZR_ROOT": ".."},
                                             name="build", haltOnFailure=True,
                                             timeout=build_timeout_seconds))
        b.addStep(ShellCommand(
                command="find ../packaging -name '*.rpm' " +
                        "-exec cp '{}' ../results ';'",
                name="copy-results"))
        b.addStep(ShellCommand(command=["tar", "czvf", "../results.tar.gz", 
                                        "../results"],
                               name="pack-results"))
        b.addStep(
            FileUpload(slavesrc="../results.tar.gz", 
                       masterdest=WithProperties("%s-%%(result_type:-results)s-%s.tar.gz" % (prj, build_arch)),
                       name="upload-results"))
        add_post_dependency_triggers(prj, build_arch, b)
        b.addStep(ShellCommand(command=["rm", "-rf", "../results"],
                               name="clear-results-dir"))

        locks = [nondep_job_lock.access('counting')]
        if repo in repo_locks:
            locks.append(repo_locks[repo].access('counting'))

        lsb_builds.append(
            { 'name': '%s-%s' % (prj, build_arch), 'slavenames': buildslaves,
              'builddir': "%s-%s" % (prj, build_arch), 'factory': b, 
              'category': 'lsb', 'locks': locks })

    # Weird builds.  These don't conform to one of the patterns, and
    # so have to be custom.

    if arch in multi_slave_archs:
        buildslaves = multi_slave_archs[arch]
    else:
        buildslaves = ["lfbuild-" + arch]

    # Builds based on appbat (appbat and libbat).

    libbat = factory.BuildFactory()

    libbat.addStep(ShellCommand(command=["df", "-h"],
                                name="report-space"))

    libbat.addStep(lfbuildbot.LSBReloadSDK(name="reload-sdk", 
                                           locks=[pkg_lock]))

    add_pre_dependency_triggers("libbat", arch, libbat)

    libbat.addStep(Bzr(baseURL=bzr_toplevel, defaultBranch="lsb/devel/appbat",
                       workdir="appbat", timeout=bzr_timeout_seconds,
                       mode="full", method="clobber"))
    libbat.addStep(ShellCommand(command=["rm", "-rf", "../tools"],
                                workdir="appbat", name="clear-old-nalfs"))
    libbat.addStep(ShellCommand(command=["mkdir", "-p", "../tools"],
                                workdir="appbat", name="create-tools-dir"))
    libbat.addStep(ShellCommand(command=["bzr", "checkout", 
                                         os.path.join(bzr_toplevel, 
                                                      "lsb/devel/nALFS")],
                                workdir="tools", name="checkout-nalfs",
                                timeout=bzr_timeout_seconds))
    libbat.addStep(ShellCommand(command=["rm", "-rf", "../results"], 
                                workdir="appbat", name="clear-old-results"))
    libbat.addStep(ShellCommand(command=["mkdir", "-p", "../results"],
                                workdir="appbat", name="create-results-dir"))
    libbat.addStep(
        ShellCommand(command=["mkdir", "-p",  
                              "%s/tmp/appbat-pkgcache" % buildbot_slave_path],
                     workdir="appbat", name="create-pkgcache"))
    libbat.addStep(
        ShellCommand(command="cp %s/tmp/appbat-pkgcache/* packages"
                     % buildbot_slave_path,
                     workdir="appbat", name="copy-pkgcache",
                     flunkOnFailure=False, haltOnFailure=False))
    libbat.addStep(
        ShellCommand(command="rm -f %s/tmp/appbat-pkgcache/*"
                     % buildbot_slave_path,
                     workdir="appbat", name="clear-pkgcache",
                     flunkOnFailure=False, haltOnFailure=False))
    libbat.addStep(
        lfbuildbot.LSBConfigureAppbat(command=["./configure.libbat"],
                                      workdir="appbat", name="configure"))
    libbat.addStep(ShellCommand(command=["extras/entitycheck.py", "-c",
                                         "--delete-bad"],
                                workdir="appbat", name="check-pkgcache",
                                flunkOnFailure=False, haltOnFailure=False))
    libbat.addStep(ShellCommand(command=["extras/entitycheck.py", "-q", "-f"],
                                workdir="appbat", name="download-pkgs",
                                flunkOnFailure=False, haltOnFailure=False))
    libbat.addStep(
        lfbuildbot.LSBBuildAppbat(command="make libbat ENTITYCHECK_QUIET=-q", 
                                  workdir="appbat", name="build-libbat",
                                  timeout=build_timeout_seconds))
    libbat.addStep(
        lfbuildbot.LSBBuildCommand(command=["make", "check-libbat"],
                                   workdir="appbat",
                                   name="check-libbat", haltOnFailure=True))
    libbat.addStep(
        lfbuildbot.LSBBuildCommand(command="cd rpm && make libbat",
                                   name="build-pkg", workdir="appbat"))
    libbat.addStep(ShellCommand(
            command="find rpm -name '*.rpm' -exec cp '{}' ../results ';'",
            workdir="appbat", name="copy-results"))
    libbat.addStep(ShellCommand(command=["tar", "czvf", "../results.tar.gz", 
                                         "../results"],
                                workdir="appbat", name="pack-results"))
    libbat.addStep(
        FileUpload(slavesrc="../results.tar.gz", 
                   masterdest=WithProperties("libbat-%%(result_type:-results)s-%s.tar.gz" % arch),
                   workdir="appbat", name="upload-results"))
    libbat.addStep(
        ShellCommand(command="cp packages/*tar* %s/tmp/appbat-pkgcache"
                     % buildbot_slave_path,
                     workdir="appbat", name="save-pkgcache",
                     flunkOnFailure=False, haltOnFailure=False,
                     alwaysRun=True))
    libbat.addStep(ShellCommand(command=["make", "clean"],
                                workdir="appbat", name="clean",
                                alwaysRun=True))

    add_post_dependency_triggers("libbat", arch, libbat)

    locks = [nondep_job_lock.access('counting')]

    lsb_builds.append(
        { 'name': 'libbat-' + arch, 'slavenames': buildslaves,
          'builddir': 'libbat-' + arch, 'factory': libbat,
          'category': 'lsb', 'locks': locks })

    appbat = factory.BuildFactory()

    appbat.addStep(ShellCommand(command=["df", "-h"],
                                name="report-space"))

    appbat.addStep(lfbuildbot.LSBReloadSDK(name="reload-sdk",
                                           locks=[pkg_lock]))

    add_pre_dependency_triggers("appbat", arch, appbat)

    appbat.addStep(Bzr(baseURL=bzr_toplevel, defaultBranch="lsb/devel/appbat",
                       workdir="appbat", mode="full", method="clobber",
                       timeout=bzr_timeout_seconds))
    appbat.addStep(ShellCommand(command=["rm", "-rf", "../tools"],
                                workdir="appbat", name="clear-old-nalfs"))
    appbat.addStep(ShellCommand(command=["mkdir", "-p", "../tools"],
                                workdir="appbat", name="create-tools-dir"))
    appbat.addStep(ShellCommand(command=["bzr", "checkout", 
                                         os.path.join(bzr_toplevel, 
                                                      "lsb/devel/nALFS")],
                                workdir="tools", name="checkout-nalfs",
                                timeout=bzr_timeout_seconds))
    appbat.addStep(ShellCommand(command=["rm", "-rf", "../results"], 
                                workdir="appbat", name="clear-old-results"))
    appbat.addStep(ShellCommand(command=["mkdir", "-p", "../results"],
                                workdir="appbat", name="create-results-dir"))
    appbat.addStep(
        ShellCommand(command=["mkdir", "-p", 
                              "%s/tmp/appbat-pkgcache" % buildbot_slave_path],
                     workdir="appbat", name="create-pkgcache"))
    appbat.addStep(
        ShellCommand(command="cp %s/tmp/appbat-pkgcache/* packages"
                     % buildbot_slave_path,
                     workdir="appbat", name="copy-pkgcache",
                     flunkOnFailure=False, haltOnFailure=False))
    appbat.addStep(
        ShellCommand(command="rm -f %s/tmp/appbat-pkgcache/*"
                     % buildbot_slave_path,
                     workdir="appbat", name="clear-pkgcache",
                     flunkOnFailure=False, haltOnFailure=False))
    appbat.addStep(
        lfbuildbot.LSBConfigureAppbat(command=["./configure"],
                                      env={"PATH": "/opt/lsb/bin:/opt/lsb/appbat/bin:/usr/local/bin:/bin:/usr/bin"},
                                      workdir="appbat", name="configure",
                                      flunkOnFailure=True, haltOnFailure=True))
    appbat.addStep(ShellCommand(command=["extras/entitycheck.py", "-c",
                                         "--delete-bad"],
                                workdir="appbat", name="check-pkgcache",
                                flunkOnFailure=False, haltOnFailure=False))
    appbat.addStep(ShellCommand(command=["extras/entitycheck.py", "-q", "-f"],
                                workdir="appbat", name="download-pkgs",
                                flunkOnFailure=False, haltOnFailure=False))
    appbat.addStep(
        lfbuildbot.LSBBuildAppbat(command="make appbat ENTITYCHECK_QUIET=-q", 
                                  env={"PATH": "/opt/lsb/bin:/opt/lsb/appbat/bin:/usr/local/bin:/bin:/usr/bin"},
                                  workdir="appbat", name="build-appbat", 
                                  timeout=build_timeout_seconds))
    appbat.addStep(
        lfbuildbot.LSBBuildAppbat(command="make desktop ENTITYCHECK_QUIET=-q", 
                                  env={"PATH": "/opt/lsb/bin:/opt/lsb/appbat/bin:/usr/local/bin:/bin:/usr/bin"},
                                  workdir="appbat", name="build-desktop", 
                                  timeout=build_timeout_seconds))
    appbat.addStep(
        lfbuildbot.LSBBuildAppbat(command="make addons ENTITYCHECK_QUIET=-q", 
                                  env={"PATH": "/opt/lsb/bin:/opt/lsb/appbat/bin:/usr/local/bin:/bin:/usr/bin"},
                                  workdir="appbat", name="build-addons",
                                  timeout=build_timeout_seconds))
    appbat.addStep(
        lfbuildbot.LSBBuildCommand(
            command=["make", "check-appbat"], workdir="appbat",
            name="check-appbat", haltOnFailure=True, alwaysRun=True))
    appbat.addStep(
        lfbuildbot.LSBBuildCommand(
            command=["make", "check-desktop"], workdir="appbat",
            name="check-desktop", haltOnFailure=True, alwaysRun=True))
    appbat.addStep(
        lfbuildbot.LSBBuildCommand(
            command=["make", "check-addons"], workdir="appbat",
            name="check-addons", haltOnFailure=True, alwaysRun=True))
    appbat.addStep(
        lfbuildbot.LSBBuildCommand(command="cd rpm && make",
                                   name="build-pkg", workdir="appbat",
                                   timeout=build_timeout_seconds))
    appbat.addStep(ShellCommand(
            command="find rpm -name '*.rpm' -exec cp '{}' ../results ';'",
            workdir="appbat", name="copy-results"))
    appbat.addStep(ShellCommand(command=["run-appbat-tests", 
                                         "../results", "../results/tests"],
                                workdir="appbat", name="run-appbat-tests"))
    appbat.addStep(ShellCommand(command=["tar", "czvf", "../results.tar.gz", 
                                         "../results"],
                                workdir="appbat", name="pack-results"))
    appbat.addStep(
        FileUpload(slavesrc="../results.tar.gz", 
                   masterdest=WithProperties("appbat-%%(result_type:-results)s-%s.tar.gz" % arch),
                   workdir="appbat", name="upload-results", 
                   flunkOnFailure=False))
    appbat.addStep(
        ShellCommand(command="cp packages/*tar* %s/tmp/appbat-pkgcache"
                     % buildbot_slave_path,
                     workdir="appbat", name="save-pkgcache",
                     flunkOnFailure=False, haltOnFailure=False,
                     alwaysRun=True))
    appbat.addStep(ShellCommand(command=["make", "clean"],
                                workdir="appbat", name="clean",
                                alwaysRun=True))

    add_post_dependency_triggers("appbat", arch, appbat)

    locks = [nondep_job_lock.access('counting')]

    lsb_builds.append(
        { 'name': 'appbat-' + arch, 'slavenames': buildslaves,
          'builddir': 'appbat-' + arch, 'factory': appbat,
          'category': 'lsb', 'locks': locks })

# Builds for devchk.  These are handled with a completely different set
# of builders and build slaves, because they depend on and test a number
# of native distro features, and can/should be run as tests against a
# diverse set of distros.

for build_slave in devchk_build_slaves:
    devchk = factory.BuildFactory()

    devchk.addStep(ShellCommand(command=["df", "-h"],
                                name="report-space"))

    devchk.addStep(lfbuildbot.LSBReloadSDK(name="reload-sdk",
                                           slave_id=build_slave))

    devchk.addStep(Bzr(baseURL=bzr_toplevel, 
                       defaultBranch="lsb/devel/devchk",
                       workdir="devchk",
                       mode="full", method="clobber",
                       timeout=bzr_timeout_seconds))
    devchk.addStep(ShellCommand(command=["rm", "-rf", "../results"], 
                                workdir="devchk", name="clear-old-results"))
    devchk.addStep(ShellCommand(command=["mkdir", "-p", "../results"],
                                workdir="devchk", 
                                name="create-results-dir"))
    devchk.addStep(ShellCommand(command="cd ts/devchk && ./dodevchk",
                                workdir="devchk", name="build",
                                flunkOnFailure=False))
    devchk.addStep(ShellCommand(command="cp ts/devchk/*.output ts/devchk/*.sql ../results || true",
                                workdir="devchk", name="collect-results",
                                flunkOnFailure=False))
    devchk.addStep(ShellCommand(command="test $(ls ../results | wc -l) -gt 0",
                                workdir="devchk", name="check-results",
                                haltOnFailure=True))
    devchk.addStep(ShellCommand(command=["tar", "czvf", "../results.tar.gz", 
                                         "../results"],
                                workdir="devchk", name="pack-results"))
    devchk.addStep(
        FileUpload(slavesrc="../results.tar.gz", 
                   masterdest=WithProperties("devchk-%%(result_type:-results)s-%s.tar.gz" % build_slave),
                   workdir="devchk", name="upload-results", 
                   flunkOnFailure=False))

    lsb_builds.append(
        { 'name': 'devchk-' + build_slave,
          'slavenames': ["devchk-" + build_slave],
          'builddir': 'devchk-' + build_slave, 'factory': devchk,
          'category': 'lsb' })

# Final build configuration

c['builders'] = lsb_builds


####### STATUS TARGETS

# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.

c['status'] = []

c['status'].append(html.WebStatus(http_port=8009, allowForce=True,
                                  auth=HTPasswdAuth(web_htpasswd_path)))

c['status'].append(words.IRC(host="irc.freenode.net", nick="lsb_bb",
                             channels=["#lsb"], allowForce=True,
                             notify_events={'exception': 1,
                                            'successToFailure': 1,
                                            'failureToSuccess': 1}))

from buildbot.status import mail
c['status'].append(mail.MailNotifier(
    fromaddr="buildbot@linuxbase.org", sendToInterestedUsers=False,
    extraRecipients=["lsb-messages@lists.linux-foundation.org"],
    mode='change'))

# from buildbot.status import client
# c['status'].append(client.PBListener(9988))


####### DEBUGGING OPTIONS

# if you set 'debugPassword', then you can connect to the buildmaster with
# the diagnostic tool in contrib/debugclient.py . From this tool, you can
# manually force builds and inject changes, which may be useful for testing
# your buildmaster without actually commiting changes to your repository (or
# before you have a functioning 'sources' set up). The debug tool uses the
# same port number as the slaves do: 'slavePortnum'.

#c['debugPassword'] = "debugpassword"

# if you set 'manhole', you can ssh into the buildmaster and get an
# interactive python shell, which may be useful for debugging buildbot
# internals. It is probably only useful for buildbot developers. You can also
# use an authorized_keys file, or plain telnet.
#from buildbot import manhole
#c['manhole'] = manhole.PasswordManhole("tcp:9999:interface=127.0.0.1",
#                                       "admin", "password")


####### PROJECT IDENTITY

# the 'projectName' string will be used to describe the project that this
# buildbot is working on. For example, it is used as the title of the
# waterfall HTML page. The 'projectURL' string will be used to provide a link
# from buildbot HTML pages to your project's home page.

c['projectName'] = "LSB"
c['projectURL'] = "http://www.linuxbase.org/"

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.Waterfall page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c['buildbotURL'] = "http://www.linuxbase.org/buildbot/"


####### MISCELLANEOUS

# Make revisions in buildbot web status link to version control
# web viewer.

c['revlink'] = revlinks.RevlinkMatch(
    [r'/var/www/bzr/(.*)'],
    r'https://bzr.linuxfoundation.org/loggerhead/\1/revision/%s')