]> git.sommitrealweird.co.uk Git - quagga-debian.git/blob - infra/buildbot/master/master.cfg
00e56dd9082695c90dcb11ccfc46f94186ede318
[quagga-debian.git] / infra / buildbot / master / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 from buildbot.plugins import *
5 from buildbot.plugins import buildslave, util
6
7 # This is a sample buildmaster config file. It must be installed as
8 # 'master.cfg' in your buildmaster's base directory.
9
10 # This is the dictionary that the buildmaster pays attention to. We also use
11 # a shorter alias to save typing.
12 c = BuildmasterConfig = {}
13
14 quaggagit = 'git://git.sv.gnu.org/quagga.git'
15
16 # password defs
17 execfile("pass.cfg")
18
19 # filter a given 'workers' entry into a property list
20 # suitable for public display
21 def workers2publicprops (worker):
22         publicprops = [ "os", "version", "vm", "pkg", "texi", "cc",
23                         "latent", ]
24         return  { k:worker[k] for k in worker if k in publicprops }
25
26 # vm: non-VM are assumed faster and used for initial build
27 # pkg: rpm, sysv, dpkg - only do test rpm builds at moment
28 # texi: True or "true" if we can use for doc building
29 # cc: List of tuples of installed compilers, with:
30 #     (<tag>, <version>, <command>)
31 #     tag: gcc, clang, sunpro
32 # latent: VM spun up on demand via LatentSlave, uses "session" for
33 #         the libvirt URI.
34 # session: libvirt URI to use for latent workers. Default will be set on
35 #          latent VMs if not specified.
36 # hd_image: libvirt image to use
37 workers = {
38         "fedora-24": { 
39           "os": "Fedora",
40           "version": "24",
41           "vm": False,
42           "pkg": "rpm",
43           "texi": True,
44           "cc": [ ("gcc", "6.3.1"),
45                   ("clang", "3.8.1"),
46                   ("gcc",  "3.4.6", "gcc34"),
47                 ],
48         }, 
49         "fedora-26": { 
50           "os": "Fedora",
51           "version": "26",
52           "vm": False,
53           "pkg": "rpm",
54           "cc": [ ("gcc", "7.0.1"),
55                   ("clang", "3.9.0"),
56                   ("gcc",  "3.4.6", "gcc34"),
57                 ],
58         }, 
59         "centos-7": {
60           "os": "CentOS",
61           "version": "7",
62           "vm": False,
63           "pkg": "rpm",
64           "cc": [ ("gcc", "4.8.5") ],
65         },
66         "debian-8": {
67           "os": "Debian",
68           "version": "8",
69           "vm": True,   
70           "pkg": "dpkg",
71           "latent": True,
72           "cc": [ ("gcc", "4.9.2") ],
73           "hd_image": "/var/lib/libvirt/images/debian8.qcow2",
74         },
75         "debian-9": { 
76           "os": "Debian",
77           "version": "9",
78           "vm": True,   
79           "pkg": "dpkg",
80           "cc": [ ("gcc", "6.3.0") ],
81           "latent": True,
82           "hd_image": "/var/lib/libvirt/images/debian9.qcow2",
83         },
84         "freebsd-10": { 
85           "os": "FreeBSD",
86           "version": "10",
87           "vm": True,   
88           "pkg": "",
89           "latent": True,
90           "cc": [ ("clang", "3.4.1") ],
91           "hd_image": "/var/lib/libvirt/images/freebsd103.qcow2",
92         }, 
93         "freebsd-11": { 
94           "os": "FreeBSD",
95           "version": "11",
96           "vm": True,   
97           "pkg": "",
98           "cc": [ ("gcc", "4.9.4"), ("clang", "3.8.0"), ],
99           "latent": True,
100           "hd_image": "/var/lib/libvirt/images/freebsd110.qcow2",
101         },
102         "oi-hipster": {
103           "os": "OpenIndiana",
104           "version": "hipster",
105           "vm": True,
106           "pkg": "sysv",
107           "latent": True,
108           "cc": [ ("gcc", "6.3.0"), ("sunpro", "12.0"), 
109                   ("gcc", "4.4.4") 
110                 ],
111           "hd_image": "/var/lib/libvirt/images/buildbot-oi-hipster.qcow2",
112         },
113 }
114
115 # ensure "latent" is set to false, where not set.
116 # add in the passwords
117 for kw in workers:
118         w = workers[kw]
119         w["bot"] = "buildbot-" + kw
120         if "latent" not in w:
121                 w["latent"] = False
122         w["pass"] = workers_pass[kw]
123
124 analyses_builders = [ "clang-analyzer" ]
125
126 # default Libvirt session
127 for w in (w for w in workers.values () if ("latent" in w and w["latent"])
128                                         and ("session" not in w)):
129         w["session"] = 'qemu+ssh://buildbot@sagan.jakma.org/system'
130
131 osbuilders = ["build-" + kw for kw in workers]
132 osfastbuilders = ["build-" + kw for kw in workers if workers[kw]["vm"] == False]
133 osslowbuilders = ["build-" + kw for kw in workers if workers[kw]["vm"] == True]
134
135 rpmbuilders = ["rpm-" + kw for kw in workers if workers[kw]["pkg"] == "rpm"]
136
137 # compilers
138 # not using yet
139 # [kw for kw in workers if len([v for (c,v) in workers[kw]["cc"] if c == "gcc"]) > 0 ]
140
141 allbuilders =  []
142 allbuilders += osbuilders
143 allbuilders += rpmbuilders
144 allbuilders += analyses_builders
145 allbuilders += ["commit-builder"]
146 allbuilders += ["build-distcheck"]
147 allbuilders += ["build-docs" ]
148
149 # Force merging of requests.
150 # c['mergeRequests'] = lambda *args, **kwargs: True
151
152 ####### BUILDSLAVES
153 c['slaves'] = []
154
155 # The 'slaves' list defines the set of recognized buildslaves. Each element is
156 # a BuildSlave object, specifying a unique slave name and password.  The same
157 # slave name and password must be configured on the slave.
158
159 for w in (w for w in workers.values() if ("latent" not in w)
160                                       or (w["latent"] == False)):
161         c['slaves'].append(buildslave.BuildSlave(w["bot"], w["pass"],
162                                 properties=workers2publicprops (w),
163         ))
164
165 for w in (w for w in workers.values()
166                   if ("latent" in w) 
167                      and w["latent"]
168                      and "hd_image" in w):
169         c['slaves'].append(buildslave.LibVirtSlave(
170                                         w["bot"],
171                                         w["pass"],
172                                         util.Connection(w["session"]),
173                                         w["hd_image"],
174                                         properties=workers2publicprops (w),
175         ))
176
177 # 'protocols' contains information about protocols which master will use for
178 # communicating with slaves.
179 # You must define at least 'port' option that slaves could connect to your master
180 # with this protocol.
181 # 'port' must match the value configured into the buildslaves (with their
182 # --master option)
183 c['protocols'] = {'pb': {'port': 9989}}
184
185 ####### CHANGESOURCES
186
187 # the 'change_source' setting tells the buildmaster how it should find out
188 # about source code changes.  Here we point to the buildbot clone of pyflakes.
189
190 c['change_source'] = []
191 c['change_source'].append(changes.GitPoller(
192         quaggagit,
193         workdir='gitpoller-workdir', 
194         branches=['master','volatile/next'],
195         pollinterval=300))
196
197 ####### REVISION LINKS
198 # associate changesouce repositories to URI templates for code view
199 #
200 c['revlink'] = util.RevlinkMatch([quaggagit + r"(.*)"],
201         r"http://git.savannah.gnu.org/cgit/quagga.git/commit/?id=%s")
202
203 ####### SCHEDULERS
204
205 # Configure the Schedulers, which decide how to react to incoming changes. 
206
207 # We want a first line of 'quick' builds, which then trigger further builds.
208 #
209 # A control-flow builder, "commit-builder", used to sequence the 'real'
210 # sets of builders, via Triggers.
211
212 c['schedulers'] = []
213 c['schedulers'].append(schedulers.SingleBranchScheduler(
214                             name="master-change",
215                             change_filter=util.ChangeFilter(branch='master'),
216                             treeStableTimer=10,
217                             builderNames=[ "commit-builder" ]))
218
219 c['schedulers'].append(schedulers.SingleBranchScheduler(
220                             name="next-change",
221                             change_filter=util.ChangeFilter(
222                                 branch='volatile/next'),
223                             treeStableTimer=10,
224                             builderNames=[ "commit-builder" ] ))
225
226 # Initial build checks on faster, non-VM
227 c['schedulers'].append(schedulers.Triggerable(
228         name="trigger-build-first",
229         builderNames=osfastbuilders))
230
231 # Build using remaining builders, after firstbuilders.
232 c['schedulers'].append(schedulers.Triggerable(
233         name="trigger-build-rest",
234         builderNames=osslowbuilders))
235
236 # Analyses tools, e.g. CLang Analyzer scan-build
237 c['schedulers'].append(schedulers.Triggerable(
238                        name="trigger-build-analyses",   
239                        builderNames=analyses_builders))
240 # Dist check
241 c['schedulers'].append(schedulers.Triggerable(
242                        name="trigger-distcheck",        
243                        builderNames=["build-distcheck"]))
244 # RPM check and build
245 c['schedulers'].append(schedulers.Triggerable(
246                        name="trigger-rpm",      
247                        builderNames=rpmbuilders))
248
249 # Doc build check (non-nightly, so no upload)
250 c['schedulers'].append(schedulers.Triggerable(
251                        name="trigger-build-docs",       
252                        builderNames=["build-docs"]))
253
254 # Try and force schedulers
255 c['schedulers'].append(schedulers.ForceScheduler(
256                        name="force",
257                        builderNames=allbuilders))
258
259 c['schedulers'].append(schedulers.Try_Userpass(
260                        name="try",
261                        builderNames=osbuilders
262                                     + rpmbuilders
263                                     + ["build-distcheck", 
264                                        "clang-analyzer",
265                                        "build-docs" ],
266                         userpass=users,
267                         port=8031))
268
269 ## nightly docs build
270 c['schedulers'].append(schedulers.Nightly(
271         name="nightly-docs",
272         branch="master",
273         builderNames=[ "build-docs" ],
274         hour=3,
275         minute=0,
276         onlyIfChanged=True,
277         properties = { "nightly": True },
278 ))
279
280
281 ####### BUILDERS
282 c['builders'] = []
283
284 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
285 # what steps, and which slaves can execute them.  Note that any particular build will
286 # only take place on one slave.
287
288 common_setup = [
289         steps.Git(repourl=quaggagit, mode='incremental'),
290         steps.ShellCommand(command=["./update-autotools"],
291                                     description="generating autoconf",
292                                     descriptionDone="autoconf"),
293         steps.Configure(command="../build/configure"),
294         steps.ShellCommand(command=["make", "clean"],
295                                     description="cleaning",
296                                     descriptionDone="make clean"),
297 ]
298
299 ### Default 'check' build, builder instantiated for each OS
300
301 factory = util.BuildFactory()
302 # check out the source
303 factory.addStep(steps.Git(repourl=quaggagit, mode='incremental'))
304 factory.addStep(steps.ShellCommand(command=["./update-autotools"],
305                                    description="generating autoconf",
306                                    descriptionDone="autoconf"))
307 factory.addStep(steps.Configure())
308 factory.addStep(steps.ShellCommand(command=["make", "clean"],
309                                    description="cleaning",
310                                    descriptionDone="clean"))
311
312 #factory.addSteps(common_setup)
313 factory.addStep(steps.Compile(command=["make", "-j", "2", "all"]))
314 factory.addStep(steps.ShellCommand(command=["make", "check"],
315                                    description="checking",
316                                    descriptionDone="make check"))
317
318 # create builder for every OS, for every buildbot
319 # XXX: at moment this assumes 1:1 OS<->bot
320 for kw in workers:
321    c['builders'].append(util.BuilderConfig(
322         name="build-" + kw,
323         slavenames=workers[kw]["bot"],
324         factory=factory))
325
326 ### distcheck Builder, executed on any available bot
327 factory = util.BuildFactory()
328 # check out the source
329 factory.addStep(steps.Git(repourl=quaggagit, mode='incremental'))
330 factory.addStep(steps.ShellCommand(command=["./update-autotools"],
331                                    description="generating autoconf",
332                                    descriptionDone="autoconf"))
333 factory.addStep(steps.Configure())
334 factory.addStep(steps.ShellCommand(command=["make", "clean"],
335                                    description="cleaning",
336                                    descriptionDone="make clean"))
337 factory.addStep(steps.ShellCommand(command=["make", "distcheck"],
338                                    description="run make distcheck",
339                                    descriptionDone="make distcheck"))
340 c['builders'].append(
341         util.BuilderConfig(name="build-distcheck",
342         slavenames=list(w["bot"] for w in workers.values()),
343         factory=factory,
344 ))
345
346 ### LLVM clang-analyzer build, executed on any available non-VM bot
347
348 f = util.BuildFactory()
349 # check out the source
350 f.addStep(steps.Git(repourl=quaggagit, mode='incremental',
351                     getDescription=True))
352 f.addStep(steps.ShellCommand(command=["./update-autotools"],
353                              description="run autotools",
354                              descriptionDone="autoconf"))
355 f.addStep(steps.Configure())
356 f.addStep(steps.ShellCommand(command=["make", "clean"],
357                              description="cleaning",
358                              descriptionDone="make clean"))
359
360 f.addStep(steps.SetProperty(property="clang-id",
361         value=util.Interpolate("%(prop:commit-description)s-%(prop:buildnumber)s")))
362
363 f.addStep(steps.SetProperty(property="clang-output-dir",
364         value=util.Interpolate("../CLANG-%(prop:clang-id)s")))
365 f.addStep(steps.SetProperty(property="clang-uri",
366         value=util.Interpolate("/clang-analyzer/%(prop:clang-id)s")))
367 # relative to buildbot master working directory
368 f.addStep(steps.SetProperty(property="clang-upload-dir",
369         value=util.Interpolate("public_html/clang-analyzer/%(prop:clang-id)s")))
370
371 f.addStep(steps.Compile(command=["scan-build",
372                                  "-analyze-headers",
373                                  "-o",
374                                  util.Interpolate("%(prop:clang-output-dir)s"),
375                                  "make", "-j", "all"]))
376 f.addStep(steps.DirectoryUpload(
377           slavesrc=util.Interpolate("%(prop:clang-output-dir)s"),
378           masterdest = util.Interpolate("%(prop:clang-upload-dir)s"),
379           compress = 'bz2',
380           name = "clang report",
381           url = util.Interpolate("%(prop:clang-uri)s"),
382 ))
383 f.addStep(steps.RemoveDirectory(
384         dir=util.Interpolate("%(prop:clang-output-dir)s")
385 ))
386
387
388 c['builders'].append(
389     util.BuilderConfig(name="clang-analyzer",
390       slavenames=list(w["bot"] for w in workers.values() if not w["vm"]),
391       factory=f))
392
393
394 ### RPM: check and build
395 f = util.BuildFactory ()
396
397 # check out the source
398 f.addStep(steps.Git(repourl=quaggagit, mode='full'))
399 f.addStep(steps.ShellCommand(command=["./update-autotools"],
400                                    description="run autotools",
401                                    descriptionDone="autotools"))
402 f.addStep(steps.Configure())
403 f.addStep(steps.ShellCommand(command=["make", "dist"],
404                                      description="run make dist",
405                                      descriptionDone="make dist"))
406 # not imported somehow
407 #f.addStep(steps.RpmLint(fileloc="redhat/quagga.spec"))
408 f.addStep(steps.ShellCommand(command=["rpmlint", "-i", "redhat/quagga.spec"],
409                                 description="run rpmlint",
410                                 descriptionDone="rpmlint"))
411 f.addStep(steps.RpmBuild(specfile="redhat/quagga.spec"))
412 #                        rpmdir=util.Interpolate("%(prop:builddir)s/rpm")))
413
414 # XXX: assuming 1:1 OS:buildbot mapping
415 for kw in (kw for kw in workers if workers[kw]["pkg"] == "rpm"):
416         c['builders'].append(
417                 util.BuilderConfig(name="rpm-" + kw,
418                                    slavenames="buildbot-" + kw,
419                                    factory=f
420                 )
421         )
422
423 ### Build documentation
424
425 def build_is_nightly (step):
426     n = step.getProperty("nightly")
427     if n == True or n == "True" or n == "true":
428        return True
429     return False
430
431 f = util.BuildFactory ()
432 f.addStep(steps.Git(repourl=quaggagit, mode='full'))
433 f.addStep(steps.ShellCommand(command=["./update-autotools"],
434                                    description="run autotools",
435                                    descriptionDone="autotools"))
436 f.addStep(steps.Configure(command=["../build/configure"],
437                           workdir="docs"))
438 f.addStep(steps.ShellCommand(command=["make", "V=99", "quagga.html"],
439                              description="making split HTML doc",
440                              descriptionDone="docs: split HTML",
441                              workdir="docs/doc",
442                              haltOnFailure=True,
443 ))
444 #f.addStep(steps.FileUpload(
445 #         slavesrc="build/doc/fig-normal-processing.png",
446 #         masterdest = "public_html/docs/nightly/quagga/",
447 #         name = "Upload Fig 1",
448 #         doStepIf=build_is_nightly,
449 #))
450 #f.addStep(steps.FileUpload(
451 #         slavesrc="build/doc/fig-rs-processing.png",
452 #         masterdest = "public_html/docs/nightly/quagga/",
453 #         name = "Upload Fig 2",
454 #         doStepIf=build_is_nightly,
455 #))
456 f.addStep(steps.MultipleFileUpload(
457           slavesrcs=[ "doc/fig-rs-processing.png",
458                       "doc/fig-normal-processing.png" ],
459           masterdest = "public_html/docs/nightly/quagga/",
460           name = "Upload Figures",
461           doStepIf=build_is_nightly,
462 ))
463 f.addStep(steps.DirectoryUpload(
464           slavesrc="quagga.html",
465           masterdest = "public_html/docs/nightly/quagga",
466           compress = 'bz2',
467           name = "Upload split HTML",
468           url = "/docs/nightly/quagga/index.html",
469           workdir="docs/doc",
470           doStepIf=build_is_nightly,
471 ))
472 f.addStep(steps.RemoveDirectory(
473           dir="docs/doc/quagga.html",
474 ))
475 f.addStep(steps.ShellCommand(command=["make", "V=99",
476                                         "MAKEINFOFLAGS=--no-split",
477                                         "quagga.html"],
478                              description="making one-page HTML doc",
479                              descriptionDone="docs: one-page HTML",
480                              workdir="docs/doc",
481                              haltOnFailure=True
482 ))
483 f.addStep(steps.FileUpload(
484           slavesrc="quagga.html",
485           masterdest = "public_html/docs/nightly/quagga/quagga.html",
486           name = "Upload single HTML",
487           url = "/docs/nightly/quagga/quagga.html",
488           workdir="docs/doc",
489           doStepIf=build_is_nightly,
490 ))
491 f.addStep(steps.ShellCommand(command=["make", "V=99", "quagga.pdf"],
492                              description="making PDF docs",
493                              descriptionDone="docs: PDF",
494                              workdir="docs/doc"
495 ))
496 f.addStep(steps.FileUpload(
497           slavesrc="quagga.pdf",
498           masterdest = "public_html/docs/nightly/quagga/quagga.pdf",
499           name = "Upload PDF",
500           url = "/docs/nightly/quagga/quagga.pdf",
501           workdir="docs/doc",
502           doStepIf=build_is_nightly,
503 ))
504
505 c['builders'].append(
506     util.BuilderConfig(name="build-docs",
507       slavenames=[w["bot"] for w in workers.values() 
508                         if "texi" in w and w["texi"] == True ],
509       factory=f
510 ))
511
512 ### Co-ordination builds used to sequence parallel builds via Triggerable
513
514 # to understand this you have to read this list and the Triggered schedulers
515 # to see what sets of builds are being sequenced. Bit clunky, but Buildbot
516 # doesn't have a way to just specify a pipeline of groups of builders more
517 # cleanly.
518
519 f = util.BuildFactory()
520 f.addStep(steps.Trigger (
521         schedulerNames = [ "trigger-build-first" ],
522         waitForFinish=True,
523         updateSourceStamp=True
524 ))
525 f.addStep(steps.Trigger (
526         schedulerNames = [ "trigger-build-rest" ],
527         waitForFinish=True,
528         updateSourceStamp=True
529 ))
530 f.addStep(steps.Trigger (
531         schedulerNames = [ "trigger-build-analyses", "trigger-distcheck",
532                            "trigger-build-docs" ],
533         waitForFinish=True,
534         updateSourceStamp=True
535 ))
536 f.addStep(steps.Trigger (
537         schedulerNames = [ "trigger-rpm" ],
538         waitForFinish=True,
539         updateSourceStamp=True
540 ))
541
542 c['builders'].append(
543     util.BuilderConfig(name="commit-builder",
544       slavenames=[w["bot"] for w in workers.values() if not w["vm"]],
545       factory=f)
546 )
547
548 ####### STATUS TARGETS
549
550 # 'status' is a list of Status Targets. The results of each build will be
551 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
552 # including web pages, email senders, and IRC bots.
553
554 c['status'] = []
555
556 from buildbot.status import html
557 from buildbot.status.web import authz, auth
558
559 authz_cfg=authz.Authz(
560     # change any of these to True to enable; see the manual for more
561     # options
562     #auth=auth.BasicAuth([("pyflakes","pyflakes")]),
563     auth=util.BasicAuth(users),
564     gracefulShutdown = False,
565     forceBuild = 'auth', # use this to test your slave once it is set up
566     forceAllBuilds = 'auth',  # ..or this
567     pingBuilder = 'auth',
568     stopBuild = 'auth',
569     stopAllBuilds = 'auth',
570     cancelPendingBuild = 'auth',
571     cancelAllPendingBuilds = 'auth',
572     pauseSlave = 'auth',    
573 )
574 c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
575
576 c['status'].append(status.MailNotifier(
577         fromaddr="buildbot@quagga.net",
578         extraRecipients=["paul@jakma.org"],
579         sendToInterestedUsers=False,
580 ))
581
582 c['status'].append (status.IRC(
583         "irc.freenode.net", "bb-quagga",
584         useColors=True,
585         channels=[{"channel": "#quagga"}],
586         notify_events={
587                 'exception': 1,
588                 'successToFailure': 1,
589                 'failureToSuccess': 1,
590         },
591 ))
592
593 ####### PROJECT IDENTITY
594
595 # the 'title' string will appear at the top of this buildbot
596 # installation's html.WebStatus home page (linked to the
597 # 'titleURL') and is embedded in the title of the waterfall HTML page.
598
599 c['title'] = "Quagga"
600 c['titleURL'] = "https://www.quagga.net/"
601
602 # the 'buildbotURL' string should point to the location where the buildbot's
603 # internal web server (usually the html.WebStatus page) is visible. This
604 # typically uses the port number set in the Waterfall 'status' entry, but
605 # with an externally-visible host name which the buildbot cannot figure out
606 # without some help.
607
608 c['buildbotURL'] = "http://buildbot.quagga.net/"
609
610 ####### DB URL
611
612 c['db'] = {
613     # This specifies what database buildbot uses to store its state.  You can leave
614     # this at its default for all but the largest installations.
615     'db_url' : "sqlite:///state.sqlite",
616 }
617
618 #### debug
619 c['debugPassword'] = debugPassword