From 3d9bc13b06c2d11431f288d7d66bca67cfe2f718 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Tue, 19 Jun 2018 14:17:12 +0200 Subject: [PATCH] First stab at tracking build results --- pgcommitfest/commitfest/admin.py | 2 + pgcommitfest/commitfest/api.py | 63 ++++++++++++++++++- .../migrations/0003_patch_build_status.py | 40 ++++++++++++ pgcommitfest/commitfest/models.py | 46 ++++++++++++++ .../commitfest/templates/commitfest.html | 8 ++- pgcommitfest/commitfest/templates/patch.html | 8 +++ .../commitfest/templatetags/commitfest.py | 17 ++++- pgcommitfest/commitfest/views.py | 7 ++- pgcommitfest/urls.py | 1 + 9 files changed, 187 insertions(+), 5 deletions(-) create mode 100644 pgcommitfest/commitfest/migrations/0003_patch_build_status.py diff --git a/pgcommitfest/commitfest/admin.py b/pgcommitfest/commitfest/admin.py index 05fe3ed..909ace3 100644 --- a/pgcommitfest/commitfest/admin.py +++ b/pgcommitfest/commitfest/admin.py @@ -22,6 +22,8 @@ admin.site.register(CommitFest) admin.site.register(Topic) admin.site.register(Patch, PatchAdmin) admin.site.register(PatchHistory) +admin.site.register(BuildProvider) +admin.site.register(PatchBuildStatus) admin.site.register(MailThread) admin.site.register(MailThreadAttachment, MailThreadAttachmentAdmin) diff --git a/pgcommitfest/commitfest/api.py b/pgcommitfest/commitfest/api.py index 1c97ef1..fec46ea 100644 --- a/pgcommitfest/commitfest/api.py +++ b/pgcommitfest/commitfest/api.py @@ -6,9 +6,10 @@ from functools import wraps from django.utils.decorators import available_attrs from django.db import connection +import datetime import json -from models import CommitFest +from models import CommitFest, Patch, PatchBuildStatus, BuildProvider def api_authenticate(view_func): def _wrapped_view(request, *args, **kwargs): @@ -74,3 +75,63 @@ GROUP BY p.id, poc.id""".format(wherestring), params) } return HttpResponse(json.dumps(res), content_type='application/json') + + +@csrf_exempt +@api_authenticate +def build_result(request, cfid, patchid): + if request.method != 'POST': + return HttpResponse('Invalid method', status=405) + if request.META['CONTENT_TYPE'] != 'application/json': + return HttpResponse("Only JSON accepted", status=415) + try: + obj = json.loads(request.body) + except ValueError: + return HttpResponse("Invalid data format", status=415) + + commitfest = get_object_or_404(CommitFest, pk=cfid) + patch = get_object_or_404(Patch, pk=patchid) + + # Mandatory fields + try: + provider = BuildProvider.objects.get(urlname=obj['provider']) + messageid = obj['messageid'] + status = obj['status'] + statustime = obj['timestamp'] + except BuildProvider.DoesNotExist: + return HttpResponse("Invalid build provider", status=422) + except KeyError, e: + return HttpResponse("Mandatory parameter {0} missing".format(e.args[0]), status=400) + + if not status in PatchBuildStatus._STATUS_MAP: + return HttpResponse("Invalid build status {0}".format(status), status=422) + try: + statustime = datetime.datetime.strptime(statustime, '%Y-%m-%dT%H:%M:%S.%fZ') + except ValueError: + return HttpResponse("Invalid timestamp {0}".format(statustime), status=422) + + # Optional parameters + url = obj.get('url', '') + commitid = obj.get('commit', '') + + (buildstatus, created) = PatchBuildStatus.objects.get_or_create(commitfest=commitfest, + patch=patch, + buildprovider=provider, + status_timestamp=statustime, + defaults={ + 'buildmessageid': messageid, + 'buildstatus': PatchBuildStatus._STATUS_MAP[status], + 'status_url': url, + 'master_commit_id': commitid, + }, + ) + if not created: + if buildstatus.buildmessageid == messageid and \ + buildstatus.buildstatus == PatchBuildStatus._STATUS_MAP[status] and \ + buildstatus.status_url == url and \ + buildstatus.master_commit_id == commitid: + return HttpResponse("Build status already stored", status=200) + return HttpResponse("Conflicting build status already stored", status=409) + + # That's it! + return HttpResponse("Stored", status=201) diff --git a/pgcommitfest/commitfest/migrations/0003_patch_build_status.py b/pgcommitfest/commitfest/migrations/0003_patch_build_status.py new file mode 100644 index 0000000..6b8a70d --- /dev/null +++ b/pgcommitfest/commitfest/migrations/0003_patch_build_status.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('commitfest', '0002_notifications'), + ] + + operations = [ + migrations.CreateModel( + name='BuildProvider', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('urlname', models.CharField(unique=True, max_length=16)), + ('name', models.CharField(max_length=100)), + ], + ), + migrations.CreateModel( + name='PatchBuildStatus', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('buildmessageid', models.CharField(max_length=1000)), + ('buildstatus', models.IntegerField(default=0, choices=[(0, b'Pending'), (1, b'Success'), (2, b'Fail')])), + ('status_timestamp', models.DateTimeField()), + ('status_url', models.URLField(blank=True)), + ('master_commit_id', models.CharField(max_length=40, blank=True)), + ('buildprovider', models.ForeignKey(to='commitfest.BuildProvider')), + ('commitfest', models.ForeignKey(to='commitfest.CommitFest')), + ('patch', models.ForeignKey(to='commitfest.Patch')), + ], + ), + migrations.AlterUniqueTogether( + name='patchbuildstatus', + unique_together=set([('patch', 'commitfest', 'buildprovider', 'status_timestamp')]), + ), + ] diff --git a/pgcommitfest/commitfest/models.py b/pgcommitfest/commitfest/models.py index 8e5a442..b136117 100644 --- a/pgcommitfest/commitfest/models.py +++ b/pgcommitfest/commitfest/models.py @@ -250,6 +250,52 @@ class PatchHistory(models.Model): if u != self.by: # Don't notify for changes we make ourselves PendingNotification(history=self, user=u).save() +class BuildProvider(models.Model): + urlname = models.CharField(max_length=16, null=False, blank=False, unique=True) + name = models.CharField(max_length=100, null=False, blank=False) + + def __unicode__(self): + return self.name + +class PatchBuildStatus(models.Model): + STATUS_PENDING=0 + STATUS_SUCCESS=1 + STATUS_FAIL=2 + _STATUS_CHOICES=( + (STATUS_PENDING, "Pending"), + (STATUS_SUCCESS, "Success"), + (STATUS_FAIL, "Fail",) + ) + _STATUS_CHOICE_MAP=dict(_STATUS_CHOICES) + _STATUS_MAP={ + 'pending': STATUS_PENDING, + 'success': STATUS_SUCCESS, + 'fail': STATUS_FAIL, + } + _STATUS_REVERSE_MAP={v:k for k,v in _STATUS_MAP.items()} + + commitfest = models.ForeignKey(CommitFest, null=False, blank=False) + patch = models.ForeignKey(Patch, null=False, blank=False) + buildprovider = models.ForeignKey(BuildProvider, null=False, blank=False) + buildmessageid = models.CharField(max_length=1000, null=False, blank=False) + buildstatus = models.IntegerField(null=False, blank=False, default=0, choices=_STATUS_CHOICES) + status_timestamp = models.DateTimeField(null=False, blank=False) + status_url = models.URLField(null=False, blank=True) + master_commit_id = models.CharField(max_length=40, null=False, blank=True) + + class Meta: + unique_together = ( + ('patch', 'commitfest', 'buildprovider', 'status_timestamp'), + ) + + @property + def textstatus(self): + return self._STATUS_CHOICE_MAP[self.buildstatus] + + @property + def urlstatus(self): + return self._STATUS_REVERSE_MAP[self.buildstatus] + class MailThread(models.Model): # This class tracks mail threads from the main postgresql.org # mailinglist archives. For each thread, we store *one* messageid. diff --git a/pgcommitfest/commitfest/templates/commitfest.html b/pgcommitfest/commitfest/templates/commitfest.html index c9d1e91..b68db09 100644 --- a/pgcommitfest/commitfest/templates/commitfest.html +++ b/pgcommitfest/commitfest/templates/commitfest.html @@ -68,6 +68,7 @@ {%if p.is_open%}Num cfs{%if sortkey == 3%}
{%endif%}{%else%}Num cfs{%endif%} {%if p.is_open%}Latest activity{%if sortkey == 1%}
{%endif%}{%else%}Latest activity{%endif%} {%if p.is_open%}Latest mail{%if sortkey == 2%}
{%endif%}{%else%}Latest mail{%endif%} + Build status {%if user.is_staff%} Select {%endif%} @@ -78,7 +79,7 @@ {%if grouping%} {%ifchanged p.topic%} - {{p.topic}} + {{p.topic}} {%endifchanged%} {%endif%} @@ -90,6 +91,11 @@ {{p.num_cfs}} {{p.modified|date:"Y-m-d"}}
{{p.modified|date:"H:i"}} {{p.lastmail|date:"Y-m-d"}}
{{p.lastmail|date:"H:i"}} + +{%for bs in p.buildstatus%} +{%if bs.url%}{%endif%}{%if bs.url%}{%endif%} +{%endfor%} + {%if user.is_staff%} Author
Reviewer {%endif%} diff --git a/pgcommitfest/commitfest/templates/patch.html b/pgcommitfest/commitfest/templates/patch.html index 3ed775e..4194ed2 100644 --- a/pgcommitfest/commitfest/templates/patch.html +++ b/pgcommitfest/commitfest/templates/patch.html @@ -107,6 +107,14 @@ + + Build status + +{%for s in buildstatuses|dictsort:"status_timestamp"%} +
from {{s.buildprovider}} at {{s.status_timestamp}} (tested patch in {{s.buildmessageid}}{%if s.master_commit_id%} against git commit {{s.master_commit_id}}{%endif%})
+{%endfor%} + + History diff --git a/pgcommitfest/commitfest/templatetags/commitfest.py b/pgcommitfest/commitfest/templatetags/commitfest.py index b8f68e4..acb49be 100644 --- a/pgcommitfest/commitfest/templatetags/commitfest.py +++ b/pgcommitfest/commitfest/templatetags/commitfest.py @@ -1,7 +1,7 @@ from django.template.defaultfilters import stringfilter from django import template -from pgcommitfest.commitfest.models import PatchOnCommitFest +from pgcommitfest.commitfest.models import PatchOnCommitFest, PatchBuildStatus register = template.Library() @@ -41,3 +41,18 @@ def alertmap(value): @stringfilter def hidemail(value): return value.replace('@', ' at ') + +@register.filter(name='buildstatus') +@stringfilter +def buildstatus(value): + return PatchBuildStatus._STATUS_CHOICE_MAP[int(value)] + +@register.filter(name='glyphbuildstatus') +@stringfilter +def glyphbuildstatus(value): + v = int(value) + if v == PatchBuildStatus.STATUS_PENDING: + return 'question-sign' + elif v == PatchBuildStatus.STATUS_SUCCESS: + return 'ok-sign' + return 'minus-sign' diff --git a/pgcommitfest/commitfest/views.py b/pgcommitfest/commitfest/views.py index 3ee60e3..3a04115 100644 --- a/pgcommitfest/commitfest/views.py +++ b/pgcommitfest/commitfest/views.py @@ -20,7 +20,7 @@ from pgcommitfest.mailqueue.util import send_mail, send_simple_mail from pgcommitfest.userprofile.util import UserWrapper from models import CommitFest, Patch, PatchOnCommitFest, PatchHistory, Committer -from models import MailThread +from models import MailThread, PatchBuildStatus from forms import PatchForm, NewPatchForm, CommentForm, CommitFestFilterForm from forms import BulkEmailForm from ajax import doAttachThread, refresh_single_thread @@ -194,7 +194,8 @@ def commitfest(request, cfid): (poc.status=ANY(%(openstatuses)s)) AS is_open, (SELECT string_agg(first_name || ' ' || last_name || ' (' || username || ')', ', ') FROM auth_user INNER JOIN commitfest_patch_authors cpa ON cpa.user_id=auth_user.id WHERE cpa.patch_id=p.id) AS author_names, (SELECT string_agg(first_name || ' ' || last_name || ' (' || username || ')', ', ') FROM auth_user INNER JOIN commitfest_patch_reviewers cpr ON cpr.user_id=auth_user.id WHERE cpr.patch_id=p.id) AS reviewer_names, -(SELECT count(1) FROM commitfest_patchoncommitfest pcf WHERE pcf.patch_id=p.id) AS num_cfs +(SELECT count(1) FROM commitfest_patchoncommitfest pcf WHERE pcf.patch_id=p.id) AS num_cfs, +(SELECT json_agg(json_build_object('provider', providername, 'status', buildstatus, 'timestamp', status_timestamp, 'url', status_url)) FROM (SELECT DISTINCT ON (buildprovider_id) *, bp.name AS providername FROM commitfest_patchbuildstatus pbs INNER JOIN commitfest_buildprovider bp ON bp.id=buildprovider_id WHERE pbs.commitfest_id=%(cid)s AND pbs.patch_id=p.id ORDER BY buildprovider_id, status_timestamp DESC) x) AS buildstatus FROM commitfest_patch p INNER JOIN commitfest_patchoncommitfest poc ON poc.patch_id=p.id INNER JOIN commitfest_topic t ON t.id=p.topic_id @@ -247,6 +248,7 @@ def patch(request, cfid, patchid): patch = get_object_or_404(Patch.objects.select_related(), pk=patchid, commitfests=cf) patch_commitfests = PatchOnCommitFest.objects.select_related('commitfest').filter(patch=patch).order_by('-commitfest__startdate') committers = Committer.objects.filter(active=True).order_by('user__last_name', 'user__first_name') + buildstatuses = PatchBuildStatus.objects.select_related('buildprovider').filter(commitfest=cf, patch=patch).order_by('buildprovider', '-status_timestamp').distinct('buildprovider') #XXX: this creates a session, so find a smarter way. Probably handle #it in the callback and just ask the user then? @@ -270,6 +272,7 @@ def patch(request, cfid, patchid): 'cf': cf, 'patch': patch, 'patch_commitfests': patch_commitfests, + 'buildstatuses': buildstatuses, 'is_committer': is_committer, 'is_this_committer': is_this_committer, 'is_reviewer': is_reviewer, diff --git a/pgcommitfest/urls.py b/pgcommitfest/urls.py index 105dd5d..dc5deb5 100644 --- a/pgcommitfest/urls.py +++ b/pgcommitfest/urls.py @@ -35,6 +35,7 @@ urlpatterns = [ url(r'^thread_notify/$', views.thread_notify), url(r'^api/active_commitfests/$', api.active_commitfests), url(r'^api/commitfest/(\d+)/$', api.commitfest), + url(r'^api/commitfest/(\d+)/(\d+)/$', api.build_result), url(r'^selectable/', include('selectable.urls')), -- 2.39.5