From nobody Sat May 4 16:48:49 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) client-ip=209.132.183.28; envelope-from=patchew-devel-bounces@redhat.com; helo=mx1.redhat.com; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) smtp.mailfrom=patchew-devel-bounces@redhat.com; dmarc=pass(p=none dis=none) header.from=redhat.com Return-Path: Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by mx.zohomail.com with SMTPS id 1528886270302213.35958846534766; Wed, 13 Jun 2018 03:37:50 -0700 (PDT) Received: from smtp.corp.redhat.com (int-mx01.intmail.prod.int.phx2.redhat.com [10.5.11.11]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id A4204C04B318; Wed, 13 Jun 2018 10:37:49 +0000 (UTC) Received: from colo-mx.corp.redhat.com (colo-mx02.intmail.prod.int.phx2.redhat.com [10.5.11.21]) by smtp.corp.redhat.com (Postfix) with ESMTPS id 97320600C8; Wed, 13 Jun 2018 10:37:49 +0000 (UTC) Received: from lists01.pubmisc.prod.ext.phx2.redhat.com (lists01.pubmisc.prod.ext.phx2.redhat.com [10.5.19.33]) by colo-mx.corp.redhat.com (Postfix) with ESMTP id 895714CA80; Wed, 13 Jun 2018 10:37:49 +0000 (UTC) Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.rdu2.redhat.com [10.11.54.4]) by lists01.pubmisc.prod.ext.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id w5DAblN4009726 for ; Wed, 13 Jun 2018 06:37:47 -0400 Received: by smtp.corp.redhat.com (Postfix) id 6DB1520244E0; Wed, 13 Jun 2018 10:37:47 +0000 (UTC) Received: from donizetti.redhat.com (unknown [10.36.118.61]) by smtp.corp.redhat.com (Postfix) with ESMTP id BBBAA2017D0F for ; Wed, 13 Jun 2018 10:37:46 +0000 (UTC) From: Paolo Bonzini To: patchew-devel@redhat.com Date: Wed, 13 Jun 2018 12:37:35 +0200 Message-Id: <20180613103744.30891-2-pbonzini@redhat.com> In-Reply-To: <20180613103744.30891-1-pbonzini@redhat.com> References: <20180613103744.30891-1-pbonzini@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.4 X-loop: patchew-devel@redhat.com Subject: [Patchew-devel] [PATCH 01/10] models: move Result log_url from constructor to renderer X-BeenThere: patchew-devel@redhat.com X-Mailman-Version: 2.1.12 Precedence: junk List-Id: Patchew development and discussion list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: patchew-devel-bounces@redhat.com Errors-To: patchew-devel-bounces@redhat.com X-Scanned-By: MIMEDefang 2.79 on 10.5.11.11 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.31]); Wed, 13 Jun 2018 10:37:49 +0000 (UTC) X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Type: text/plain; charset="utf-8" The log_url is currently provided at construction time. However, the URL is not going to be stored in the database and the constructor concept will disappear altogether once Results will be fetched from the database. In preparation for changing Result to a model, retrieve it through a method in the renderer. This has other benefits too. It also removes the request argument parameter from the Result constructor (which would not have any equivalent for database-stored Results) and from the rest_results_hook; and it makes log_url a read-only field in the REST API, which will be the right thing once PUT of Results will be supported. Signed-off-by: Paolo Bonzini --- api/models.py | 16 +++++++++++----- api/rest.py | 9 ++++++--- mods/git.py | 15 ++++++++------- mods/testing.py | 19 +++++++++++-------- www/views.py | 4 ++-- 5 files changed, 38 insertions(+), 25 deletions(-) diff --git a/api/models.py b/api/models.py index b4f8468..1753c6b 100644 --- a/api/models.py +++ b/api/models.py @@ -645,7 +645,7 @@ class Module(models.Model): def __str__(self): return self.name =20 -class Result(namedtuple("Result", "name status log log_url obj data render= er")): +class Result(namedtuple("Result", "name status log obj data renderer")): __slots__ =3D () PENDING =3D 'pending' SUCCESS =3D 'success' @@ -653,12 +653,10 @@ class Result(namedtuple("Result", "name status log lo= g_url obj data renderer")): RUNNING =3D 'running' VALID_STATUSES =3D (PENDING, SUCCESS, FAILURE, RUNNING) =20 - def __new__(cls, name, status, obj, log=3DNone, log_url=3DNone, data= =3DNone, request=3DNone, renderer=3DNone): - if log_url is not None and request is not None: - log_url =3D request.build_absolute_uri(log_url) + def __new__(cls, name, status, obj, log=3DNone, data=3DNone, renderer= =3DNone): if status not in cls.VALID_STATUSES: raise ValueError("invalid value '%s' for status field" % statu= s) - return super(cls, Result).__new__(cls, status=3Dstatus, log=3Dlog,= log_url=3Dlog_url, + return super(cls, Result).__new__(cls, status=3Dstatus, log=3Dlog, obj=3Dobj, data=3Ddata, name=3Dn= ame, renderer=3Drenderer) =20 def is_success(self): @@ -680,3 +678,11 @@ class Result(namedtuple("Result", "name status log log= _url obj data renderer")): if self.renderer is None: return None return self.renderer.render_result(self) + + def get_log_url(self, request=3DNone): + if not self.is_completed() or self.renderer is None: + return None + log_url =3D self.renderer.get_result_log_url(self) + if log_url is not None and request is not None: + log_url =3D request.build_absolute_uri(log_url) + return log_url diff --git a/api/rest.py b/api/rest.py index 084dbbc..f853964 100644 --- a/api/rest.py +++ b/api/rest.py @@ -450,9 +450,13 @@ class ResultSerializer(serializers.Serializer): resource_uri =3D HyperlinkedResultField(view_name=3D'results-detail') name =3D CharField() status =3D CharField() # one of 'failure', 'success', 'pending', 'runn= ing' - log_url =3D CharField(required=3DFalse) + log_url =3D SerializerMethodField(required=3DFalse) data =3D JSONField(required=3DFalse) =20 + def get_log_url(self, obj): + request =3D self.context['request'] + return obj.get_log_url(request) + class ResultSerializerFull(ResultSerializer): log =3D CharField(required=3DFalse) =20 @@ -473,8 +477,7 @@ class ResultsViewSet(viewsets.ViewSet, generics.Generic= APIView): except IndexError: raise Http404 results =3D [] - dispatch_module_hook("rest_results_hook", request=3Dself.request, - obj=3Dobj, results=3Dresults, + dispatch_module_hook("rest_results_hook", obj=3Dobj, results=3Dres= ults, detailed=3Ddetailed) return {x.name: x for x in results} =20 diff --git a/mods/git.py b/mods/git.py index fe10423..4bfb5c6 100644 --- a/mods/git.py +++ b/mods/git.py @@ -126,7 +126,7 @@ class GitModule(PatchewModule): def rest_series_fields_hook(self, request, fields, detailed): fields['based_on'] =3D PluginMethodField(obj=3Dself, required=3DFa= lse) =20 - def rest_results_hook(self, request, obj, results, detailed=3DFalse): + def rest_results_hook(self, obj, results, detailed=3DFalse): if not isinstance(obj, Message): return log =3D obj.get_property("git.apply-log") @@ -148,13 +148,10 @@ class GitModule(PatchewModule): if git_base: data['base'] =3D git_base status =3D Result.SUCCESS - log_url =3D reverse("git-log", kwargs=3D{'series': obj.message= _id}) else: status =3D Result.PENDING - log_url =3D None results.append(Result(name=3D'git', obj=3Dobj, status=3Dstatus, - log=3Dlog, log_url=3Dlog_url, data=3Ddata, - request=3Drequest, renderer=3Dself)) + log=3Dlog, data=3Ddata, renderer=3Dself)) =20 def prepare_message_hook(self, request, message, detailed): if not message.is_series_head: @@ -199,9 +196,10 @@ class GitModule(PatchewModule): if not result.is_completed(): return None =20 - html_log_url =3D result.log_url + "?html=3D1" + log_url =3D result.get_log_url() + html_log_url =3D log_url + "?html=3D1" colorbox_a =3D format_html('apply log', - html_log_url, result.log_url) + html_log_url, log_url) if result.is_failure(): return format_html('Failed in applying to current master ({})'= , colorbox_a) else: @@ -218,6 +216,9 @@ class GitModule(PatchewModule): s +=3D format_html('
git fetch {} {}', gi= t_repo, git_tag) return s =20 + def get_result_log_url(self, result): + return reverse("git-log", kwargs=3D{'series': result.obj.message_i= d}) + def prepare_project_hook(self, request, project): if not project.maintained_by(request.user): return diff --git a/mods/testing.py b/mods/testing.py index 824df15..efa2d82 100644 --- a/mods/testing.py +++ b/mods/testing.py @@ -165,8 +165,8 @@ class TestingModule(PatchewModule): "testing_name": test}) + "?type=3Dpr= oject" if html: log_url +=3D "&html=3D1" - # Generate a full URL, including the host and port, for use in ema= il - # notifications and REST API responses. + # Generate a full URL, including the host and port, for use in + # email notifications if request: log_url =3D request.build_absolute_uri(log_url) return log_url @@ -251,7 +251,7 @@ class TestingModule(PatchewModule): }) return ret =20 - def rest_results_hook(self, request, obj, results, detailed=3DFalse): + def rest_results_hook(self, obj, results, detailed=3DFalse): all_tests =3D set([k for k, v in _instance.get_tests(obj).items() = if v["enabled"]]) for pn, p in obj.get_properties().items(): if not pn.startswith("testing.report."): @@ -262,7 +262,6 @@ class TestingModule(PatchewModule): except: pass failed =3D not p["passed"] - log_url =3D self.reverse_testing_log(obj, tn, request=3Dreques= t, html=3DFalse) passed_str =3D Result.FAILURE if failed else Result.SUCCESS if detailed: log =3D obj.get_property("testing.log." + tn) @@ -272,8 +271,7 @@ class TestingModule(PatchewModule): data =3D p.copy() del data['passed'] results.append(Result(name=3D'testing.' + tn, obj=3Dobj, statu= s=3Dpassed_str, - log=3Dlog, log_url=3Dlog_url, request=3D= request, data=3Ddata, - renderer=3Dself)) + log=3Dlog, data=3Ddata, renderer=3Dself)) =20 if obj.get_property("testing.ready"): for tn in all_tests: @@ -305,15 +303,20 @@ class TestingModule(PatchewModule): "char": "T", }) =20 + def get_result_log_url(self, result): + tn =3D result.name[len("testing."):] + return self.reverse_testing_log(result.obj, tn, html=3DFalse) + def render_result(self, result): if not result.is_completed(): return None pn =3D result.name tn =3D pn[len("testing."):] - html_log_url =3D result.log_url + '&html=3D1' + log_url =3D result.get_log_url() + html_log_url =3D log_url + '&html=3D1' passed_str =3D "failed" if result.is_failure() else "passed" return format_html('Test {} {}', - tn, html_log_url, result.log_url, passed_str) + tn, html_log_url, log_url, passed_str) =20 def check_active_testers(self, project): at =3D [] diff --git a/www/views.py b/www/views.py index 1eb6c3c..c2010c5 100644 --- a/www/views.py +++ b/www/views.py @@ -93,8 +93,8 @@ def prepare_series(request, s, skip_patches=3DFalse): =20 def prepare_results(request, obj): results =3D [] - dispatch_module_hook("rest_results_hook", request=3Drequest, - obj=3Dobj, results=3Dresults, detailed=3DFalse) + dispatch_module_hook("rest_results_hook", obj=3Dobj, + results=3Dresults, detailed=3DFalse) =20 results_dicts =3D [] for result in results: --=20 2.17.0 _______________________________________________ Patchew-devel mailing list Patchew-devel@redhat.com https://www.redhat.com/mailman/listinfo/patchew-devel From nobody Sat May 4 16:48:49 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) client-ip=209.132.183.28; envelope-from=patchew-devel-bounces@redhat.com; helo=mx1.redhat.com; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) smtp.mailfrom=patchew-devel-bounces@redhat.com; dmarc=pass(p=none dis=none) header.from=redhat.com Return-Path: Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by mx.zohomail.com with SMTPS id 1528886270324613.8013597300177; Wed, 13 Jun 2018 03:37:50 -0700 (PDT) Received: from smtp.corp.redhat.com (int-mx11.intmail.prod.int.phx2.redhat.com [10.5.11.26]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id C2663308FB83; Wed, 13 Jun 2018 10:37:49 +0000 (UTC) Received: from colo-mx.corp.redhat.com (colo-mx02.intmail.prod.int.phx2.redhat.com [10.5.11.21]) by smtp.corp.redhat.com (Postfix) with ESMTPS id B1461309128B; Wed, 13 Jun 2018 10:37:49 +0000 (UTC) Received: from lists01.pubmisc.prod.ext.phx2.redhat.com (lists01.pubmisc.prod.ext.phx2.redhat.com [10.5.19.33]) by colo-mx.corp.redhat.com (Postfix) with ESMTP id 9BE3A4CA82; Wed, 13 Jun 2018 10:37:49 +0000 (UTC) Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.rdu2.redhat.com [10.11.54.4]) by lists01.pubmisc.prod.ext.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id w5DAbmVY009731 for ; Wed, 13 Jun 2018 06:37:48 -0400 Received: by smtp.corp.redhat.com (Postfix) id 2E3332026987; Wed, 13 Jun 2018 10:37:48 +0000 (UTC) Received: from donizetti.redhat.com (unknown [10.36.118.61]) by smtp.corp.redhat.com (Postfix) with ESMTP id B445E20244E1 for ; Wed, 13 Jun 2018 10:37:47 +0000 (UTC) From: Paolo Bonzini To: patchew-devel@redhat.com Date: Wed, 13 Jun 2018 12:37:36 +0200 Message-Id: <20180613103744.30891-3-pbonzini@redhat.com> In-Reply-To: <20180613103744.30891-1-pbonzini@redhat.com> References: <20180613103744.30891-1-pbonzini@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.4 X-loop: patchew-devel@redhat.com Subject: [Patchew-devel] [PATCH 02/10] models: extract blobs functions to a separate file X-BeenThere: patchew-devel@redhat.com X-Mailman-Version: 2.1.12 Precedence: junk List-Id: Patchew development and discussion list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: patchew-devel-bounces@redhat.com Errors-To: patchew-devel-bounces@redhat.com X-Scanned-By: MIMEDefang 2.84 on 10.5.11.26 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.43]); Wed, 13 Jun 2018 10:37:49 +0000 (UTC) X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Type: text/plain; charset="utf-8" They will be used by migrations, and we don't want to import api.models there. --- api/blobs.py | 37 +++++++++++++++++++++++++++++++++++++ api/models.py | 27 ++++----------------------- 2 files changed, 41 insertions(+), 23 deletions(-) create mode 100644 api/blobs.py diff --git a/api/blobs.py b/api/blobs.py new file mode 100644 index 0000000..b1c7d34 --- /dev/null +++ b/api/blobs.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# +# Copyright 2016, 2018 Red Hat, Inc. +# +# Authors: +# Fam Zheng +# Paolo Bonzini +# +# This work is licensed under the MIT License. Please see the LICENSE fil= e or +# http://opensource.org/licenses/MIT. + + +import os +import json +import uuid +import logging + +from django.conf import settings +import lzma + +def save_blob(data, name=3DNone): + if not name: + name =3D str(uuid.uuid4()) + fn =3D os.path.join(settings.DATA_DIR, "blob", name + ".xz") + lzma.open(fn, 'w').write(data.encode("utf-8")) + return name + +def load_blob(name): + fn =3D os.path.join(settings.DATA_DIR, "blob", name + ".xz") + return lzma.open(fn, 'r').read().decode("utf-8") + +def load_blob_json(name): + try: + return json.loads(load_blob(name)) + except json.decoder.JSONDecodeError as e: + logging.error('Failed to load blob %s: %s' %(name, e)) + return None diff --git a/api/models.py b/api/models.py index 1753c6b..0545b80 100644 --- a/api/models.py +++ b/api/models.py @@ -10,39 +10,20 @@ =20 =20 from collections import namedtuple -import os import json import datetime import re -import uuid -import logging =20 -from django.conf import settings from django.db import models from django.contrib.auth.models import User from django.urls import reverse import jsonfield -from mbox import MboxMessage -from event import emit_event, declare_event import lzma =20 -def save_blob(data, name=3DNone): - if not name: - name =3D str(uuid.uuid4()) - fn =3D os.path.join(settings.DATA_DIR, "blob", name + ".xz") - lzma.open(fn, 'w').write(data.encode("utf-8")) - return name - -def load_blob(name): - fn =3D os.path.join(settings.DATA_DIR, "blob", name + ".xz") - return lzma.open(fn, 'r').read().decode("utf-8") - -def load_blob_json(name): - try: - return json.loads(load_blob(name)) - except json.decoder.JSONDecodeError as e: - logging.error('Failed to load blob %s: %s' %(name, e)) - return None +from mbox import MboxMessage +from event import emit_event, declare_event +from .blobs import save_blob, load_blob, load_blob_json +import mod =20 class Project(models.Model): name =3D models.CharField(max_length=3D1024, db_index=3DTrue, unique= =3DTrue, --=20 2.17.0 _______________________________________________ Patchew-devel mailing list Patchew-devel@redhat.com https://www.redhat.com/mailman/listinfo/patchew-devel From nobody Sat May 4 16:48:49 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) client-ip=209.132.183.28; envelope-from=patchew-devel-bounces@redhat.com; helo=mx1.redhat.com; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) smtp.mailfrom=patchew-devel-bounces@redhat.com; dmarc=pass(p=none dis=none) header.from=redhat.com Return-Path: Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by mx.zohomail.com with SMTPS id 1528886273788981.9990383074677; Wed, 13 Jun 2018 03:37:53 -0700 (PDT) Received: from smtp.corp.redhat.com (int-mx10.intmail.prod.int.phx2.redhat.com [10.5.11.25]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 0D67EC057F4E; Wed, 13 Jun 2018 10:37:52 +0000 (UTC) Received: from colo-mx.corp.redhat.com (colo-mx01.intmail.prod.int.phx2.redhat.com [10.5.11.20]) by smtp.corp.redhat.com (Postfix) with ESMTPS id EDD9A2010CB0; Wed, 13 Jun 2018 10:37:51 +0000 (UTC) Received: from lists01.pubmisc.prod.ext.phx2.redhat.com (lists01.pubmisc.prod.ext.phx2.redhat.com [10.5.19.33]) by colo-mx.corp.redhat.com (Postfix) with ESMTP id DFC0E1809547; Wed, 13 Jun 2018 10:37:51 +0000 (UTC) Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.rdu2.redhat.com [10.11.54.4]) by lists01.pubmisc.prod.ext.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id w5DAbnbk009741 for ; Wed, 13 Jun 2018 06:37:49 -0400 Received: by smtp.corp.redhat.com (Postfix) id E732C20244E1; Wed, 13 Jun 2018 10:37:48 +0000 (UTC) Received: from donizetti.redhat.com (unknown [10.36.118.61]) by smtp.corp.redhat.com (Postfix) with ESMTP id 74C1220244E0 for ; Wed, 13 Jun 2018 10:37:48 +0000 (UTC) From: Paolo Bonzini To: patchew-devel@redhat.com Date: Wed, 13 Jun 2018 12:37:37 +0200 Message-Id: <20180613103744.30891-4-pbonzini@redhat.com> In-Reply-To: <20180613103744.30891-1-pbonzini@redhat.com> References: <20180613103744.30891-1-pbonzini@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.4 X-loop: patchew-devel@redhat.com Subject: [Patchew-devel] [PATCH 03/10] update jsonfield to 3.0.x X-BeenThere: patchew-devel@redhat.com X-Mailman-Version: 2.1.12 Precedence: junk List-Id: Patchew development and discussion list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: patchew-devel-bounces@redhat.com Errors-To: patchew-devel-bounces@redhat.com X-Scanned-By: MIMEDefang 2.84 on 10.5.11.25 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.32]); Wed, 13 Jun 2018 10:37:52 +0000 (UTC) X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Type: text/plain; charset="utf-8" This fork is required to support multi-table inheritance. The original has= a bug (https://github.com/dmkoch/django-jsonfield/issues/101). Signed-off-by: Paolo Bonzini --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ac668e5..1b40ecd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,6 @@ Markdown>=3D2.6.6 django-debug-toolbar=3D=3D1.8 gunicorn>=3D19 djangorestframework>=3D3.0.0 -jsonfield >=3D 2.0.2, <3.0 +jsonfield2 >=3D 3.0, < 4.0 drf-nested-routers coreapi-cli \ No newline at end of file --=20 2.17.0 _______________________________________________ Patchew-devel mailing list Patchew-devel@redhat.com https://www.redhat.com/mailman/listinfo/patchew-devel From nobody Sat May 4 16:48:49 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) client-ip=209.132.183.28; envelope-from=patchew-devel-bounces@redhat.com; helo=mx1.redhat.com; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) smtp.mailfrom=patchew-devel-bounces@redhat.com; dmarc=pass(p=none dis=none) header.from=redhat.com Return-Path: Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by mx.zohomail.com with SMTPS id 1528886273817733.7685040572759; Wed, 13 Jun 2018 03:37:53 -0700 (PDT) Received: from smtp.corp.redhat.com (int-mx01.intmail.prod.int.phx2.redhat.com [10.5.11.11]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id CDDC981DF5; Wed, 13 Jun 2018 10:37:51 +0000 (UTC) Received: from colo-mx.corp.redhat.com (colo-mx02.intmail.prod.int.phx2.redhat.com [10.5.11.21]) by smtp.corp.redhat.com (Postfix) with ESMTPS id BA727600C8; Wed, 13 Jun 2018 10:37:51 +0000 (UTC) Received: from lists01.pubmisc.prod.ext.phx2.redhat.com (lists01.pubmisc.prod.ext.phx2.redhat.com [10.5.19.33]) by colo-mx.corp.redhat.com (Postfix) with ESMTP id AFF904BB78; Wed, 13 Jun 2018 10:37:51 +0000 (UTC) Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.rdu2.redhat.com [10.11.54.4]) by lists01.pubmisc.prod.ext.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id w5DAbnUV009756 for ; Wed, 13 Jun 2018 06:37:50 -0400 Received: by smtp.corp.redhat.com (Postfix) id DF7A9210C6DC; Wed, 13 Jun 2018 10:37:49 +0000 (UTC) Received: from donizetti.redhat.com (unknown [10.36.118.61]) by smtp.corp.redhat.com (Postfix) with ESMTP id 39B482026987 for ; Wed, 13 Jun 2018 10:37:49 +0000 (UTC) From: Paolo Bonzini To: patchew-devel@redhat.com Date: Wed, 13 Jun 2018 12:37:38 +0200 Message-Id: <20180613103744.30891-5-pbonzini@redhat.com> In-Reply-To: <20180613103744.30891-1-pbonzini@redhat.com> References: <20180613103744.30891-1-pbonzini@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.4 X-loop: patchew-devel@redhat.com Subject: [Patchew-devel] [PATCH 04/10] models: create Result model X-BeenThere: patchew-devel@redhat.com X-Mailman-Version: 2.1.12 Precedence: junk List-Id: Patchew development and discussion list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: patchew-devel-bounces@redhat.com Errors-To: patchew-devel-bounces@redhat.com X-Scanned-By: MIMEDefang 2.79 on 10.5.11.11 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.25]); Wed, 13 Jun 2018 10:37:51 +0000 (UTC) X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Type: text/plain; charset="utf-8" The Result model has more or less a 1:1 correspondence with the fields in the REST results detail view, and thus with ResultSerializerFull. Until all plugins are converted, everything will still go through namedtuple results, so the namedtuple is kept, renamed to ResultTuple. The duplicated code will disappear at the end of this series. There is an extra field tracking the time of the last update, which will be used by the testing module; for simplicity it is not yet presented by the serializers, though it will be added when ResultTuple is dropped. The external interface of Result and ResultTuple is more or less the same, except that there is no more renderer. The renderer is now discovered by convention: it is always a PatchewModule, and the name of the result up to the first period (if any) identifies it. As for obj, the parent object, the Result object has no clue whether it comes from a project or a message, but the subclasses (ProjectResult and MessageResult) do, so no change is needed. Compared to properties, logs are always stored in the database, independent of the size. However, they are always stored in xz format compared to the JSON string format used by properties. Using xz will complicate migrations a little, since the log property won't be available there, but not as much as handling blobs. Signed-off-by: Paolo Bonzini --- api/migrations/0027_auto_20180521_0152.py | 61 +++++++++ api/models.py | 150 ++++++++++++++++++++-- mods/git.py | 4 +- mods/testing.py | 6 +- 4 files changed, 204 insertions(+), 17 deletions(-) create mode 100644 api/migrations/0027_auto_20180521_0152.py diff --git a/api/migrations/0027_auto_20180521_0152.py b/api/migrations/002= 7_auto_20180521_0152.py new file mode 100644 index 0000000..d6833ef --- /dev/null +++ b/api/migrations/0027_auto_20180521_0152.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.13 on 2018-05-31 05:44 +from __future__ import unicode_literals + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import jsonfield.encoder +import jsonfield.fields + + +class Migration(migrations.Migration): + + dependencies =3D [ + ('api', '0026_auto_20180426_0829'), + ] + + operations =3D [ + migrations.CreateModel( + name=3D'LogEntry', + fields=3D[ + ('id', models.AutoField(auto_created=3DTrue, primary_key= =3DTrue, serialize=3DFalse, verbose_name=3D'ID')), + ('data_xz', models.BinaryField()), + ], + ), + migrations.CreateModel( + name=3D'Result', + fields=3D[ + ('id', models.AutoField(auto_created=3DTrue, primary_key= =3DTrue, serialize=3DFalse, verbose_name=3D'ID')), + ('name', models.CharField(max_length=3D256)), + ('last_update', models.DateTimeField()), + ('status', models.CharField(max_length=3D7, validators=3D[= django.core.validators.RegexValidator(code=3D'invalid', message=3D'status m= ust be one of pending, success, failure, running', regex=3D'pending|success= |failure|running')])), + ('data', jsonfield.fields.JSONField(dump_kwargs=3D{'cls': = jsonfield.encoder.JSONEncoder, 'separators': (',', ':')}, load_kwargs=3D{})= ), + ], + ), + migrations.CreateModel( + name=3D'MessageResult', + fields=3D[ + ('result_ptr', models.OneToOneField(auto_created=3DTrue, o= n_delete=3Ddjango.db.models.deletion.CASCADE, parent_link=3DTrue, primary_k= ey=3DTrue, serialize=3DFalse, to=3D'api.Result')), + ('message', models.ForeignKey(on_delete=3Ddjango.db.models= .deletion.CASCADE, related_name=3D'results', to=3D'api.Message')), + ], + bases=3D('api.result',), + ), + migrations.CreateModel( + name=3D'ProjectResult', + fields=3D[ + ('result_ptr', models.OneToOneField(auto_created=3DTrue, o= n_delete=3Ddjango.db.models.deletion.CASCADE, parent_link=3DTrue, primary_k= ey=3DTrue, serialize=3DFalse, to=3D'api.Result')), + ('project', models.ForeignKey(on_delete=3Ddjango.db.models= .deletion.CASCADE, related_name=3D'results', to=3D'api.Project')), + ], + bases=3D('api.result',), + ), + migrations.AddField( + model_name=3D'result', + name=3D'log_entry', + field=3Dmodels.OneToOneField(null=3DTrue, on_delete=3Ddjango.d= b.models.deletion.CASCADE, to=3D'api.LogEntry'), + ), + migrations.AlterIndexTogether( + name=3D'result', + index_together=3Dset([('status', 'name')]), + ), + ] diff --git a/api/models.py b/api/models.py index 0545b80..6288d8d 100644 --- a/api/models.py +++ b/api/models.py @@ -14,7 +14,9 @@ import json import datetime import re =20 +from django.core import validators from django.db import models +from django.db.models import Q from django.contrib.auth.models import User from django.urls import reverse import jsonfield @@ -25,6 +27,115 @@ from event import emit_event, declare_event from .blobs import save_blob, load_blob, load_blob_json import mod =20 +class LogEntry(models.Model): + data_xz =3D models.BinaryField() + + @property + def data(self): + if not hasattr(self, "_data"): + self._data =3D lzma.decompress(self.data_xz).decode("utf-8") + return self._data + + @data.setter + def data(self, value): + self._data =3D value + self.data_xz =3D lzma.compress(value.encode("utf-8")) + +class Result(models.Model): + PENDING =3D 'pending' + SUCCESS =3D 'success' + FAILURE =3D 'failure' + RUNNING =3D 'running' + VALID_STATUSES =3D (PENDING, SUCCESS, FAILURE, RUNNING) + VALID_STATUSES_RE =3D '|'.join(VALID_STATUSES) + + name =3D models.CharField(max_length=3D256) + last_update =3D models.DateTimeField() + status =3D models.CharField(max_length=3D7, validators=3D[ + validators.RegexValidator(regex=3DVALID_STATUSES_RE, + message=3D'status must be one of ' + ', = '.join(VALID_STATUSES), + code=3D'invalid')]) + log_entry =3D models.OneToOneField(LogEntry, on_delete=3Dmodels.CASCAD= E, + null=3DTrue) + data =3D jsonfield.JSONField() + + class Meta: + index_together =3D [('status', 'name')] + + def is_success(self): + return self.status =3D=3D self.SUCCESS + + def is_failure(self): + return self.status =3D=3D self.FAILURE + + def is_completed(self): + return self.is_success() or self.is_failure() + + def is_pending(self): + return self.status =3D=3D self.PENDING + + def is_running(self): + return self.status =3D=3D self.RUNNING + + def save(self): + self.last_update =3D datetime.datetime.utcnow() + return super(Result, self).save() + + @property + def renderer(self): + found =3D re.match("^[^.]*", self.name) + return mod.get_module(found.group(0)) if found else None + + @property + def obj(self): + return None + + def render(self): + if self.renderer is None: + return None + return self.renderer.render_result(self) + + @property + def log(self): + if self.log_entry is None: + return None + else: + return self.log_entry.data + + @log.setter + def log(self, value): + entry =3D self.log_entry + if value is None: + if entry is not None: + self.log_entry =3D None + entry.delete() + else: + if entry is None: + entry =3D LogEntry() + entry.data =3D value + entry.save() + if self.log_entry is None: + self.log_entry =3D entry + + def get_log_url(self, request=3DNone): + if not self.is_completed() or self.renderer is None: + return None + log_url =3D self.renderer.get_result_log_url(self) + if log_url is not None and request is not None: + log_url =3D request.build_absolute_uri(log_url) + return log_url + + @staticmethod + def get_result_tuples(obj, module, results): + name_filter =3D Q(name=3Dmodule) | Q(name__startswith=3Dmodule + '= .') + renderer =3D mod.get_module(module) + for r in obj.results.filter(name_filter): + results.append(ResultTuple(name=3Dr.name, obj=3Dobj, status=3D= r.status, + log=3Dr.log, data=3Dr.data, rendere= r=3Drenderer)) + + def __str__(self): + return '%s (%s)' % (self.name, self.status) + class Project(models.Model): name =3D models.CharField(max_length=3D1024, db_index=3DTrue, unique= =3DTrue, help_text=3D"""The name of the project""") @@ -183,6 +294,16 @@ class Project(models.Model): series.save() return len(updated_series) =20 + def create_result(self, **kwargs): + return ProjectResult(project=3Dself, **kwargs) + +class ProjectResult(Result): + project =3D models.ForeignKey(Project, related_name=3D'results') + + @property + def obj(self): + return self.project + class ProjectProperty(models.Model): project =3D models.ForeignKey('Project', on_delete=3Dmodels.CASCADE) name =3D models.CharField(max_length=3D1024, db_index=3DTrue) @@ -592,12 +713,22 @@ class Message(models.Model): self.save() emit_event("SeriesComplete", project=3Dself.project, series=3Dself) =20 + def create_result(self, **kwargs): + return MessageResult(message=3Dself, **kwargs) + def __str__(self): return self.subject =20 class Meta: unique_together =3D ('project', 'message_id',) =20 +class MessageResult(Result): + message =3D models.ForeignKey(Message, related_name=3D'results') + + @property + def obj(self): + return self.message + class MessageProperty(models.Model): message =3D models.ForeignKey('Message', on_delete=3Dmodels.CASCADE, related_name=3D'properties') @@ -626,34 +757,29 @@ class Module(models.Model): def __str__(self): return self.name =20 -class Result(namedtuple("Result", "name status log obj data renderer")): +class ResultTuple(namedtuple("ResultTuple", "name status log obj data rend= erer")): __slots__ =3D () - PENDING =3D 'pending' - SUCCESS =3D 'success' - FAILURE =3D 'failure' - RUNNING =3D 'running' - VALID_STATUSES =3D (PENDING, SUCCESS, FAILURE, RUNNING) =20 def __new__(cls, name, status, obj, log=3DNone, data=3DNone, renderer= =3DNone): - if status not in cls.VALID_STATUSES: + if status not in Result.VALID_STATUSES: raise ValueError("invalid value '%s' for status field" % statu= s) - return super(cls, Result).__new__(cls, status=3Dstatus, log=3Dlog, + return super(cls, ResultTuple).__new__(cls, status=3Dstatus, log= =3Dlog, obj=3Dobj, data=3Ddata, name=3Dn= ame, renderer=3Drenderer) =20 def is_success(self): - return self.status =3D=3D self.SUCCESS + return self.status =3D=3D Result.SUCCESS =20 def is_failure(self): - return self.status =3D=3D self.FAILURE + return self.status =3D=3D Result.FAILURE =20 def is_completed(self): return self.is_success() or self.is_failure() =20 def is_pending(self): - return self.status =3D=3D self.PENDING + return self.status =3D=3D Result.PENDING =20 def is_running(self): - return self.status =3D=3D self.RUNNING + return self.status =3D=3D Result.RUNNING =20 def render(self): if self.renderer is None: diff --git a/mods/git.py b/mods/git.py index 4bfb5c6..fadce4c 100644 --- a/mods/git.py +++ b/mods/git.py @@ -18,7 +18,7 @@ from django.core.exceptions import PermissionDenied from django.utils.html import format_html from mod import PatchewModule from event import declare_event, register_handler, emit_event -from api.models import Message, MessageProperty, Result +from api.models import Message, MessageProperty, Result, ResultTuple from api.rest import PluginMethodField from api.views import APILoginRequiredView, prepare_series from patchew.logviewer import LogView @@ -150,7 +150,7 @@ class GitModule(PatchewModule): status =3D Result.SUCCESS else: status =3D Result.PENDING - results.append(Result(name=3D'git', obj=3Dobj, status=3Dstatus, + results.append(ResultTuple(name=3D'git', obj=3Dobj, status=3Dstatu= s, log=3Dlog, data=3Ddata, renderer=3Dself)) =20 def prepare_message_hook(self, request, message, detailed): diff --git a/mods/testing.py b/mods/testing.py index efa2d82..ca3d60d 100644 --- a/mods/testing.py +++ b/mods/testing.py @@ -17,7 +17,7 @@ from mod import PatchewModule import time import math from api.views import APILoginRequiredView -from api.models import Message, MessageProperty, Project, Result +from api.models import Message, MessageProperty, Project, Result, ResultTu= ple from api.search import SearchEngine from event import emit_event, declare_event, register_handler from patchew.logviewer import LogView @@ -270,12 +270,12 @@ class TestingModule(PatchewModule): =20 data =3D p.copy() del data['passed'] - results.append(Result(name=3D'testing.' + tn, obj=3Dobj, statu= s=3Dpassed_str, + results.append(ResultTuple(name=3D'testing.' + tn, obj=3Dobj, = status=3Dpassed_str, log=3Dlog, data=3Ddata, renderer=3Dself)) =20 if obj.get_property("testing.ready"): for tn in all_tests: - results.append(Result(name=3D'testing.' + tn, obj=3Dobj, s= tatus=3D'pending')) + results.append(ResultTuple(name=3D'testing.' + tn, obj=3Do= bj, status=3D'pending')) =20 def prepare_message_hook(self, request, message, detailed): if not message.is_series_head: --=20 2.17.0 _______________________________________________ Patchew-devel mailing list Patchew-devel@redhat.com https://www.redhat.com/mailman/listinfo/patchew-devel From nobody Sat May 4 16:48:49 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) client-ip=209.132.183.28; envelope-from=patchew-devel-bounces@redhat.com; helo=mx1.redhat.com; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) smtp.mailfrom=patchew-devel-bounces@redhat.com; dmarc=pass(p=none dis=none) header.from=redhat.com Return-Path: Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by mx.zohomail.com with SMTPS id 1528886273827332.18972804526254; Wed, 13 Jun 2018 03:37:53 -0700 (PDT) Received: from smtp.corp.redhat.com (int-mx07.intmail.prod.int.phx2.redhat.com [10.5.11.22]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id AE382307D85D; Wed, 13 Jun 2018 10:37:52 +0000 (UTC) Received: from colo-mx.corp.redhat.com (colo-mx02.intmail.prod.int.phx2.redhat.com [10.5.11.21]) by smtp.corp.redhat.com (Postfix) with ESMTPS id 9C94F1001938; Wed, 13 Jun 2018 10:37:52 +0000 (UTC) Received: from lists01.pubmisc.prod.ext.phx2.redhat.com (lists01.pubmisc.prod.ext.phx2.redhat.com [10.5.19.33]) by colo-mx.corp.redhat.com (Postfix) with ESMTP id 8C9854BB78; Wed, 13 Jun 2018 10:37:52 +0000 (UTC) Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.rdu2.redhat.com [10.11.54.4]) by lists01.pubmisc.prod.ext.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id w5DAbpp1009764 for ; Wed, 13 Jun 2018 06:37:51 -0400 Received: by smtp.corp.redhat.com (Postfix) id ECB8920244E0; Wed, 13 Jun 2018 10:37:50 +0000 (UTC) Received: from donizetti.redhat.com (unknown [10.36.118.61]) by smtp.corp.redhat.com (Postfix) with ESMTP id 3591C2017D0F for ; Wed, 13 Jun 2018 10:37:50 +0000 (UTC) From: Paolo Bonzini To: patchew-devel@redhat.com Date: Wed, 13 Jun 2018 12:37:39 +0200 Message-Id: <20180613103744.30891-6-pbonzini@redhat.com> In-Reply-To: <20180613103744.30891-1-pbonzini@redhat.com> References: <20180613103744.30891-1-pbonzini@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.4 X-loop: patchew-devel@redhat.com Subject: [Patchew-devel] [PATCH 05/10] git: switch to Result model X-BeenThere: patchew-devel@redhat.com X-Mailman-Version: 2.1.12 Precedence: junk List-Id: Patchew development and discussion list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: patchew-devel-bounces@redhat.com Errors-To: patchew-devel-bounces@redhat.com X-Scanned-By: MIMEDefang 2.84 on 10.5.11.22 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.48]); Wed, 13 Jun 2018 10:37:52 +0000 (UTC) X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Type: text/plain; charset="utf-8" Operate on a Result named 'git' instead of using multiple properties. Only the hook has to convert the Result to namedtuple format, until the REST API and web views are converted to access the database directly. --- api/blobs.py | 8 ++ api/migrations/0028_populate_git_results.py | 104 ++++++++++++++++ api/migrations/__init__.py | 49 ++++++++ mods/git.py | 124 +++++++++----------- tests/test_git.py | 34 +++--- 5 files changed, 235 insertions(+), 84 deletions(-) create mode 100644 api/migrations/0028_populate_git_results.py diff --git a/api/blobs.py b/api/blobs.py index b1c7d34..20d4567 100644 --- a/api/blobs.py +++ b/api/blobs.py @@ -35,3 +35,11 @@ def load_blob_json(name): except json.decoder.JSONDecodeError as e: logging.error('Failed to load blob %s: %s' %(name, e)) return None + +def delete_blob(name): + fn =3D os.path.join(settings.DATA_DIR, "blob", name + ".xz") + try: + os.remove(fn) + except FileNotFoundError: + pass + diff --git a/api/migrations/0028_populate_git_results.py b/api/migrations/0= 028_populate_git_results.py new file mode 100644 index 0000000..482641a --- /dev/null +++ b/api/migrations/0028_populate_git_results.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations +from django.db.models import Count +from api.migrations import get_property, set_property, delete_property_blob + +import datetime +import lzma + +# For Result's constant status values +import api.models + +def result_from_properties(apps, schema_editor): + # We can't import the models directly as they may be a newer + # version than this migration expects. We use the historical version. + # The code is mostly based on the implementation of the old + # "rest_results_hook" method in GitModule, which used to build + # a Result namedtuple from the properties + Message =3D apps.get_model('api', 'Message') + MessageProperty =3D apps.get_model('api', 'MessageProperty') + MessageResult =3D apps.get_model('api', 'MessageResult') + LogEntry =3D apps.get_model('api', 'LogEntry') + messages =3D Message.objects.filter(properties__name__startswith=3D'gi= t.').distinct() + for m in messages: + need_apply =3D get_property(MessageProperty, 'git.need-apply', mes= sage=3Dm) + if need_apply is None: + continue + log =3D get_property(MessageProperty, "git.apply-log", message=3Dm) + r =3D MessageResult(name=3D'git', message=3Dm) + if log: + log_xz =3D lzma.compress(log.encode("utf-8")) + log_entry =3D LogEntry(data_xz=3Dlog_xz) + log_entry.save() + r.log_entry =3D log_entry + if get_property(MessageProperty, "git.apply-failed", message= =3Dm): + r.status =3D api.models.Result.FAILURE + else: + git_repo =3D get_property(MessageProperty, "git.repo", mes= sage=3Dm) + git_tag =3D get_property(MessageProperty, "git.tag", messa= ge=3Dm) + git_url =3D get_property(MessageProperty, "git.url", messa= ge=3Dm) + git_base =3D get_property(MessageProperty, "git.base", mes= sage=3Dm) + data =3D {} + if git_repo and git_tag: + data['repo'] =3D git_repo + data['tag'] =3D 'refs/tags/' + git_tag + if git_url: + data['url'] =3D git_url + if git_base: + data['base'] =3D git_base + r.data =3D data + r.status =3D api.models.Result.SUCCESS + else: + status =3D api.models.Result.PENDING + r.last_update =3D datetime.datetime.utcnow() + r.save() + messages =3D Message.objects.filter(properties__name=3D'git.apply-log'= , properties__blob=3DTrue) + for m in messages: + delete_property_blob(MessageProperty, "git.apply-log", message=3Dm) + MessageProperty.objects.filter(name__startswith=3D'git.').delete() + +def result_to_properties(apps, schema_editor): + # We can't import the models directly as they may be a newer + # version than this migration expects. We use the historical version. + Message =3D apps.get_model('api', 'Message') + MessageProperty =3D apps.get_model('api', 'MessageProperty') + MessageResult =3D apps.get_model('api', 'MessageResult') + LogEntry =3D apps.get_model('api', 'LogEntry') + messages =3D Message.objects.filter(results__name=3D'git') + for m in messages: + r =3D MessageResult.objects.get(name=3D'git', message=3Dm) + if not r: + continue + if r.status =3D=3D api.models.Result.PENDING: + set_property(MessageProperty, 'git.need-apply', True, message= =3Dm) + else: + log =3D lzma.decompress(r.log_entry.data_xz).decode("utf-8") + set_property(MessageProperty, 'git.need-apply', False, message= =3Dm) + set_property(MessageProperty, 'git.apply-log', log, message=3D= m) + if r.status =3D=3D api.models.Result.FAILURE: + set_property(MessageProperty, "git.apply-failed", True, me= ssage=3Dm) + else: + set_property(MessageProperty, "git.apply-failed", False, m= essage=3Dm) + if 'repo' in r.data: + set_property(MessageProperty, "git.repo", r.data['repo= '], message=3Dm) + if 'tag' in r.data: + set_property(MessageProperty, "git.tag", r.data['repo'= ][len('refs/tags/'):], message=3Dm) + if 'url' in r.data: + set_property(MessageProperty, "git.url", r.data['url']= , message=3Dm) + if 'base' in r.data: + set_property(MessageProperty, "git.base", r.data['base= '], message=3Dm) + MessageResult.objects.filter(message=3Dm, name=3D'git').delete() + +class Migration(migrations.Migration): + + dependencies =3D [ + ('api', '0027_auto_20180521_0152'), + ] + + operations =3D [ + migrations.RunPython(result_from_properties, + reverse_code=3Dresult_to_properties), + ] diff --git a/api/migrations/__init__.py b/api/migrations/__init__.py index e69de29..f4b7eb7 100644 --- a/api/migrations/__init__.py +++ b/api/migrations/__init__.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# +# Copyright 2018 Red Hat, Inc. +# +# Authors: +# Paolo Bonzini +# +# This work is licensed under the MIT License. Please see the LICENSE fil= e or +# http://opensource.org/licenses/MIT. + + +import json +from api import blobs + +def load_blob_json_safe(name): + try: + return json.loads(blobs.load_blob(name)) + except Exception as e: + return 'Failed to load blob %s: %s' % (name, e) + +def get_property_raw(model, name, **kwargs): + try: + mp =3D model.objects.get(name=3Dname, **kwargs) + return mp + except model.DoesNotExist: + return None + +def load_property(mp): + if mp.blob: + return load_blob_json_safe(mp.value) + else: + return json.loads(mp.value) + +def get_property(model, name, **kwargs): + mp =3D get_property_raw(model, name, **kwargs) + return load_property(mp) if mp else None + +def delete_property_blob(model, name, **kwargs): + mp =3D get_property_raw(model, name, **kwargs) + if mp.blob: + blobs.delete_blob(mp.value) + +def set_property(model, name, value, **kwargs): + if value is not None: + value =3D json.dumps(value) + mp, created =3D model.objects.get_or_create(name=3Dname, **kwargs) + mp.value =3D value + mp.blob =3D False + mp.save() diff --git a/mods/git.py b/mods/git.py index fadce4c..b8c2ff0 100644 --- a/mods/git.py +++ b/mods/git.py @@ -18,7 +18,7 @@ from django.core.exceptions import PermissionDenied from django.utils.html import format_html from mod import PatchewModule from event import declare_event, register_handler, emit_event -from api.models import Message, MessageProperty, Result, ResultTuple +from api.models import Message, MessageProperty, Project, Result, ResultTu= ple from api.rest import PluginMethodField from api.views import APILoginRequiredView, prepare_series from patchew.logviewer import LogView @@ -26,16 +26,24 @@ from schema import * =20 _instance =3D None =20 +def _get_git_result(msg): + try: + return msg.results.get(name=3D"git") + except: + return None +Message.git_result =3D property(_get_git_result) + + class GitLogViewer(LogView): def content(self, request, **kwargs): series =3D kwargs['series'] obj =3D Message.objects.find_series(series) if not obj: raise Http404("Object not found: " + series) - log =3D obj.get_property("git.apply-log") - if log is None: + r =3D obj.git_result + if r is None or not r.is_completed(): raise Http404("Git apply log not found") - return log + return r.log =20 =20 class GitModule(PatchewModule): @@ -66,9 +74,16 @@ class GitModule(PatchewModule): register_handler("SeriesComplete", self.on_series_update) register_handler("TagsUpdate", self.on_series_update) =20 + def mark_as_pending_apply(self, series): + r =3D series.git_result or series.create_result(name=3D'git') + r.log =3D None + r.status =3D Result.PENDING + r.data =3D {} + r.save() + def on_series_update(self, event, series, **params): if series.is_complete: - series.set_property("git.need-apply", True) + self.mark_as_pending_apply(series) =20 def get_project_config(self, project, what): return project.get_property("git." + what) @@ -114,50 +129,20 @@ class GitModule(PatchewModule): =20 def get_based_on(self, message, request, format): git_base =3D self.get_base(message) - if not git_base: - return None - - return { - "repo": git_base.get_property("git.repo"), - "tag": 'refs/tags/' + git_base.get_property("git.tag") - } - + return git_base.data if git_base else None =20 def rest_series_fields_hook(self, request, fields, detailed): fields['based_on'] =3D PluginMethodField(obj=3Dself, required=3DFa= lse) =20 def rest_results_hook(self, obj, results, detailed=3DFalse): - if not isinstance(obj, Message): - return - log =3D obj.get_property("git.apply-log") - data =3D None - if log: - if obj.get_property("git.apply-failed"): - status =3D Result.FAILURE - else: - git_repo =3D obj.get_property("git.repo") - git_tag =3D obj.get_property("git.tag") - git_url =3D obj.get_property("git.url") - git_base =3D obj.get_property("git.base") - data =3D {} - if git_repo and git_tag: - data['repo'] =3D git_repo - data['tag'] =3D 'refs/tags/' + git_tag - if git_url: - data['url'] =3D git_url - if git_base: - data['base'] =3D git_base - status =3D Result.SUCCESS - else: - status =3D Result.PENDING - results.append(ResultTuple(name=3D'git', obj=3Dobj, status=3Dstatu= s, - log=3Dlog, data=3Ddata, renderer=3Dself)) + Result.get_result_tuples(obj, "git", results) =20 def prepare_message_hook(self, request, message, detailed): if not message.is_series_head: return - if message.get_property("git.apply-log"): - if message.get_property("git.apply-failed"): + r =3D message.git_result + if r and r.is_completed(): + if r.is_failure(): title =3D "Failed in applying to current master" message.status_tags.append({ "title": title, @@ -165,10 +150,10 @@ class GitModule(PatchewModule): "char": "G", }) else: - git_url =3D message.get_property("git.url") - git_repo =3D message.get_property("git.repo") - git_tag =3D message.get_property("git.tag") + git_url =3D r.data.get('url') if git_url: + git_repo =3D r.data['repo'] + git_tag =3D r.data['tag'] message.status_tags.append({ "url": git_url, "title": format_html("Applied as tag {} in repo {}= ", git_tag, git_repo), @@ -181,9 +166,7 @@ class GitModule(PatchewModule): "type": "info", "char": "G", }) - if request.user.is_authenticated: - if message.get_property("git.apply-failed") !=3D None or \ - message.get_property("git.need-apply") =3D=3D None: + if request.user.is_authenticated: url =3D reverse("git_reset", kwargs=3D{"series": message.message_id}) message.extra_ops.append({"url": url, @@ -238,7 +221,8 @@ class GitModule(PatchewModule): filter(project=3Dseries.project, message_id=3Dbase_id)= .first() if not base: return None - return base if base.get_property("git.repo") else None + r =3D base.git_result + return r if r and r.data.get("repo") else None =20 def prepare_series_hook(self, request, series, response): po =3D series.project @@ -247,8 +231,8 @@ class GitModule(PatchewModule): response[prop] =3D po.get_property(prop) base =3D self.get_base(series) if base: - response["git.repo"] =3D base.get_property("git.repo") - response["git.base"] =3D base.get_property("git.tag") + response["git.repo"] =3D base.data["repo"] + response["git.base"] =3D base.data["tag"] =20 def _poll_project(self, po): repo, branch =3D self._get_project_repo_and_branch(po) @@ -268,10 +252,7 @@ class GitModule(PatchewModule): obj =3D Message.objects.find_series(series) if not obj: raise Http404("Not found: " + series) - for p in obj.get_properties(): - if p.startswith("git.") and p !=3D "git.need-apply": - obj.set_property(p, None) - obj.set_property("git.need-apply", True) + self.mark_as_pending_apply(obj) return HttpResponseRedirect(request.META.get('HTTP_REFERER')) =20 def www_url_hook(self, urlpatterns): @@ -287,11 +268,9 @@ class ApplierGetView(APILoginRequiredView): allowed_groups =3D ["importers"] =20 def handle(self, request): - mp =3D MessageProperty.objects.filter(name=3D"git.need-apply", - value=3D'true', - message__is_complete=3DTrue).f= irst() - if mp: - return prepare_series(request, mp.message) + m =3D Message.objects.filter(results__name=3D"git", results__statu= s=3D"pending").first() + if m: + return prepare_series(request, m) =20 class ApplierReportView(APILoginRequiredView): name =3D "applier-report" @@ -299,12 +278,23 @@ class ApplierReportView(APILoginRequiredView): =20 def handle(self, request, project, message_id, tag, url, base, repo, failed, log): - s =3D Message.objects.series_heads().get(project__name=3Dproject, - message_id=3Dmessage_id) - s.set_property("git.tag", tag) - s.set_property("git.url", url) - s.set_property("git.base", base) - s.set_property("git.repo", repo) - s.set_property("git.apply-failed", failed) - s.set_property("git.apply-log", log) - s.set_property("git.need-apply", False) + p =3D Project.objects.get(name=3Dproject) + r =3D Message.objects.series_heads().get(project=3Dp, + message_id=3Dmessage_id).gi= t_result + r.log =3D log + if failed: + r.status =3D Result.FAILURE + else: + data =3D {} + data['repo'] =3D repo + data['tag'] =3D 'refs/tags/' + tag + if url: + data['url'] =3D url + elif url_template and tag: + url_template =3D p.get_property("git.url_template") + data['url'] =3D url_template.replace("%t", tag) + if base: + data['base'] =3D base + r.data =3D data + r.status =3D Result.SUCCESS + r.save() diff --git a/tests/test_git.py b/tests/test_git.py index d64f901..509949a 100755 --- a/tests/test_git.py +++ b/tests/test_git.py @@ -14,7 +14,7 @@ sys.path.append(os.path.dirname(__file__)) from patchewtest import PatchewTestCase, main import shutil import subprocess -from api.models import Message +from api.models import Message, Result =20 class GitTest(PatchewTestCase): =20 @@ -34,37 +34,37 @@ class GitTest(PatchewTestCase): def do_apply(self): self.cli(["apply", "--applier-mode"]) for s in Message.objects.series_heads(): - self.assertFalse(s.get_property("git.need-apply")) + self.assertNotEqual(s.git_result.status, Result.PENDING) =20 def test_need_apply(self): self.cli_import("0001-simple-patch.mbox.gz") s =3D Message.objects.series_heads()[0] self.assertEqual(s.is_complete, True) - self.assertEqual(s.get_property("git.need-apply"), True) + self.assertEqual(s.git_result.status, Result.PENDING) self.do_apply() =20 def test_need_apply_multiple(self): self.cli_import("0004-multiple-patch-reviewed.mbox.gz") s =3D Message.objects.series_heads()[0] self.assertEqual(s.is_complete, True) - self.assertEqual(s.get_property("git.need-apply"), True) + self.assertEqual(s.git_result.status, Result.PENDING) self.do_apply() =20 def test_need_apply_incomplete(self): self.cli_import("0012-incomplete-series.mbox.gz") s =3D Message.objects.series_heads()[0] self.assertEqual(s.is_complete, False) - self.assertEqual(s.get_property("git.need-apply"), None) + self.assertEqual(s.git_result is None, True) =20 def test_apply(self): self.cli_import("0013-foo-patch.mbox.gz") self.do_apply() s =3D Message.objects.series_heads()[0] self.assertEqual(s.is_complete, True) - self.assertEqual(s.get_property("git.repo"), self.repo) - self.assertEqual(s.get_property("git.tag"), - "patchew/20160628014747.20971-1-famz@redhat.com") - self.assertEqual(s.get_property("git.url"), + self.assertEqual(s.git_result.data['repo'], self.repo) + self.assertEqual(s.git_result.data['tag'], + "refs/tags/patchew/20160628014747.20971-1-famz@re= dhat.com") + self.assertEqual(s.git_result.data['url'], self.repo + " patchew/20160628014747.20971-1-famz= @redhat.com") =20 def test_apply_with_base(self): @@ -74,10 +74,10 @@ class GitTest(PatchewTestCase): self.do_apply() s =3D Message.objects.series_heads().filter(message_id=3D"20160628= 014747.20971-2-famz@redhat.com")[0] self.assertEqual(s.is_complete, True) - self.assertEqual(s.get_property("git.repo"), self.repo) - self.assertEqual(s.get_property("git.tag"), - "patchew/20160628014747.20971-2-famz@redhat.com") - self.assertEqual(s.get_property("git.url"), + self.assertEqual(s.git_result.data['repo'], self.repo) + self.assertEqual(s.git_result.data['tag'], + "refs/tags/patchew/20160628014747.20971-2-famz@re= dhat.com") + self.assertEqual(s.git_result.data['url'], self.repo + " patchew/20160628014747.20971-2-famz= @redhat.com") =20 def test_apply_with_base_and_brackets(self): @@ -87,10 +87,10 @@ class GitTest(PatchewTestCase): self.do_apply() s =3D Message.objects.series_heads().filter(message_id=3D"20160628= 014747.20971-2-famz@redhat.com")[0] self.assertEqual(s.is_complete, True) - self.assertEqual(s.get_property("git.repo"), self.repo) - self.assertEqual(s.get_property("git.tag"), - "patchew/20160628014747.20971-2-famz@redhat.com") - self.assertEqual(s.get_property("git.url"), + self.assertEqual(s.git_result.data['repo'], self.repo) + self.assertEqual(s.git_result.data['tag'], + "refs/tags/patchew/20160628014747.20971-2-famz@re= dhat.com") + self.assertEqual(s.git_result.data['url'], self.repo + " patchew/20160628014747.20971-2-famz= @redhat.com") =20 def test_rest_need_apply(self): --=20 2.17.0 _______________________________________________ Patchew-devel mailing list Patchew-devel@redhat.com https://www.redhat.com/mailman/listinfo/patchew-devel From nobody Sat May 4 16:48:49 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) client-ip=209.132.183.28; envelope-from=patchew-devel-bounces@redhat.com; helo=mx1.redhat.com; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) smtp.mailfrom=patchew-devel-bounces@redhat.com; dmarc=pass(p=none dis=none) header.from=redhat.com Return-Path: Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by mx.zohomail.com with SMTPS id 15288862743864.651242257381114; Wed, 13 Jun 2018 03:37:54 -0700 (PDT) Received: from smtp.corp.redhat.com (int-mx09.intmail.prod.int.phx2.redhat.com [10.5.11.24]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id DCFB88123D; Wed, 13 Jun 2018 10:37:53 +0000 (UTC) Received: from colo-mx.corp.redhat.com (colo-mx02.intmail.prod.int.phx2.redhat.com [10.5.11.21]) by smtp.corp.redhat.com (Postfix) with ESMTPS id CDEFE308BDB0; Wed, 13 Jun 2018 10:37:53 +0000 (UTC) Received: from lists01.pubmisc.prod.ext.phx2.redhat.com (lists01.pubmisc.prod.ext.phx2.redhat.com [10.5.19.33]) by colo-mx.corp.redhat.com (Postfix) with ESMTP id C3E904BB78; Wed, 13 Jun 2018 10:37:53 +0000 (UTC) Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.rdu2.redhat.com [10.11.54.4]) by lists01.pubmisc.prod.ext.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id w5DAbp3P009776 for ; Wed, 13 Jun 2018 06:37:52 -0400 Received: by smtp.corp.redhat.com (Postfix) id B70A52026987; Wed, 13 Jun 2018 10:37:51 +0000 (UTC) Received: from donizetti.redhat.com (unknown [10.36.118.61]) by smtp.corp.redhat.com (Postfix) with ESMTP id 4501B20244E1 for ; Wed, 13 Jun 2018 10:37:51 +0000 (UTC) From: Paolo Bonzini To: patchew-devel@redhat.com Date: Wed, 13 Jun 2018 12:37:40 +0200 Message-Id: <20180613103744.30891-7-pbonzini@redhat.com> In-Reply-To: <20180613103744.30891-1-pbonzini@redhat.com> References: <20180613103744.30891-1-pbonzini@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.4 X-loop: patchew-devel@redhat.com Subject: [Patchew-devel] [PATCH 06/10] testing: track changes to the set of tests X-BeenThere: patchew-devel@redhat.com X-Mailman-Version: 2.1.12 Precedence: junk List-Id: Patchew development and discussion list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: patchew-devel-bounces@redhat.com Errors-To: patchew-devel-bounces@redhat.com X-Scanned-By: MIMEDefang 2.84 on 10.5.11.24 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.25]); Wed, 13 Jun 2018 10:37:53 +0000 (UTC) X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Type: text/plain; charset="utf-8" Use the new SetProperty event to recompute testing.done and testing.ready every time the set of tests changes. This removes the need to do so from the testing-get API, and it will also provide a perfect place to create PENDING records in the result table. --- mods/testing.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/mods/testing.py b/mods/testing.py index ca3d60d..8328a7c 100644 --- a/mods/testing.py +++ b/mods/testing.py @@ -118,6 +118,24 @@ class TestingModule(PatchewModule): elif isinstance(obj, Project) and name =3D=3D "git.head" \ and old_value !=3D value: self.clear_and_start_testing(obj) + elif isinstance(obj, Project) and name.startswith("testing.tests."= ) \ + and old_value !=3D value: + self.recalc_pending_tests(obj) + + def is_testing_done(self, obj, tn): + return obj.get_property("testing.report." + tn) + + def recalc_pending_tests(self, obj): + test_dict =3D self.get_tests(obj) + all_tests =3D set((k for k, v in test_dict.items() if v.get("enabl= ed", False))) + if len(all_tests): + done_tests =3D set((tn for tn in all_tests if self.is_testing_= done(obj, tn))) + if len(done_tests) < len(all_tests): + obj.set_property("testing.done", None) + obj.set_property("testing.ready", 1) + return + obj.set_property("testing.done", True) + obj.set_property("testing.ready", None) =20 def clear_and_start_testing(self, obj, test=3D""): for k in list(obj.get_properties().keys()): @@ -129,7 +147,7 @@ class TestingModule(PatchewModule): k.startswith("testing.report." + test) or \ k.startswith("testing.log." + test): obj.set_property(k, None) - obj.set_property("testing.ready", 1) + self.recalc_pending_tests(obj) =20 def www_view_testing_reset(self, request, project_or_series): if not request.user.is_authenticated: @@ -404,14 +422,10 @@ class TestingGetView(APILoginRequiredView): test=3Dtest) =20 def _find_applicable_test(self, user, project, tester, capabilities, o= bj): - all_tests =3D set([k for k, v in _instance.get_tests(obj).items() = if v["enabled"]]) - done_tests =3D set() for tn, t in _instance.get_tests(project).items(): if not t.get("enabled"): continue - all_tests.add(tn) - if obj.get_property("testing.report." + tn): - done_tests.add(tn) + if _instance.is_testing_done(obj, tn): continue # TODO: group? ok =3D True @@ -423,8 +437,6 @@ class TestingGetView(APILoginRequiredView): if not ok: continue return t - if len(all_tests) and all_tests.issubset(done_tests): - obj.set_property("testing.done", True) =20 def _find_project_test(self, request, po, tester, capabilities): if not po.get_property("testing.ready"): --=20 2.17.0 _______________________________________________ Patchew-devel mailing list Patchew-devel@redhat.com https://www.redhat.com/mailman/listinfo/patchew-devel From nobody Sat May 4 16:48:49 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) client-ip=209.132.183.28; envelope-from=patchew-devel-bounces@redhat.com; helo=mx1.redhat.com; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) smtp.mailfrom=patchew-devel-bounces@redhat.com; dmarc=pass(p=none dis=none) header.from=redhat.com Return-Path: Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by mx.zohomail.com with SMTPS id 1528886276518770.1336702787905; Wed, 13 Jun 2018 03:37:56 -0700 (PDT) Received: from smtp.corp.redhat.com (int-mx05.intmail.prod.int.phx2.redhat.com [10.5.11.15]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 09CAD3097266; Wed, 13 Jun 2018 10:37:56 +0000 (UTC) Received: from colo-mx.corp.redhat.com (colo-mx01.intmail.prod.int.phx2.redhat.com [10.5.11.20]) by smtp.corp.redhat.com (Postfix) with ESMTPS id E435862971; Wed, 13 Jun 2018 10:37:55 +0000 (UTC) Received: from lists01.pubmisc.prod.ext.phx2.redhat.com (lists01.pubmisc.prod.ext.phx2.redhat.com [10.5.19.33]) by colo-mx.corp.redhat.com (Postfix) with ESMTP id D5A691809547; Wed, 13 Jun 2018 10:37:55 +0000 (UTC) Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.rdu2.redhat.com [10.11.54.4]) by lists01.pubmisc.prod.ext.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id w5DAbrIp009786 for ; Wed, 13 Jun 2018 06:37:53 -0400 Received: by smtp.corp.redhat.com (Postfix) id 695EE20244E1; Wed, 13 Jun 2018 10:37:53 +0000 (UTC) Received: from donizetti.redhat.com (unknown [10.36.118.61]) by smtp.corp.redhat.com (Postfix) with ESMTP id 14D9D20244E0 for ; Wed, 13 Jun 2018 10:37:51 +0000 (UTC) From: Paolo Bonzini To: patchew-devel@redhat.com Date: Wed, 13 Jun 2018 12:37:41 +0200 Message-Id: <20180613103744.30891-8-pbonzini@redhat.com> In-Reply-To: <20180613103744.30891-1-pbonzini@redhat.com> References: <20180613103744.30891-1-pbonzini@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.4 X-loop: patchew-devel@redhat.com Subject: [Patchew-devel] [PATCH 07/10] testing: switch to Result model X-BeenThere: patchew-devel@redhat.com X-Mailman-Version: 2.1.12 Precedence: junk List-Id: Patchew development and discussion list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: patchew-devel-bounces@redhat.com Errors-To: patchew-devel-bounces@redhat.com X-Scanned-By: MIMEDefang 2.79 on 10.5.11.15 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.43]); Wed, 13 Jun 2018 10:37:56 +0000 (UTC) X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Type: text/plain; charset="utf-8" --- .../0029_populate_testing_results.py | 220 ++++++++++++++++++ mods/testing.py | 182 ++++++--------- tests/test_testing.py | 53 +++-- 3 files changed, 325 insertions(+), 130 deletions(-) create mode 100644 api/migrations/0029_populate_testing_results.py diff --git a/api/migrations/0029_populate_testing_results.py b/api/migratio= ns/0029_populate_testing_results.py new file mode 100644 index 0000000..0607936 --- /dev/null +++ b/api/migrations/0029_populate_testing_results.py @@ -0,0 +1,220 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations +from django.db.models import Count +from api.migrations import (get_property, get_property_raw, + load_property, set_property, delete_property_blob) + +import abc +from collections import defaultdict +from copy import copy +import datetime +import lzma + +# For Result's constant status values +import api.models + +class Converter(object, metaclass=3Dabc.ABCMeta): + def __init__(self, apps, schema_editor): + # We can't import the models directly as they may be a newer + # version than this migration expects. We use the historical versi= on. + self.Project =3D apps.get_model('api', 'Project') + self.Message =3D apps.get_model('api', 'Message') + self.MessageProperty =3D apps.get_model('api', 'MessageProperty') + self.ProjectProperty =3D apps.get_model('api', 'ProjectProperty') + self.MessageResult =3D apps.get_model('api', 'MessageResult') + self.ProjectResult =3D apps.get_model('api', 'ProjectResult') + self.LogEntry =3D apps.get_model('api', 'LogEntry') + + @abc.abstractmethod + def get_objects_for_project(self, po): + pass + + @abc.abstractmethod + def set_property(self, obj, name, value): + pass + + @abc.abstractmethod + def get_property(self, obj, name): + pass + + @abc.abstractmethod + def get_properties(self, obj, **kwargs): + pass + + @abc.abstractmethod + def delete_property(self, obj, name): + pass + + @abc.abstractmethod + def create_result(self, obj, **kwargs): + pass + + def get_projects_with_tests(self): + return self.Project.objects.filter( + projectproperty__name__startswith=3D'testing.').distinct() + + def get_tests(self, po): + # Based on TestingModule.get_tests + ret =3D {} + props =3D self.ProjectProperty.objects.filter(name__startswith=3D'= testing.tests.', + project=3Dpo) + for p in props: + k =3D p.name + v =3D load_property(p) + tn =3D k[len("testing.tests."):] + if "." not in tn: + continue + an =3D tn[tn.find(".") + 1:] + tn =3D tn[:tn.find(".")] + ret.setdefault(tn, {}) + ret[tn][an] =3D v + return ret + + def do_result_from_properties(self): + for po in self.Project.objects.all(): + tests =3D self.get_tests(po) + for obj in self.get_objects_for_project(po): + pending_status =3D api.models.Result.RUNNING \ + if self.get_property(obj, "testing.started") \ + else api.models.Result.PENDING + done_tests =3D set() + results =3D [] + for prop in self.get_properties(obj, name__startswith=3D't= esting.report.'): + tn =3D prop.name[len('testing.report.'):] + done_tests.add(tn) + r =3D self.create_result(obj) + report =3D load_property(prop) + passed =3D report["passed"] + del report["passed"] + log =3D self.get_property(obj, "testing.log." + tn) + r.name =3D 'testing.' + tn + r.status =3D api.models.Result.SUCCESS if passed else = api.models.Result.FAILURE + r.last_update =3D datetime.datetime.utcnow() + r.data =3D report + if log: + log_xz =3D lzma.compress(log.encode("utf-8")) + log_entry =3D self.LogEntry(data_xz=3Dlog_xz) + log_entry.save() + r.log_entry =3D log_entry + self.delete_property(obj, "testing.log." + tn) + r.save() + results.append(r) + self.delete_property(obj, "testing.report." + tn) + if self.get_property(obj, "testing.ready"): + for tn, test in tests.items(): + if tn in done_tests: + continue + r =3D self.create_result(obj) + r.name =3D 'testing.' + tn + r.status =3D pending_status + r.last_update =3D datetime.datetime.utcnow() + r.save() + results.append(r) + self.delete_property(obj, "testing.ready") + #print(obj, len(done_tests), len(tests) - len(done_tests)) + obj.results.add(*results) + try: + self.delete_property(obj, "testing.started") + self.delete_property(obj, "testing.failed") + self.delete_property(obj, "testing.start-time") + except: + pass + + def do_result_to_properties(self): + for po in self.Project.objects.all(): + for obj in self.get_objects_for_project(po): + by_status =3D defaultdict(lambda: 0) + start_time =3D datetime.datetime.utcnow() + for r in obj.results.filter(name__startswith=3D'testing.'): + by_status[r.status] +=3D 1 + if r.status in (api.models.Result.SUCCESS, api.models.= Result.FAILURE): + tn =3D r.name[len('testing.'):] + report =3D copy(r.data) + report['passed'] =3D (r.status =3D=3D api.models.R= esult.SUCCESS) + self.set_property(obj, "testing.report." + tn, rep= ort) + if r.log_entry: + log =3D lzma.decompress(r.log_entry.data_xz).d= ecode("utf-8") + self.set_property(obj, "testing.log." + tn, lo= g) + else: + started =3D started or r.status =3D=3D api.models.= Result.RUNNING + if r.last_update < start_time: + start_time =3D r.last_update + #print(obj, dict(by_status)) + if by_status[api.models.Result.FAILURE]: + self.set_property(obj, "testing.failed", True) + if by_status[api.models.Result.RUNNING]: + self.set_property(obj, "testing.started", True) + self.set_property(obj, "testing.start-time", d.timesta= mp()) + if by_status[api.models.Result.RUNNING] + by_status[api.mo= dels.Result.PENDING]: + self.set_property(obj, "testing.ready", 1) + else: + self.set_property(obj, "testing.done", True) + obj.results.filter(name__startswith=3D'testing.').delete() + +class ProjectConverter(Converter): + def get_objects_for_project(self, po): + yield po + + def set_property(self, obj, name, value): + set_property(self.ProjectProperty, name, value, project=3Dobj) + + def get_property(self, obj, name): + try: + return get_property(self.ProjectProperty, name, project=3Dobj) + except: + return None + + def get_properties(self, obj, **kwargs): + return self.ProjectProperty.objects.filter(project=3Dobj, **kwargs) + + def delete_property(self, obj, name): + delete_property_blob(self.ProjectProperty, name, project=3Dobj) + get_property_raw(self.ProjectProperty, name, project=3Dobj).delete= () + + def create_result(self, obj, **kwargs): + return self.ProjectResult(project=3Dobj, **kwargs) + +class MessageConverter(Converter): + def get_objects_for_project(self, po): + yield from self.Message.objects.filter(is_series_head=3DTrue, proj= ect=3Dpo) + + def set_property(self, obj, name, value): + set_property(self.MessageProperty, name, value, message=3Dobj) + + def get_property(self, obj, name): + try: + return get_property(self.MessageProperty, name, message=3Dobj) + except: + return None + + def get_properties(self, obj, **kwargs): + return self.MessageProperty.objects.filter(message=3Dobj, **kwargs) + + def delete_property(self, obj, name): + delete_property_blob(self.MessageProperty, name, message=3Dobj) + get_property_raw(self.MessageProperty, name, message=3Dobj).delete= () + + def create_result(self, obj, **kwargs): + return self.MessageResult(message=3Dobj, **kwargs) + +def result_from_properties(apps, schema_editor): + ProjectConverter(apps, schema_editor).do_result_from_properties() + MessageConverter(apps, schema_editor).do_result_from_properties() + +def result_to_properties(apps, schema_editor): + ProjectConverter(apps, schema_editor).do_result_to_properties() + MessageConverter(apps, schema_editor).do_result_to_properties() + +class Migration(migrations.Migration): + + dependencies =3D [ + ('api', '0028_populate_git_results'), + ] + + operations =3D [ + migrations.RunPython(result_from_properties, + reverse_code=3Dresult_to_properties), + ] diff --git a/mods/testing.py b/mods/testing.py index 8328a7c..9c053a7 100644 --- a/mods/testing.py +++ b/mods/testing.py @@ -11,13 +11,15 @@ from django.conf.urls import url from django.http import HttpResponseForbidden, Http404, HttpResponseRedire= ct from django.core.exceptions import PermissionDenied +from django.db.models import Q from django.urls import reverse from django.utils.html import format_html from mod import PatchewModule import time import math from api.views import APILoginRequiredView -from api.models import Message, MessageProperty, Project, Result, ResultTu= ple +from api.models import (Message, MessageProperty, MessageResult, + Project, ProjectResult, Result, ResultTuple) from api.search import SearchEngine from event import emit_event, declare_event, register_handler from patchew.logviewer import LogView @@ -42,10 +44,8 @@ class TestingLogViewer(LogView): obj =3D Message.objects.find_series(project_or_series) if not obj: raise Http404("Object not found: " + project_or_series) - log =3D obj.get_property("testing.log." + testing_name) - if log is None: - raise Http404("Testing log not found: " + testing_name) - return log + r =3D _instance.get_testing_result(obj, testing_name) + return r.log =20 =20 class TestingModule(PatchewModule): @@ -122,31 +122,40 @@ class TestingModule(PatchewModule): and old_value !=3D value: self.recalc_pending_tests(obj) =20 - def is_testing_done(self, obj, tn): - return obj.get_property("testing.report." + tn) + def get_testing_results(self, obj, *args, **kwargs): + return obj.results.filter(name__startswith=3D'testing.', *args, **= kwargs) + + def get_testing_result(self, obj, name): + try: + return obj.results.get(name=3D'testing.' + name) + except: + raise Http404("Test doesn't exist") + + def get_test_name(self, result): + return result.name[len('testing.'):] =20 def recalc_pending_tests(self, obj): test_dict =3D self.get_tests(obj) all_tests =3D set((k for k, v in test_dict.items() if v.get("enabl= ed", False))) + for r in self.get_testing_results(obj, status=3DResult.PENDING): + r.delete() if len(all_tests): - done_tests =3D set((tn for tn in all_tests if self.is_testing_= done(obj, tn))) + done_tests =3D [self.get_test_name(r) for r in self.get_testin= g_results(obj)] + for tn in all_tests: + if not tn in done_tests: + obj.create_result(name=3D'testing.' + tn, status=3DRes= ult.PENDING).save() if len(done_tests) < len(all_tests): obj.set_property("testing.done", None) - obj.set_property("testing.ready", 1) return obj.set_property("testing.done", True) - obj.set_property("testing.ready", None) =20 def clear_and_start_testing(self, obj, test=3D""): for k in list(obj.get_properties().keys()): - if (not test and k =3D=3D "testing.started") or \ - (not test and k =3D=3D "testing.start-time") or \ - (not test and k =3D=3D "testing.failed") or \ - k =3D=3D "testing.done" or \ - k =3D=3D "testing.tested-head" or \ - k.startswith("testing.report." + test) or \ - k.startswith("testing.log." + test): + if k =3D=3D "testing.done" or \ + k =3D=3D "testing.tested-head": obj.set_property(k, None) + for r in self.get_testing_results(obj): + r.delete() self.recalc_pending_tests(obj) =20 def www_view_testing_reset(self, request, project_or_series): @@ -202,24 +211,21 @@ class TestingModule(PatchewModule): raise Exception("Series doesn't exist") project =3D obj.project.name user =3D request.user - log_url =3D self.reverse_testing_log(obj, test, request=3Drequest) - html_log_url =3D self.reverse_testing_log(obj, test, request=3Dreq= uest, html=3DTrue) - obj.set_property("testing.report." + test, - {"passed": passed, - "is_timeout": is_timeout, - "user": user.username, - "tester": tester or user.username, - }) - obj.set_property("testing.log." + test, log) - if not passed: - obj.set_property("testing.failed", True) - reports =3D [x for x in obj.get_properties() if x.startswith("test= ing.report.")] - done_tests =3D set([x[len("testing.report."):] for x in reports]) - all_tests =3D set([k for k, v in self.get_tests(obj).items() if v[= "enabled"]]) - if all_tests.issubset(done_tests): + + r =3D self.get_testing_result(obj, test) + r.data =3D {"is_timeout": is_timeout, + "user": user.username, + "tester": tester or user.username} + r.log =3D log + r.status =3D Result.SUCCESS if passed else Result.FAILURE + r.save() + if not self.get_testing_results(obj, + status__in=3D(Result.PENDING, Result.RUNNING)).exis= ts(): obj.set_property("testing.done", True) - obj.set_property("testing.ready", None) obj.set_property("testing.tested-head", head) + + log_url =3D self.reverse_testing_log(obj, test, request=3Drequest) + html_log_url =3D self.reverse_testing_log(obj, test, request=3Dreq= uest, html=3DTrue) emit_event("TestingReport", tester=3Dtester, user=3Duser.username, obj=3Dobj, passed=3Dpassed, test=3Dtest, log=3Dlog, lo= g_url=3Dlog_url, html_log_url=3Dhtml_log_url, is_timeout=3Dis_timeout) @@ -257,11 +263,8 @@ class TestingModule(PatchewModule): "class": "warning", "icon": "refresh", }] - for pn, p in obj.get_properties().items(): - if not pn.startswith("testing.report."): - continue - tn =3D pn[len("testing.report."):] - failed =3D not p["passed"] + for r in self.get_testing_results(obj, ~Q(status=3DResult.PENDING)= ): + tn =3D self.get_test_name(r) ret.append({"url": url + "&test=3D" + tn, "title": format_html("Reset {} testing stat= e", tn), "class": "warning", @@ -270,39 +273,16 @@ class TestingModule(PatchewModule): return ret =20 def rest_results_hook(self, obj, results, detailed=3DFalse): - all_tests =3D set([k for k, v in _instance.get_tests(obj).items() = if v["enabled"]]) - for pn, p in obj.get_properties().items(): - if not pn.startswith("testing.report."): - continue - tn =3D pn[len("testing.report."):] - try: - all_tests.remove(tn) - except: - pass - failed =3D not p["passed"] - passed_str =3D Result.FAILURE if failed else Result.SUCCESS - if detailed: - log =3D obj.get_property("testing.log." + tn) - else: - log =3D None - - data =3D p.copy() - del data['passed'] - results.append(ResultTuple(name=3D'testing.' + tn, obj=3Dobj, = status=3Dpassed_str, - log=3Dlog, data=3Ddata, renderer=3Dself)) - - if obj.get_property("testing.ready"): - for tn in all_tests: - results.append(ResultTuple(name=3D'testing.' + tn, obj=3Do= bj, status=3D'pending')) + Result.get_result_tuples(obj, "testing", results) =20 def prepare_message_hook(self, request, message, detailed): if not message.is_series_head: return if message.project.maintained_by(request.user) \ - and message.get_property("testing.started"): + and self.get_testing_results(message, ~Q(status=3DResult.P= ENDING)).exists(): message.extra_ops +=3D self._build_reset_ops(message) =20 - if message.get_property("testing.failed"): + if self.get_testing_results(message, status=3DResult.FAILURE).exis= ts(): message.status_tags.append({ "title": "Testing failed", "url": reverse("series_detail", @@ -421,61 +401,45 @@ class TestingGetView(APILoginRequiredView): }, test=3Dtest) =20 - def _find_applicable_test(self, user, project, tester, capabilities, o= bj): - for tn, t in _instance.get_tests(project).items(): - if not t.get("enabled"): - continue - if _instance.is_testing_done(obj, tn): + def _find_applicable_test(self, queryset, user, po, tester, capabiliti= es): + # Prefer non-running tests, or tests that started the earliest + q =3D queryset.filter(status__in=3D(Result.PENDING, Result.RUNNING= ), + name__startswith=3D'testing.').order_by('statu= s', 'last_update') + tests =3D _instance.get_tests(po) + for r in q: + tn =3D _instance.get_test_name(r) + t =3D tests.get(tn, None) + # Shouldn't happen, but let's protect against it + if not t: continue - # TODO: group? - ok =3D True reqs =3D t.get("requirements", "") for r in [x.strip() for x in reqs.split(",") if x]: if r not in capabilities: - ok =3D False break - if not ok: - continue - return t + else: + yield r, t =20 def _find_project_test(self, request, po, tester, capabilities): - if not po.get_property("testing.ready"): - return head =3D po.get_property("git.head") repo =3D po.git tested =3D po.get_property("testing.tested-head") if not head or not repo: - return - test =3D self._find_applicable_test(request.user, po, - tester, capabilities, po) - if not test: - return - td =3D self._generate_project_test_data(po.name, repo, head, teste= d, test) - return po, td + return None + candidates =3D self._find_applicable_test(ProjectResult.objects.fi= lter(project=3Dpo), + request.user, po, tester, = capabilities) + for r, test in candidates: + td =3D self._generate_project_test_data(po.name, repo, head, t= ested, test) + return r, po, td + return None =20 def _find_series_test(self, request, po, tester, capabilities): - q =3D MessageProperty.objects.filter(name=3D"testing.ready", - value=3D1, - message__project=3Dpo) - candidate =3D None - for prop in q: - s =3D prop.message - test =3D self._find_applicable_test(request.user, po, - tester, capabilities, s) - if not test: - continue - if not s.get_property("testing.started"): - candidate =3D s, test - break - # Pick one series that started test the earliest - if not candidate or \ - s.get_property("testing.start-time") < \ - candidate[0].get_property("testing.start-time"): - candidate =3D s, test - if not candidate: - return None - return candidate[0], \ - self._generate_series_test_data(candidate[0], candidate[1]) + candidates =3D self._find_applicable_test(MessageResult.objects.fi= lter(message__project=3Dpo), + request.user, po, tester, = capabilities) + for r, test in candidates: + s =3D r.message + td =3D self._generate_series_test_data(s, test) + return r, s, td + return None =20 def handle(self, request, project, tester, capabilities): # Try project head test first @@ -486,9 +450,9 @@ class TestingGetView(APILoginRequiredView): candidate =3D self._find_series_test(request, po, tester, capa= bilities) if not candidate: return - obj, test_data =3D candidate - obj.set_property("testing.started", True) - obj.set_property("testing.start-time", time.time()) + r, obj, test_data =3D candidate + r.status =3D Result.RUNNING + r.save() return test_data =20 class TestingReportView(APILoginRequiredView): diff --git a/tests/test_testing.py b/tests/test_testing.py index 8f831a1..9e61a45 100755 --- a/tests/test_testing.py +++ b/tests/test_testing.py @@ -14,7 +14,7 @@ import os import subprocess sys.path.append(os.path.dirname(__file__)) from patchewtest import PatchewTestCase, main -from api.models import Message +from api.models import Message, Result =20 def create_test(project, name): prefix =3D "testing.tests." + name + "." @@ -35,14 +35,24 @@ class TestingTestCase(PatchewTestCase, metaclass=3Dabc.= ABCMeta): =20 create_test(self.p, "a") =20 - def _do_testing_done(self, obj, log, report): - if not 'passed' in report: - report['passed'] =3D True - obj.set_property("testing.report.tests", report) - if log is not None: - obj.set_property("testing.log.tests", log) + def modify_test_result(self, obj, **kwargs): + try: + r =3D obj.results.get(name=3D'testing.a') + except: + r =3D obj.create_result(name=3D'testing.a') + if not 'status' in kwargs: + kwargs['status'] =3D Result.PENDING + + if len(kwargs): + for k, v in kwargs.items(): + setattr(r, k, v) + r.save() + + def _do_testing_done(self, obj, **kwargs): + if not 'status' in kwargs: + kwargs['status'] =3D Result.SUCCESS + self.modify_test_result(obj, **kwargs) obj.set_property("testing.done", True) - obj.set_property("testing.ready", None) =20 def do_testing_report(self, **report): self.api_login() @@ -81,6 +91,8 @@ class TestingTestCase(PatchewTestCase, metaclass=3Dabc.AB= CMeta): tester=3D"dummy tester", capabilities=3D[]) self.assertIn("head", td) + resp =3D self.get_test_result('a') + self.assertEquals(resp.data['status'], 'running') =20 def test_done(self): self.do_testing_done() @@ -96,8 +108,8 @@ class TestingTestCase(PatchewTestCase, metaclass=3Dabc.A= BCMeta): self.assertEquals(resp.data['status'], 'pending') =20 def test_rest_done_success(self): - self.do_testing_done(log=3D'everything good!', passed=3DTrue) - resp =3D self.get_test_result('tests') + self.do_testing_done(log=3D'everything good!', status=3DResult.SUC= CESS) + resp =3D self.get_test_result('a') self.assertEquals(resp.data['status'], 'success') self.assertEquals(resp.data['log'], 'everything good!') log =3D self.client.get(resp.data['log_url']) @@ -105,10 +117,9 @@ class TestingTestCase(PatchewTestCase, metaclass=3Dabc= .ABCMeta): self.assertEquals(log.content, b'everything good!') =20 def test_rest_done_failure(self): - self.do_testing_done(log=3D'sorry no good', passed=3DFalse, random= _stuff=3D'xyz') - resp =3D self.get_test_result('tests') + self.do_testing_done(log=3D'sorry no good', status=3DResult.FAILUR= E) + resp =3D self.get_test_result('a') self.assertEquals(resp.data['status'], 'failure') - self.assertEquals(resp.data['data']['random_stuff'], 'xyz') self.assertEquals(resp.data['log'], 'sorry no good') log =3D self.client.get(resp.data['log_url']) self.assertEquals(log.status_code, 200) @@ -151,8 +162,8 @@ class MessageTestingTest(TestingTestCase): self.msg.set_property("git.tag", "dummy tag") self.msg.set_property("git.base", "dummy base") =20 - def do_testing_done(self, log=3DNone, **report): - self._do_testing_done(self.msg, log, report) + def do_testing_done(self, **kwargs): + self._do_testing_done(self.msg, **kwargs) =20 def do_testing_report(self, **report): r =3D super(MessageTestingTest, self).do_testing_report(**report) @@ -164,17 +175,18 @@ class MessageTestingTest(TestingTestCase): self.PROJECT_BASE, self.msg.message= _id, test_name)) =20 def test_testing_ready(self): - self.assertTrue(self.msg.get_property("testing.ready")) # Set property through series_heads elements must be handled the s= ame self.msg.set_property("git.repo", None) self.msg.set_property("git.tag", None) - self.msg.set_property("testing.ready", None) + self.assertEqual(self.msg.results.filter(name=3D'testing.a').first= ().status, + Result.PENDING) msg =3D Message.objects.series_heads()[0] self.assertEqual(self.msg.message_id, msg.message_id) msg.set_property("git.repo", "dummy repo") msg.set_property("git.tag", "dummy tag") msg.set_property("git.base", "dummy base") - self.assertTrue(msg.get_property("testing.ready")) + self.assertEqual(self.msg.results.filter(name=3D'testing.a').first= ().status, + Result.PENDING) =20 class ProjectTestingTest(TestingTestCase): =20 @@ -182,10 +194,9 @@ class ProjectTestingTest(TestingTestCase): super(ProjectTestingTest, self).setUp() self.p.set_property("git.head", "5678") self.p.set_property("testing.tested-head", "1234") - self.p.set_property("testing.ready", 1) =20 - def do_testing_done(self, log=3DNone, **report): - self._do_testing_done(self.p, log, report) + def do_testing_done(self, **kwargs): + self._do_testing_done(self.p, **kwargs) =20 def do_testing_report(self, **report): r =3D super(ProjectTestingTest, self).do_testing_report(**report) --=20 2.17.0 _______________________________________________ Patchew-devel mailing list Patchew-devel@redhat.com https://www.redhat.com/mailman/listinfo/patchew-devel From nobody Sat May 4 16:48:49 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) client-ip=209.132.183.28; envelope-from=patchew-devel-bounces@redhat.com; helo=mx1.redhat.com; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) smtp.mailfrom=patchew-devel-bounces@redhat.com; dmarc=pass(p=none dis=none) header.from=redhat.com Return-Path: Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by mx.zohomail.com with SMTPS id 1528886276561329.34060858888347; Wed, 13 Jun 2018 03:37:56 -0700 (PDT) Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.phx2.redhat.com [10.5.11.14]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 1664F81E03; Wed, 13 Jun 2018 10:37:56 +0000 (UTC) Received: from colo-mx.corp.redhat.com (colo-mx02.intmail.prod.int.phx2.redhat.com [10.5.11.21]) by smtp.corp.redhat.com (Postfix) with ESMTPS id 0560391F50; Wed, 13 Jun 2018 10:37:56 +0000 (UTC) Received: from lists01.pubmisc.prod.ext.phx2.redhat.com (lists01.pubmisc.prod.ext.phx2.redhat.com [10.5.19.33]) by colo-mx.corp.redhat.com (Postfix) with ESMTP id EAAD74BB78; Wed, 13 Jun 2018 10:37:55 +0000 (UTC) Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.rdu2.redhat.com [10.11.54.4]) by lists01.pubmisc.prod.ext.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id w5DAbsf0009797 for ; Wed, 13 Jun 2018 06:37:54 -0400 Received: by smtp.corp.redhat.com (Postfix) id 3AC32210C6DC; Wed, 13 Jun 2018 10:37:54 +0000 (UTC) Received: from donizetti.redhat.com (unknown [10.36.118.61]) by smtp.corp.redhat.com (Postfix) with ESMTP id B02DD2026987 for ; Wed, 13 Jun 2018 10:37:53 +0000 (UTC) From: Paolo Bonzini To: patchew-devel@redhat.com Date: Wed, 13 Jun 2018 12:37:42 +0200 Message-Id: <20180613103744.30891-9-pbonzini@redhat.com> In-Reply-To: <20180613103744.30891-1-pbonzini@redhat.com> References: <20180613103744.30891-1-pbonzini@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.4 X-loop: patchew-devel@redhat.com Subject: [Patchew-devel] [PATCH 08/10] complete switch to database-based Results X-BeenThere: patchew-devel@redhat.com X-Mailman-Version: 2.1.12 Precedence: junk List-Id: Patchew development and discussion list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: patchew-devel-bounces@redhat.com Errors-To: patchew-devel-bounces@redhat.com X-Scanned-By: MIMEDefang 2.79 on 10.5.11.14 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.25]); Wed, 13 Jun 2018 10:37:56 +0000 (UTC) X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Type: text/plain; charset="utf-8" All results are stored in the database now and there is no need for the ResultTuple infrastructure. Remove all the code invoking the rest_results_hook and switch ResultsViewSet and ResultSerializer to be model-based. Signed-off-by: Paolo Bonzini --- api/models.py | 46 --------------------------------------- api/rest.py | 58 ++++++++++++++++--------------------------------- mods/git.py | 5 +---- mods/testing.py | 5 +---- www/views.py | 15 +++++-------- 5 files changed, 26 insertions(+), 103 deletions(-) diff --git a/api/models.py b/api/models.py index 6288d8d..3ca87a6 100644 --- a/api/models.py +++ b/api/models.py @@ -16,7 +16,6 @@ import re =20 from django.core import validators from django.db import models -from django.db.models import Q from django.contrib.auth.models import User from django.urls import reverse import jsonfield @@ -125,14 +124,6 @@ class Result(models.Model): log_url =3D request.build_absolute_uri(log_url) return log_url =20 - @staticmethod - def get_result_tuples(obj, module, results): - name_filter =3D Q(name=3Dmodule) | Q(name__startswith=3Dmodule + '= .') - renderer =3D mod.get_module(module) - for r in obj.results.filter(name_filter): - results.append(ResultTuple(name=3Dr.name, obj=3Dobj, status=3D= r.status, - log=3Dr.log, data=3Dr.data, rendere= r=3Drenderer)) - def __str__(self): return '%s (%s)' % (self.name, self.status) =20 @@ -756,40 +747,3 @@ class Module(models.Model): =20 def __str__(self): return self.name - -class ResultTuple(namedtuple("ResultTuple", "name status log obj data rend= erer")): - __slots__ =3D () - - def __new__(cls, name, status, obj, log=3DNone, data=3DNone, renderer= =3DNone): - if status not in Result.VALID_STATUSES: - raise ValueError("invalid value '%s' for status field" % statu= s) - return super(cls, ResultTuple).__new__(cls, status=3Dstatus, log= =3Dlog, - obj=3Dobj, data=3Ddata, name=3Dn= ame, renderer=3Drenderer) - - def is_success(self): - return self.status =3D=3D Result.SUCCESS - - def is_failure(self): - return self.status =3D=3D Result.FAILURE - - def is_completed(self): - return self.is_success() or self.is_failure() - - def is_pending(self): - return self.status =3D=3D Result.PENDING - - def is_running(self): - return self.status =3D=3D Result.RUNNING - - def render(self): - if self.renderer is None: - return None - return self.renderer.render_result(self) - - def get_log_url(self, request=3DNone): - if not self.is_completed() or self.renderer is None: - return None - log_url =3D self.renderer.get_result_log_url(self) - if log_url is not None and request is not None: - log_url =3D request.build_absolute_uri(log_url) - return log_url diff --git a/api/rest.py b/api/rest.py index f853964..9ec0ae8 100644 --- a/api/rest.py +++ b/api/rest.py @@ -14,7 +14,7 @@ from django.http import Http404 from django.template import loader =20 from mod import dispatch_module_hook -from .models import Project, Message +from .models import Project, ProjectResult, Message, MessageResult, Result from .search import SearchEngine from rest_framework import (permissions, serializers, viewsets, filters, mixins, generics, renderers, status) @@ -446,10 +446,12 @@ class HyperlinkedResultField(HyperlinkedIdentityField= ): kwargs['projects_pk'] =3D obj.id return self.reverse(view_name, kwargs=3Dkwargs, request=3Drequest,= format=3Dformat) =20 -class ResultSerializer(serializers.Serializer): +class ResultSerializer(serializers.ModelSerializer): + class Meta: + model =3D Result + fields =3D ('resource_uri', 'name', 'status', 'last_update', 'data= ', 'log_url') + resource_uri =3D HyperlinkedResultField(view_name=3D'results-detail') - name =3D CharField() - status =3D CharField() # one of 'failure', 'success', 'pending', 'runn= ing' log_url =3D SerializerMethodField(required=3DFalse) data =3D JSONField(required=3DFalse) =20 @@ -458,53 +460,31 @@ class ResultSerializer(serializers.Serializer): return obj.get_log_url(request) =20 class ResultSerializerFull(ResultSerializer): + class Meta: + model =3D Result + fields =3D ResultSerializer.Meta.fields + ('log',) + + # The database field is log_xz, so this is needed here log =3D CharField(required=3DFalse) =20 -class ResultsViewSet(viewsets.ViewSet, generics.GenericAPIView): +class ResultsViewSet(mixins.ListModelMixin, mixins.RetrieveModelMixin, + viewsets.GenericViewSet): lookup_field =3D 'name' lookup_value_regex =3D '[^/]+' - permission_classes =3D (PatchewPermission,) + filter_backends =3D (filters.OrderingFilter,) + ordering_fields =3D ('name',) + ordering =3D ('name',) =20 def get_serializer_class(self, *args, **kwargs): if self.lookup_field in self.kwargs: return ResultSerializerFull return ResultSerializer =20 - def get_results(self, detailed): - queryset =3D self.get_queryset() - try: - obj =3D queryset[0] - except IndexError: - raise Http404 - results =3D [] - dispatch_module_hook("rest_results_hook", obj=3Dobj, results=3Dres= ults, - detailed=3Ddetailed) - return {x.name: x for x in results} - - def list(self, request, *args, **kwargs): - results =3D self.get_results(detailed=3DFalse).values() - serializer =3D self.get_serializer(results, many=3DTrue) - # Fake paginator response for forwards-compatibility, in case - # this ViewSet becomes model-based - return Response(OrderedDict([ - ('count', len(results)), - ('results', serializer.data) - ])) - - def retrieve(self, request, name, *args, **kwargs): - results =3D self.get_results(detailed=3DTrue) - try: - result =3D results[name] - except KeyError: - raise Http404 - serializer =3D self.get_serializer(result) - return Response(serializer.data) - class ProjectResultsViewSet(ResultsViewSet): def get_queryset(self): - return Project.objects.filter(id=3Dself.kwargs['projects_pk']) + return ProjectResult.objects.filter(project=3Dself.kwargs['project= s_pk']) =20 class SeriesResultsViewSet(ResultsViewSet): def get_queryset(self): - return Message.objects.filter(project=3Dself.kwargs['projects_pk'], - message_id=3Dself.kwargs['series_mes= sage_id']) + return MessageResult.objects.filter(message__project=3Dself.kwargs= ['projects_pk'], + message__message_id=3Dself.kwa= rgs['series_message_id']) diff --git a/mods/git.py b/mods/git.py index b8c2ff0..2b24b24 100644 --- a/mods/git.py +++ b/mods/git.py @@ -18,7 +18,7 @@ from django.core.exceptions import PermissionDenied from django.utils.html import format_html from mod import PatchewModule from event import declare_event, register_handler, emit_event -from api.models import Message, MessageProperty, Project, Result, ResultTu= ple +from api.models import Message, MessageProperty, Project, Result from api.rest import PluginMethodField from api.views import APILoginRequiredView, prepare_series from patchew.logviewer import LogView @@ -134,9 +134,6 @@ class GitModule(PatchewModule): def rest_series_fields_hook(self, request, fields, detailed): fields['based_on'] =3D PluginMethodField(obj=3Dself, required=3DFa= lse) =20 - def rest_results_hook(self, obj, results, detailed=3DFalse): - Result.get_result_tuples(obj, "git", results) - def prepare_message_hook(self, request, message, detailed): if not message.is_series_head: return diff --git a/mods/testing.py b/mods/testing.py index 9c053a7..7dfc5a2 100644 --- a/mods/testing.py +++ b/mods/testing.py @@ -19,7 +19,7 @@ import time import math from api.views import APILoginRequiredView from api.models import (Message, MessageProperty, MessageResult, - Project, ProjectResult, Result, ResultTuple) + Project, ProjectResult, Result) from api.search import SearchEngine from event import emit_event, declare_event, register_handler from patchew.logviewer import LogView @@ -272,9 +272,6 @@ class TestingModule(PatchewModule): }) return ret =20 - def rest_results_hook(self, obj, results, detailed=3DFalse): - Result.get_result_tuples(obj, "testing", results) - def prepare_message_hook(self, request, message, detailed): if not message.is_series_head: return diff --git a/www/views.py b/www/views.py index c2010c5..d7c60dd 100644 --- a/www/views.py +++ b/www/views.py @@ -92,19 +92,14 @@ def prepare_series(request, s, skip_patches=3DFalse): return r =20 def prepare_results(request, obj): - results =3D [] - dispatch_module_hook("rest_results_hook", obj=3Dobj, - results=3Dresults, detailed=3DFalse) - - results_dicts =3D [] - for result in results: + rendered_results =3D [] + for result in obj.results.all(): html =3D result.render() if html is None: continue - d =3D result._asdict() - d['html'] =3D html - results_dicts.append(d) - return results_dicts + result.html =3D html + rendered_results.append(result) + return rendered_results =20 def prepare_series_list(request, sl): return [prepare_message(request, s.project, s, False) for s in sl] --=20 2.17.0 _______________________________________________ Patchew-devel mailing list Patchew-devel@redhat.com https://www.redhat.com/mailman/listinfo/patchew-devel From nobody Sat May 4 16:48:49 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) client-ip=209.132.183.28; envelope-from=patchew-devel-bounces@redhat.com; helo=mx1.redhat.com; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) smtp.mailfrom=patchew-devel-bounces@redhat.com; dmarc=pass(p=none dis=none) header.from=redhat.com Return-Path: Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by mx.zohomail.com with SMTPS id 1528886279647754.2044316691284; Wed, 13 Jun 2018 03:37:59 -0700 (PDT) Received: from smtp.corp.redhat.com (int-mx03.intmail.prod.int.phx2.redhat.com [10.5.11.13]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 267A081DF5; Wed, 13 Jun 2018 10:37:59 +0000 (UTC) Received: from colo-mx.corp.redhat.com (colo-mx01.intmail.prod.int.phx2.redhat.com [10.5.11.20]) by smtp.corp.redhat.com (Postfix) with ESMTPS id 0EE444387; Wed, 13 Jun 2018 10:37:59 +0000 (UTC) Received: from lists01.pubmisc.prod.ext.phx2.redhat.com (lists01.pubmisc.prod.ext.phx2.redhat.com [10.5.19.33]) by colo-mx.corp.redhat.com (Postfix) with ESMTP id 024A21809547; Wed, 13 Jun 2018 10:37:59 +0000 (UTC) Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.rdu2.redhat.com [10.11.54.4]) by lists01.pubmisc.prod.ext.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id w5DAbtlt009802 for ; Wed, 13 Jun 2018 06:37:55 -0400 Received: by smtp.corp.redhat.com (Postfix) id 0461020244E0; Wed, 13 Jun 2018 10:37:55 +0000 (UTC) Received: from donizetti.redhat.com (unknown [10.36.118.61]) by smtp.corp.redhat.com (Postfix) with ESMTP id 816792017D0F for ; Wed, 13 Jun 2018 10:37:54 +0000 (UTC) From: Paolo Bonzini To: patchew-devel@redhat.com Date: Wed, 13 Jun 2018 12:37:43 +0200 Message-Id: <20180613103744.30891-10-pbonzini@redhat.com> In-Reply-To: <20180613103744.30891-1-pbonzini@redhat.com> References: <20180613103744.30891-1-pbonzini@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.4 X-loop: patchew-devel@redhat.com Subject: [Patchew-devel] [PATCH 09/10] models: convert MessageProperty and ProjectProperty to JSONField X-BeenThere: patchew-devel@redhat.com X-Mailman-Version: 2.1.12 Precedence: junk List-Id: Patchew development and discussion list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: patchew-devel-bounces@redhat.com Errors-To: patchew-devel-bounces@redhat.com X-Scanned-By: MIMEDefang 2.79 on 10.5.11.13 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.25]); Wed, 13 Jun 2018 10:37:59 +0000 (UTC) X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Type: text/plain; charset="utf-8" The huge logs are not stored anymore as properties, this means that the pro= perty tables can just use JSONField and the blob field can be removed. Signed-off-by: Paolo Bonzini --- api/blobs.py | 7 ---- api/migrations/0030_deblob_properties.py | 34 ++++++++++++++++++ api/migrations/0031_auto_20180520_1654.py | 34 ++++++++++++++++++ api/migrations/__init__.py | 7 ++-- api/models.py | 42 ++++------------------- 5 files changed, 79 insertions(+), 45 deletions(-) create mode 100644 api/migrations/0030_deblob_properties.py create mode 100644 api/migrations/0031_auto_20180520_1654.py diff --git a/api/blobs.py b/api/blobs.py index 20d4567..4927952 100644 --- a/api/blobs.py +++ b/api/blobs.py @@ -29,13 +29,6 @@ def load_blob(name): fn =3D os.path.join(settings.DATA_DIR, "blob", name + ".xz") return lzma.open(fn, 'r').read().decode("utf-8") =20 -def load_blob_json(name): - try: - return json.loads(load_blob(name)) - except json.decoder.JSONDecodeError as e: - logging.error('Failed to load blob %s: %s' %(name, e)) - return None - def delete_blob(name): fn =3D os.path.join(settings.DATA_DIR, "blob", name + ".xz") try: diff --git a/api/migrations/0030_deblob_properties.py b/api/migrations/0030= _deblob_properties.py new file mode 100644 index 0000000..801f7a2 --- /dev/null +++ b/api/migrations/0030_deblob_properties.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations +from django.db.models import Count + +import json +from api import blobs + +def deblob_properties(apps, schema_editor): + def do_deblob_properties(model): + objects =3D model.objects.filter(blob=3DTrue) + for obj in objects: + obj.blob =3D False + if obj.value is not None: + obj.value =3D blobs.load_blob(obj.value) + obj.save() + + # We can't import the models directly as they may be a newer + # version than this migration expects. We use the historical version. + do_deblob_properties(apps.get_model('api', 'MessageProperty')) + do_deblob_properties(apps.get_model('api', 'ProjectProperty')) + +class Migration(migrations.Migration): + + dependencies =3D [ + ('api', '0029_populate_testing_results'), + ] + + operations =3D [ + migrations.RunPython(deblob_properties, + reverse_code=3Dmigrations.RunPython.noop), + ] diff --git a/api/migrations/0031_auto_20180520_1654.py b/api/migrations/003= 1_auto_20180520_1654.py new file mode 100644 index 0000000..edb0c4f --- /dev/null +++ b/api/migrations/0031_auto_20180520_1654.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.12 on 2018-05-20 16:54 +from __future__ import unicode_literals + +from django.db import migrations +import jsonfield.fields + + +class Migration(migrations.Migration): + + dependencies =3D [ + ('api', '0030_deblob_properties'), + ] + + operations =3D [ + migrations.RemoveField( + model_name=3D'messageproperty', + name=3D'blob', + ), + migrations.RemoveField( + model_name=3D'projectproperty', + name=3D'blob', + ), + migrations.AlterField( + model_name=3D'messageproperty', + name=3D'value', + field=3Djsonfield.fields.JSONField(), + ), + migrations.AlterField( + model_name=3D'projectproperty', + name=3D'value', + field=3Djsonfield.fields.JSONField(), + ), + ] diff --git a/api/migrations/__init__.py b/api/migrations/__init__.py index f4b7eb7..510b26a 100644 --- a/api/migrations/__init__.py +++ b/api/migrations/__init__.py @@ -26,7 +26,7 @@ def get_property_raw(model, name, **kwargs): return None =20 def load_property(mp): - if mp.blob: + if hasattr(mp, 'blob') and mp.blob: return load_blob_json_safe(mp.value) else: return json.loads(mp.value) @@ -37,7 +37,7 @@ def get_property(model, name, **kwargs): =20 def delete_property_blob(model, name, **kwargs): mp =3D get_property_raw(model, name, **kwargs) - if mp.blob: + if hasattr(mp, 'blob') and mp.blob: blobs.delete_blob(mp.value) =20 def set_property(model, name, value, **kwargs): @@ -45,5 +45,6 @@ def set_property(model, name, value, **kwargs): value =3D json.dumps(value) mp, created =3D model.objects.get_or_create(name=3Dname, **kwargs) mp.value =3D value - mp.blob =3D False + if hasattr(mp, 'blob'): + mp.blob =3D False mp.save() diff --git a/api/models.py b/api/models.py index 3ca87a6..c8a5396 100644 --- a/api/models.py +++ b/api/models.py @@ -10,7 +10,6 @@ =20 =20 from collections import namedtuple -import json import datetime import re =20 @@ -23,8 +22,9 @@ import lzma =20 from mbox import MboxMessage from event import emit_event, declare_event -from .blobs import save_blob, load_blob, load_blob_json +from .blobs import save_blob, load_blob import mod +import lzma =20 class LogEntry(models.Model): data_xz =3D models.BinaryField() @@ -171,37 +171,23 @@ class Project(models.Model): def get_property(self, prop, default=3DNone): a =3D ProjectProperty.objects.filter(project=3Dself, name=3Dprop).= first() if a: - if a.blob: - return load_blob_json(a.value) - else: - return json.loads(a.value) + return a.value else: return default =20 def get_properties(self): r =3D {} for m in ProjectProperty.objects.filter(project=3Dself): - if m.blob: - r[m.name] =3D load_blob_json(m.value) - else: - r[m.name] =3D json.loads(m.value) + r[m.name] =3D m.value return r =20 def _do_set_property(self, prop, value): if value =3D=3D None: ProjectProperty.objects.filter(project=3Dself, name=3Dprop).de= lete() return - # TODO: drop old blob - json_data =3D json.dumps(value) - blob =3D len(json_data) > 1024 - if blob: - value =3D save_blob(json_data) - else: - value =3D json.dumps(value) pp, created =3D ProjectProperty.objects.get_or_create(project=3Dse= lf, name=3Dprop) pp.value =3D value - pp.blob =3D blob pp.save() =20 def set_property(self, prop, value): @@ -298,8 +284,7 @@ class ProjectResult(Result): class ProjectProperty(models.Model): project =3D models.ForeignKey('Project', on_delete=3Dmodels.CASCADE) name =3D models.CharField(max_length=3D1024, db_index=3DTrue) - value =3D models.CharField(max_length=3D1024) - blob =3D models.BooleanField(blank=3DTrue, default=3DFalse) + value =3D jsonfield.JSONField() =20 class Meta: unique_together =3D ('project', 'name',) @@ -562,10 +547,7 @@ class Message(models.Model): all_props =3D self.properties.all() r =3D {} for m in all_props: - if m.blob: - r[m.name] =3D load_blob_json(m.value) - else: - r[m.name] =3D json.loads(m.value) + r[m.name] =3D m.value self._properties =3D r return r =20 @@ -573,17 +555,9 @@ class Message(models.Model): if value =3D=3D None: MessageProperty.objects.filter(message=3Dself, name=3Dprop).de= lete() return - json_data =3D json.dumps(value) - blob =3D len(json_data) > 1024 mp, created =3D MessageProperty.objects.get_or_create(message=3Dse= lf, name=3Dprop) - # TODO: drop old blob - if blob: - value =3D save_blob(json_data) - else: - value =3D json_data mp.value =3D value - mp.blob =3D blob mp.save() # Invalidate cache self._properties =3D None @@ -724,9 +698,7 @@ class MessageProperty(models.Model): message =3D models.ForeignKey('Message', on_delete=3Dmodels.CASCADE, related_name=3D'properties') name =3D models.CharField(max_length=3D256) - # JSON encoded value - value =3D models.CharField(max_length=3D1024) - blob =3D models.BooleanField(blank=3DTrue, default=3DFalse) + value =3D jsonfield.JSONField() =20 def __str__(self): if len(self.value) > 30: --=20 2.17.0 _______________________________________________ Patchew-devel mailing list Patchew-devel@redhat.com https://www.redhat.com/mailman/listinfo/patchew-devel From nobody Sat May 4 16:48:49 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) client-ip=209.132.183.28; envelope-from=patchew-devel-bounces@redhat.com; helo=mx1.redhat.com; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) smtp.mailfrom=patchew-devel-bounces@redhat.com; dmarc=pass(p=none dis=none) header.from=redhat.com Return-Path: Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by mx.zohomail.com with SMTPS id 1528886279672967.5634328784012; Wed, 13 Jun 2018 03:37:59 -0700 (PDT) Received: from smtp.corp.redhat.com (int-mx08.intmail.prod.int.phx2.redhat.com [10.5.11.23]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 30716308FB9A; Wed, 13 Jun 2018 10:37:59 +0000 (UTC) Received: from colo-mx.corp.redhat.com (colo-mx01.intmail.prod.int.phx2.redhat.com [10.5.11.20]) by smtp.corp.redhat.com (Postfix) with ESMTPS id 1E98C17B40; Wed, 13 Jun 2018 10:37:59 +0000 (UTC) Received: from lists01.pubmisc.prod.ext.phx2.redhat.com (lists01.pubmisc.prod.ext.phx2.redhat.com [10.5.19.33]) by colo-mx.corp.redhat.com (Postfix) with ESMTP id 136361808800; Wed, 13 Jun 2018 10:37:59 +0000 (UTC) Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.rdu2.redhat.com [10.11.54.4]) by lists01.pubmisc.prod.ext.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id w5DAbtQx009810 for ; Wed, 13 Jun 2018 06:37:56 -0400 Received: by smtp.corp.redhat.com (Postfix) id C4FE92026987; Wed, 13 Jun 2018 10:37:55 +0000 (UTC) Received: from donizetti.redhat.com (unknown [10.36.118.61]) by smtp.corp.redhat.com (Postfix) with ESMTP id 4B79220244E1 for ; Wed, 13 Jun 2018 10:37:55 +0000 (UTC) From: Paolo Bonzini To: patchew-devel@redhat.com Date: Wed, 13 Jun 2018 12:37:44 +0200 Message-Id: <20180613103744.30891-11-pbonzini@redhat.com> In-Reply-To: <20180613103744.30891-1-pbonzini@redhat.com> References: <20180613103744.30891-1-pbonzini@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.4 X-loop: patchew-devel@redhat.com Subject: [Patchew-devel] [PATCH 10/10] logviewer: let subclasses return a Result X-BeenThere: patchew-devel@redhat.com X-Mailman-Version: 2.1.12 Precedence: junk List-Id: Patchew development and discussion list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: patchew-devel-bounces@redhat.com Errors-To: patchew-devel-bounces@redhat.com X-Scanned-By: MIMEDefang 2.84 on 10.5.11.23 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.43]); Wed, 13 Jun 2018 10:37:59 +0000 (UTC) X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Type: text/plain; charset="utf-8" There is no need for subclasses to return the log directly; they can just get the result, and the log viewer can check whether the result has a log attached to it. Signed-off-by: Paolo Bonzini --- mods/git.py | 7 ++----- mods/testing.py | 5 ++--- patchew/logviewer.py | 14 ++++++++------ 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/mods/git.py b/mods/git.py index 2b24b24..a2b6d86 100644 --- a/mods/git.py +++ b/mods/git.py @@ -35,15 +35,12 @@ Message.git_result =3D property(_get_git_result) =20 =20 class GitLogViewer(LogView): - def content(self, request, **kwargs): + def get_result(self, request, **kwargs): series =3D kwargs['series'] obj =3D Message.objects.find_series(series) if not obj: raise Http404("Object not found: " + series) - r =3D obj.git_result - if r is None or not r.is_completed(): - raise Http404("Git apply log not found") - return r.log + return obj.git_result =20 =20 class GitModule(PatchewModule): diff --git a/mods/testing.py b/mods/testing.py index 7dfc5a2..0a80b3d 100644 --- a/mods/testing.py +++ b/mods/testing.py @@ -35,7 +35,7 @@ exit 0 """ =20 class TestingLogViewer(LogView): - def content(self, request, **kwargs): + def get_result(self, request, **kwargs): project_or_series =3D kwargs['project_or_series'] testing_name =3D kwargs['testing_name'] if request.GET.get("type") =3D=3D "project": @@ -44,8 +44,7 @@ class TestingLogViewer(LogView): obj =3D Message.objects.find_series(project_or_series) if not obj: raise Http404("Object not found: " + project_or_series) - r =3D _instance.get_testing_result(obj, testing_name) - return r.log + return _instance.get_testing_result(obj, testing_name) =20 =20 class TestingModule(PatchewModule): diff --git a/patchew/logviewer.py b/patchew/logviewer.py index fe0f3ca..f986727 100644 --- a/patchew/logviewer.py +++ b/patchew/logviewer.py @@ -436,7 +436,7 @@ def ansi2html(input, white_bg=3DFalse): =20 class LogView(View, metaclass=3Dabc.ABCMeta): @abc.abstractmethod - def content(request, **kwargs): + def get_result(request, **kwargs): return None =20 # Unfortunately 
 items cannot be focused; for arrow keys to
@@ -463,16 +463,18 @@ if (parent.jQuery && parent.jQuery.colorbox) {
     });
 }""")
=20
-    def generate_html(self):
+    def generate_html(self, log):
         yield self.HTML_PROLOG
-        yield from ansi2html(self.text)
+        yield from ansi2html(log)
=20
     def get(self, request, **kwargs):
-        self.text =3D self.content(request, **kwargs)
+        result =3D self.get_result(request, **kwargs)
+        if result is None or not result.is_completed() or result.log is No=
ne:
+            raise Http404("No log found")
         if request.GET.get('html', None) !=3D '1':
-            return HttpResponse(self.text, content_type=3D'text/plain')
+            return HttpResponse(result.log, content_type=3D'text/plain')
=20
-        return StreamingHttpResponse(self.generate_html())
+        return StreamingHttpResponse(self.generate_html(result.log))
=20
 if __name__ =3D=3D "__main__":
     import io
--=20
2.17.0

_______________________________________________
Patchew-devel mailing list
Patchew-devel@redhat.com
https://www.redhat.com/mailman/listinfo/patchew-devel