From nobody Thu May 2 01:24:24 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) client-ip=209.132.183.28; envelope-from=libvir-list-bounces@redhat.com; helo=mx1.redhat.com; Authentication-Results: mx.zoho.com; spf=pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) smtp.mailfrom=libvir-list-bounces@redhat.com; Return-Path: Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by mx.zohomail.com with SMTPS id 1495549717905890.3669914782093; Tue, 23 May 2017 07:28:37 -0700 (PDT) Received: from smtp.corp.redhat.com (int-mx05.intmail.prod.int.phx2.redhat.com [10.5.11.15]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 7215F81229; Tue, 23 May 2017 14:28:35 +0000 (UTC) Received: from colo-mx.corp.redhat.com (colo-mx01.intmail.prod.int.phx2.redhat.com [10.5.11.20]) by smtp.corp.redhat.com (Postfix) with ESMTPS id 4EEA27D682; Tue, 23 May 2017 14:28:35 +0000 (UTC) Received: from lists01.pubmisc.prod.ext.phx2.redhat.com (lists01.pubmisc.prod.ext.phx2.redhat.com [10.5.19.33]) by colo-mx.corp.redhat.com (Postfix) with ESMTP id 02B53180BAFA; Tue, 23 May 2017 14:28:35 +0000 (UTC) Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.phx2.redhat.com [10.5.11.14]) by lists01.pubmisc.prod.ext.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id v4NEQKQe028662 for ; Tue, 23 May 2017 10:26:20 -0400 Received: by smtp.corp.redhat.com (Postfix) id 945EE7BB1D; Tue, 23 May 2017 14:26:20 +0000 (UTC) Received: from moe.brq.redhat.com (dhcp129-131.brq.redhat.com [10.34.129.131]) by smtp.corp.redhat.com (Postfix) with ESMTP id 1CECD7BCB1 for ; Tue, 23 May 2017 14:26:19 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mx1.redhat.com 7215F81229 Authentication-Results: ext-mx01.extmail.prod.ext.phx2.redhat.com; dmarc=none (p=none dis=none) header.from=redhat.com Authentication-Results: ext-mx01.extmail.prod.ext.phx2.redhat.com; spf=pass smtp.mailfrom=libvir-list-bounces@redhat.com DKIM-Filter: OpenDKIM Filter v2.11.0 mx1.redhat.com 7215F81229 From: Michal Privoznik To: libvir-list@redhat.com Date: Tue, 23 May 2017 16:26:08 +0200 Message-Id: <1f42d8629f7fa516ebe8e02b74cbeba1e46aab4e.1495549496.git.mprivozn@redhat.com> In-Reply-To: References: In-Reply-To: References: X-Scanned-By: MIMEDefang 2.79 on 10.5.11.14 X-loop: libvir-list@redhat.com Subject: [libvirt] [libvirt-python][PATCH v2 1/4] Implement virStreamSendHole/virStreamRecvHole X-BeenThere: libvir-list@redhat.com X-Mailman-Version: 2.1.12 Precedence: junk List-Id: Development discussions about the libvirt library & tools List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: libvir-list-bounces@redhat.com Errors-To: libvir-list-bounces@redhat.com X-Scanned-By: MIMEDefang 2.79 on 10.5.11.15 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.25]); Tue, 23 May 2017 14:28:36 +0000 (UTC) X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Type: text/plain; charset="utf-8" The return value for virStreamRecvHole is slightly different to its C counterpart. In python, either it returns the hole size or None if C API fails. Signed-off-by: Michal Privoznik --- generator.py | 2 ++ libvirt-override-virStream.py | 21 +++++++++++++++ libvirt-override.c | 63 +++++++++++++++++++++++++++++++++++++++= ++++ 3 files changed, 86 insertions(+) diff --git a/generator.py b/generator.py index 5dfa73e..ca1df35 100755 --- a/generator.py +++ b/generator.py @@ -543,6 +543,8 @@ skip_function =3D ( 'virStreamSendAll', # Pure python libvirt-override-virStream.py 'virStreamRecv', # overridden in libvirt-override-virStream.py 'virStreamSend', # overridden in libvirt-override-virStream.py + 'virStreamRecvHole', # overridden in libvirt-override-virStream.py + 'virStreamSendHole', # overridden in libvirt-override-virStream.py =20 'virConnectUnregisterCloseCallback', # overridden in virConnect.py 'virConnectRegisterCloseCallback', # overridden in virConnect.py diff --git a/libvirt-override-virStream.py b/libvirt-override-virStream.py index 2e77cc7..62c1328 100644 --- a/libvirt-override-virStream.py +++ b/libvirt-override-virStream.py @@ -125,3 +125,24 @@ ret =3D libvirtmod.virStreamSend(self._o, data) if ret =3D=3D -1: raise libvirtError ('virStreamSend() failed') return ret + + def recvHole(self, flags =3D 0): + """This method is used to determine the length in bytes + of the empty space to be created in a stream's target + file when uploading or downloading sparsely populated + files. This is the counterpart to sendHole. + """ + ret =3D libvirtmod.virStreamRecvHole(self._o, flags) + if ret is None: raise libvirtError ('virStreamRecvHole() failed') + return ret + + def sendHole(self, length, flags =3D 0): + """Rather than transmitting empty file space, this method + directs the stream target to create length bytes of empty + space. This method would be used when uploading or + downloading sparsely populated files to avoid the + needless copy of empty file space. + """ + ret =3D libvirtmod.virStreamSendHole(self._o, length, flags) + if ret =3D=3D -1: raise libvirtError('virStreamSendHole() failed') + return ret diff --git a/libvirt-override.c b/libvirt-override.c index a762941..a6ef49e 100644 --- a/libvirt-override.c +++ b/libvirt-override.c @@ -9463,6 +9463,65 @@ libvirt_virConnectSecretEventDeregisterAny(PyObject = *self ATTRIBUTE_UNUSED, } #endif /* LIBVIR_CHECK_VERSION(3, 0, 0)*/ =20 + +#if LIBVIR_CHECK_VERSION(3, 4, 0) +static PyObject * +libvirt_virStreamRecvHole(PyObject *self ATTRIBUTE_UNUSED, + PyObject *args) +{ + PyObject *pyobj_stream; + virStreamPtr stream; + long long length =3D -1; + unsigned int flags; + int ret; + + if (!PyArg_ParseTuple(args, (char *) "OI:virStreamRecvHole", + &pyobj_stream, &flags)) + return NULL; + + stream =3D PyvirStream_Get(pyobj_stream); + + LIBVIRT_BEGIN_ALLOW_THREADS; + ret =3D virStreamRecvHole(stream, &length, flags); + LIBVIRT_END_ALLOW_THREADS; + + DEBUG("StreamRecvHole ret=3D%d length=3D%lld\n", ret, length); + + if (ret < 0) + return VIR_PY_NONE; + + return libvirt_longlongWrap(length); +} + + +static PyObject * +libvirt_virStreamSendHole(PyObject *self ATTRIBUTE_UNUSED, + PyObject *args) +{ + PyObject *pyobj_stream; + virStreamPtr stream; + long long length; + unsigned int flags; + int ret; + + if (!PyArg_ParseTuple(args, (char *) "OLI:virStreamSendHole", + &pyobj_stream, &length, &flags)) + return NULL; + + stream =3D PyvirStream_Get(pyobj_stream); + + LIBVIRT_BEGIN_ALLOW_THREADS; + ret =3D virStreamSendHole(stream, length, flags); + LIBVIRT_END_ALLOW_THREADS; + + DEBUG("StreamSendHole ret=3D%d\n", ret); + + return libvirt_intWrap(ret); +} + +#endif /* LIBVIR_CHECK_VERSION(3, 4, 0) */ + + /************************************************************************ * * * The registration stuff * @@ -9687,6 +9746,10 @@ static PyMethodDef libvirtMethods[] =3D { {(char *) "virConnectSecretEventRegisterAny", libvirt_virConnectSecret= EventRegisterAny, METH_VARARGS, NULL}, {(char *) "virConnectSecretEventDeregisterAny", libvirt_virConnectSecr= etEventDeregisterAny, METH_VARARGS, NULL}, #endif /* LIBVIR_CHECK_VERSION(3, 0, 0) */ +#if LIBVIR_CHECK_VERSION(3, 4, 0) + {(char *) "virStreamRecvHole", libvirt_virStreamRecvHole, METH_VARARGS= , NULL}, + {(char *) "virStreamSendHole", libvirt_virStreamSendHole, METH_VARARGS= , NULL}, +#endif /* LIBVIR_CHECK_VERSION(3, 4, 0) */ {NULL, NULL, 0, NULL} }; =20 --=20 2.13.0 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list From nobody Thu May 2 01:24:24 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) client-ip=209.132.183.28; envelope-from=libvir-list-bounces@redhat.com; helo=mx1.redhat.com; Authentication-Results: mx.zoho.com; spf=pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) smtp.mailfrom=libvir-list-bounces@redhat.com; Return-Path: Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by mx.zohomail.com with SMTPS id 1495549700454305.0485111895731; Tue, 23 May 2017 07:28:20 -0700 (PDT) Received: from smtp.corp.redhat.com (int-mx02.intmail.prod.int.phx2.redhat.com [10.5.11.12]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 0FB87285B1; Tue, 23 May 2017 14:28:18 +0000 (UTC) Received: from colo-mx.corp.redhat.com (colo-mx02.intmail.prod.int.phx2.redhat.com [10.5.11.21]) by smtp.corp.redhat.com (Postfix) with ESMTPS id 8BAD251DF8; Tue, 23 May 2017 14:28:16 +0000 (UTC) Received: from lists01.pubmisc.prod.ext.phx2.redhat.com (lists01.pubmisc.prod.ext.phx2.redhat.com [10.5.19.33]) by colo-mx.corp.redhat.com (Postfix) with ESMTP id 30FF14A48C; Tue, 23 May 2017 14:28:15 +0000 (UTC) Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.phx2.redhat.com [10.5.11.14]) by lists01.pubmisc.prod.ext.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id v4NEQLID028670 for ; Tue, 23 May 2017 10:26:21 -0400 Received: by smtp.corp.redhat.com (Postfix) id 6C3137814F; Tue, 23 May 2017 14:26:21 +0000 (UTC) Received: from moe.brq.redhat.com (dhcp129-131.brq.redhat.com [10.34.129.131]) by smtp.corp.redhat.com (Postfix) with ESMTP id E9C137BCB0 for ; Tue, 23 May 2017 14:26:20 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mx1.redhat.com 0FB87285B1 Authentication-Results: ext-mx06.extmail.prod.ext.phx2.redhat.com; dmarc=none (p=none dis=none) header.from=redhat.com Authentication-Results: ext-mx06.extmail.prod.ext.phx2.redhat.com; spf=pass smtp.mailfrom=libvir-list-bounces@redhat.com DKIM-Filter: OpenDKIM Filter v2.11.0 mx1.redhat.com 0FB87285B1 From: Michal Privoznik To: libvir-list@redhat.com Date: Tue, 23 May 2017 16:26:09 +0200 Message-Id: <2e4cb221228e4927aa4eca90be87708921c85f11.1495549496.git.mprivozn@redhat.com> In-Reply-To: References: In-Reply-To: References: X-Scanned-By: MIMEDefang 2.79 on 10.5.11.14 X-loop: libvir-list@redhat.com Subject: [libvirt] [libvirt-python][PATCH v2 2/4] virStream: Introduce virStreamRecvFlags X-BeenThere: libvir-list@redhat.com X-Mailman-Version: 2.1.12 Precedence: junk List-Id: Development discussions about the libvirt library & tools List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: libvir-list-bounces@redhat.com Errors-To: libvir-list-bounces@redhat.com X-Scanned-By: MIMEDefang 2.79 on 10.5.11.12 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.30]); Tue, 23 May 2017 14:28:19 +0000 (UTC) X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Type: text/plain; charset="utf-8" Yet again, we need a custom wrapper over virStreamRecvFlags because our generator is not capable of generating it. Signed-off-by: Michal Privoznik --- generator.py | 1 + libvirt-override-virStream.py | 18 ++++++++++++++++++ libvirt-override.c | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/generator.py b/generator.py index ca1df35..0e07fc8 100755 --- a/generator.py +++ b/generator.py @@ -545,6 +545,7 @@ skip_function =3D ( 'virStreamSend', # overridden in libvirt-override-virStream.py 'virStreamRecvHole', # overridden in libvirt-override-virStream.py 'virStreamSendHole', # overridden in libvirt-override-virStream.py + 'virStreamRecvFlags', # overridden in libvirt-override-virStream.py =20 'virConnectUnregisterCloseCallback', # overridden in virConnect.py 'virConnectRegisterCloseCallback', # overridden in virConnect.py diff --git a/libvirt-override-virStream.py b/libvirt-override-virStream.py index 62c1328..66d2bf6 100644 --- a/libvirt-override-virStream.py +++ b/libvirt-override-virStream.py @@ -146,3 +146,21 @@ ret =3D libvirtmod.virStreamSendHole(self._o, length, flags) if ret =3D=3D -1: raise libvirtError('virStreamSendHole() failed') return ret + + def recvFlags(self, nbytes, flags =3D 0): + """Reads a series of bytes from the stream. This method may + block the calling application for an arbitrary amount + of time. This is just like recv except it has flags + argument. + + Errors are not guaranteed to be reported synchronously + with the call, but may instead be delayed until a + subsequent call. + + On success, the received data is returned. On failure, an + exception is raised. If the stream is a NONBLOCK stream and + the request would block, integer -2 is returned. + """ + ret =3D libvirtmod.virStreamRecvFlags(self._o, nbytes, flags) + if ret is None: raise libvirtError ('virStreamRecvFlags() failed') + return ret diff --git a/libvirt-override.c b/libvirt-override.c index a6ef49e..0abfc37 100644 --- a/libvirt-override.c +++ b/libvirt-override.c @@ -9519,6 +9519,44 @@ libvirt_virStreamSendHole(PyObject *self ATTRIBUTE_U= NUSED, return libvirt_intWrap(ret); } =20 + +static PyObject * +libvirt_virStreamRecvFlags(PyObject *self ATTRIBUTE_UNUSED, + PyObject *args) +{ + PyObject *pyobj_stream; + PyObject *rv; + virStreamPtr stream; + char *buf =3D NULL; + size_t nbytes; + unsigned int flags; + int ret; + + if (!PyArg_ParseTuple(args, (char *) "OkI:virStreamRecvFlags", + &pyobj_stream, &nbytes, &flags)) + return NULL; + + stream =3D PyvirStream_Get(pyobj_stream); + + if (VIR_ALLOC_N(buf, nbytes + 1) < 0) + return PyErr_NoMemory(); + + LIBVIRT_BEGIN_ALLOW_THREADS; + ret =3D virStreamRecvFlags(stream, buf, nbytes, flags); + LIBVIRT_END_ALLOW_THREADS; + + buf[ret > -1 ? ret : 0] =3D '\0'; + DEBUG("StreamRecvFlags ret=3D%d strlen=3D%d\n", ret, (int) strlen(buf)= ); + + if (ret =3D=3D -2 || ret =3D=3D -3) + return libvirt_intWrap(ret); + if (ret < 0) + return VIR_PY_NONE; + rv =3D libvirt_charPtrSizeWrap((char *) buf, (Py_ssize_t) ret); + VIR_FREE(buf); + return rv; +} + #endif /* LIBVIR_CHECK_VERSION(3, 4, 0) */ =20 =20 @@ -9749,6 +9787,7 @@ static PyMethodDef libvirtMethods[] =3D { #if LIBVIR_CHECK_VERSION(3, 4, 0) {(char *) "virStreamRecvHole", libvirt_virStreamRecvHole, METH_VARARGS= , NULL}, {(char *) "virStreamSendHole", libvirt_virStreamSendHole, METH_VARARGS= , NULL}, + {(char *) "virStreamRecvFlags", libvirt_virStreamRecvFlags, METH_VARAR= GS, NULL}, #endif /* LIBVIR_CHECK_VERSION(3, 4, 0) */ {NULL, NULL, 0, NULL} }; --=20 2.13.0 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list From nobody Thu May 2 01:24:24 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) client-ip=209.132.183.28; envelope-from=libvir-list-bounces@redhat.com; helo=mx1.redhat.com; Authentication-Results: mx.zoho.com; spf=pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) smtp.mailfrom=libvir-list-bounces@redhat.com; Return-Path: Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by mx.zohomail.com with SMTPS id 1495549729298685.5386322437714; Tue, 23 May 2017 07:28:49 -0700 (PDT) Received: from smtp.corp.redhat.com (int-mx05.intmail.prod.int.phx2.redhat.com [10.5.11.15]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 63004C059732; Tue, 23 May 2017 14:28:47 +0000 (UTC) Received: from colo-mx.corp.redhat.com (colo-mx02.intmail.prod.int.phx2.redhat.com [10.5.11.21]) by smtp.corp.redhat.com (Postfix) with ESMTPS id 168B17D65E; Tue, 23 May 2017 14:28:46 +0000 (UTC) Received: from lists01.pubmisc.prod.ext.phx2.redhat.com (lists01.pubmisc.prod.ext.phx2.redhat.com [10.5.19.33]) by colo-mx.corp.redhat.com (Postfix) with ESMTP id AA3174A48D; Tue, 23 May 2017 14:28:44 +0000 (UTC) Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.phx2.redhat.com [10.5.11.14]) by lists01.pubmisc.prod.ext.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id v4NEQM2R028677 for ; Tue, 23 May 2017 10:26:22 -0400 Received: by smtp.corp.redhat.com (Postfix) id 3E63E7BCA1; Tue, 23 May 2017 14:26:22 +0000 (UTC) Received: from moe.brq.redhat.com (dhcp129-131.brq.redhat.com [10.34.129.131]) by smtp.corp.redhat.com (Postfix) with ESMTP id BA1277BB1D for ; Tue, 23 May 2017 14:26:21 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mx1.redhat.com 63004C059732 Authentication-Results: ext-mx08.extmail.prod.ext.phx2.redhat.com; dmarc=none (p=none dis=none) header.from=redhat.com Authentication-Results: ext-mx08.extmail.prod.ext.phx2.redhat.com; spf=pass smtp.mailfrom=libvir-list-bounces@redhat.com DKIM-Filter: OpenDKIM Filter v2.11.0 mx1.redhat.com 63004C059732 From: Michal Privoznik To: libvir-list@redhat.com Date: Tue, 23 May 2017 16:26:10 +0200 Message-Id: <6d3dc2933c887ea910fa735a9711efe2e7cf1426.1495549496.git.mprivozn@redhat.com> In-Reply-To: References: In-Reply-To: References: X-Scanned-By: MIMEDefang 2.79 on 10.5.11.14 X-loop: libvir-list@redhat.com Subject: [libvirt] [libvirt-python][PATCH v2 3/4] virStream: Introduce virStreamSparse{Recv, Send}All X-BeenThere: libvir-list@redhat.com X-Mailman-Version: 2.1.12 Precedence: junk List-Id: Development discussions about the libvirt library & tools List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: libvir-list-bounces@redhat.com Errors-To: libvir-list-bounces@redhat.com X-Scanned-By: MIMEDefang 2.79 on 10.5.11.15 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.32]); Tue, 23 May 2017 14:28:48 +0000 (UTC) X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Type: text/plain; charset="utf-8" Yet again, our parser is not capable of generating proper wrapper. To be fair, this one wold be really tough anyway. Signed-off-by: Michal Privoznik --- generator.py | 2 + libvirt-override-virStream.py | 107 ++++++++++++++++++++++++++++++++++++++= ++++ sanitytest.py | 6 ++- 3 files changed, 113 insertions(+), 2 deletions(-) diff --git a/generator.py b/generator.py index 0e07fc8..93d1dc3 100755 --- a/generator.py +++ b/generator.py @@ -546,6 +546,8 @@ skip_function =3D ( 'virStreamRecvHole', # overridden in libvirt-override-virStream.py 'virStreamSendHole', # overridden in libvirt-override-virStream.py 'virStreamRecvFlags', # overridden in libvirt-override-virStream.py + 'virStreamSparseRecvAll', # overridden in libvirt-override-virStream.py + 'virStreamSparseSendAll', # overridden in libvirt-override-virStream.py =20 'virConnectUnregisterCloseCallback', # overridden in virConnect.py 'virConnectRegisterCloseCallback', # overridden in virConnect.py diff --git a/libvirt-override-virStream.py b/libvirt-override-virStream.py index 66d2bf6..0ab7815 100644 --- a/libvirt-override-virStream.py +++ b/libvirt-override-virStream.py @@ -164,3 +164,110 @@ ret =3D libvirtmod.virStreamRecvFlags(self._o, nbytes, flags) if ret is None: raise libvirtError ('virStreamRecvFlags() failed') return ret + + def sparseRecvAll(self, handler, holeHandler, opaque): + """Receive the entire data stream, sending the data to + the requested data sink handler and calling the skip + holeHandler to generate holes for sparse stream targets. + This is simply a convenient alternative to recvFlags, for + apps that do blocking-I/O and want to preserve sparseness. + + Hypothetical callbacks can look like this: + + def handler(stream, # virStream instance + buf, # string containing received data + opaque): # extra data passed to sparseRecvAll as o= paque + fd =3D opaque + return os.write(fd, buf) + + def holeHandler(stream, # virStream instance + length, # number of bytes to skip + opaque): # extra data passed to sparseRecvAll = as opaque + fd =3D opaque + cur =3D os.lseek(fd, length, os.SEEK_CUR) + return os.ftruncate(fd, cur) # take this extra step to + # actually allocate the hole + """ + while True: + want =3D 64 * 1024 + got =3D self.recvFlags(want, VIR_STREAM_RECV_STOP_AT_HOLE) + if got =3D=3D -2: + raise libvirtError("cannot use sparseRecvAll with " + "nonblocking stream") + if got =3D=3D -3: + length =3D self.recvHole() + if length is None: + self.abort() + raise RuntimeError("recvHole handler failed") + ret =3D holeHandler(self, length, opaque) + if type(ret) is int and ret < 0: + self.abort() + raise RuntimeError("holeHandler handler returned %d" %= ret) + continue + + if len(got) =3D=3D 0: + break + + try: + ret =3D handler(self, got, opaque) + if type(ret) is int and ret < 0: + raise RuntimeError("sparseRecvAll handler returned %d"= % ret) + except Exception as e: + self.abort() + raise e + + def sparseSendAll(self, handler, holeHandler, skipHandler, opaque): + """Send the entire data stream, reading the data from the + requested data source. This is simply a convenient + alternative to virStreamSend, for apps that do + blocking-I/O and want to preserve sparseness. + + Hypothetical callbacks can look like this: + + def handler(stream, # virStream instance + nbytes, # int amt of data to read + opaque): # extra data passed to sparseSendAll as o= paque + fd =3D opaque + return os.read(fd, nbytes) + + def holeHandler(stream, # virStream instance + opaque): # extra data passed to sparseSendAll = as opaque + fd =3D opaque + cur =3D os.lseek(fd, 0, os.SEEK_CUR) + # ... find out current section and its boundaries + # and set inData =3D True/False and sectionLen correspondi= ngly + os.lseek(fd, cur, os.SEEK_SET) + return [inData, sectionLen] + + def skipHandler(stream, # virStream instance + length, # number of bytes to skip + opaque): # extra data passed to sparseSendAll = as opaque + fd =3D opaque + return os.lseek(fd, length, os.SEEK_CUR) + + """ + while True: + [inData, sectionLen] =3D holeHandler(self, opaque) + if (inData =3D=3D False and sectionLen > 0): + if (self.sendHole(sectionLen) < 0 or + skipHandler(self, sectionLen, opaque) < 0): + self.abort() + continue + + want =3D 64 * 1024 + if (want > sectionLen): + want =3D sectionLen + + try: + got =3D handler(self, want, opaque) + except Exception as e: + self.abort() + raise e + + if not got: + break + + ret =3D self.send(got) + if ret =3D=3D -2: + raise libvirtError("cannot use sparseSendAll with " + "nonblocking stream") diff --git a/sanitytest.py b/sanitytest.py index 7183baa..deec200 100644 --- a/sanitytest.py +++ b/sanitytest.py @@ -167,7 +167,8 @@ for cname in wantfunctions: # These aren't functions, they're callback signatures if name in ["virConnectAuthCallbackPtr", "virConnectCloseFunc", "virStreamSinkFunc", "virStreamSourceFunc", "virStreamEven= tCallback", - "virEventHandleCallback", "virEventTimeoutCallback", "virF= reeCallback"]: + "virEventHandleCallback", "virEventTimeoutCallback", "virF= reeCallback", + "virStreamSinkHoleFunc", "virStreamSourceHoleFunc", "virSt= reamSourceSkipFunc"]: continue if name[0:21] =3D=3D "virConnectDomainEvent" and name[-8:] =3D=3D "Cal= lback": continue @@ -373,7 +374,8 @@ for name in sorted(finalklassmap): =20 # These exist in C and exist in python, but we've got # a pure-python impl so don't check them - if name in ["virStreamRecvAll", "virStreamSendAll"]: + if name in ["virStreamRecvAll", "virStreamSendAll", + "virStreamSparseRecvAll", "virStreamSparseSendAll"]: continue =20 try: --=20 2.13.0 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list From nobody Thu May 2 01:24:24 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) client-ip=209.132.183.28; envelope-from=libvir-list-bounces@redhat.com; helo=mx1.redhat.com; Authentication-Results: mx.zoho.com; spf=pass (zoho.com: domain of redhat.com designates 209.132.183.28 as permitted sender) smtp.mailfrom=libvir-list-bounces@redhat.com; Return-Path: Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by mx.zohomail.com with SMTPS id 1495549704627390.51471060101244; Tue, 23 May 2017 07:28:24 -0700 (PDT) Received: from smtp.corp.redhat.com (int-mx06.intmail.prod.int.phx2.redhat.com [10.5.11.16]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id D831E8123A; Tue, 23 May 2017 14:28:21 +0000 (UTC) Received: from colo-mx.corp.redhat.com (colo-mx01.intmail.prod.int.phx2.redhat.com [10.5.11.20]) by smtp.corp.redhat.com (Postfix) with ESMTPS id AC1E64DA3B; Tue, 23 May 2017 14:28:21 +0000 (UTC) Received: from lists01.pubmisc.prod.ext.phx2.redhat.com (lists01.pubmisc.prod.ext.phx2.redhat.com [10.5.19.33]) by colo-mx.corp.redhat.com (Postfix) with ESMTP id 5BAA1180BAF9; Tue, 23 May 2017 14:28:21 +0000 (UTC) Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.phx2.redhat.com [10.5.11.14]) by lists01.pubmisc.prod.ext.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id v4NEQN14028685 for ; Tue, 23 May 2017 10:26:23 -0400 Received: by smtp.corp.redhat.com (Postfix) id 34C317814F; Tue, 23 May 2017 14:26:23 +0000 (UTC) Received: from moe.brq.redhat.com (dhcp129-131.brq.redhat.com [10.34.129.131]) by smtp.corp.redhat.com (Postfix) with ESMTP id B024B7C749 for ; Tue, 23 May 2017 14:26:22 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mx1.redhat.com D831E8123A Authentication-Results: ext-mx01.extmail.prod.ext.phx2.redhat.com; dmarc=none (p=none dis=none) header.from=redhat.com Authentication-Results: ext-mx01.extmail.prod.ext.phx2.redhat.com; spf=pass smtp.mailfrom=libvir-list-bounces@redhat.com DKIM-Filter: OpenDKIM Filter v2.11.0 mx1.redhat.com D831E8123A From: Michal Privoznik To: libvir-list@redhat.com Date: Tue, 23 May 2017 16:26:11 +0200 Message-Id: In-Reply-To: References: In-Reply-To: References: X-Scanned-By: MIMEDefang 2.79 on 10.5.11.14 X-loop: libvir-list@redhat.com Subject: [libvirt] [libvirt-python][PATCH v2 4/4] examples: Introduce sparsestream.py X-BeenThere: libvir-list@redhat.com X-Mailman-Version: 2.1.12 Precedence: junk List-Id: Development discussions about the libvirt library & tools List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Sender: libvir-list-bounces@redhat.com Errors-To: libvir-list-bounces@redhat.com X-Scanned-By: MIMEDefang 2.79 on 10.5.11.16 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.25]); Tue, 23 May 2017 14:28:22 +0000 (UTC) X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Type: text/plain; charset="utf-8" Sparse streams are not that straight forward to use for the very first time. Especially the sparseRecvAll() and sparseSendAll() methods which expects callbacks. What we can do to make it easier for developers is to have an example where they can take an inspiration from. Signed-off-by: Michal Privoznik --- examples/sparsestream.py | 117 +++++++++++++++++++++++++++++++++++++++++++= ++++ 1 file changed, 117 insertions(+) create mode 100755 examples/sparsestream.py diff --git a/examples/sparsestream.py b/examples/sparsestream.py new file mode 100755 index 0000000..e960c40 --- /dev/null +++ b/examples/sparsestream.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +# Example of sparse streams usage +# +# Authors: +# Michal Privoznik + +import libvirt, sys, os + +def bytesWriteHandler(stream, buf, opaque): + fd =3D opaque + return os.write(fd, buf) + +def bytesReadHandler(stream, nbytes, opaque): + fd =3D opaque + return os.read(fd, nbytes) + +def recvSkipHandler(stream, length, opaque): + fd =3D opaque + cur =3D os.lseek(fd, length, os.SEEK_CUR) + return os.ftruncate(fd, cur) + +def sendSkipHandler(stream, length, opaque): + fd =3D opaque + return os.lseek(fd, length, os.SEEK_CUR) + +def holeHandler(stream, opaque): + fd =3D opaque + cur =3D os.lseek(fd, 0, os.SEEK_CUR) + + try: + data =3D os.lseek(fd, cur, os.SEEK_DATA) + except OSError as e: + if e.errno !=3D 6: + raise e + else: + data =3D -1; + # There are three options: + # 1) data =3D=3D cur; @cur is in data + # 2) data > cur; @cur is in a hole, next data at @data + # 3) data < 0; either @cur is in trailing hole, or @cur is beyond EOF. + if data < 0: + # case 3 + inData =3D False + eof =3D os.lseek(fd, 0, os.SEEK_END) + if (eof < cur): + raise RuntimeError("Current position in file after EOF: %d" % = cur) + sectionLen =3D eof - cur + else: + if (data > cur): + # case 2 + inData =3D False + sectionLen =3D data - cur + else: + # case 1 + inData =3D True + + # We don't know where does the next hole start. Let's find out. + # Here we get the same options as above + hole =3D os.lseek(fd, data, os.SEEK_HOLE) + if hole < 0: + # case 3. But wait a second. There is always a trailing ho= le. + # Do the best what we can here + raise RuntimeError("No trailing hole") + + if (hole =3D=3D data): + # case 1. Again, this is suspicious. The reason we are her= e is + # because we are in data. But at the same time we are in a + # hole. WAT? + raise RuntimeError("Impossible happened") + else: + # case 2 + sectionLen =3D hole - data + os.lseek(fd, cur, os.SEEK_SET) + return [inData, sectionLen] + +def download(vol, st, filename): + offset =3D 0 + length =3D 0 + + fd =3D os.open(filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode= =3D0o0660) + vol.download(st, offset, length, libvirt.VIR_STORAGE_VOL_DOWNLOAD_SPAR= SE_STREAM) + st.sparseRecvAll(bytesWriteHandler, recvSkipHandler, fd) + + os.close(fd) + +def upload(vol, st, filename): + offset =3D 0 + length =3D 0 + + fd =3D os.open(filename, os.O_RDONLY) + vol.upload(st, offset, length, libvirt.VIR_STORAGE_VOL_UPLOAD_SPARSE_S= TREAM) + st.sparseSendAll(bytesReadHandler, holeHandler, sendSkipHandler, fd) + + os.close(fd) + +# main +if len(sys.argv) !=3D 5: + print("Usage: ", sys.argv[0], " URI --upload/--download VOLUME FILE") + print("Either uploads local FILE to libvirt VOLUME, or downloads libvi= rt ") + print("VOLUME into local FILE while preserving FILE/VOLUME sparseness") + sys.exit(1) + +conn =3D libvirt.open(sys.argv[1]) +vol =3D conn.storageVolLookupByKey(sys.argv[3]) + +st =3D conn.newStream() + +if sys.argv[2] =3D=3D "--download": + download(vol, st, sys.argv[4]) +elif sys.argv[2] =3D=3D "--upload": + upload(vol, st, sys.argv[4]) +else: + print("Unknown operation: %s " % sys.argv[1]) + sys.exit(1) + +st.finish() +conn.close() --=20 2.13.0 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list