Merge pull request #475 from catborise/master

rbd support enhanced

conf/requirements.txt

@@ -1,13 +1,13 @@
 Django==3.2.10
-django_bootstrap5==21.1
+django_bootstrap5==21.2
 django-icons==21.1
 django-login-required-middleware==0.7
-django-otp==1.1.1
+django-otp==1.1.3
 django-qr-code==2.3.0
 gunicorn==20.1.0
 libsass==0.21.0
-libvirt-python==7.9.0
+libvirt-python==7.10.0
-lxml==4.6.5
+lxml==4.7.1
 qrcode==7.3.1
 rwlock==0.0.7
 websockify==0.10.0

dev/requirements.txt

@@ -1,7 +1,7 @@
 -r ../conf/requirements.txt
-coverage==6.1.2
+coverage==6.2
-django-debug-toolbar==3.2.2
+django-debug-toolbar==3.2.4
 pycodestyle==2.8.0
 pyflakes==2.4.0
-pylint==2.11.1
+pylint==2.12.2
 yapf==0.31.0

instances/templates/instance.html

@@ -37,6 +37,7 @@
             |
             {% if instance.snapshots %}
                 <i class="fa fa-camera link-primary" title="There are {{ instance.snapshots|length }} snapshot(s)"></i>
+            |
             {% endif %}
             {% if instance.cur_vcpu %}
                 {{ instance.cur_vcpu }} {% trans "VCPU" %}

instances/views.py

@@ -549,7 +549,24 @@ def add_new_vol(request, pk):
             int(app_settings.INSTANCE_VOLUME_DEFAULT_OWNER_UID),
             int(app_settings.INSTANCE_VOLUME_DEFAULT_OWNER_GID),
         )
-        instance.proxy.attach_disk(target_dev, source, target_bus=bus, driver_type=format, cache_mode=cache)
+
+        conn_pool = wvmStorage(
+            instance.compute.hostname,
+            instance.compute.login,
+            instance.compute.password,
+            instance.compute.type,
+            storage,
+        )
+
+        pool_type = conn_pool.get_type()
+        disk_type = conn_pool.get_volume_type(os.path.basename(source))
+
+        if pool_type == 'rbd':
+            source_info = conn_pool.get_rbd_source()
+        else: # add more disk types to handle different pool and disk types
+            source_info = None
+
+        instance.proxy.attach_disk(target_dev, source, source_info=source_info, pool_type=pool_type, disk_type=disk_type, target_bus=bus, format_type=format, cache_mode=cache)
         msg = _("Attach new disk: %(name)s (%(format)s)") % {"name": name, "format": format}
         addlogmsg(request.user.username, instance.compute.name, instance.name, msg)
     return redirect(request.META.get("HTTP_REFERER") + "#disks")
@@ -575,12 +592,20 @@ def add_existing_vol(request, pk):
             storage,
         )
 
-        driver_type = conn_create.get_volume_type(name)
+        format_type = conn_create.get_volume_format_type(name)
+        disk_type = conn_create.get_volume_type(name)
+        pool_type = conn_create.get_type()
+        if pool_type == 'rbd':
+            source_info = conn_create.get_rbd_source()
+            path = conn_create.get_source_name()
+        else:
+            source_info = None
             path = conn_create.get_target_path()
+
         target_dev = utils.get_new_disk_dev(media, disks, bus)
         source = f"{path}/{name}"
 
-        instance.proxy.attach_disk(target_dev, source, target_bus=bus, driver_type=driver_type, cache_mode=cache)
+        instance.proxy.attach_disk(target_dev, source, source_info=source_info, pool_type=pool_type, disk_type=disk_type, target_bus=bus, format_type=format_type, cache_mode=cache)
         msg = _("Attach Existing disk: %(target_dev)s") % {"target_dev": target_dev}
         addlogmsg(request.user.username, instance.compute.name, instance.name, msg)
     return redirect(request.META.get("HTTP_REFERER") + "#disks")
@@ -793,9 +818,9 @@ def set_vcpu(request, pk):
 @superuser_only
 def set_vcpu_hotplug(request, pk):
     instance = get_instance(request.user, pk)
-    status = request.POST.get("vcpu_hotplug", "")
+    status = True if request.POST.get("vcpu_hotplug", "False") == 'True' else False
     msg = _("VCPU Hot-plug is enabled=%(status)s") % {"status": status}
-    instance.proxy.set_vcpu_hotplug(eval(status))
+    instance.proxy.set_vcpu_hotplug(status)
     addlogmsg(request.user.username, instance.compute.name, instance.name, msg)
     return redirect(request.META.get("HTTP_REFERER") + "#resize")
 
@@ -1381,7 +1406,7 @@ def create_instance(request, compute_id, arch, machine):
                                 volume = dict()
                                 volume["device"] = "disk"
                                 volume["path"] = path
-                                volume["type"] = conn.get_volume_type(path)
+                                volume["type"] = conn.get_volume_format_type(path)
                                 volume["cache_mode"] = data["cache_mode"]
                                 volume["bus"] = default_bus
                                 if volume["bus"] == "scsi":
@@ -1411,7 +1436,7 @@ def create_instance(request, compute_id, arch, machine):
                                 )
                                 volume = dict()
                                 volume["path"] = clone_path
-                                volume["type"] = conn.get_volume_type(clone_path)
+                                volume["type"] = conn.get_volume_format_type(clone_path)
                                 volume["device"] = "disk"
                                 volume["cache_mode"] = data["cache_mode"]
                                 volume["bus"] = default_bus
@@ -1431,7 +1456,7 @@ def create_instance(request, compute_id, arch, machine):
                                     path = conn.get_volume_path(vol)
                                     volume = dict()
                                     volume["path"] = path
-                                    volume["type"] = conn.get_volume_type(path)
+                                    volume["type"] = conn.get_volume_format_type(path)
                                     volume["device"] = request.POST.get("device" + str(idx), "")
                                     volume["bus"] = request.POST.get("bus" + str(idx), "")
                                     if volume["bus"] == "scsi":

storages/views.py

@@ -1,4 +1,5 @@
 import json
+import os
 
 from django.contrib import messages
 from django.http import HttpResponse, HttpResponseRedirect
@@ -90,7 +91,10 @@ def storage(request, compute_id, pool):
     """
 
     def handle_uploaded_file(path, f_name):
-        target = path + "/" + str(f_name)
+        target = os.path.normpath(os.path.join(path, f_name))
+        if not target.startswith(path):
+            raise Exception("Security Issues with file uploading")
+        
         destination = open(target, "wb+")
         for chunk in f_name.chunks():
             destination.write(chunk)
@@ -141,7 +145,7 @@ def storage(request, compute_id, pool):
             volname = request.POST.get("volname", "")
             vol = conn.get_volume(volname)
             vol.delete(0)
-            messages.success(request, _("Volume: %(volume)s is deleted.") % {"vol": volname})
+            messages.success(request, _("Volume: %(vol)s is deleted.") % {"vol": volname})
             return redirect(reverse("storage", args=[compute.id, pool]))
             # return HttpResponseRedirect(request.get_full_path())
         if "iso_upload" in request.POST:

vrtManager/create.py

@@ -11,9 +11,13 @@ def get_rbd_storage_data(stg):
     def get_ceph_hosts(doc):
         hosts = list()
         for host in doc.xpath("/pool/source/host"):
-            name = host.prop("name")
+            name = host.get('name')
             if name:
-                hosts.append({"name": name, "port": host.prop("port")})
+                port = host.get('port')
+                if port:
+                    hosts.append({"name": name, "port": port})
+                else:
+                    hosts.append({"name": name})
         return hosts
 
     ceph_hosts = util.get_xml_path(xml, func=get_ceph_hosts)
@@ -60,6 +64,7 @@ class wvmCreate(wvmConnect):
                 name += ".img"
             alloc = 0
         else:
+            image_format = 'raw'
             alloc = size
             metadata = False
         xml = f"""
@@ -89,7 +94,7 @@ class wvmCreate(wvmConnect):
         vol = stg.storageVolLookupByName(name)
         return vol.path()
 
-    def get_volume_type(self, path):
+    def get_volume_format_type(self, path):
         vol = self.get_volume_by_path(path)
         vol_type = util.get_xml_path(vol.XMLDesc(0), "/volume/target/format/@type")
         if vol_type == "unknown" or vol_type == "iso":
@@ -276,37 +281,24 @@ class wvmCreate(wvmConnect):
 
             if stg_type == "rbd":
                 ceph_user, secret_uuid, ceph_hosts = get_rbd_storage_data(stg)
-                xml += """<disk type='network' device='disk'>
+                xml += f"""<disk type='network' device='disk'>
-                            <driver name='qemu' type='%s' %s />""" % (
+                            <driver name='qemu' type='{volume["type"]}' {disk_opts} />"""
-                    volume["type"],
+                xml += f"""  <auth username='{ceph_user}'>
-                    disk_opts,
+                                <secret type='ceph' uuid='{secret_uuid}'/>
-                )
-                xml += """  <auth username='%s'>
-                                <secret type='ceph' uuid='%s'/>
                             </auth>
-                            <source protocol='rbd' name='%s'>""" % (
+                            <source protocol='rbd' name='{volume["path"]}'>"""
-                    ceph_user,
-                    secret_uuid,
-                    volume["path"],
-                )
                 if isinstance(ceph_hosts, list):
                     for host in ceph_hosts:
                         if host.get("port"):
-                            xml += """
+                            xml += f"""
-                                   <host name='%s' port='%s'/>""" % (
+                                   <host name='{host.get("name")}' port='{host.get("port")}'/>"""
-                                host.get("name"),
-                                host.get("port"),
-                            )
                         else:
-                            xml += """
+                            xml += f"""<host name='{host.get("name")}'/>"""
-                                   <host name='%s'/>""" % host.get(
-                                "name"
-                            )
                 xml += """</source>"""
             else:
-                xml += """<disk type='file' device='%s'>""" % volume["device"]
+                xml += f"""<disk type='file' device='{volume["device"]}'>"""
-                xml += """ <driver name='qemu' type='%s' %s/>""" % (volume["type"], disk_opts)
+                xml += f""" <driver name='qemu' type='{volume["type"]}' {disk_opts}/>"""
-                xml += """ <source file='%s'/>""" % volume["path"]
+                xml += f""" <source file='{volume["path"]}'/>"""
 
             if volume.get("bus") == "virtio":
                 xml += """<target dev='vd%s' bus='%s'/>""" % (vd_disk_letters.pop(0), volume.get("bus"))

vrtManager/instance.py

@@ -713,11 +713,13 @@ class wvmInstance(wvmConnect):
         self,
         target_dev,
         source,
+        source_info = None,
+        pool_type="dir",
         target_bus="ide",
         disk_type="file",
         disk_device="disk",
         driver_name="qemu",
-        driver_type="raw",
+        format_type="raw",
         readonly=False,
         shareable=False,
         serial=None,
@@ -739,11 +741,33 @@ class wvmInstance(wvmConnect):
 
         xml_disk = f"<disk type='{disk_type}' device='{disk_device}'>"
         if disk_device == "cdrom":
-            xml_disk += f"<driver name='{driver_name}' type='{driver_type}'/>"
+            xml_disk += f"<driver name='{driver_name}' type='{format_type}'/>"
         elif disk_device == "disk":
-            xml_disk += f"<driver name='{driver_name}' type='{driver_type}' {additionals}/>"
+            xml_disk += f"<driver name='{driver_name}' type='{format_type}' {additionals}/>"
-        xml_disk += f"""<source file='{source}'/>
+
-          <target dev='{target_dev}' bus='{target_bus}'/>"""
+        if disk_type == 'file':
+            xml_disk += f"<source file='{source}'/>"
+        elif disk_type == 'network':
+            if pool_type == 'rbd':
+                auth_type = source_info.get('auth_type')
+                auth_user = source_info.get('auth_user')
+                auth_uuid = source_info.get("auth_uuid")
+                xml_disk += f"""<auth username='{auth_user}'>
+                                <secret type='{auth_type}' uuid='{auth_uuid}'/>
+                            </auth>"""
+                xml_disk += f"""<source protocol='{pool_type}' name='{source}'>"""
+                for host in source_info.get("hosts"):
+                    if host.get('hostport'):
+                        xml_disk += f"""<host name="{host.get('hostname')}" port='{host.get('hostport')}'/>"""
+                    else:
+                        xml_disk += f"""<host name="{host.get('hostname')}"/>"""
+                xml_disk +="""</source>"""
+            else:
+                raise Exception("Not implemented disk type")
+        else:
+            raise Exception("Not implemented disk type")
+
+        xml_disk +=f"<target dev='{target_dev}' bus='{target_bus}'/>"
         if readonly or disk_device == "cdrom":
             xml_disk += """<readonly/>"""
         if shareable:

vrtManager/storage.py

@@ -145,6 +145,9 @@ class wvmStorage(wvmConnect):
     def get_target_path(self):
         return util.get_xml_path(self._XMLDesc(0), "/pool/target/path")
 
+    def get_source_name(self):
+        return util.get_xml_path(self._XMLDesc(0), "/pool/source/name")
+
     def get_allocation(self):
         return int(util.get_xml_path(self._XMLDesc(0), "/pool/allocation"))
 
@@ -154,6 +157,34 @@ class wvmStorage(wvmConnect):
     def get_capacity(self):
         return int(util.get_xml_path(self._XMLDesc(0), "/pool/capacity"))
 
+    def get_rbd_source(self):
+        def hosts(doc):
+            hosts_array = []
+
+            for host in doc.xpath("/pool/source/host"):
+                name = host.get('name')
+                if name:
+                    port = host.get('port')
+                    if port:
+                        hosts_array.append({"hostname": name, "hostport": port})
+                    else:
+                        hosts_array.append({"hostname": name})
+
+            name = doc.get('name')
+            auth = doc.xpath("/pool/source/auth")
+            auth_type = auth[0].get("type")
+            auth_user = auth[0].get("username")
+            auth_uuid = auth[0].xpath("secret/@uuid")[0]
+
+            return({
+                "name": name,
+                "auth_type": auth_type,
+                "auth_user": auth_user,
+                "auth_uuid": auth_uuid,
+                "hosts": hosts_array
+            })
+        return util.get_xml_path(self._XMLDesc(0), func=hosts)
+
     def get_pretty_allocation(self):
         return util.pretty_bytes(self.get_allocation())
 
@@ -185,10 +216,14 @@ class wvmStorage(wvmConnect):
         vol = self.pool.storageVolLookupByName(name)
         vol.delete(0)
 
-    def get_volume_type(self, name):
+    def get_volume_format_type(self, name):
         vol_xml = self._vol_XMLDesc(name)
         return util.get_xml_path(vol_xml, "/volume/target/format/@type")
 
+    def get_volume_type(self, name):
+        vol_xml = self._vol_XMLDesc(name)
+        return util.get_xml_path(vol_xml, "/volume/@type")
+
     def refresh(self):
         self.pool.refresh(0)
 
@@ -206,7 +241,7 @@ class wvmStorage(wvmConnect):
                     "name": volname,
                     "size": self.get_volume_size(volname),
                     "allocation": self.get_volume_allocation(volname),
-                    "type": self.get_volume_type(volname),
+                    "type": self.get_volume_format_type(volname),
                 }
             )
         return vol_list
@@ -260,7 +295,7 @@ class wvmStorage(wvmConnect):
     ):
         vol = self.get_volume(name)
         if not vol_fmt:
-            vol_fmt = self.get_volume_type(name)
+            vol_fmt = self.get_volume_format_type(name)
 
         storage_type = self.get_type()
         if storage_type == "dir":