j3d1/webvirtcloud
1
0
Fork 0
mirror of https://github.com/retspen/webvirtcloud synced 2026-03-23 11:04:49 +00:00
Code Issues Projects Releases Wiki Activity

Added V2 from scratch

Browse source
This commit is contained in:
retspen 2018-09-23 13:17:48 +03:00
parent 5c2232f4e8
commit 6c2925a35d
478 changed files with 21437 additions and 134206 deletions
Download patch file Download diff file Expand all files Collapse all files

71
devenv/README.md Normal file
View file

@ -0,0 +1,71 @@
# Setup HubGridCloud
### Requirements for hypervisor:
* docker 18.06
* vagrant 2.x
* VirtualBox 5.x
## Setup dev environment
### Docker compose
Build and run docker-compose (run only first time)
```bash
docker-compose up -d
docker exec -it $(docker-compose ps -q mariadb) \
mysql -uroot -proot -e "CREATE DATABASE webvirtcloud CHARACTER SET utf8 COLLATE utf8_general_ci;"
docker exec -it $(docker-compose ps -q app) python3.6 manage.py migrate
```
Stop docker-compose
```bash
docker-compose stop
```
Start docker-compose
```bash
docker-compose start
```
Delete docker-compose
```bash
docker-compose down
```
Rebuild app container for new requirements
```bash
docker-compose stop
docker build -t wvcapp .
docker-compose up -d --no-deps --build app
docker-compose start
```
### Scripts for running services
Run django dev server
```bash
devenv/run_django.sh
```
Run celery dev server
```bash
devenv/run_celery.sh
```
Run stmpd dev server
```bash
devenv/run_smtpd.sh
```
### Vagrant
Deploy dev hypervisor (run only first time)
```bash
vagrant up --provider=virtualbox
```
Run dev hypervisor
```bash
vagrant up
```
Stop dev hypervisor
```bash
vagrant halt
```

6
devenv/docker_app_migrate.sh Executable file
View file

@ -0,0 +1,6 @@
#!/bin/bash
set -e
docker exec -it \
-e DJANGO_SETTINGS_MODULE=webvirtcloud.settings.dev \
$(docker-compose ps -q app) python3.6 manage.py migrate

7
devenv/docker_app_rebuld.sh Executable file
View file

@ -0,0 +1,7 @@
#!/bin/bash
set -e
docker-compose stop
docker build -t wvcapp .
docker-compose up -d --no-deps --build app
docker-compose start

6
devenv/docker_run_bash.sh Executable file
View file

@ -0,0 +1,6 @@
#!/bin/bash
set -e
docker exec -it \
-e DJANGO_SETTINGS_MODULE=webvirtcloud.settings.dev \
$(docker-compose ps -q app) bash

7
devenv/run_celery.sh Executable file
View file

@ -0,0 +1,7 @@
#!/bin/bash
set -e
docker exec -it \
-e PYTHONOPTIMIZE=1 \
-e DJANGO_SETTINGS_MODULE=webvirtcloud.settings.dev \
$(docker-compose ps -q app) celery -A webvirtcloud worker -E -l INFO

6
devenv/run_django.sh Executable file
View file

@ -0,0 +1,6 @@
#!/bin/bash
set -e
docker exec -it \
-e DJANGO_SETTINGS_MODULE=webvirtcloud.settings.dev \
$(docker-compose ps -q app) python3.6 manage.py runserver 0:8000

4
devenv/run_smtpd.sh Executable file
View file

@ -0,0 +1,4 @@
#!/bin/bash
set -e
docker exec -it $(docker-compose ps -q app) python3.6 -m smtpd -n -c DebuggingServer localhost:1025

101
devenv/vagrant/bootstrap.sh Executable file
View file

@ -0,0 +1,101 @@
#!/usr/bin/env bash
set -e
echo 'centos' | sudo tee -a /etc/yum/vars/contentdir
sudo yum -y install bash-completion net-tools telnet
# Install libvirt packages
sudo yum -y install epel-release centos-release-qemu-ev
sudo yum -y install qemu-kvm libvirt bridge-utils xmlstarlet python-libguestfs libguestfs-tools libguestfs-rescue libguestfs-winsupport libguestfs-bash-completion cyrus-sasl-md5
sudo systemctl start libvirtd
sudo systemctl enable libvirtd
# Setup libvirt
sudo sed -i 's/#LIBVIRTD_ARGS/LIBVIRTD_ARGS/g' /etc/sysconfig/libvirtd
sudo sed -i 's/#listen_tls/listen_tls/g' /etc/libvirt/libvirtd.conf
sudo sed -i 's/#listen_tcp/listen_tcp/g' /etc/libvirt/libvirtd.conf
sudo sed -i 's/#auth_tcp = \"sasl\"/auth_tcp = \"none\"/g' /etc/libvirt/libvirtd.conf
sudo sed -i 's/#LIBVIRTD_ARGS/LIBVIRTD_ARGS/g' /etc/sysconfig/libvirtd
sudo sed -i 's/: gssapi/: digest-md5/g' /etc/sasl2/libvirt.conf
sudo sed -i 's/#sasldb_path/sasldb_path/g' /etc/sasl2/libvirt.conf
sudo cp -rf /vagrant/devenv/vagrant/etc/libvirt/hooks /etc/libvirt/
sudo chmod +x /etc/libvirt/hooks/qemu
sudo mkdir /mnt/backups
sudo mkdir /var/lib/libvirt/isos
sudo virsh pool-define /vagrant/devenv/vagrant/etc/libvirt/pools/images.xml
sudo virsh pool-define /vagrant/devenv/vagrant/etc/libvirt/pools/backups.xml
sudo virsh pool-define /vagrant/devenv/vagrant/etc/libvirt/pools/isos.xml
sudo virsh pool-start images
sudo virsh pool-start backups
sudo virsh pool-start isos
sudo virsh pool-autostart images
sudo virsh pool-autostart backups
sudo virsh pool-autostart isos
sudo virsh net-destroy default
sudo virsh net-undefine default
sudo virsh net-define /vagrant/devenv/vagrant/etc/libvirt/networks/private.xml
sudo virsh net-define /vagrant/devenv/vagrant/etc/libvirt/networks/public.xml
sudo virsh net-start private
sudo virsh net-start public
sudo virsh net-autostart private
sudo virsh net-autostart public
sudo virsh nwfilter-define /vagrant/devenv/vagrant/etc/libvirt/nwfilters/allow-incoming-ipv6.xml
sudo virsh nwfilter-define /vagrant/devenv/vagrant/etc/libvirt/nwfilters/no-ipv6-spoofing.xml
sudo virsh nwfilter-define /vagrant/devenv/vagrant/etc/libvirt/nwfilters/clean-traffic-ipv6.xml
sudo systemctl restart libvirtd
# Sysctl
sudo cp /vagrant/devenv/vagrant/etc/sysctl.d/99-libvirt.conf /etc/sysctl.d/99-libvirt.conf
sudo sysctl -p
# Networking
sudo cp /vagrant/devenv/vagrant/etc/sysconfig/network-scripts/* /etc/sysconfig/network-scripts/
sudo brctl addbr br-ext
sudo ifconfig br-ext up
sudo brctl addbr br-int
sudo ifconfig br-int up
sudo systemctl restart network
# FirewallD
sudo systemctl enable firewalld
sudo systemctl restart firewalld
sudo firewall-cmd --permanent --direct --add-rule ipv4 filter FORWARD 1 -m physdev --physdev-is-bridged -j ACCEPT
sudo firewall-cmd --permanent --direct --add-rule ipv4 nat POSTROUTING 0 -i br-ext -d 10.255.0.0/16 -j MASQUERADE
sudo firewall-cmd --permanent --direct --add-rule ipv4 nat PREROUTING 0 -i br-ext -s 169.254.0.0/16 -d 169.254.169.254 -p tcp --dport 80 -j REDIRECT --to-ports 8887
sudo firewall-cmd --permanent --direct --add-rule ipv4 nat PREROUTING 0 -i br-ext ! -s 169.254.0.0/16 -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.0.2.2:8000
sudo firewall-cmd --permanent --zone=trusted --add-source=10.0.2.2/24
sudo firewall-cmd --permanent --zone=trusted --add-source=169.254.0.0/16
sudo firewall-cmd --reload
# Prometheus Server
echo "Downloading and installing prometheus server..."
curl -L https://github.com/prometheus/prometheus/releases/download/v2.3.2/prometheus-2.3.2.linux-amd64.tar.gz -o /tmp/prometheus-2.3.2.linux-amd64.tar.gz > /dev/null 2>&1
tar -zxf /tmp/prometheus-2.3.2.linux-amd64.tar.gz -C /opt/
sudo cp /vagrant/devenv/vagrant/opt/prometheus/prometheus.yml /opt/prometheus-2.3.2.linux-amd64/
sudo cp /vagrant/devenv/vagrant/opt/prometheus/prometheus.service /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable prometheus
sudo systemctl start prometheus
# Prometheus Libvirt Exporter
echo "Downloading and installing prometheus libvirt exporter..."
curl -L https://github.com/retspen/libvirt_exporter/releases/download/0.1.0/libvirt_exporter-0.1.0.linux-amd64.tar.gz -o /tmp/libvirt_exporter-0.1.0.linux-amd64.tar.gz > /dev/null 2>&1
tar -zxf /tmp/libvirt_exporter-0.1.0.linux-amd64.tar.gz -C /opt/
sudo cp /vagrant/devenv/vagrant/opt/libvirt_exporter/libvirt_exporter.service /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable libvirt_exporter
sudo systemctl start libvirt_exporter
# NTP
sudo yum -y install ntp ntpdate ntp-doc
sudo systemctl enable ntpd
sudo systemctl start ntpd
# Upgrade system
sudo yum -y upgrade

39
devenv/vagrant/etc/libvirt/hooks/qemu Normal file
View file

@ -0,0 +1,39 @@
#!/bin/bash
#
# HubGridCloud libvirt qemu hook
#
domain_name="$1"
domain_task="$2"
network="169.254.0.0/16"
zeroconf_add() {
# Get VNET interface
target=$(xmlstarlet select -t -m "//devices/interface[@type='bridge']" -v "concat(target/@dev, ' ')" -n | head -1)
iface=$(echo ${target} | cut -d- -f2)
# Add rule for 169.254.0.0/16
ebtables -t nat -I "I-${iface}-ipv4-ip" 2 -p IPv4 --ip-src "${network}" -j RETURN
ebtables -t nat -I "I-${iface}-arp-ip" -p ARP --arp-ip-src "${network}" -j RETURN
}
case "${domain_task}" in
prepare)
;;
started)
zeroconf_add
;;
stopped)
;;
reconnect)
zeroconf_add
;;
*)
exit 0
echo "qemu hook called with unexpected options $*" >&2
;;
esac
exit 0

5
devenv/vagrant/etc/libvirt/networks/private.xml Normal file
View file

@ -0,0 +1,5 @@
<network>
<name>private</name>
<forward mode='bridge'/>
<bridge name='br-int'/>
</network>

5
devenv/vagrant/etc/libvirt/networks/public.xml Normal file
View file

@ -0,0 +1,5 @@
<network>
<name>public</name>
<forward mode='bridge'/>
<bridge name='br-ext'/>
</network>

3
devenv/vagrant/etc/libvirt/nwfilters/allow-incoming-ipv6.xml Normal file
View file

@ -0,0 +1,3 @@
<filter name='allow-incoming-ipv6' chain='ipv6' priority='-600'>
<rule action='accept' direction='in' priority='500'/>
</filter>

19
devenv/vagrant/etc/libvirt/nwfilters/clean-traffic-ipv6.xml Normal file
View file

@ -0,0 +1,19 @@
<filter name='clean-traffic-ipv6' chain='root'>
<filterref filter='no-mac-spoofing'/>
<filterref filter='no-ip-spoofing'/>
<rule action='accept' direction='out' priority='-650'>
<mac protocolid='ipv4'/>
</rule>
<filterref filter='no-ipv6-spoofing'/>
<rule action='accept' direction='out' priority='-650'>
<mac protocolid='ipv6'/>
</rule>
<filterref filter='allow-incoming-ipv4'/>
<filterref filter='allow-incoming-ipv6'/>
<filterref filter='no-arp-spoofing'/>
<rule action='accept' direction='inout' priority='-500'>
<mac protocolid='arp'/>
</rule>
<filterref filter='no-other-l2-traffic'/>
<filterref filter='qemu-announce-self'/>
</filter>

5
devenv/vagrant/etc/libvirt/nwfilters/no-ipv6-spoofing.xml Normal file
View file

@ -0,0 +1,5 @@
<filter name='no-ipv6-spoofing' chain='ipv6' priority='-600'>
<rule action='drop' direction='out' priority='500'>
<ipv6 match='no' srcipaddr='$IPV6'/>
</rule>
</filter>

6
devenv/vagrant/etc/libvirt/pools/backups.xml Normal file
View file

@ -0,0 +1,6 @@
<pool type='dir'>
<name>backups</name>
<target>
<path>/mnt/backups</path>
</target>
</pool>

6
devenv/vagrant/etc/libvirt/pools/images.xml Normal file
View file

@ -0,0 +1,6 @@
<pool type='dir'>
<name>images</name>
<target>
<path>/var/lib/libvirt/images</path>
</target>
</pool>

6
devenv/vagrant/etc/libvirt/pools/isos.xml Normal file
View file

@ -0,0 +1,6 @@
<pool type='dir'>
<name>isos</name>
<target>
<path>/var/lib/libvirt/isos</path>
</target>
</pool>

10
devenv/vagrant/etc/sysconfig/network-scripts/ifcfg-br-ext Normal file
View file

@ -0,0 +1,10 @@
DEVICE=br-ext
TYPE=Bridge
IPADDR=10.255.0.1
PREFIX=16
IPADDR1=169.254.169.254
PREFIX1=16
ONBOOT=yes
BOOTPROTO=none
NM_CONTROLLED=yes
DELAY=0

6
devenv/vagrant/etc/sysconfig/network-scripts/ifcfg-br-int Normal file
View file

@ -0,0 +1,6 @@
DEVICE=br-int
TYPE=Bridge
ONBOOT=yes
BOOTPROTO=none
NM_CONTROLLED=no
DELAY=0

6
devenv/vagrant/etc/sysconfig/network-scripts/ifcfg-eth1 Normal file
View file

@ -0,0 +1,6 @@
DEVICE=eth1
TYPE=Ethernet
BOOTPROTO=none
ONBOOT=yes
NM_CONTROLLED=no
BRIDGE=br-ext

9
devenv/vagrant/etc/sysconfig/network-scripts/ifcfg-eth2 Normal file
View file

@ -0,0 +1,9 @@
DEVICE=eth2
TYPE=Ethernet
IPADDR=172.16.0.1
PREFIX=16
MTU=1600
BOOTPROTO=none
ONBOOT=yes
NM_CONTROLLED=no
BRIDGE=br-int

7
devenv/vagrant/etc/sysctl.d/99-libvirt.conf Normal file
View file

@ -0,0 +1,7 @@
# Libvirt settings
net.ipv4.ip_forward=1
net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.default.rp_filter=0
net.bridge.bridge-nf-call-arptables=1
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1

12
devenv/vagrant/opt/libvirt_exporter/libvirt_exporter.service Normal file
View file

@ -0,0 +1,12 @@
[Unit]
Description=LibVirt Exporter for VM metrics
Documentation=https://github.com/retspen/libvirt_exporter
After=network-online.target
[Service]
Restart=always
WorkingDirectory=/opt/libvirt_exporter-0.1.0.linux-amd64/
ExecStart=/opt/libvirt_exporter-0.1.0.linux-amd64/libvirt_exporter --web.listen-address=localhost:9177
[Install]
WantedBy=multi-user.target

12
devenv/vagrant/opt/prometheus/prometheus.service Normal file
View file

@ -0,0 +1,12 @@
[Unit]
Description=Prometheus Server 2.3.2
Documentation=https://prometheus.io/docs/introduction/overview/
After=network-online.target
[Service]
Restart=always
WorkingDirectory=/opt/prometheus-2.3.2.linux-amd64/
ExecStart=/opt/prometheus-2.3.2.linux-amd64/prometheus
[Install]
WantedBy=multi-user.target

25
devenv/vagrant/opt/prometheus/prometheus.yml Normal file
View file

@ -0,0 +1,25 @@
# my global config
global:
scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
# scrape_timeout is set to the global default (10s).
# Alertmanager configuration
alerting:
alertmanagers:
- static_configs:
- targets:
# - alertmanager:9093
# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
# - "first_rules.yml"
# - "second_rules.yml"
# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
# The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
- job_name: 'libvirt'
static_configs:
- targets: ['localhost:9177']