From d59b2c970592170fdfa31b33ef42337232d1c2d0 Mon Sep 17 00:00:00 2001
From: Remy Moll <me@moll.re>
Date: Tue, 26 Mar 2024 15:14:23 +0100
Subject: [PATCH] better fedora setup
---
plays/fedora-k3s-setup.yml | 19 ----
plays/k3s-fedora-lxc.yml | 58 ++++++++++
...a-server-install.yml => k3s-fedora-vm.yml} | 34 +++++-
plays/k3s-full-deployment.yaml | 5 -
plays/proxmox-lxc-wireguard.yaml | 16 +++
plays/proxmox-setup.yml | 1 -
plays/raspberry-setup.yml | 2 +-
roles/docker-install/main.yml | 22 ++++
roles/k3s-ha/tasks/main.yml | 13 ++-
roles/k3s-ha/tasks/prerequisites.yml | 33 +++---
roles/k3s-ha/templates/config.yml.j2 | 5 +-
roles/proxmox-fedora-vm/tasks/main.yml | 101 +++++++++++++-----
roles/proxmox-fedora-vm/templates/vm.conf | 0
roles/proxmox-lxc-fedora/tasks/main.yml | 72 +++++++++++++
roles/proxmox-win10-vm/tasks/main.yml | 10 --
.../proxmox-win10-vm/templates/blacklist.conf | 1 -
roles/proxmox-win10-vm/templates/vm.conf.j2 | 5 -
17 files changed, 304 insertions(+), 93 deletions(-)
delete mode 100644 plays/fedora-k3s-setup.yml
create mode 100644 plays/k3s-fedora-lxc.yml
rename plays/{fedora-server-install.yml => k3s-fedora-vm.yml} (55%)
delete mode 100644 plays/k3s-full-deployment.yaml
create mode 100644 plays/proxmox-lxc-wireguard.yaml
create mode 100644 roles/docker-install/main.yml
delete mode 100644 roles/proxmox-fedora-vm/templates/vm.conf
create mode 100644 roles/proxmox-lxc-fedora/tasks/main.yml
delete mode 100644 roles/proxmox-win10-vm/tasks/main.yml
delete mode 100644 roles/proxmox-win10-vm/templates/blacklist.conf
delete mode 100644 roles/proxmox-win10-vm/templates/vm.conf.j2
diff --git a/plays/fedora-k3s-setup.yml b/plays/fedora-k3s-setup.yml
deleted file mode 100644
index 3d5e921..0000000
--- a/plays/fedora-k3s-setup.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-- name: Customization and hardening of fresh fedora install running on a VM
- hosts: fedora_server
- become: yes
- # reuse password for sudo
- become_method: sudo
-
- user: ansible
-
- vars_prompt:
- - name: k3s_token
- prompt: "Enter a (single use) k3s token"
- private: yes
-
- vars:
- - master_ip: "{{ hostvars['fedora-node-1']['ansible_default_ipv4']['address'] }}"
- roles:
- - role: ssh-key-copy
- - role: fail2ban
- - role: k3s-ha
diff --git a/plays/k3s-fedora-lxc.yml b/plays/k3s-fedora-lxc.yml
new file mode 100644
index 0000000..aeb5b34
--- /dev/null
+++ b/plays/k3s-fedora-lxc.yml
@@ -0,0 +1,58 @@
+- name: Setup of fedora LXC containers
+ hosts: proxmox
+ become: yes
+ user: root
+
+ vars:
+ - proxmox_user: root@pam
+ proxmox_token: !vault |
+ $ANSIBLE_VAULT;1.1;AES256
+ 31376165343635623536393936353032333638636564646665646464386635383761623632373036
+ 3433306233633465313737303039346635353363326562320a336532666632613663303730643136
+ 36353430353638616239313766666233396339643431636161373965383664663230616664336162
+ 3730353264643434390a386132383238613165653037623466383732323835356365326439656565
+ 38666466343932356137393333623964333463303564623666656533626331646535306531343836
+ 3136313232333265303336613235336334323963316439383230
+ proxmox_node: proxmox
+ proxmox_token_id: ansible
+
+
+
+ tasks:
+ - name: Create 3 fedora LXC containers
+ include_role:
+ name: proxmox-lxc-fedora
+ vars:
+ lxc_id: "{{ 150 + item.index }}"
+ lxc_ip: "192.168.1.{{ 200 + item.index }}"
+ lxc_gateway: "192.168.1.1"
+ name: "{{ item.name }}"
+
+ loop:
+ - name: k3s-1
+ index: 1
+ - name: k3s-2
+ index: 2
+ - name: k3s-3
+ index: 3
+ loop_control:
+ index_var: loop_index
+
+
+- name: Setup of k3s cluster
+ hosts: lxc_fedora
+ become: yes
+ user: root
+
+
+ vars_prompt:
+ - name: k3s_token
+ prompt: "Enter a (single use) k3s token"
+ private: yes
+
+
+ roles:
+ - role: k3s-ha
+
+
+# CAUTION - THIS IS BROKEN
\ No newline at end of file
diff --git a/plays/fedora-server-install.yml b/plays/k3s-fedora-vm.yml
similarity index 55%
rename from plays/fedora-server-install.yml
rename to plays/k3s-fedora-vm.yml
index 6d46890..052dbd7 100644
--- a/plays/fedora-server-install.yml
+++ b/plays/k3s-fedora-vm.yml
@@ -16,8 +16,34 @@
proxmox_node: proxmox
proxmox_token_id: ansible
+ tasks:
+ - name: Create 2 fedora vms
+ include_role:
+ name: proxmox-fedora-vm
+ vars:
+ name: "{{ item.name }}"
+ vm_id: "{{ 200 + item.index }}"
+ vm_ip: "192.168.1.{{ 200 + item.index }}"
+ loop:
+ - name: k3s-1
+ index: 1
+ - name: k3s-2
+ index: 2
+ loop_control:
+ index_var: loop_index
+
+
+- name: Setup of k3s cluster
+ hosts: kvm_fedora # this group is created by the previous task
+ become: yes
+ user: root
+
+
+ vars_prompt:
+ - name: k3s_token
+ prompt: "Enter a (single use) k3s token"
+ private: yes
+
+
roles:
- - role: proxmox-fedora-vm
- vms:
- - name: fedora-node-1
- - name: fedora-node-2
+ - role: k3s-ha
diff --git a/plays/k3s-full-deployment.yaml b/plays/k3s-full-deployment.yaml
deleted file mode 100644
index fa24bf9..0000000
--- a/plays/k3s-full-deployment.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-- name: Full k3s deployment
- hosts: localhost
-
- roles:
- - role: role-k3s-deployments
diff --git a/plays/proxmox-lxc-wireguard.yaml b/plays/proxmox-lxc-wireguard.yaml
new file mode 100644
index 0000000..2fbcc17
--- /dev/null
+++ b/plays/proxmox-lxc-wireguard.yaml
@@ -0,0 +1,16 @@
+- name: WG-easy setup on a fresh Proxmox lxc container
+ hosts: ubuntu-wireguard
+ user: ubuntu
+
+ vars_prompt:
+ - name: vpn_admin_password
+ prompt: "Enter the http password for the vpn admin interface"
+ private: yes
+
+ roles:
+ - role: ssh-key-copy
+ - role: docker-install
+ - role: wireguard-easy-docker
+ vars:
+ - vpn_hostname: vpn.kluster.moll.re
+ # - vpn_admin_password
diff --git a/plays/proxmox-setup.yml b/plays/proxmox-setup.yml
index f222277..8ad3c28 100644
--- a/plays/proxmox-setup.yml
+++ b/plays/proxmox-setup.yml
@@ -1,7 +1,6 @@
- name: Proxmox configuration
hosts: proxmox
user: root
-
roles:
- role: proxmox-basic-setup
diff --git a/plays/raspberry-setup.yml b/plays/raspberry-setup.yml
index b64f3d0..93d135e 100644
--- a/plays/raspberry-setup.yml
+++ b/plays/raspberry-setup.yml
@@ -4,4 +4,4 @@
user: pi
roles:
- role: ssh-key-copy
- - role: raspberry
\ No newline at end of file
+ - role: raspberry
diff --git a/roles/docker-install/main.yml b/roles/docker-install/main.yml
new file mode 100644
index 0000000..0ba2e45
--- /dev/null
+++ b/roles/docker-install/main.yml
@@ -0,0 +1,22 @@
+- name: Install docker
+ package:
+ name: docker-ce
+ state: present
+ update_cache: yes
+ become: true
+
+- name: Add user to docker group
+ user:
+ name: "{{ ansible_user }}"
+ groups: docker
+ append: yes
+ become: true
+
+- name: Start docker service
+ service:
+ name: docker
+ state: started
+ enabled: yes
+ become: true
+
+
\ No newline at end of file
diff --git a/roles/k3s-ha/tasks/main.yml b/roles/k3s-ha/tasks/main.yml
index ebf3a9d..bda7c75 100644
--- a/roles/k3s-ha/tasks/main.yml
+++ b/roles/k3s-ha/tasks/main.yml
@@ -9,13 +9,20 @@
mode: 0755
+- name: Create k3s config directory
+ file:
+ path: "/var/config/k3s"
+ state: directory
+ mode: 0755
+
+
- name: Create k3s config file
template:
src: ../templates/config.yml.j2
- dest: "/home/{{ ansible_user }}/k3s_config.yml"
+ dest: "/var/config/k3s/config.yml"
-- name: Only run uninstall script if it exists
+- name: Only run uninstall script if it exists (clean install)
become: true
shell: /usr/local/bin/k3s-uninstall.sh
args:
@@ -23,7 +30,7 @@
- name: Execute k3s install script providing a config.yml
- shell: "/tmp/install-k3s.sh --config /home/{{ ansible_user }}/k3s_config.yml server"
+ shell: "/tmp/install-k3s.sh --config /var/config/k3s/config.yml server"
- name: Copy kube config file back to local machine
diff --git a/roles/k3s-ha/tasks/prerequisites.yml b/roles/k3s-ha/tasks/prerequisites.yml
index ca9fa17..ba74c78 100644
--- a/roles/k3s-ha/tasks/prerequisites.yml
+++ b/roles/k3s-ha/tasks/prerequisites.yml
@@ -1,3 +1,19 @@
+- name: Install required packages
+ ansible.builtin.package:
+ name: "{{ item }}"
+ state: present
+ loop:
+ - firewalld
+ - curl
+
+
+- name: Enable and start firewalld
+ ansible.builtin.service:
+ name: firewalld
+ state: started
+ enabled: yes
+
+
- name: Set firewalld rule for required ports
firewalld:
port: "{{ item }}"
@@ -5,6 +21,8 @@
state: enabled
immediate: yes
loop:
+ - 22/tcp # ssh
+ # k3s ports
- 2379-2380/tcp
- 6443/tcp
- 8472/udp
@@ -12,8 +30,9 @@
# tcp and udb for metallb
- 7946/udp
- 7946/tcp
-
-- name: Set firewalld rule for rquired zones
+
+
+- name: Set firewalld rule for required zones
firewalld:
zone: trusted
source: "{{ item }}"
@@ -23,13 +42,3 @@
loop:
- 10.42.0.0/16 # pods
- 10.43.0.0/16 # services
-
-- name: Update hostname to be unique
- ansible.builtin.replace:
- path: /etc/hosts
- regexp: '(\s+)localhost(\s+.*)?$'
- replace: '\{{ inventory_hostname }}\2'
-
-- name: Set hostname
- ansible.builtin.hostname:
- name: "{{ inventory_hostname }}"
diff --git a/roles/k3s-ha/templates/config.yml.j2 b/roles/k3s-ha/templates/config.yml.j2
index 7b672bc..db66034 100644
--- a/roles/k3s-ha/templates/config.yml.j2
+++ b/roles/k3s-ha/templates/config.yml.j2
@@ -4,9 +4,8 @@ disable:
- traefik
- servicelb
-{% if ansible_host == master_ip %}
+{% if inventory_hostname == ansible_play_hosts[0] %}
cluster-init: true
{% else %}
-server: https://{{ master_ip }}:6443
+server: https://{{ hostvars['k3s-1'].ansible_default_ipv4.address }}:6443
{% endif %}
-
diff --git a/roles/proxmox-fedora-vm/tasks/main.yml b/roles/proxmox-fedora-vm/tasks/main.yml
index f3f5987..ebebcfb 100644
--- a/roles/proxmox-fedora-vm/tasks/main.yml
+++ b/roles/proxmox-fedora-vm/tasks/main.yml
@@ -1,32 +1,75 @@
-- name: Proxmox VM provisioning
- community.general.proxmox_kvm:
- api_host: "{{ ansible_host }}"
- api_user: "{{ proxmox_user }}"
- api_token_id: "{{ proxmox_token_id }}"
- api_token_secret: "{{ proxmox_token }}"
- node: "{{ proxmox_node }}"
-
- # source:
- clone: fedora-server-template
- # created manually already installed
- vmid: 100
- # newid: "{{ 110 + loop_index }}"
- name: "{{ item.name }}"
- storage: VM_storage
- agent: true
- timeout: 60
+# - name: Download fedora qcow2 template
+# get_url:
+# url: "https://download.fedoraproject.org/pub/fedora/linux/releases/39/Cloud/x86_64/images/Fedora-Cloud-Base-39-1.5.x86_64.qcow2"
+# dest: "/var/lib/vz/template/fedora-cloud-39-template.qcow2"
+# mode: 0644
- loop: "{{ vms }}"
+# # export the path:
+# register: fedora_dowload
+# run_once: true
-- name: Ensure VMs are booted
- community.general.proxmox_kvm:
- api_host: "{{ ansible_host }}"
- api_user: "{{ proxmox_user }}"
- api_token_id: "{{ proxmox_token_id }}"
- api_token_secret: "{{ proxmox_token }}"
- node: "{{ proxmox_node }}"
- # set manually:
- name: "{{ item.name }}"
- state: started
- loop: "{{ vms }}"
+# - name: Provision a new VM (empty for now)
+# community.general.proxmox_kvm:
+# api_host: "{{ ansible_host }}"
+# api_user: "{{ proxmox_user }}"
+# api_token_id: "{{ proxmox_token_id }}"
+# api_token_secret: "{{ proxmox_token }}"
+# node: "{{ proxmox_node }}"
+
+# storage: VM_storage
+# ostype: l26
+# cores: 4
+# memory: 500
+# net:
+# net0: 'virtio,bridge=vmbr0'
+# vmid: "{{ vm_id }}"
+# name: "{{ name }}"
+# onboot: true
+# agent: true
+
+# # graphics
+# vga: serial0
+# serial:
+# serial0: socket
+
+# # required for cloud-init
+# scsihw: virtio-scsi-pci
+# bootdisk: scsi0
+# boot: "order=scsi0"
+
+# # cloud init config
+# ide:
+# ide2: VM_storage:cloudinit
+# sshkeys: "{{ lookup('file', '~/.ssh/id_ed25519.pub') }}"
+# ipconfig:
+# ipconfig0: 'ip={{ vm_ip }}/24,gw=192.168.1.1'
+# ciuser: ansible
+
+# timeout: 60
+
+
+# - name: Import fedora qcow2 template
+# shell: |
+# qm disk import "{{ vm_id }}" "{{ fedora_dowload.dest }}" VM_storage --format qcow2
+# qm set "{{ vm_id }}" --scsi0 VM_storage:vm-{{ vm_id }}-disk-0,discard=on
+# qm resize "{{ vm_id }}" scsi0 50G
+
+
+# - name: Boot VM
+# community.general.proxmox_kvm:
+# api_host: "{{ ansible_host }}"
+# api_user: "{{ proxmox_user }}"
+# api_token_id: "{{ proxmox_token_id }}"
+# api_token_secret: "{{ proxmox_token }}"
+# node: "{{ proxmox_node }}"
+# vmid: "{{ vm_id }}"
+# state: started
+
+
+- name: Add host to group 'kvm_fedora' with variable
+ ansible.builtin.add_host:
+ groups: kvm_fedora
+ name: "{{ name }}"
+ ansible_host: "{{ vm_ip }}"
+ ansible_user: ansible
diff --git a/roles/proxmox-fedora-vm/templates/vm.conf b/roles/proxmox-fedora-vm/templates/vm.conf
deleted file mode 100644
index e69de29..0000000
diff --git a/roles/proxmox-lxc-fedora/tasks/main.yml b/roles/proxmox-lxc-fedora/tasks/main.yml
new file mode 100644
index 0000000..cea0d4d
--- /dev/null
+++ b/roles/proxmox-lxc-fedora/tasks/main.yml
@@ -0,0 +1,72 @@
+- name: Add fedora LXC container
+ proxmox:
+ vmid: "{{ lxc_id }}"
+ hostname: "{{ name }}"
+ state: present
+ node: "{{ proxmox_node }}"
+ ostemplate: local:vztmpl/fedora-39-default_20231118_amd64.tar.xz
+ ostype: fedora
+ memory: 9216 # 9GB
+ swap: 0
+ cores: 5
+ netif: '{"net0":"name=eth0,gw={{ lxc_gateway }},ip={{ lxc_ip }}/24,bridge=vmbr0"}'
+ onboot: 1
+ pubkey: "{{ lookup('file', '~/.ssh/id_ed25519.pub') }}"
+ unprivileged: false
+ disk: local-lvm:20
+
+ api_user: "{{ proxmox_user }}"
+ api_host: "{{ proxmox_node }}"
+ api_token_secret: "{{ proxmox_token }}"
+ api_token_id: "{{ proxmox_token_id }}"
+ timeout: 30
+
+
+- name: Add k3s specific config to LXC config file
+ lineinfile:
+ state: present
+ path: "/etc/pve/lxc/{{ lxc_id }}.conf"
+ line: |
+ lxc.apparmor.profile: unconfined
+ lxc.cgroup.devices.allow: a
+ lxc.cap.drop:
+ lxc.mount.auto: proc:rw sys:rw
+ lxc.autodev: 1
+ lxc.mount.entry: /dev/dri dev/dri none bind,optional,create=dir
+ lxc.mount.entry: /lib/modules /lib/modules none bind,optional,create=dir
+
+
+- name: Add host to group 'lxc_fedora' with variable
+ ansible.builtin.add_host:
+ groups: lxc_fedora
+ name: "{{ name }}"
+ ansible_host: "{{ lxc_ip }}"
+ ansible_user: root
+
+
+- name: Start LXC container
+ proxmox:
+ vmid: "{{ lxc_id }}"
+ state: started
+ node: "{{ proxmox_node }}"
+ api_user: "{{ proxmox_user }}"
+ api_host: "{{ proxmox_node }}"
+ api_token_secret: "{{ proxmox_token }}"
+ api_token_id: "{{ proxmox_token_id }}"
+ timeout: 30
+
+
+# Note: at this point the LXC container is running but cannot be accessed via ssh
+- name: Use lxc to install openssh-server
+ ansible.builtin.command: |
+ lxc-attach -n {{ lxc_id }} -- dnf install -y openssh-server
+
+
+- name: Push boot config to LXC container
+ ansible.builtin.command: |
+ pct push {{ lxc_id }} /boot/config-{{ ansible_kernel }} /boot/config-{{ ansible_kernel }}
+
+
+- name: Use lxc to enable openssh-server
+ ansible.builtin.command: |
+ lxc-attach -n {{ lxc_id }} -- systemctl enable --now sshd
diff --git a/roles/proxmox-win10-vm/tasks/main.yml b/roles/proxmox-win10-vm/tasks/main.yml
deleted file mode 100644
index a9c3112..0000000
--- a/roles/proxmox-win10-vm/tasks/main.yml
+++ /dev/null
@@ -1,10 +0,0 @@
-- name: Blacklist GPU drivers
- ansible.builtin.lineinfile:
- path: /etc/modprobe.d/blacklist.conf
- line: blacklist amdgpu
-
-- name: Add Windows 10 VM
- template:
- src: ../templates/vm.conf.j2
- dest: /etc/pve/qm/106.conf
-
diff --git a/roles/proxmox-win10-vm/templates/blacklist.conf b/roles/proxmox-win10-vm/templates/blacklist.conf
deleted file mode 100644
index fda53f2..0000000
--- a/roles/proxmox-win10-vm/templates/blacklist.conf
+++ /dev/null
@@ -1 +0,0 @@
-blacklist amdgpu
diff --git a/roles/proxmox-win10-vm/templates/vm.conf.j2 b/roles/proxmox-win10-vm/templates/vm.conf.j2
deleted file mode 100644
index c3c145f..0000000
--- a/roles/proxmox-win10-vm/templates/vm.conf.j2
+++ /dev/null
@@ -1,5 +0,0 @@
-bios: ovmf
-scsihw: virtio-scsi-pci
-bootdisk: scsi0
-scsi0: .....
-hostpci0: 34:00:00,x-vga=on