$ export CLOUD=hetzner
$ export STORAGE_PLUGIN=openebs-mayastor
$ make provision k8s-install test-dbench cleanup

server-reset: require-target
ifeq ("true",$(SKIP_SERVER_RESET))
    @echo "[info] SKIP_SERVER_RESET=true, skipping server reset step..."
else
    @echo "[info] resetting server @ [$(TARGET)]..."
    @$(ANSIBLE_PLAYBOOK) server-reset.yml \
        -i $(INVENTORY_PATH)  \
        --limit=$(TARGET) \
        $(ANSIBLE_ARGS)
endif

#
# Playbook for resetting a (hetzner) server
#
# Before this playbook is run, the server is expected to be in a state where ansible is
# functional and usable (after fresh machine allocation or after some other run has finished), and normally accessible.
#
# The installimage bits are heavily inspired by
# https://github.com/andrelohmann/ansible-role-hetzner_installimage
#
---
- name: k8s-setup all-in-one server setup
  hosts: "{{ ansible_limit | default(omit) }}"
  remote_user: root
  # pre-server reset ansible is expected to be usable
  # (*should* also work if this playbook is run while machine is in rescue mode)
  gather_facts: yes
  vars:
    public_key: ~/.ssh/id_rsa.pub
    # host key for the machine is going to change during this playbook
    # (during rescue mode entry)
    # so we need to disable strict host key checking
    ansible_ssh_common_args: |
      -o StrictHostKeyChecking=no -o userknownhostsfile=/dev/null      

    known_hosts_path: ~/.ssh/known_hosts

    # Hetzner install image options
    hetzner_installimage_drives:
      - DRIVE1 /dev/sda
      - DRIVE2 /dev/sdb
    hetzner_installimage_raid:
      - SWRAID 0
      - SWRAIDLEVEL 0 # doesn't matter if SWRAID is 0
    hetzner_installimage_hostname: "{{ inventory_hostname.split('.') | first }}"
    hetzner_installimage_partitions:
      # We'll be running with swap even though kubernetes suggests against it.
      - PART swap swap 32G
      - PART /boot ext4 1G
      - PART / ext4 30G
      - PART /root-disk-remaining ext4 all
    hetzner_installimage_image: /root/.oldroot/nfs/images/Ubuntu-2004-focal-64-minimal.tar.gz
    hetzner_installimage_bootloader: grub

  tasks:
    #################
    # Machine reset #
    #################

    - name: Ensure required required variables are provided
      ansible.builtin.fail:
        msg: "hetzner_webservice_(username|password) variables are empty/not defined"
      no_log: true
      when: 'item == ""'
      with_items:
        - "{{ hetzner_webservice_username }}"
        - "{{ hetzner_webservice_password }}"

    - name: Generate local SSH key fingerprint
      delegate_to: localhost
      shell: "ssh-keygen -E md5 -lf ~/.ssh/id_rsa.pub | cut -d' ' -f2 | cut -d':' -f2-"
      register: ssh_key_fingerprint

    - name: Trigger rescue mode on server
      ansible.builtin.uri:
        method: POST
        url: "https://robot-ws.your-server.de/boot/{{ hostvars[inventory_hostname]['ansible_env'].SSH_CONNECTION.split(' ')[2] }}/rescue"
        user: "{{ lookup('env', 'HETZNER_WEBSERVICE_USERNAME') | default(hetzner_webservice_username) }}"
        password: "{{ lookup('env', 'HETZNER_WEBSERVICE_PASSWORD') | default(hetzner_webservice_password) }}"
        status_code: [200, 409] # 409 returned when it is already set
        body_format: form-urlencoded
        body:
          os: linux
          arch: 64
          authorized_key:
            - "{{ ssh_key_fingerprint.stdout }}"

    ###############
    # Rescue mode #
    ###############

    - name: Restart server to enter rescue mode
      delegate_to: localhost
      ansible.builtin.uri:
        method: POST
        url: "https://robot-ws.your-server.de/reset/{{ hostvars[inventory_hostname]['ansible_env'].SSH_CONNECTION.split(' ')[2] }}"
        user: "{{ hetzner_webservice_username }}"
        password: "{{ hetzner_webservice_password }}"
        status_code: [200, 503] # 503 is returned when rescue mode is already enabled
        body_format: form-urlencoded
        body:
          type: hw # "power button"

    - name: Wait for SSH to come back up (enter rescue mode) -- this will take a while
      ansible.builtin.wait_for_connection:
        sleep: 10 # wait 30 seconds between checks
        delay: 30 # wait 30 seconds by default
        timeout: 600 # 10 minutes max wait

    - name: Get MOTD from rescue system shell
      shell: cat /etc/motd
      register: motd_output

    - name: Ensure MOTD is expected value for Hetzner
      fail:
        msg: MOTD not similar to Hetzner rescue mode MOTD
      when: motd_output.stdout.find('Welcome to the Hetzner Rescue System.') == -1

    ###################
    # Drive discovery #
    ###################

    - name: Automatically determine first disk device
      shell: |
        lsblk | grep disk | awk '{split($0, a, " "); print a[1]}' | sort | head -1 | tail -1        
      register: first_disk_device

    - name: Print first disk
      debug:
        var: first_disk_device.stdout

    - name: Automatically determine second disk device
      shell: |
        lsblk | grep disk | awk '{split($0, a, " "); print a[1]}' | sort | head -2 | tail -1        
      register: second_disk_device

    - name: Print second disk
      debug:
        var: second_disk_device.stdout
      when: second_disk_device.stdout != ""

    - name: Set hetzner_installimage_drives for single disk ({{ first_disk_device.stdout }})
      set_fact:
        hetzner_installimage_drives:
          - DRIVE1 /dev/{{ first_disk_device.stdout }}

    - name: Set hetzner_installimage_drives for multiple disks
      set_fact:
        hetzner_installimage_drives:
          - DRIVE1 /dev/{{ first_disk_device.stdout }}
          - DRIVE2 /dev/{{ second_disk_device.stdout }}
      when: second_disk_device.stdout != ""

    ########################
    # Hetzner installimage #
    ########################

    - name: Copy current authorized keys into file for installimage
      shell: |
        /usr/bin/tail -1 /root/.ssh/authorized_keys > /root/ssh-authorized-keys        

    - name: Create installimage utility configuration file
      template:
        src: installimage.j2
        dest: /autosetup
        owner: root
        group: root
        mode: 0644

    - name: Run Hetzner installimage
      command: |
        /root/.oldroot/nfs/install/installimage -g -K /root/ssh-authorized-keys        
      register: installimage_result

    - name: Remove server entries from known_hosts file ({{ hostname }})
      tags: [ "known_hosts:delete" ]
      delegate_to: localhost
      ansible.builtin.known_hosts:
        path: "{{ known_hosts_path }}"
        name: "{{ hostname }}"
        state: absent
      vars:
        hostname: "{{ inventory_hostname }}"

    - name: Remove server entries from known_hosts file ({{ ip }})
      tags: [ "known_hosts:delete" ]
      delegate_to: localhost
      ansible.builtin.known_hosts:
        path: "{{ known_hosts_path }}"
        name: "{{ ip }}"
        state: absent
      vars:
        ip: "{{ hostvars[inventory_hostname]['ansible_env'].SSH_CONNECTION.split(' ')[2] }}"

    - name: Reboot the machine to get out of rescue mode
      ansible.builtin.reboot:

    - name: Wait for SSH to the new server to be up
      connection: local
      wait_for:
        host: '{{ inventory_hostname }}'
        search_regex: OpenSSH
        delay: 10
        port: 22

    - name: Add server entries from known_hosts file ({{ hostname }})
      tags: [ "known_hosts:add" ]
      delegate_to: localhost
      ansible.builtin.known_hosts:
        path: "{{ known_hosts_path }}"
        name: "{{ hostname }}"
        state: present
        key: "{{ lookup('pipe', 'ssh-keyscan {{ hostname }}') }}"
      vars:
        hostname: "{{  inventory_hostname }}"

    - name: Add server entries from known_hosts file ({{ ip }})
      tags: [ "known_hosts:add" ]
      delegate_to: localhost
      ansible.builtin.known_hosts:
        path: "{{ known_hosts_path }}"
        name: "{{ ip }}"
        state: present
        key: "{{ lookup('pipe', 'ssh-keyscan {{ ip }}') }}"
      vars:
        ip: "{{ hostvars[inventory_hostname]['ansible_env'].SSH_CONNECTION.split(' ')[2] }}"

#
# Play for executing tasks to set up ansible on a server that doesn't
# necessarily have Python/ansible requirements installed
#
---
- name: pre-ansible setup
  hosts: "{{ ansible_limit | default(omit) }}"
  remote_user: root
  gather_facts: no # would fail since python may not necessarily be installed
  vars:
    username: ubuntu
    public_key: ~/.ssh/id_rsa.pub
  tasks:
    - name: "Set hostname ({{ generated_hostname }})"
      ansible.builtin.hostname:
        name: "{{ generated_hostname }}"
      vars:
        generated_hostname: "{{ inventory_hostname.split('.') | first }}"

    #####################
    # Passwordless Sudo #
    #####################

    - name: check for passwordless sudo
      raw: "timeout 1s sudo echo 'check'"
      register: passwordless_sudo_check
      ignore_errors: yes
      no_log: true

    - name: create admin group
      when: passwordless_sudo_check["rc"] != 0
      raw: |
        echo {{ ssh_initial_password }} | sudo -Ss &&
        sudo groupadd admins --system || true        

    - name: add user to admin group
      when: passwordless_sudo_check["rc"] != 0
      raw: |
        echo {{ ssh_initial_password }} | sudo -Ss &&
        sudo usermod -a -G admins {{ ssh_user }}        

    - name: copy sudoers file, make temporary editable
      when: passwordless_sudo_check["rc"] != 0
      raw: |
        echo {{ ssh_initial_password }} | sudo -Ss &&
        sudo cp /etc/sudoers /etc/sudoers.bak &&
        sudo cp /etc/sudoers /etc/sudoers.tmp &&
        sudo chmod 777 /etc/sudoers.tmp        

    - name: add admins no passwd rule for sudoers file
      when: passwordless_sudo_check["rc"] != 0
      raw: |
        echo {{ ssh_initial_password }} | sudo -Ss &&
        sudo echo -e "\n%admins ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers.tmp &&
        sudo chmod 440 /etc/sudoers.tmp        

    - name: check and install new sudoers
      when: passwordless_sudo_check["rc"] != 0
      raw: |
        echo {{ ssh_initial_password }} | sudo -Ss &&
        sudo visudo -q -c -f /etc/sudoers.tmp &&c
        sudo cp -f /etc/sudoers.tmp /etc/sudoers        

    ###################
    # Ansible install #
    ###################

    - name: check for installed ansible (apt)
      register: ansible_check
      ignore_errors: yes
      no_log: true
      shell: |
        dpkg -s ansible        

    # see: https://stackoverflow.com/questions/33563425/ansible-1-9-4-failed-to-lock-apt-for-exclusive-operation
    - name: (apt) Ensure apt list dir exists
      when: ansible_check["rc"] != 0
      file:
        path: /var/lib/apt/lists/
        state: directory
        mode: 0755

    - name: (apt) Install software-properties-common
      when: ansible_check["rc"] != 0
      ansible.builtin.apt:
        update_cache: yes
        name:
          - software-properties-common

    - name: (apt) Enable universe repository
      become: yes
      when: ansible_check["rc"] != 0
      ansible.builtin.command: add-apt-repository universe

    - name: apt-get install software-properties-common
      when: ansible_check["rc"] != 0
      ansible.builtin.apt:
        name:
          - software-properties-common

    - name: add apt repo for ansible
      when: ansible_check["rc"] != 0
      shell: |
        apt-add-repository -y ppa:ansible/ansible        

    - name: apt-get update and install ansible
      when: ansible_check["rc"] != 0
      ansible.builtin.apt:
        update_cache: yes
        name:
          - ansible

#
# Play for executing tasks to set up ansible on a server that doesn't
# necessarily have Python/ansible requirements installed
#
---
- name: post-ansible setup
  hosts: "{{ ansible_limit | default(omit) }}"
  remote_user: root
  gather_facts: yes
  tasks:
    - name: Install generally useful packages
      become: yes
      ansible.builtin.apt:
        name: "{{ packages }}"
        update_cache: yes
        state: present
      vars:
        packages:
        - make
        - libseccomp2
        - apt-transport-https
        - ufw

    - name: Enable UFW, default reject
      tags: [ "ufw" ]
      become: yes
      community.general.ufw:
        state: enabled
        policy: reject

    - name: (ufw) Allow SSH access
      tags: [ "ufw" ]
      become: yes
      community.general.ufw:
        rule: allow
        name: OpenSSH

    - name: (ufw) Limit SSH
      tags: [ "ufw" ]
      become: yes
      community.general.ufw:
        rule: limit
        port: ssh
        proto: tcp

    ###################
    # Default Tooling #
    ###################

    - name: Install performance measurement tooling
      ansible.builtin.apt:
        name: "{{ packages }}"
        update_cache: yes
        state: present
      vars:
        packages:
        - iotop
        - htop
        - sysstat # contains iostat and others
        - unattended-upgrades

    - name: Install some creature comforts
      ansible.builtin.apt:
        name: "{{ packages }}"
        update_cache: yes
        state: present
      vars:
        packages:
        - tree

    #################################
    # Unatttended security upgrades #
    #################################

    - name: Install performance measurement tooling
      ansible.builtin.apt:
        name: "{{ packages }}"
        update_cache: yes
        state: present
      vars:
        packages:
        - unattended-upgrades

    - name: Ensure unattended upgrades service is running
      ansible.builtin.systemd:
        name: unattended-upgrades
        state: started

    - name: Install unattended upgrade config w/ blacklist and email notifications
      template:
        src: 50-unattended-upgrades.conf.j2
        dest: /etc/apt/apt.conf.d/50-unattended-upgrades.conf
        owner: root
        group: root
        mode: 0644

    - name: Install automatic upgrades
      template:
        src: 20-auto-upgrades.conf.j2
        dest: /etc/apt/apt.conf.d/20-auto-upgrades.conf
        owner: root
        group: root
        mode: 0644

    - name: Hold all grub-related packages
      shell: |
        apt-mark hold grub*
        apt-mark hold grub
        apt-mark hold grub-common
        apt-mark hold grub2
        apt-mark hold grub2-common
        apt-mark hold grub-pc
        apt-mark hold grub-pc-bin        

---
- name: Storage plugin setup
  hosts: "{{ ansible_limit | default(omit) }}"
  remote_user: root
  vars:
    supported_storage_plugins:
      - all
      - rook-ceph-lvm
      - linstor-rbd9
      - openebs-mayastor
      - openebs-cstor
      - openebs-jiva
      - openebs-localpv-hostpath
      - openebs-localpv-device
      - openebs-localpv-zfs
  tasks:
    # ... elided ... #

    - name: Do a thing that is useful for only OpenEBS cStor and Jiva
      when: storage_plugin == "all" or storage_plugin in target_plugins
      thing:
        args: that is being done
      vars:
        target_plugins:
          - openebs-cstor
          - openebs-jiva

Unattended-Upgrade::Allowed-Origins {
  "${distro_id}:${distro_codename}";
  "${distro_id}:${distro_codename}-security";
  "${distro_id}ESM:${distro_codename}";
}

Unattended-Upgrade::Package-Blacklist {
  "grub";
}

{% if unattended_upgrade_email is defined %}
Unattended-Upgrade::Mail "{{ unattended_upgrade_email }}";
Unattended-Upgrade::MailOnlyOnError "true";
{% endif %}

APT::Periodic::Update-Package-Lists "1";
APT::Periodic::Unattended-Upgrade "1";
APT::Periodic::AutocleanInterval "7";

{% for drive in hetzner_installimage_drives %}
{{ drive }}
{% endfor %}

{% for raid in hetzner_installimage_raid %}
{{ raid }}
{% endfor %}

BOOTLOADER {{ hetzner_installimage_bootloader }}
HOSTNAME {{ hetzner_installimage_hostname }}

{% for partition in hetzner_installimage_partitions %}
{{ partition }}
{% endfor %}

IMAGE {{ hetzner_installimage_image }}

    - name: Install LVM
      when: storage_plugin == "all" or storage_plugin in target_plugins
      block:
        - name: Install lvm2
          ansible.builtin.apt:
            name: lvm2
            update_cache: yes
            state: present
        - name: Ensure rbd kernel module is installed
          community.general.modprobe:
            name: rbd
            state: present
      vars:
        target_plugins:
          - rook-ceph-lvm
          - linstor-rbd9

    - name: Install ZFS
      when: storage_plugin == "all" or storage_plugin in target_plugins
      ansible.builtin.apt:
        name:
          - zfs-linux
          - zfsutils-linux
        update_cache: yes
        state: present
      vars:
        target_plugins:
          - rook-ceph-zfs
          - openebs-localpv-zfs
          - linstor-bd9

    - name: Enable huge page support
      when: storage_plugin == "all" or storage_plugin in target_plugins
      block:
        - name: set nr_hugepages
          ansible.builtin.shell: |
            echo 512 | tee /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages            
        - name: set nr_hugepages via sysctl
          ansible.builtin.shell: |
            echo vm.nr_hugepages = 512 | tee -a /etc/sysctl.conf            
      vars:
        target_plugins:
          - openebs-mayastor

    - name: Ensure nvme-oF over TCP support
      when: storage_plugin == "all" or storage_plugin in target_plugins
      community.general.modprobe:
        name: nvme-tcp
        state: present
      vars:
        target_plugins:
          - openebs-mayastor

    - name: Install iscsi
      when: storage_plugin == "all" or storage_plugin in target_plugins
      block:
        - name: Install apt package
          ansible.builtin.apt:
            name: open-iscsi
            update_cache: yes
            state: present
        - name: Enable iscsid
          ansible.builtin.systemd:
            name: iscsid
            state: started
            enabled: yes
      vars:
        target_plugins:
          - openebs-mayastor
          - openebs-cstor
          - openebs-jiva

    - name: Add drbd9 apt repositories
      when: storage_plugin == "all" or storage_plugin in target_plugins
      ansible.builtin.apt_repository:
        repo: ppa:linbit/linbit-drbd9-stack
        state: present
      vars:
        target_plugins:
          - linstor-rbd9

    - name: Install LINSTOR components
      when: storage_plugin == "all" or storage_plugin in target_plugins
      block:
        - name: Install drbd packages
          ansible.builtin.apt:
            name:
              - drbd-dkms
              - drbd-utils
            update_cache: yes
            state: present
        - name: Install linstor components
          ansible.builtin.apt:
            name:
              - linstor-controller
              - linstor-satellite
              - linstor-client
            update_cache: yes
            state: present
        - name: Ensure rbd kernel module is installed
          community.general.modprobe:
            name: rbd
            state: present
      vars:
        target_plugins:
          - linstor-rbd9

k8s-install: require-target
    @echo "[info] performing k8s install..."
    $(ANSIBLE_PLAYBOOK) k8s-install.yml \
        -i $(INVENTORY_PATH) \
        --limit=$(TARGET) \
        $(ANSIBLE_ARGS)

#
# Playbook for setting up kubernetes
#
---
- name: k8s-setup all-in-one server setup
  hosts: "{{ ansible_limit | default(omit) }}"
  remote_user: root
  vars:
    k0s_version: v0.12.0
    k0s_checksum: sha256:0a3ead8f8e5f950390eeb76bd39611d1754b282536e8d9dbbaa0676550c2edbf
  tasks:
    - name: Populate service facts
      ansible.builtin.service_facts:

    - name: Download k0s
      ansible.builtin.get_url:
        url: |
          https://github.com/k0sproject/k0s/releases/download/{{ k0s_version }}/k0s-{{ k0s_version }}-amd64          
        checksum: "{{ k0s_checksum }}"
        mode: 0755
        dest: /usr/bin/k0s
      when: ansible_facts.services["k0scontroller.service"] is not defined

    - name: Create /var/lib/k0s folder
      ansible.builtin.file:
        path: /var/lib/k0s
        state: directory
      when: ansible_facts.services["k0scontroller.service"] is not defined

    - name: Add k0s config file
      ansible.builtin.template:
        src: k0s-config.yaml.j2
        dest: /var/lib/k0s/config.yaml
        owner: root
        group: root
        mode: 0644
      when: ansible_facts.services["k0scontroller.service"] is not defined

    - name: Install k0s
      ansible.builtin.command: |
        k0s install controller -c /var/lib/k0s/config.yaml --single        
      when: ansible_facts.services["k0scontroller.service"] is not defined

    - name: Start the k0s service
      ansible.builtin.systemd:
        name: k0scontroller
        state: started
        enabled: yes
      when: ansible_facts.services["k0scontroller.service"] is not defined

    - name: Create worker join token (saved @ /tmp/worker-token)
      shell: |
        k0s token create --role=worker --expiry=168h > /tmp/worker-token        
      when: ansible_facts.services["k0scontroller.service"] is not defined

    - name: Copy out worker token
      ansible.builtin.fetch:
        src: /tmp/worker-token
        dest: output

    - name: Copy out cluster configuration
      ansible.builtin.fetch:
        src: /var/lib/k0s/pki/admin.conf
        dest: output

    - name: Replace localhost in cluster configuration
      delegate_to: localhost
      ansible.builtin.replace:
        path: "output/{{ inventory_hostname }}/var/lib/k0s/pki/admin.conf"
        regexp: 'https://localhost:6443'
        replace: "https://{{ cluster_external_address | default(inventory_hostname) }}:6443"

    - name: (ufw) Allow TCP access on port 6443
      tags: [ "ufw" ]
      become: yes
      community.general.ufw:
        rule: allow
        port: '6443'
        proto: tcp

    - name: (ufw) Allow UDP access on port 6443
      tags: [ "ufw" ]
      become: yes
      community.general.ufw:
        rule: allow
        port: '6443'
        proto: udp

---
apiVersion: k0s.k0sproject.io/v1beta1
kind: Cluster
metadata:
  name: k0s
spec:
  api:
    externalAddress: {{ cluster_external_address | default(inventory_hostname) }}
    address: {{ hostvars[inventory_hostname]['ansible_env'].SSH_CONNECTION.split(' ')[2] }}
    sans:
      - {{ hostvars[inventory_hostname]['ansible_env'].SSH_CONNECTION.split(' ')[2] }}
      - {{ cluster_external_address | default(inventory_hostname) }}
      - {{ inventory_hostname }}
  storage:
    type: etcd
    etcd:
      peerAddress: {{ hostvars[inventory_hostname]['ansible_env'].SSH_CONNECTION.split(' ')[2] }}
  network:
    podCIDR: 10.244.0.0/16
    serviceCIDR: 10.96.0.0/12
    provider: calico
    calico:
      mode: vxlan
      vxlanPort: 4789
      vxlanVNI: 4096
      mtu: 1450
      wireguard: false
      flexVolumeDriverPath: /usr/libexec/k0s/kubelet-plugins/volume/exec/nodeagent~uds
      withWindowsNodes: false
      overlay: Always
  podSecurityPolicy:
    defaultPolicy: 00-k0s-privileged
  telemetry:
    interval: 10m0s
    enabled: true
  installConfig:
    users:
      etcdUser: etcd
      kineUser: kube-apiserver
      konnectivityUser: konnectivity-server
      kubeAPIserverUser: kube-apiserver
      kubeSchedulerUser: kube-scheduler
  images:
    default_pull_policy: IfNotPresent
    konnectivity:
      image: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent
      version: v0.0.13
    metricsserver:
      image: gcr.io/k8s-staging-metrics-server/metrics-server
      version: v0.3.7
    kubeproxy:
      image: k8s.gcr.io/kube-proxy
      version: v1.20.5
    coredns:
      image: docker.io/coredns/coredns
      version: 1.7.0
    calico:
      cni:
        image: calico/cni
        version: v3.16.2
      flexvolume:
        image: calico/pod2daemon-flexvol
        version: v3.16.2
      node:
        image: calico/node
        version: v3.16.2
      kubecontrollers:
        image: calico/kube-controllers
        version: v3.16.2