#!/bin/bash
# shellcheck disable=SC1091 # Runtime-resolved path; intentionally not followed.
. "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/../local-snap.sh" || return 1
set -eux

# Install dependencies
install_deps genisoimage

# Install LXD
install_lxd

IMAGE="${TEST_IMG:-ubuntu-minimal-daily:24.04}"

poolDriverList="${1:-dir btrfs lvm lvm-thin zfs ceph}"

# Configure LXD
lxc project switch default
lxc network create lxdbr0
lxc project create test -c features.images=false
lxc project switch test
lxc profile device add default eth0 nic network=lxdbr0

poolName="vmpool$$"

# subtest for instance snapshots with multiple attached volumes
instance_multi_volume_snapshot_test() {
  echo "==> Instance multi-volume snapshot and restore"

  echo "==> Create custom volume and attach it to VM"
  if [ "${poolDriver}" != "powerflex" ] && [ "${poolDriver}" != "alletra" ]; then
    lxc storage volume create "${poolName}" vol7 size=4MiB
    lxc storage volume create "${poolName}" vol8 --type=block size=4MiB
  else
    lxc storage volume create "${poolName}" vol7 size=8GiB
    lxc storage volume create "${poolName}" vol8 --type=block size=8GiB
  fi
  lxc storage volume attach "${poolName}" vol7 v1 /mnt/vol7
  lxc storage volume attach "${poolName}" vol8 v1 customblock

  echo "==> Start VM"
  lxc start v1
  waitInstanceReady v1

  echo "==> Create test files"
  lxc exec v1 -- touch /mnt/vol7/snap1 snap1
  echo foo | lxc file push - v1/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_customblock
  lxc exec v1 -- sync

  echo "==> Check multi-volume snapshot"
  lxc snapshot v1 v1-snap1 --disk-volumes=all-exclusive

  echo "==> Check attached volume snapshots inherit expiry from instance snapshot"
  [ "$(lxc storage volume get "${poolName}" vol7/snap0 expires_at --property)" = "$(lxc config get v1/v1-snap1 expires_at --property)" ]
  [ "$(lxc storage volume get "${poolName}" vol8/snap0 expires_at --property)" = "$(lxc config get v1/v1-snap1 expires_at --property)" ]

  lxc exec v1 -- rm /mnt/vol7/snap1 snap1
  echo bar | lxc file push - v1/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_customblock
  lxc exec v1 -- sync

  echo "==> Check root volume restore"
  lxc restore v1 v1-snap1 --disk-volumes=root
  waitInstanceReady v1
  lxc exec v1 -- test -f snap1
  ! lxc exec v1 -- test -f /mnt/vol7/snap1 || false
  lxc exec v1 -- grep -Fxm1 bar /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_customblock

  lxc exec v1 -- rm snap1
  lxc exec v1 -- sync

  echo "==> Check multi-volume restore"
  lxc restore v1 v1-snap1 --disk-volumes=all-exclusive
  waitInstanceReady v1
  lxc exec v1 -- test -f snap1
  lxc exec v1 -- test -f /mnt/vol7/snap1
  lxc exec v1 -- grep -Fxm1 foo /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_customblock

  echo "==> Check deleting multi-volume snapshot also deletes attached volume snapshots when requested"
  [ "$(lxc storage volume list -f csv | grep -cF "Created alongside virtual-machine v1/v1-snap1 snapshot in project test")" = 2 ]
  lxc delete v1/v1-snap1 --disk-volumes=all-exclusive
  [ "$(lxc storage volume list -f csv | grep -cF "Created alongside virtual-machine v1/v1-snap1 snapshot in project test")" = 0 ]

  lxc stop -f v1
}

for poolDriver in $poolDriverList
do
	echo "==> Create storage pool using driver ${poolDriver}"
	if [ "${poolDriver}" = "dir" ]; then
		lxc storage create "${poolName}" "${poolDriver}" volume.size=3584MiB
	elif [ "${poolDriver}" = "ceph" ]; then
		# In case the storage_remote_drop_source extension isn't present, set the pool name
		# using the deprecated 'source' config key.
		sourceKey="ceph.osd.pool_name"
		if ! hasNeededAPIExtension storage_remote_drop_source; then
			sourceKey="source"
		fi

		lxc storage create "${poolName}" "${poolDriver}" "${sourceKey}=${poolName}" volume.size=3584MiB
	elif [ "${poolDriver}" = "lvm" ]; then
		lxc storage create "${poolName}" "${poolDriver}" size=40GiB lvm.use_thinpool=false volume.size=3584MiB
	elif [ "${poolDriver}" = "lvm-thin" ]; then
		lxc storage create "${poolName}" lvm size=20GiB volume.size=3584MiB
	elif [ "${poolDriver}" = "powerflex" ]; then
		createPowerFlexPool "${poolName}"
	elif [ "${poolDriver}" = "powerstore" ]; then
		createPowerStorePool "${poolName}"
	elif [ "${poolDriver}" = "pure" ]; then
		createPureStoragePool "${poolName}"
	elif [ "${poolDriver}" = "alletra" ]; then
		createAlletraStoragePool "${poolName}"
	else
		lxc storage create "${poolName}" "${poolDriver}" size=20GiB volume.size=3584MiB
	fi

	echo "==> Create VM"
	lxc init "${IMAGE}" v1 --vm -s "${poolName}"
	lxc init "${IMAGE}" v2 --vm -s "${poolName}"

	echo "==> Create custom block volume and attach it to VM"
	if [ "${poolDriver}" != "powerflex" ] && [ "${poolDriver}" != "alletra" ]; then
		lxc storage volume create "${poolName}" vol1 --type=block size=4MiB
	else
		lxc storage volume create "${poolName}" vol1 --type=block size=8GiB
	fi
	lxc storage volume attach "${poolName}" vol1 v1

	echo "==> Create custom volume and attach it to VM"
	if [ "${poolDriver}" != "powerflex" ] && [ "${poolDriver}" != "alletra" ]; then
		lxc storage volume create "${poolName}" vol4 size=4MiB
	else
		lxc storage volume create "${poolName}" vol4 size=8GiB
	fi
	lxc storage volume attach "${poolName}" vol4 v1 /foo

	echo "==> Start VM and add content to custom block volume"
	lxc start v1
	waitInstanceReady v1
	setup_instance_gocoverage v1
	lxc exec v1 -- /bin/sh -c "mkfs.ext4 /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_vol1 && mount /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_vol1 /mnt && echo foo > /mnt/bar && umount /mnt"

	echo "==> Stop VM and detach custom volumes"
	prepare_vm_for_hard_stop v1
	lxc stop -f v1
	lxc storage volume detach "${poolName}" vol1 v1
	lxc storage volume detach "${poolName}" vol4 v1

	echo "==> Test custom block volume hotplugging"
	lxc start v1
	waitInstanceReady v1
	lxc storage volume attach "${poolName}" vol1 v1
	sleep 3
	BLOCK_SPECIAL_FILE="$(lxc exec v1 -- readlink -f /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_vol1)"
	lxc exec v1 -- test -b "$BLOCK_SPECIAL_FILE"
	lxc storage volume detach "${poolName}" vol1 v1
	sleep 3
	! lxc exec v1 -- stat "$BLOCK_SPECIAL_FILE" || false
	! lxc exec v1 -- stat /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_vol1 || false
	prepare_vm_for_hard_stop v1
	lxc stop -f v1

	echo "==> Backup custom block volume"
	lxc storage volume export "${poolName}" vol1 vol1.tar.gz
	lxc storage volume export "${poolName}" vol1 vol1-optimized.tar.gz --optimized-storage

	echo "==> Import custom block volume"
	lxc storage volume import "${poolName}" vol1.tar.gz vol2
	lxc storage volume import "${poolName}" vol1-optimized.tar.gz vol3
	rm vol1.tar.gz
	rm vol1-optimized.tar.gz

    if hasNeededAPIExtension custom_volume_iso; then
	  echo "==> Import custom ISO volume"
	  tmp_iso_dir="$(mktemp -d)"
	  echo foo > "${tmp_iso_dir}/foo"
	  genisoimage -o vol5.iso "${tmp_iso_dir}"
	  rm -f "${tmp_iso_dir}/foo"
	  echo bar > "${tmp_iso_dir}/bar"
	  genisoimage -o vol6.iso "${tmp_iso_dir}"
	  rm -rf "${tmp_iso_dir}"
	  lxc storage volume import "${poolName}" vol5.iso vol5
	  lxc storage volume import "${poolName}" vol6.iso vol6
	  rm -f vol5.iso vol6.iso
    else
	  echo "==> Skipping custom ISO volume tests, not supported"
    fi

	echo "==> Attach custom block volumes to VM"
	# Both volumes can be attached at the same time. The VM will have /dev/sdb and /dev/sdc.
	# It doesn't matter which volume is which device as they contain the same content.
	lxc storage volume attach "${poolName}" vol2 v1
	lxc storage volume attach "${poolName}" vol3 v1

	echo "==> Share a custom block volume"
	if hasNeededAPIExtension shared_custom_block_volumes; then
		! lxc storage volume attach "${poolName}" vol2 v2 || false
		lxc storage volume set "${poolName}" vol2 security.shared true
		lxc storage volume attach "${poolName}" vol2 v2
		! lxc storage volume set "${poolName}" vol2 security.shared false || false
		lxc storage volume detach "${poolName}" vol2 v2
		lxc storage volume set "${poolName}" vol2 security.shared false
	else
		# Sharing is allowed by default without shared_custom_block_volumes extension
		lxc storage volume attach "${poolName}" vol2 v2
		lxc storage volume detach "${poolName}" vol2 v2
	fi

    if hasNeededAPIExtension custom_volume_iso; then
	  echo "==> Attach custom ISO volumes to VM"
	  lxc storage volume attach "${poolName}" vol5 v1
	  lxc storage volume attach "${poolName}" vol6 v1
	  lxc storage volume attach "${poolName}" vol5 v2
	  lxc storage volume attach "${poolName}" vol6 v2

	  echo "===> Check if /dev/disk/by-id representations are accurate"
	  lxc start v1 v2
	  waitInstanceReady v1
	  waitInstanceReady v2
	  setup_instance_gocoverage v2
	  lxc exec v1 -- test -b /dev/disk/by-id/scsi-0QEMU_QEMU_CD-ROM_lxd_vol5
	  lxc exec v1 -- test -b /dev/disk/by-id/scsi-0QEMU_QEMU_CD-ROM_lxd_vol6
	  lxc exec v2 -- test -b /dev/disk/by-id/scsi-0QEMU_QEMU_CD-ROM_lxd_vol5
	  lxc exec v2 -- test -b /dev/disk/by-id/scsi-0QEMU_QEMU_CD-ROM_lxd_vol6
	  prepare_vm_for_hard_stop v1
	  prepare_vm_for_hard_stop v2
	  lxc stop -f v1 v2
    else
	  echo "==> Skipping custom ISO volume tests, not supported"
    fi

	echo "==> Start VM and check content"
	lxc storage volume attach "${poolName}" vol1 v1
	lxc start v1
	waitInstanceReady v1

	echo "==> Verify that block volumes vol1 and vol2 (export/import) are bit for bit identical"
	lxc exec v1 -- cmp /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_vol1 /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_vol2
	echo "==> Verify that block volumes vol1 and vol3 (optimized export/import) are bit for bit identical"
	lxc exec v1 -- cmp /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_vol1 /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_vol3

	# shellcheck disable=2016
	lxc exec v1 -- /bin/sh -c 'mount /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_vol2 /mnt && [ $(cat /mnt/bar) = foo ] && umount /mnt'
	# shellcheck disable=2016
	lxc exec v1 -- /bin/sh -c 'mount /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_vol3 /mnt && [ $(cat /mnt/bar) = foo ] && umount /mnt'

    if hasNeededAPIExtension custom_volume_iso; then
	  lxc start v2
	  waitInstanceReady v2
	  # mount ISOs and check content
	  # shellcheck disable=2016
	  lxc exec v1 -- /bin/sh -c 'mount /dev/disk/by-id/scsi-0QEMU_QEMU_CD-ROM_lxd_vol5 /mnt && [ $(cat /mnt/foo) = foo ] && ! touch /mnt/foo && umount /mnt'
	  # shellcheck disable=2016
	  lxc exec v1 -- /bin/sh -c 'mount /dev/disk/by-id/scsi-0QEMU_QEMU_CD-ROM_lxd_vol6 /mnt && [ $(cat /mnt/bar) = bar ] && ! touch /mnt/bar && umount /mnt'

	  # concurrent readonly ISO mounts
	  # shellcheck disable=2016
	  lxc exec v1 -- /bin/sh -c 'mount /dev/disk/by-id/scsi-0QEMU_QEMU_CD-ROM_lxd_vol5 /mnt && [ $(cat /mnt/foo) = foo ] && ! touch /mnt/foo'
	  # shellcheck disable=2016
	  lxc exec v2 -- /bin/sh -c 'mount /dev/disk/by-id/scsi-0QEMU_QEMU_CD-ROM_lxd_vol5 /mnt && [ $(cat /mnt/foo) = foo ] && ! touch /mnt/foo'
	  # umount'ing last to ensure both are mounted at the same time
	  lxc exec v1 -- umount /mnt
	  lxc exec v2 -- umount /mnt
    else
	  echo "==> Skipping custom ISO volume tests, not supported"
    fi

	echo "==> Readonly custom block volumes are readonly"
	if [ "${poolDriver}" != "powerflex" ] && [ "${poolDriver}" != "alletra" ]; then
		lxc storage volume create "${poolName}" volblro --type block size=4MiB
	else
		lxc storage volume create "${poolName}" volblro --type block size=8GiB
	fi

	lxc config device add v1 volblro disk pool="${poolName}" source=volblro readonly=true

	# shellcheck disable=SC2016
	lxc exec v1 -- sh -c "for i in \$(seq 10); do test -e /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_volblro && break; echo \"Waiting for disk-by-id file to appear (\${i}s)\"; sleep 1; done"
	DISK_DEV="$(basename "$(lxc exec v1 -- readlink -f /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_volblro)")"
	lxc exec v1 -- sh -c "for i in \$(seq 10); do test -e /sys/block/${DISK_DEV}/ro && break; echo \"Waiting for sys file to appear (\${i}s)\"; sleep 1; done"
	[ "$(lxc exec v1 -- cat /sys/block/"${DISK_DEV}"/ro)" -eq 1 ]

	echo "==> Detaching volumes"
	lxc exec v1 -- sync
	prepare_vm_for_hard_stop v1
	lxc stop -f v1
	prepare_vm_for_hard_stop v2
	lxc delete -f v2
	lxc storage volume detach "${poolName}" vol1 v1
	lxc storage volume detach "${poolName}" vol2 v1
	lxc storage volume detach "${poolName}" vol3 v1
	lxc storage volume detach "${poolName}" vol6 v1 || true  # optional ISO
	lxc storage volume detach "${poolName}" volblro v1

	# attach VM root volumes
	if hasNeededAPIExtension vm_root_volume_attachment; then
		echo "==> Instance root & snapshot volumes"

		empty_vm_size=8KiB
		if [ "${poolDriver}" = "powerflex" ] || [ "${poolDriver}" = "alletra" ]; then
			empty_vm_size=8GiB
		fi

		lxc init "${IMAGE}" --vm v2 --storage "${poolName}"
		lxc init --empty --vm v3 --storage "${poolName}" --device root,size="${empty_vm_size}"

		# Requires either security.shared or security.protection.start
		! lxc storage volume attach "${poolName}" virtual-machine/v2 v1 || false

		lxc config set v2 security.protection.start=true

		# security.protection.start on a VM allows exactly one other attachment
		lxc storage volume attach "${poolName}" virtual-machine/v2 v1
		! lxc storage volume attach "${poolName}" virtual-machine/v2 v3 || false

		# Deleting the instance will fail while it's root volume is in use
		! lxc delete v2 || false

		# Make sure used_by is calculated correctly
		lxc storage volume show "${poolName}" virtual-machine/v2 | grep -F '/1.0/instances/v1'

		# Can't unset security.protection.start when v2's root volume is attached to vm1
		! lxc config unset v2 security.protection.start || false

		lxc storage volume detach "${poolName}" virtual-machine/v2 v1

		# Unset security.protection.start works when not attached
		lxc config unset v2 security.protection.start

		lxc config set v2 security.protection.start=true
		lxc storage volume attach "${poolName}" virtual-machine/v2 v1

		lxc storage volume set "${poolName}" virtual-machine/v2 security.shared=true

		# Unset security.shared works when security.protection.start=true
		lxc storage volume unset "${poolName}" virtual-machine/v2 security.shared
		lxc storage volume set "${poolName}" virtual-machine/v2 security.shared=true

		# security.shared allows many attachments
		lxc storage volume attach "${poolName}" virtual-machine/v2 v3

		lxc storage volume show "${poolName}" virtual-machine/v2 | grep -F '/1.0/instances/v1'
		lxc storage volume show "${poolName}" virtual-machine/v2 | grep -F '/1.0/instances/v3'

		lxc config unset v2 security.protection.start

		lxc snapshot v2

		# Snapshots can be attached to profiles without security.shared because they
		# will be mounted read-only
		lxc profile create v2-snap0
		lxc storage volume attach-profile "${poolName}" virtual-machine/v2/snap0 v2-snap0 v2-rs

		# Deleting an attached snapshot should fail
		! lxc delete v2/snap0 || false

		# Detach so that we can double-check hotplug
		lxc storage volume detach "${poolName}" virtual-machine/v2 v1

		# Make sure that the devices actually show up and can be mounted
		lxc start v1
		waitInstanceReady v1

		# Specify a different device name here; udev appears to be truncating the
		# default name when it creates the /dev/disk/by-id/scsi* symlinks, so use
		# a shorter name to prevent truncation.
		lxc storage volume attach "${poolName}" virtual-machine/v2 v1 v2-root
		lxc storage volume attach "${poolName}" virtual-machine/v2/snap0 v1 v2-rs

		sleep 3

		snap0_path=/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_v2--rs

		lxc exec v1 -- test -L /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_v2--root
		lxc exec v1 -- test -L "${snap0_path}"

		# Make sure snapshots are mounted read-only
		snap0_devname="$(basename "$(lxc exec v1 -- readlink "${snap0_path}")")"
		[ "$(lxc exec v1 -- cat "/sys/block/${snap0_devname}/ro")" = "1" ]

		# Start v2 to ensure that reference counts are decremented correctly on
		# device detach.
		lxc start v2
		waitInstanceReady v2
		setup_instance_gocoverage v2

		# `lxc stop [-f]` won't fail if unmounting the device fails
		# `detach` will fail if the unmount fails, so just detach/reattach
		lxc storage volume detach "${poolName}" virtual-machine/v2 v1
		lxc storage volume attach "${poolName}" virtual-machine/v2 v1 v2-root

		# This detach should come after detaching the parent volume to verify that
		# LXD's lvm reference counts allow v2-root to be detached (above)
		lxc storage volume detach "${poolName}" virtual-machine/v2/snap0 v1

		prepare_vm_for_hard_stop v2
		lxc stop v2 --force

		# Snapshots attached via profile
		lxc profile add v1 v2-snap0
		sleep 2

		lxc exec v1 -- test -L "${snap0_path}"
		lxc profile remove v1 v2-snap0

		prepare_vm_for_hard_stop v1
		lxc stop --force v1

		# Can't unset security.shared when v1's root volume is attached elsewhere
		! lxc storage volume unset "${poolName}" virtual-machine/v2 security.shared || false

		# Instances cannot be moved when their root disk is attached to another instance
		! lxc move v2 v4 || false

		lxc storage volume detach "${poolName}" virtual-machine/v2 v1
		lxc storage volume detach "${poolName}" virtual-machine/v2 v3

		lxc storage volume unset "${poolName}" virtual-machine/v2 security.shared

		lxc profile delete v2-snap0
		lxc delete v2 v3
	else
		echo "==> Skipping instance root attachment tests, not supported"
	fi

	if hasNeededAPIExtension instance_snapshots_multi_volume; then
		instance_multi_volume_snapshot_test
	else
		echo "==> Skipping instance multi-volume snapshot and restore tests, not supported"
	fi

	echo "==> Deleting VM"
	lxc delete v1

	echo "==> Deleting storage pool and volumes"
	lxc storage volume rm "${poolName}" vol1
	lxc storage volume rm "${poolName}" vol2
	lxc storage volume rm "${poolName}" vol3
	lxc storage volume rm "${poolName}" vol4
	lxc storage volume rm "${poolName}" vol5 || true  # optional ISO
	lxc storage volume rm "${poolName}" vol6 || true  # optional ISO
	lxc storage volume rm "${poolName}" volblro
	lxc storage volume rm "${poolName}" vol7 || true  # optional multi-volume
	lxc storage volume rm "${poolName}" vol8 || true  # optional multi-volume
	lxc storage rm "${poolName}"
done

# Cleanup
echo "==> Cleaning up"
lxc profile device remove default eth0
lxc project switch default
lxc project delete test
lxc network delete lxdbr0

# shellcheck disable=SC2034
FAIL=0
