Symptoms

Several container migration attempts were performed, some of them were unsuccessful due to various reasons/misconfigurations. Finally the container was migrated. There is a number of CT_ID.xxxx folders in "/vz/private":

# cd /vz/private/
# ls -la
insgesamt 0
drwx------ 1 root root 0  9. Jan 2013  .
drwx------ 1 root root 0 17. Jul 06:58 ..
drwx------ 1 root root 0 20. Jul 12:16 101
drwx------ 1 root root 0 19. Jul 21:51 101.8208Ef
drwx------ 1 root root 0 19. Jul 13:37 101.u7euXi
drwx------ 1 root root 0 19. Jul 00:52 101.wnIEGR
drwx------ 1 root root 0 19. Jul 07:23 101.WzxCKi
drwx------ 1 root root 0 20. Jul 07:09 101.XYGNiD
drwx------ 1 root root 0 19. Jul 14:35 101.z0nrUn

When trying to delete these leftovers I get the "Operation not permitted" error message.

Cause

The *.hds files left from previous migration attempts are still mounted:

# grep '/101\.' /sys/block/ploop*/pdelta/*/image
/sys/block/ploop0/pdelta/0/image:/pstorage/cluster/private/101.wnIEGR/root.hdd/root.hds
/sys/block/ploop13021/pdelta/0/image:/pstorage/cluster/private/101.XYGNiD/root.hdd/root.hds
/sys/block/ploop17543/pdelta/0/image:/pstorage/cluster/private/101.z0nrUn/root.hdd/root.hds
/sys/block/ploop27629/pdelta/0/image:/pstorage/cluster/private/101.8208Ef_ren/root.hdd/root.hds
/sys/block/ploop53561/pdelta/0/image:/pstorage/cluster/private/101.u7euXi/root.hdd/root.hds
/sys/block/ploop54602/pdelta/0/image:/pstorage/cluster/private/101.WzxCKi/root.hdd/root.hds

It is not possible to release them as they all referring to the same "/vz/root/101"

 # grep 101 /proc/mounts
/dev/ploop0p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
/dev/ploop54602p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
/dev/ploop53561p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
/dev/ploop17543p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
/dev/ploop27629p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
/dev/ploop13021p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
/dev/ploop16067p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
proc /vz/root/101/proc proc rw,relatime 0 0
sysfs /vz/root/101/sys sysfs rw,relatime 0 0
none /vz/root/101/dev tmpfs rw,relatime,size=5242880k,nr_inodes=1310720,mode=755 0 0
none /vz/root/101/dev/pts devpts rw,relatime,mode=600,ptmxmode=000 0 0
none /vz/root/101/proc/sys/fs/binfmt_misc binfmt_misc rw,relatime 0 0
proc /vz/root/101/var/named/run-root/proc proc rw,relatime 0 0
/dev/ploop16067p1 /vz/root/101/var/named/run-root/var/run/dbus ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0

Resolution

1. Stop the original container (101).

2. Get the list of ploops related to this cotnainer, e.g.:

   # grep 101 /proc/mounts
   /dev/ploop0p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
   /dev/ploop54602p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
   /dev/ploop53561p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
   /dev/ploop17543p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
   /dev/ploop27629p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
   /dev/ploop13021p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
   

3. Unmount the directory under "/vz/root/CT_ID":

   # umount /vz/root/101/
   

   In the example we have 6 ploop images mounted over this directory, so the command has to be repeated 6 times.

   After that make sure nothing can be found in /proc/mounts regarding the container:

   # grep 101 /proc/mounts
   #
   

4. Unmount all ploop devices by names (listed in step 2 ). Example:

   # ploop umount -d /dev/ploop0
   Unmounting device /dev/ploop0
   # ploop umount -d /dev/ploop54602
   Unmounting device /dev/ploop54602
   # ploop umount -d /dev/ploop53561
   Unmounting device /dev/ploop53561
   # ploop umount -d /dev/ploop17543
   Unmounting device /dev/ploop17543
   # ploop umount -d /dev/ploop27629
   Unmounting device /dev/ploop27629
   # ploop umount -d /dev/ploop13021
   Unmounting device /dev/ploop13021
   

After that original container can be started and excessive directories can be removed.