HI ,
I have multiple FS showing in df -h and all of them are showing same usage which I suspect because they all are same ? but I have gone inside multiple FS and cant find soft links that point to others , they are all overlay FS.
I am getting pod eviction due to high FS usage in /var and when I check inside /var most space is occupied by some files but they are on separate FS which is overlay and this overlay FS all of them have same usage as /var.
Can someone explain what is this kind of setup in which all FS are showing same usage.
posting lsblk and df -h o/p
lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sdb 8:16 0 64G 0 disk /var/lib/kubelet/pods/3ef9eb41-2888-4d26-94fa-3fea2f2086d9/volumes/xxxx/xxxxx.volume.xx.phx.abyhqljrd
sdc 8:32 0 64G 0 disk /var/lib/kubelet/pods/3ef9eb41-2888-4d26-94fa-3fea2f2086d9/volumes/xxxx/xxxxvolume.xxx.phx.abyhqljrz
sda 8:0 0 128G 0 disk
├─sda2 8:2 0 512M 0 part /boot
├─sda3 8:3 0 49G 0 part
│ ├─vg00-var_log 252:1 0 4G 0 lvm /var/log
│ ├─vg00-root 252:6 0 10G 0 lvm /
│ ├─vg00-opt 252:4 0 5G 0 lvm /opt
│ ├─vg00-var_tmp 252:2 0 1G 0 lvm
│ ├─vg00-var_log_audit
252:0 0 2G 0 lvm /var/log/audit
│ ├─vg00-swap 252:7 0 1G 0 lvm
│ ├─vg00-home 252:5 0 1G 0 lvm /home
│ └─vg00-var 252:3 0 20.3G 0 lvm /var
└─sda1 8:1 0 128M 0 part /boot/efi
df -h o/p
Filesystem Size Used Avail Use% Mounted on
devtmpfs 63G 4.0K 63G 1% /dev
tmpfs 63G 28K 63G 1% /dev/shm
tmpfs 63G 91M 63G 1% /run
tmpfs 63G 0 63G 0% /sys/fs/cgroup
/dev/mapper/vg00-root 9.6G 3.8G 5.3G 42% /
/dev/mapper/vg00-home 958M 1.4M 889M 1% /home
/dev/mapper/vg00-opt 4.7G 1.5G 3.0G 33% /opt
/dev/sda2 488M 170M 282M 38% /boot
/dev/mapper/vg00-var 20G 14G 4.7G 75% /var
/dev/mapper/vg00-var_log 3.8G 177M 3.4G 5% /var/log
/dev/mapper/vg00-var_log_audit 1.9G 38M 1.8G 3% /var/log/audit
/dev/sda1 128M 7.3M 121M 6% /boot/efi
tmpfs 63G 44K 63G 1% /var/tmp
tmpfs 63G 4.0K 63G 1% /tmp
tmpfs 126G 12K 126G 1% /var/lib/kubelet/pods/1bbce14c-5f37-43df-91d9-a3a48afc10df/volumes/kubernetes.io~projected/kube-api-access-r47tf
tmpfs 300M 12K 300M 1% /var/lib/kubelet/pods/aa6a0da0-8f48-4e0d-a54a-31bf53d80218/volumes/kubernetes.io~projected/kube-api-access-785zz
tmpfs 126G 12K 126G 1% /var/lib/kubelet/pods/e880e700-c3d6-40f1-a4cd-59b4ccc216e8/volumes/kubernetes.io~projected/kube-api-access-pn7rz
tmpfs 256M 12K 256M 1% /var/lib/kubelet/pods/363c40d0-fc03-4fe4-8d71-c2927b3de2db/volumes/kubernetes.io~projected/kube-api-access-9pgg6
shm 64M 0 64M 0% /run/containers/storage/overlay-containers/91cfa6f0ab2d66d90bc8cff7414f950c500ef7ae3d1968f20cdf1d6607192db0/userdata/shm
shm 64M 0 64M 0% /run/containers/storage/overlay-containers/5b3289a9e740d0199294c9c4183246417e8cf649cac9f681dc1cf8fc6de203b1/userdata/shm
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/cb8ab0d377b02850c8c886b9c48b681e4fc586bc2ebfe430d708cb0a234e1cdb/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/2ad1020161399478149966bc681b4d687bd50fcb50783746c90e03ec51f31dfe/merged
shm 64M 0 64M 0% /run/containers/storage/overlay-containers/6881d4384847f611e5c41fed3356514f515ed01454a89a4e25082a4edc4541fb/userdata/shm
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/6baff0e8fbdc7854daa82942c70fdfbdcd6a18aeaddf809d6964503b5f809123/merged
shm 64M 0 64M 0% /run/containers/storage/overlay-containers/a4e15384dea6964e41c22da5de538cc7c0a723b236217643663f2a99a1a463ac/userdata/shm
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/e1f739a9e32ac23bad55348ea2e428ef78708b0441b7a6ef93c267802f093614/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/2d506a24841dcdabe527b863b7c62dd4e0d16ea9fca026cf23c0ab88d3166035/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/93da4ddec2416b6b5b8fc7c10fc974dd97b4a65377ea88f5a4cadc7c948cddcf/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/9dee24d53d07c7a6d467b5d5f96acd6e130e899a29272c87893b7ae1073f8fc4/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/686b32df3fe52912ab801174947084c3e39e0ccae641d8a9381c8c536046bff9/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/61454ee22a3047db373d80763e4fe521237aeaf6c95110d935bea84a5be75799/merged
tmpfs 126G 12K 126G 1% /var/lib/kubelet/pods/516ae111-d2c5-49c1-a345-9e6d7894aaa9/volumes/kubernetes.io~projected/kube-api-access-hwpwb
tmpfs 1.1G 24K 1.1G 1% /var/lib/kubelet/pods/04e363d9-aa35-4c60-a47a-d96e2c2990c2/volumes/kubernetes.io~empty-dir/istio-envoy
tmpfs 1.2G 24K 1.2G 1% /var/lib/kubelet/pods/865c8a66-81f4-4972-aaf0-55bcdbdd784c/volumes/kubernetes.io~empty-dir/istio-envoy
tmpfs 1.1G 8.0K 1.1G 1% /var/lib/kubelet/pods/04e363d9-aa35-4c60-a47a-d96e2c2990c2/volumes/kubernetes.io~downward-api/istio-podinfo
tmpfs 1.2G 8.0K 1.2G 1% /var/lib/kubelet/pods/865c8a66-81f4-4972-aaf0-55bcdbdd784c/volumes/kubernetes.io~downward-api/istio-podinfo
tmpfs 1.2G 12K 1.2G 1% /var/lib/kubelet/pods/865c8a66-81f4-4972-aaf0-55bcdbdd784c/volumes/kubernetes.io~projected/kube-api-access-9jnpm
tmpfs 1.1G 4.0K 1.1G 1% /var/lib/kubelet/pods/04e363d9-aa35-4c60-a47a-d96e2c2990c2/volumes/kubernetes.io~projected/istio-token
tmpfs 1.1G 12K 1.1G 1% /var/lib/kubelet/pods/04e363d9-aa35-4c60-a47a-d96e2c2990c2/volumes/kubernetes.io~projected/kube-api-access-spzbv
tmpfs 1.2G 4.0K 1.2G 1% /var/lib/kubelet/pods/865c8a66-81f4-4972-aaf0-55bcdbdd784c/volumes/kubernetes.io~projected/istio-token
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/9e4afc2e9b75e1fc43b63538e8f736c934d12cbcacc5008a6fa832adf322c08d/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/f98ae69eeb1db9b6dfbace1e346ff85713bb172200c675cd77ea7dcce129e136/merged
shm 64M 0 64M 0% /run/containers/storage/overlay-containers/aa5092705c239931fdbe2dc075cca5c8fb87bb97c9f81fd389d75a73acbf9b49/userdata/shm
shm 64M 0 64M 0% /run/containers/storage/overlay-containers/b555a1237b0b76e775e250cd8fdc82b0fc178cb19c830caa1cbcb336e406863c/userdata/shm
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/dcdac130902a34ccf11957f0703fdbb2fa434ad6a2ee02cfd1f5ac0f3b762c59/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/fdcef75777132dc4684b263995ec52d09e12c6f93977dad15dac1f82d98577f4/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/5bcd5c8450e962109a0a523e9905a2c24d1937f60cb266c88b2571873c6020ca/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/77c3e3f23a45869ad6d4fa2e16e15d6c03552c1b729368f715984c7e617c3ac9/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/5f4addc393fed389388d8acb08f4b0eada80179e571296cb04005986f74f699a/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/814f94b1215514e1f68bbfc0068add940182e9cfbdb687e1e44ff0db060016b7/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/910044c349a07338f1ae3eeea9319400bc29e9e505b04f3892911a630a3f4661/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/cc269a3ff2e47684c4dba6f41c1a7cba0a9cdbfc89478147976a5b3c3d322c2f/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/39c19fde4c44820f2a089e4d237dac471051239debbec827a0aa0e771847fa05/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/bdc5d60b767271411bae751489c577b6ad030863e300138f5a82da7013a06655/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/04c4f937c03fe8507c6f0ef6fadac120705bba3dbae07df4c3a7d3ac40211f38/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/a5ed8a80759b036a589cd79ac4d22cdf6b1b2a95c523bb6b60b9ae95e07d3fa0/merged
tmpfs 13G 0 13G 0% /run/user/995
tmpfs 13G 0 13G 0% /run/user/103067
tmpfs 5.5G 24K 5.5G 1% /var/lib/kubelet/pods/858c6a74-29ad-438b-985b-ee0a89160189/volumes/kubernetes.io~empty-dir/istio-envoy
tmpfs 5.5G 8.0K 5.5G 1% /var/lib/kubelet/pods/858c6a74-29ad-438b-985b-ee0a89160189/volumes/kubernetes.io~downward-api/istio-podinfo
tmpfs 3.0G 24K 3.0G 1% /var/lib/kubelet/pods/ecfe457f-8367-4f08-8d84-3c56938e362c/volumes/kubernetes.io~empty-dir/istio-envoy
tmpfs 3.0G 8.0K 3.0G 1% /var/lib/kubelet/pods/ecfe457f-8367-4f08-8d84-3c56938e362c/volumes/kubernetes.io~downward-api/istio-podinfo
tmpfs 3.0G 24K 3.0G 1% /var/lib/kubelet/pods/d117fd56-9389-46c6-9541-0c0d17c6fe06/volumes/kubernetes.io~empty-dir/istio-envoy
tmpfs 3.0G 8.0K 3.0G 1% /var/lib/kubelet/pods/d117fd56-9389-46c6-9541-0c0d17c6fe06/volumes/kubernetes.io~downward-api/istio-podinfo
tmpfs 3.0G 24K 3.0G 1% /var/lib/kubelet/pods/3ef9eb41-2888-4d26-94fa-3fea2f2086d9/volumes/kubernetes.io~empty-dir/istio-envoy
tmpfs 3.0G 8.0K 3.0G 1% /var/lib/kubelet/pods/3ef9eb41-2888-4d26-94fa-3fea2f2086d9/volumes/kubernetes.io~downward-api/istio-podinfo
tmpfs 5.5G 4.0K 5.5G 1% /var/lib/kubelet/pods/858c6a74-29ad-438b-985b-ee0a89160189/volumes/kubernetes.io~projected/istio-token
tmpfs 5.5G 12K 5.5G 1% /var/lib/kubelet/pods/858c6a74-29ad-438b-985b-ee0a89160189/volumes/kubernetes.io~projected/kube-api-access-5mtg4
shm 64M 0 64M 0% /run/containers/storage/overlay-containers/3571ff2fb0e8b9d98875b93b736834493ee5c2a5007defba5659aabdebedf906/userdata/shm
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/50739667d04950b6bbf4d84f7cad950b24a3421ce510648eed4483ac76f55584/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/aa36fce5c78fd603b015b15370eb992a62d850ffa1599d8a9bd2a066c204fb14/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/509462ce2af0bc2c5a56e255e26053c5c79cd0bc84dabb8d509f6bff17ed0998/merged
tmpfs 3.0G 4.0K 3.0G 1% /var/lib/kubelet/pods/ecfe457f-8367-4f08-8d84-3c56938e362c/volumes/kubernetes.io~projected/istio-token
tmpfs 3.0G 12K 3.0G 1% /var/lib/kubelet/pods/ecfe457f-8367-4f08-8d84-3c56938e362c/volumes/kubernetes.io~projected/kube-api-access-qn2xz
tmpfs 3.0G 4.0K 3.0G 1% /var/lib/kubelet/pods/d117fd56-9389-46c6-9541-0c0d17c6fe06/volumes/kubernetes.io~projected/istio-token
shm 64M 0 64M 0% /run/containers/storage/overlay-containers/e7fda535d4162317ce5016a7b867d8877f5be1a4e6aeb869cba75a03d5b28cff/userdata/shm
tmpfs 3.0G 12K 3.0G 1% /var/lib/kubelet/pods/d117fd56-9389-46c6-9541-0c0d17c6fe06/volumes/kubernetes.io~projected/kube-api-access-h92qh
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/5113120dc8185f40a0f5d62b2bdf8211dfc694ee2874db9761feca0131414b60/merged
shm 64M 0 64M 0% /run/containers/storage/overlay-containers/1acbdcf4b0680096f6a8fde530df38ae82f17405ef2230a106f067310ad89cdc/userdata/shm
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/fad7bbdabe51d8af21a683543a22f214eeab15350911f07ccf537d0f734c3968/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/b3de5af9d2d29075fe42dfd335006d0d386c59ea82ea377c2d96940fd4794d93/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/497ae3fb9758f41e2ee713a5f4b688fb42e88e2cee794828de2a0cdb62a999e0/merged
tmpfs 3.0G 4.0K 3.0G 1% /var/lib/kubelet/pods/3ef9eb41-2888-4d26-94fa-3fea2f2086d9/volumes/kubernetes.io~projected/istio-token
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/3dc70af4ee6655d5db9b738f475f5f92e8c7ffd69998ec7569bbc3f7d8a8c11c/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/0cb1d1b5870893a7d77945396f44d5ebb70e6748fc43473b27e9ad79ae1b6d52/merged
tmpfs 3.0G 24K 3.0G 1% /var/lib/kubelet/pods/e6904d88-c592-49da-b123-50b5450d7a26/volumes/kubernetes.io~empty-dir/istio-envoy
tmpfs 3.0G 24K 3.0G 1% /var/lib/kubelet/pods/00848337-5e99-44c5-9b72-9a384855d6dc/volumes/kubernetes.io~empty-dir/istio-envoy
tmpfs 3.0G 24K 3.0G 1% /var/lib/kubelet/pods/3ee0e65f-5a5b-4dae-ae31-35993e09d2ea/volumes/kubernetes.io~empty-dir/istio-envoy
tmpfs 3.0G 8.0K 3.0G 1% /var/lib/kubelet/pods/00848337-5e99-44c5-9b72-9a384855d6dc/volumes/kubernetes.io~downward-api/istio-podinfo
tmpfs 3.0G 8.0K 3.0G 1% /var/lib/kubelet/pods/e6904d88-c592-49da-b123-50b5450d7a26/volumes/kubernetes.io~downward-api/istio-podinfo
tmpfs 3.0G 8.0K 3.0G 1% /var/lib/kubelet/pods/3ee0e65f-5a5b-4dae-ae31-35993e09d2ea/volumes/kubernetes.io~downward-api/istio-podinfo
tmpfs 3.0G 12K 3.0G 1% /var/lib/kubelet/pods/3ef9eb41-2888-4d26-94fa-3fea2f2086d9/volumes/kubernetes.io~projected/kube-api-access-cqkj6
tmpfs 3.0G 4.0K 3.0G 1% /var/lib/kubelet/pods/3ee0e65f-5a5b-4dae-ae31-35993e09d2ea/volumes/kubernetes.io~projected/istio-token
tmpfs 3.0G 12K 3.0G 1% /var/lib/kubelet/pods/e6904d88-c592-49da-b123-50b5450d7a26/volumes/kubernetes.io~projected/kube-api-access-9vlp2
tmpfs 3.0G 12K 3.0G 1% /var/lib/kubelet/pods/3ee0e65f-5a5b-4dae-ae31-35993e09d2ea/volumes/kubernetes.io~projected/kube-api-access-dtj82
shm 64M 0 64M 0% /run/containers/storage/overlay-containers/2541b95410d0655fe6e25494701506c6b2820d18adb3310e485a29c441298c92/userdata/shm
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/65889e3a4071e00fbc2902a71201bd5bb81b644b9a3e7eb9073caa7251fc8e34/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/46191ca9b47aaf4ee4b979191dea26a9260faf2e9fd1a1501705c8eb6ef4c27d/merged
tmpfs 3.0G 12K 3.0G 1% /var/lib/kubelet/pods/00848337-5e99-44c5-9b72-9a384855d6dc/volumes/kubernetes.io~projected/kube-api-access-7b92m
tmpfs 3.0G 4.0K 3.0G 1% /var/lib/kubelet/pods/00848337-5e99-44c5-9b72-9a384855d6dc/volumes/kubernetes.io~projected/istio-token
tmpfs 3.0G 4.0K 3.0G 1% /var/lib/kubelet/pods/e6904d88-c592-49da-b123-50b5450d7a26/volumes/kubernetes.io~projected/istio-token
shm 64M 0 64M 0% /run/containers/storage/overlay-containers/6b2ceb5932300800d9aa9f6044d5096c9a55e8ebb368d373d156ebc116a25d64/userdata/shm
/dev/sdb 63G 152M 60G 1% /var/lib/kubelet/plugins/kubernetes.io/flexvolume/oracle/oci/mounts/ocid1.volume.oc1.phx.abyhqljrd34pcrewmbev55eqbtuwd63dhpdngekky54dk4rilbvjvjpbqwua
shm 64M 0 64M 0% /run/containers/storage/overlay-containers/db2f44408e980724e817099f683cfab1d23678f331928aea2bf938887493dbbd/userdata/shm
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/fb0b002d88e8c72b6d5eecbbd70da6b73eac557cb231c88d816a79b3d51ae6b2/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/eb72d534a5bd7f610d2064eeb7fada7a6fc6274655ef7b2a04f0661c7fde0a05/merged
/dev/sdc 63G 68M 60G 1% /var/lib/kubelet/plugins/kubernetes.io/flexvolume/oracle/oci/mounts/ocid1.volume.oc1.phx.abyhqljrzojk7o3hcpiels72ymcn6vmh4ufqfwa3ofannkzmu4qpsegu6bva
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/7affba7b970447762e2c55b06c5e4a32282c8d82947769093769f0723212baa5/merged
shm 64M 0 64M 0% /run/containers/storage/overlay-containers/d24f4abdb123023f1441250963bd6718f86922f4cd7a086aa9245dd8a2b95f04/userdata/shm
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/89886a1cee7bc22ff178516788881660f49cd288480abbda8a461474981b8c70/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/2df2e462b03a891a04c7540648c0c9b6143a6c34326db233873ce1f3154c2950/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/f340fc9ecc1d8f292b279653b7f5e0b37ea6f95b25349022bd9d8871fed5ec90/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/d164b4393eeadf8179822f6dd6c82ed52a954c1bdbe394caf91b54d884cbb404/merged
overlay 20G 14G 4.7G 75% /var/lib/containers/storage/overlay/330dcdad7d79898f22ec7b4f318d9b1fcb4e1233ccea91a29412c91ffeab12c4/merged