[ceph-users] List directory in cephfs blocking very long time

David C dcsysengineer at gmail.com
Tue Dec 5 04:15:43 PST 2017


Not seen this myself but you should update to at least CentOS 7.3, ideally
7.4. I believe a lot of cephfs fixes went into those kernels. If you still
have the issue with the CentOS kernels, test with the latest upstream
kernel. And/or test with latest Fuse client.

On Tue, Dec 5, 2017 at 12:01 PM, 张建 <jian.zhang at sky-data.cn> wrote:

> Hello,
>
> My problem description:
> On cephfs client 1:
> Copy a directory "base" which contains hundreds of rpm files to the
> mounted cephfs directory.
> then , change to cephfs directory and execute "ls -l base", it returns
> quickly. This is no problem.
>
> On the other cephfs client,
> Change to the mounted cephfs directory and execute "ls -l base", it blocks
> for long time.
> Execute "strace ls -l base", we can find it blocks in lstat files (about 5
> seconds blocking for each file).
>
> Is there anybody met this problem?
>
> ==========================================================
> cephfs client information:
> OS: CentOS Linux release 7.2.1511 (Core)
> kernel: Linux 3.10.0-327.el7.x86_64
> mount type: with ceph kernel drvier
>
> ==========================================================
> ceph cluster information:
> OS: CentOS Linux release 7.2.1511 (Core)
> kernel: Linux 3.10.0-327.el7.x86_64
> ceph version 12.2.2 (cf0baeeeeba3b47f9427c6c97e2144b094b7e5ba) luminous
> (stable)
> ceph cluster node1:  1 mon, 1 mgr, 1 mds, 4 osd
> ceph cluster node2:  1 mon, 1 mgr, 1 mds, 4 osd
> ceph cluster node3:  1 mon, 1 mgr, 1 mds, 4 osd
> Each osd is deployed with bluestore type (4TB HDD block device + 240GB SSD
> block.db device).
> public network:  port 1 of 56Gb/s Dual Port Infiniband, IPoIB
> cluster network: port 2 of 56Gb/s Dual Port Infiniband, IPoIB
>
> ceph.conf:
> -------------------------
> [global]
> fsid = 86887b08-6ff4-4ec1-a622-64bc688d1f2f
> mon_initial_members = storage-0, storage-1, storage-2
> mon_host = 10.0.30.11,10.0.30.12,10.0.30.13
> auth_cluster_required = cephx
> auth_service_required = cephx
> auth_client_required = cephx
>
> public_network = 10.0.30.0/24
> cluster_network = 10.0.40.0/24
>
> osd_pool_default_size = 2
> osd_pool_default_min_size = 1
>
> osd_objectstore = bluestore
> bluestore_block_db_size = 236223201280
>
> ==========================================================
> strace log:
> -------------------------
> [root at computing-1 cephfs]# strace ls -l base
> execve("/usr/bin/ls", ["ls", "-l", "base"], [/* 24 vars */]) = 0
> brk(0)                                  = 0x155f000
> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
> 0x7f8c65b91000
> access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or
> directory)
> open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
> fstat(3, {st_mode=S_IFREG|0644, st_size=53575, ...}) = 0
> mmap(NULL, 53575, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f8c65b83000
> close(3)                                = 0
> open("/lib64/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
> read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300j\0\0\0\0\0\0"...,
> 832) = 832
> fstat(3, {st_mode=S_IFREG|0755, st_size=155744, ...}) = 0
> mmap(NULL, 2255216, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0)
> = 0x7f8c6574a000
> mprotect(0x7f8c6576e000, 2093056, PROT_NONE) = 0
> mmap(0x7f8c6596d000, 8192, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x23000) = 0x7f8c6596d000
> mmap(0x7f8c6596f000, 6512, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f8c6596f000
> close(3)                                = 0
> open("/lib64/libcap.so.2", O_RDONLY|O_CLOEXEC) = 3
> read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0
> \26\0\0\0\0\0\0"..., 832) = 832
> fstat(3, {st_mode=S_IFREG|0755, st_size=20024, ...}) = 0
> mmap(NULL, 2114112, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0)
> = 0x7f8c65545000
> mprotect(0x7f8c65549000, 2093056, PROT_NONE) = 0
> mmap(0x7f8c65748000, 8192, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f8c65748000
> close(3)                                = 0
> open("/lib64/libacl.so.1", O_RDONLY|O_CLOEXEC) = 3
> read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200\37\0\0\0\0\0\0"...,
> 832) = 832
> fstat(3, {st_mode=S_IFREG|0755, st_size=37056, ...}) = 0
> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
> 0x7f8c65b82000
> mmap(NULL, 2130560, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0)
> = 0x7f8c6533c000
> mprotect(0x7f8c65343000, 2097152, PROT_NONE) = 0
> mmap(0x7f8c65543000, 8192, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7f8c65543000
> close(3)                                = 0
> open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
> read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0
> \34\2\0\0\0\0\0"..., 832) = 832
> fstat(3, {st_mode=S_IFREG|0755, st_size=2107816, ...}) = 0
> mmap(NULL, 3932736, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0)
> = 0x7f8c64f7b000
> mprotect(0x7f8c65131000, 2097152, PROT_NONE) = 0
> mmap(0x7f8c65331000, 24576, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b6000) = 0x7f8c65331000
> mmap(0x7f8c65337000, 16960, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f8c65337000
> close(3)                                = 0
> open("/lib64/libpcre.so.1", O_RDONLY|O_CLOEXEC) = 3
> read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360\25\0\0\0\0\0\0"...,
> 832) = 832
> fstat(3, {st_mode=S_IFREG|0755, st_size=398272, ...}) = 0
> mmap(NULL, 2490888, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0)
> = 0x7f8c64d1a000
> mprotect(0x7f8c64d7a000, 2093056, PROT_NONE) = 0
> mmap(0x7f8c64f79000, 8192, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5f000) = 0x7f8c64f79000
> close(3)                                = 0
> open("/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
> read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\16\0\0\0\0\0\0"...,
> 832) = 832
> fstat(3, {st_mode=S_IFREG|0755, st_size=19512, ...}) = 0
> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
> 0x7f8c65b81000
> mmap(NULL, 2109744, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0)
> = 0x7f8c64b16000
> mprotect(0x7f8c64b19000, 2093056, PROT_NONE) = 0
> mmap(0x7f8c64d18000, 8192, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f8c64d18000
> close(3)                                = 0
> open("/lib64/libattr.so.1", O_RDONLY|O_CLOEXEC) = 3
> read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\23\0\0\0\0\0\0"...,
> 832) = 832
> fstat(3, {st_mode=S_IFREG|0755, st_size=19888, ...}) = 0
> mmap(NULL, 2113904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0)
> = 0x7f8c64911000
> mprotect(0x7f8c64915000, 2093056, PROT_NONE) = 0
> mmap(0x7f8c64b14000, 8192, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f8c64b14000
> close(3)                                = 0
> open("/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
> read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240l\0\0\0\0\0\0"...,
> 832) = 832
> fstat(3, {st_mode=S_IFREG|0755, st_size=142296, ...}) = 0
> mmap(NULL, 2208864, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0)
> = 0x7f8c646f5000
> mprotect(0x7f8c6470b000, 2097152, PROT_NONE) = 0
> mmap(0x7f8c6490b000, 8192, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7f8c6490b000
> mmap(0x7f8c6490d000, 13408, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f8c6490d000
> close(3)                                = 0
> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
> 0x7f8c65b80000
> mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
> 0x7f8c65b7e000
> arch_prctl(ARCH_SET_FS, 0x7f8c65b7e800) = 0
> mprotect(0x7f8c65331000, 16384, PROT_READ) = 0
> mprotect(0x7f8c6490b000, 4096, PROT_READ) = 0
> mprotect(0x7f8c64b14000, 4096, PROT_READ) = 0
> mprotect(0x7f8c64d18000, 4096, PROT_READ) = 0
> mprotect(0x7f8c64f79000, 4096, PROT_READ) = 0
> mprotect(0x7f8c65543000, 4096, PROT_READ) = 0
> mprotect(0x7f8c65748000, 4096, PROT_READ) = 0
> mprotect(0x7f8c6596d000, 4096, PROT_READ) = 0
> mprotect(0x61a000, 4096, PROT_READ)     = 0
> mprotect(0x7f8c65b92000, 4096, PROT_READ) = 0
> munmap(0x7f8c65b83000, 53575)           = 0
> set_tid_address(0x7f8c65b7ead0)         = 121484
> set_robust_list(0x7f8c65b7eae0, 24)     = 0
> rt_sigaction(SIGRTMIN, {0x7f8c646fb780, [], SA_RESTORER|SA_SIGINFO,
> 0x7f8c64704100}, NULL, 8) = 0
> rt_sigaction(SIGRT_1, {0x7f8c646fb810, [], SA_RESTORER|SA_RESTART|SA_SIGINFO,
> 0x7f8c64704100}, NULL, 8) = 0
> rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
> getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
> statfs("/sys/fs/selinux", 0x7ffeb752e770) = -1 ENOENT (No such file or
> directory)
> statfs("/selinux", 0x7ffeb752e770)      = -1 ENOENT (No such file or
> directory)
> brk(0)                                  = 0x155f000
> brk(0x1580000)                          = 0x1580000
> open("/proc/filesystems", O_RDONLY)     = 3
> fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
> 0x7f8c65b90000
> read(3, "nodev\tsysfs\nnodev\trootfs\nnodev\tb"..., 1024) = 377
> stat("/etc/sysconfig/64bit_strstr_via_64bit_strstr_sse2_unaligned",
> 0x7ffeb752e650) = -1 ENOENT (No such file or directory)
> read(3, "", 1024)                       = 0
> close(3)                                = 0
> munmap(0x7f8c65b90000, 4096)            = 0
> access("/etc/selinux/config", F_OK)     = 0
> open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
> fstat(3, {st_mode=S_IFREG|0644, st_size=106065056, ...}) = 0
> mmap(NULL, 106065056, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f8c5e1ce000
> close(3)                                = 0
> ioctl(1, SNDCTL_TMR_TIMEBASE or SNDRV_TIMER_IOCTL_NEXT_DEVICE or TCGETS,
> {B38400 opost isig icanon echo ...}) = 0
> ioctl(1, TIOCGWINSZ, {ws_row=68, ws_col=252, ws_xpixel=0, ws_ypixel=0}) = 0
> open("/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
> fstat(3, {st_mode=S_IFREG|0644, st_size=2502, ...}) = 0
> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
> 0x7f8c65b90000
> read(3, "# Locale name alias data base.\n#"..., 4096) = 2502
> read(3, "", 4096)                       = 0
> close(3)                                = 0
> munmap(0x7f8c65b90000, 4096)            = 0
> open("/usr/share/locale/en_US.UTF-8/LC_TIME/coreutils.mo", O_RDONLY) = -1
> ENOENT (No such file or directory)
> open("/usr/share/locale/en_US.utf8/LC_TIME/coreutils.mo", O_RDONLY) = -1
> ENOENT (No such file or directory)
> open("/usr/share/locale/en_US/LC_TIME/coreutils.mo", O_RDONLY) = -1
> ENOENT (No such file or directory)
> open("/usr/share/locale/en.UTF-8/LC_TIME/coreutils.mo", O_RDONLY) = -1
> ENOENT (No such file or directory)
> open("/usr/share/locale/en.utf8/LC_TIME/coreutils.mo", O_RDONLY) = -1
> ENOENT (No such file or directory)
> open("/usr/share/locale/en/LC_TIME/coreutils.mo", O_RDONLY) = -1 ENOENT
> (No such file or directory)
> stat("/etc/sysconfig/64bit_strstr_via_64bit_strstr_sse2_unaligned",
> 0x7ffeb752e2b0) = -1 ENOENT (No such file or directory)
> open("/usr/lib64/gconv/gconv-modules.cache", O_RDONLY) = 3
> fstat(3, {st_mode=S_IFREG|0644, st_size=26254, ...}) = 0
> mmap(NULL, 26254, PROT_READ, MAP_SHARED, 3, 0) = 0x7f8c65b8a000
> close(3)                                = 0
> futex(0x7f8c65336a80, FUTEX_WAKE_PRIVATE, 2147483647) = 0
> lstat("base", {st_mode=S_IFDIR|0777, st_size=3929293109, ...}) = 0
> socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
> connect(3, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) =
> -1 ENOENT (No such file or directory)
> close(3)                                = 0
> socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
> connect(3, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) =
> -1 ENOENT (No such file or directory)
> close(3)                                = 0
> open("/etc/nsswitch.conf", O_RDONLY|O_CLOEXEC) = 3
> fstat(3, {st_mode=S_IFREG|0644, st_size=1717, ...}) = 0
> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
> 0x7f8c65b89000
> read(3, "#\n# /etc/nsswitch.conf\n#\n# An ex"..., 4096) = 1717
> read(3, "", 4096)                       = 0
> close(3)                                = 0
> munmap(0x7f8c65b89000, 4096)            = 0
> open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
> fstat(3, {st_mode=S_IFREG|0644, st_size=53575, ...}) = 0
> mmap(NULL, 53575, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f8c65b70000
> close(3)                                = 0
> open("/lib64/libnss_files.so.2", O_RDONLY|O_CLOEXEC) = 3
> read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@\"\0\0\0\0\0\0"...,
> 832) = 832
> fstat(3, {st_mode=S_IFREG|0755, st_size=57824, ...}) = 0
> mmap(NULL, 2168952, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0)
> = 0x7f8c5dfbc000
> mprotect(0x7f8c5dfc7000, 2093056, PROT_NONE) = 0
> mmap(0x7f8c5e1c6000, 8192, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xa000) = 0x7f8c5e1c6000
> mmap(0x7f8c5e1c8000, 22648, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f8c5e1c8000
> close(3)                                = 0
> mprotect(0x7f8c5e1c6000, 4096, PROT_READ) = 0
> munmap(0x7f8c65b70000, 53575)           = 0
> open("/etc/passwd", O_RDONLY|O_CLOEXEC) = 3
> fstat(3, {st_mode=S_IFREG|0644, st_size=1330, ...}) = 0
> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
> 0x7f8c65b89000
> read(3, "root:x:0:0:root:/root:/bin/bash\n"..., 4096) = 1330
> close(3)                                = 0
> munmap(0x7f8c65b89000, 4096)            = 0
> socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
> connect(3, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) =
> -1 ENOENT (No such file or directory)
> close(3)                                = 0
> socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
> connect(3, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) =
> -1 ENOENT (No such file or directory)
> close(3)                                = 0
> open("/etc/group", O_RDONLY|O_CLOEXEC)  = 3
> fstat(3, {st_mode=S_IFREG|0644, st_size=597, ...}) = 0
> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
> 0x7f8c65b89000
> read(3, "root:x:0:\nbin:x:1:\ndaemon:x:2:\ns"..., 4096) = 597
> close(3)                                = 0
> munmap(0x7f8c65b89000, 4096)            = 0
> openat(AT_FDCWD, "base", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3
> getdents(3, /* 533 entries */, 32768)   = 32720
> lstat("base/dbus-glib-0.100-7.el7.x86_64.rpm", {st_mode=S_IFREG|0777,
> st_size=104756, ...}) = 0
> lstat("base/tomcat-admin-webapps-7.0.54-2.el7_1.noarch.rpm",
> {st_mode=S_IFREG|0777, st_size=38872, ...}) = 0
> lstat("base/hunspell-gl-0.20080515-8.el7.noarch.rpm",
> {st_mode=S_IFREG|0777, st_size=212276, ...}) = 0
> lstat("base/libreoffice-langpack-de-4.3.7.2-5.el7.x86_64.rpm",
> {st_mode=S_IFREG|0777, st_size=7570712, ...}) = 0
> lstat("base/paktype-naskh-basic-fonts-4.1-3.el7.noarch.rpm",
> {st_mode=S_IFREG|0777, st_size=436532, ...}) = 0
> lstat("base/openlmi-networking-0.3.0-3.el7.x86_64.rpm",
> {st_mode=S_IFREG|0777, st_size=148344, ...}) = 0
> lstat("base/kdenetwork-kopete-libs-4.10.5-8.el7_0.x86_64.rpm",
> {st_mode=S_IFREG|0777, st_size=1191908, ...}) = 0
> lstat("base/libwmf-0.2.8.4-41.el7_1.x86_64.rpm", {st_mode=S_IFREG|0777,
> st_size=137716, ...}) = 0
> lstat("base/texlive-texconfig-bin-svn27344.0-38.20130427_r30134.el7.noarch.rpm",
> {st_mode=S_IFREG|0777, st_size=16908, ...}) = 0
> lstat("base/gcc-c++-4.8.5-4.el7.x86_64.rpm", {st_mode=S_IFREG|0777,
> st_size=7508064, ...}) = 0
> ==========================================================
>
> --
> Best Regards
> Jian Zhang
>
> _______________________________________________
> ceph-users mailing list
> ceph-users at lists.ceph.com
> http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.ceph.com/pipermail/ceph-users-ceph.com/attachments/20171205/78ef62f8/attachment.html>


More information about the ceph-users mailing list