Hello Group,
We recently upgraded our existing Lustre system from 1.6.7.2 to 1.8.4.
One of the hoped-for features is "execute-only" binaries on the
Lustre file system. According to Bug 22376 (
https://bugzilla.lustre.org/show_bug.cgi?id=22376 ) this execute-only
feature was available in the patch for Lustre 1.8.2. I had assumed
the patch would be incorporated upstream (i.e. to 1.8.4). The
behavior I am seeing on the Lustre 1.8.4 is shown below with an a.out
executable file of the common "hello world" C program.
icecube:/mnt/lustre # ls -l a.out
---x--x--x 1 root mygrp 9027 Dec 2 13:57 a.out
larko at icecube:/mnt/lustre> ./a.out
-bash: ./a.out: Permission denied
larko at icecube:/mnt/lustre> strace a.out
execve("/mnt/lustre/a.out", ["a.out"], [/* 73 vars */]) = -1
EACCES
(Permission denied)
dup(2) = 3
fcntl(3, F_GETFL) = 0x8002 (flags O_RDWR|O_LARGEFILE)
fstat(3, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 2), ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x2b2df42b3000
lseek(3, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek)
write(3, "strace: exec: Permission denied\n", 32strace: exec:
Permission denied
) = 32
close(3) = 0
munmap(0x2b2df42b3000, 4096) = 0
exit_group(1) = ?
larko at icecube:/mnt/lustre> whoami
larko
larko at icecube:/mnt/lustre> cat /etc/passwd | grep larko
larko:x:11111:11111:Catherine M Larko
(MYGRP96090RAY):/usr/people/larko:/bin/bash # where 11111 is "mygrp"
The execute-only does work for the root users:
icecube:/mnt/lustre # whoami
root
icecube:/mnt/lustre # ./a.out
Hello World
icecube:/mnt/lustre # strace ./a.out
execve("./a.out", ["./a.out"], [/* 69 vars */]) = 0
brk(0) = 0x501000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x2b6427bdf000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x2b6427be0000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or
directory)
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=131880, ...}) = 0
mmap(NULL, 131880, PROT_READ, MAP_PRIVATE, 3, 0) = 0x2b6427be1000
close(3) = 0
open("/lib64/libc.so.6", O_RDONLY) = 3
read(3,
"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\324\1"..., 832) =
832
fstat(3, {st_mode=S_IFREG|0755, st_size=1570761, ...}) = 0
mmap(NULL, 2355560, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x2b6427ce0000
madvise(0x2b6427ce0000, 2355560, MADV_SEQUENTIAL|0x1) = 0
mprotect(0x2b6427e16000, 1048576, PROT_NONE) = 0
mmap(0x2b6427f16000, 20480, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x136000) = 0x2b6427f16000
mmap(0x2b6427f1b000, 16744, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x2b6427f1b000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x2b6427f20000
arch_prctl(ARCH_SET_FS, 0x2b6427f206d0) = 0
mprotect(0x2b6427f16000, 12288, PROT_READ) = 0
munmap(0x2b6427be1000, 131880) = 0
fstat(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 2), ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x2b6427be1000
write(1, "Hello World\n", 12Hello World
) = 12
exit_group(0) = ?
Regarding the default value of the drop cache:
larko at icecube:/mnt/lustre> cat /proc/sys/vm/drop_caches 0
If I try the other suggestion in the bugzilla URL referenced above
about sendiing the contents once to /dev/null there is no change in
resulting behavior.
icecube:/mnt/lustre # cat ./a.out > /dev/null
larko at icecube:/mnt/lustre> ./a.out
-bash: ./a.out: Permission denied
There are no unusual lines what-so-ever on the MGS/MDT /var/log/messages file.
Any tips? Settings??
Thank you,
Megan Larko
On 2010-12-02, at 12:23, Ms. Megan Larko wrote:> We recently upgraded our existing Lustre system from 1.6.7.2 to 1.8.4. > One of the hoped-for features is "execute-only" binaries on the > Lustre file system.Are you running a SLES10 kernel? If yes, please see bug 23025.> According to Bug 22376 ( > https://bugzilla.lustre.org/show_bug.cgi?id=22376 ) this execute-only > feature was available in the patch for Lustre 1.8.2. I had assumed > the patch would be incorporated upstream (i.e. to 1.8.4). The > behavior I am seeing on the Lustre 1.8.4 is shown below with an a.out > executable file of the common "hello world" C program.I''ve added a simple test to bug 22376 to verify if this is working correctly in current versions of Lustre. Please CC yourself to that bug to track its progress.> icecube:/mnt/lustre # ls -l a.out > ---x--x--x 1 root mygrp 9027 Dec 2 13:57 a.out > larko at icecube:/mnt/lustre> ./a.out > -bash: ./a.out: Permission denied > larko at icecube:/mnt/lustre> strace a.out > execve("/mnt/lustre/a.out", ["a.out"], [/* 73 vars */]) = -1 EACCES > (Permission denied) > dup(2) = 3 > fcntl(3, F_GETFL) = 0x8002 (flags O_RDWR|O_LARGEFILE) > fstat(3, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 2), ...}) = 0 > mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, > 0) = 0x2b2df42b3000 > lseek(3, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek) > write(3, "strace: exec: Permission denied\n", 32strace: exec: Permission denied > ) = 32 > close(3) = 0 > munmap(0x2b2df42b3000, 4096) = 0 > exit_group(1) = ? > larko at icecube:/mnt/lustre> whoami > larko > larko at icecube:/mnt/lustre> cat /etc/passwd | grep larko > larko:x:11111:11111:Catherine M Larko > (MYGRP96090RAY):/usr/people/larko:/bin/bash # where 11111 is "mygrp" > > The execute-only does work for the root users: > icecube:/mnt/lustre # whoami > root > icecube:/mnt/lustre # ./a.out > Hello World > icecube:/mnt/lustre # strace ./a.out > execve("./a.out", ["./a.out"], [/* 69 vars */]) = 0 > brk(0) = 0x501000 > mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, > 0) = 0x2b6427bdf000 > mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, > 0) = 0x2b6427be0000 > access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory) > open("/etc/ld.so.cache", O_RDONLY) = 3 > fstat(3, {st_mode=S_IFREG|0644, st_size=131880, ...}) = 0 > mmap(NULL, 131880, PROT_READ, MAP_PRIVATE, 3, 0) = 0x2b6427be1000 > close(3) = 0 > open("/lib64/libc.so.6", O_RDONLY) = 3 > read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\324\1"..., 832) = 832 > fstat(3, {st_mode=S_IFREG|0755, st_size=1570761, ...}) = 0 > mmap(NULL, 2355560, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, > 0) = 0x2b6427ce0000 > madvise(0x2b6427ce0000, 2355560, MADV_SEQUENTIAL|0x1) = 0 > mprotect(0x2b6427e16000, 1048576, PROT_NONE) = 0 > mmap(0x2b6427f16000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x136000) = 0x2b6427f16000 > mmap(0x2b6427f1b000, 16744, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x2b6427f1b000 > close(3) = 0 > mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, > 0) = 0x2b6427f20000 > arch_prctl(ARCH_SET_FS, 0x2b6427f206d0) = 0 > mprotect(0x2b6427f16000, 12288, PROT_READ) = 0 > munmap(0x2b6427be1000, 131880) = 0 > fstat(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 2), ...}) = 0 > mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, > 0) = 0x2b6427be1000 > write(1, "Hello World\n", 12Hello World > ) = 12 > exit_group(0) = ? > > Regarding the default value of the drop cache: > larko at icecube:/mnt/lustre> cat /proc/sys/vm/drop_caches 0 > > If I try the other suggestion in the bugzilla URL referenced above > about sendiing the contents once to /dev/null there is no change in > resulting behavior. > icecube:/mnt/lustre # cat ./a.out > /dev/null > larko at icecube:/mnt/lustre> ./a.out > -bash: ./a.out: Permission denied > > > There are no unusual lines what-so-ever on the MGS/MDT /var/log/messages file. > > Any tips? Settings?? > > Thank you, > Megan Larko > _______________________________________________ > Lustre-discuss mailing list > Lustre-discuss at lists.lustre.org > http://lists.lustre.org/mailman/listinfo/lustre-discussCheers, Andreas -- Andreas Dilger Lustre Technical Lead Oracle Corporation Canada Inc.
Greetings Andreas and Folks, I read the info on Bug 23025. Comment #12 at the bottom indicates: ------- Comment #12 From Sam Chan 2010-06-29 13:49:43 ------- problem is fixed in Novell bugzilla 379057. kernel 2.6.16.60-0.60.1 and newer should resolve the problem. i tested 2.6.16.60-0.66.1+lustre.1.8.3 and things work fine now. _________________________________________________________ That''s sounds cool, but I am seeing this on SLES10SP3 with linux kernel 2.6.16.60-0.69.1-smp on the clients and kernel 2.6.16.60-0.69.1+lustre1.8.4-smp on the Lustre servers using lustre 1.8.4 on both sides of the conversation. Cheers! megan On Thu, Dec 2, 2010 at 3:10 PM, Andreas Dilger <andreas.dilger at oracle.com> wrote:> On 2010-12-02, at 12:23, Ms. Megan Larko wrote: >> We recently upgraded our existing Lustre system from 1.6.7.2 to 1.8.4. >> ?One of the hoped-for features is "execute-only" binaries on the >> Lustre file system. > > Are you running a SLES10 kernel? ?If yes, please see bug 23025. > >> ?According to Bug 22376 ( >> https://bugzilla.lustre.org/show_bug.cgi?id=22376 ) ?this execute-only >> feature was available in the patch for Lustre 1.8.2. ? I had assumed >> the patch would be incorporated upstream (i.e. to 1.8.4). ? The >> behavior I am seeing on the Lustre 1.8.4 is shown below with an a.out >> executable file of the common "hello world" C program. > > I''ve added a simple test to bug 22376 to verify if this is working correctly in current versions of Lustre. ?Please CC yourself to that bug to track its progress. > >> icecube:/mnt/lustre # ls -l a.out >> ---x--x--x 1 root mygrp 9027 Dec ?2 13:57 a.out >> larko at icecube:/mnt/lustre> ./a.out >> -bash: ./a.out: Permission denied >> larko at icecube:/mnt/lustre> strace a.out >> execve("/mnt/lustre/a.out", ["a.out"], [/* 73 vars */]) = -1 EACCES >> (Permission denied) >> dup(2) ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?= 3 >> fcntl(3, F_GETFL) ? ? ? ? ? ? ? ? ? ? ? = 0x8002 (flags O_RDWR|O_LARGEFILE) >> fstat(3, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 2), ...}) = 0 >> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, >> 0) = 0x2b2df42b3000 >> lseek(3, 0, SEEK_CUR) ? ? ? ? ? ? ? ? ? = -1 ESPIPE (Illegal seek) >> write(3, "strace: exec: Permission denied\n", 32strace: exec: Permission denied >> ) = 32 >> close(3) ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?= 0 >> munmap(0x2b2df42b3000, 4096) ? ? ? ? ? ?= 0 >> exit_group(1) ? ? ? ? ? ? ? ? ? ? ? ? ? = ? >> larko at icecube:/mnt/lustre> whoami >> larko >> larko at icecube:/mnt/lustre> cat /etc/passwd | grep larko >> larko:x:11111:11111:Catherine M Larko >> (MYGRP96090RAY):/usr/people/larko:/bin/bash ?# where 11111 is "mygrp" >> >> The execute-only does work for the root users: >> icecube:/mnt/lustre # whoami >> root >> icecube:/mnt/lustre # ./a.out >> Hello World >> icecube:/mnt/lustre # strace ./a.out >> execve("./a.out", ["./a.out"], [/* 69 vars */]) = 0 >> brk(0) ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?= 0x501000 >> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, >> 0) = 0x2b6427bdf000 >> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, >> 0) = 0x2b6427be0000 >> access("/etc/ld.so.preload", R_OK) ? ? ?= -1 ENOENT (No such file or directory) >> open("/etc/ld.so.cache", O_RDONLY) ? ? ?= 3 >> fstat(3, {st_mode=S_IFREG|0644, st_size=131880, ...}) = 0 >> mmap(NULL, 131880, PROT_READ, MAP_PRIVATE, 3, 0) = 0x2b6427be1000 >> close(3) ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?= 0 >> open("/lib64/libc.so.6", O_RDONLY) ? ? ?= 3 >> read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\324\1"..., 832) = 832 >> fstat(3, {st_mode=S_IFREG|0755, st_size=1570761, ...}) = 0 >> mmap(NULL, 2355560, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, >> 0) = 0x2b6427ce0000 >> madvise(0x2b6427ce0000, 2355560, MADV_SEQUENTIAL|0x1) = 0 >> mprotect(0x2b6427e16000, 1048576, PROT_NONE) = 0 >> mmap(0x2b6427f16000, 20480, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x136000) = 0x2b6427f16000 >> mmap(0x2b6427f1b000, 16744, PROT_READ|PROT_WRITE, >> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x2b6427f1b000 >> close(3) ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?= 0 >> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, >> 0) = 0x2b6427f20000 >> arch_prctl(ARCH_SET_FS, 0x2b6427f206d0) = 0 >> mprotect(0x2b6427f16000, 12288, PROT_READ) = 0 >> munmap(0x2b6427be1000, 131880) ? ? ? ? ?= 0 >> fstat(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 2), ...}) = 0 >> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, >> 0) = 0x2b6427be1000 >> write(1, "Hello World\n", 12Hello World >> ) ? ? ? ? ? = 12 >> exit_group(0) ? ? ? ? ? ? ? ? ? ? ? ? ? = ? >> >> Regarding the default value of the drop cache: >> larko at icecube:/mnt/lustre> cat /proc/sys/vm/drop_caches 0 >> >> If I try the other suggestion in the bugzilla URL referenced above >> about sendiing the contents once to /dev/null there is no change in >> resulting behavior. >> icecube:/mnt/lustre # cat ./a.out > /dev/null >> larko at icecube:/mnt/lustre> ./a.out >> -bash: ./a.out: Permission denied >> >> >> There are no unusual lines what-so-ever on the MGS/MDT /var/log/messages file. >> >> Any tips? ? Settings?? >> >> Thank you, >> Megan Larko >> _______________________________________________ >> Lustre-discuss mailing list >> Lustre-discuss at lists.lustre.org >> http://lists.lustre.org/mailman/listinfo/lustre-discuss > > > Cheers, Andreas > -- > Andreas Dilger > Lustre Technical Lead > Oracle Corporation Canada Inc. > >