R devel - May 2009 - file descriptor leak in getSrcLines in R 2.10.0 svn 48590

I noticed the following file descriptor leak when I couldn't remove
a package unless I shut down the R session that had loaded and
used it.  The function that triggered the problem printed the output
of a call to parse().  Each time one prints a srcref a connection is
opened and not closed.  It looks like it happens in
as.character.srcref's
call to getSrcLines, which has some logic I don't understand about
closing 'srcfile' on exit only if !.is.Open(srcfile):
> getSrcLines
function (srcfile, first, last)
{
    if (first > last)
        return(character(0L))
    if (!.isOpen(srcfile))
        on.exit(close(srcfile))
    conn <- open(srcfile, first)
    lines <- readLines(conn, n = last - first + 1L, warn = FALSE)
    srcfile$line <- first + length(lines)
    return(lines)
}

(It looks like the srcref stuff is not finished yet, as there are other
problems, like print(parse(file)) not showing what it used to.)

This is on Linux, compiled by me with an ancient version of gcc.

% R
R version 2.10.0 Under development (unstable) (2009-05-21 r48590)
...
> showConnections()
     description class mode text isopen can read can
write
> tf<-tempfile()
> cat(file=tf, "1:7\nlog(pi)\n")
> showConnections()
     description class mode text isopen can read can
write
> p<-parse(tf)
> showConnections()
     description class mode text isopen can read can
write
> p
expression(<srcref: file "/tmp/RtmpZ1llo5/file327b23c6" chars 1:1
to
1:3>,
    <srcref: file "/tmp/RtmpZ1llo5/file327b23c6" chars 2:1 to
2:7>)
attr(,"srcfile")
/tmp/RtmpZ1llo5/file327b23c6
> showConnections()
  description                    class  mode text   isopen   can read
can write
3 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
"no"
4 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
"no"
5 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
"no"
6 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
"no"
> tf
[1] "/tmp/RtmpZ1llo5/file327b23c6"
> z<-attr(p,"srcref")[[2]]
> showConnections()
  description                    class  mode text   isopen   can read
can write
3 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
"no"
4 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
"no"
5 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
"no"
6 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
"no"
> zz<-as.character(z)
> showConnections()
  description                    class  mode text   isopen   can read
can write
3 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
"no"
4 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
"no"
5 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
"no"
6 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
"no"
7 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
"no"
> zz
[1] "<srcref: file \"/tmp/RtmpZ1llo5/file327b23c6\" chars 2:1
to 2:7>"

Bill Dunlap
TIBCO Software Inc - Spotfire Division
wdunlap tibco.com

On 5/21/2009 2:17 PM, William Dunlap wrote:
> I noticed the following file descriptor leak when I couldn't remove
> a package unless I shut down the R session that had loaded and
> used it.  The function that triggered the problem printed the output
> of a call to parse().  Each time one prints a srcref a connection is
> opened and not closed.  It looks like it happens in
> as.character.srcref's
> call to getSrcLines, which has some logic I don't understand about
> closing 'srcfile' on exit only if !.is.Open(srcfile):
Thanks, will look into it.

Duncan Murdoch
> 
>> getSrcLines
> function (srcfile, first, last)
> {
>     if (first > last)
>         return(character(0L))
>     if (!.isOpen(srcfile))
>         on.exit(close(srcfile))
>     conn <- open(srcfile, first)
>     lines <- readLines(conn, n = last - first + 1L, warn = FALSE)
>     srcfile$line <- first + length(lines)
>     return(lines)
> }
> 
> (It looks like the srcref stuff is not finished yet, as there are other
> problems, like print(parse(file)) not showing what it used to.)
> 
> This is on Linux, compiled by me with an ancient version of gcc.
> 
> % R
> R version 2.10.0 Under development (unstable) (2009-05-21 r48590)
> ...
>> showConnections()
>      description class mode text isopen can read can write
>> tf<-tempfile()
>> cat(file=tf, "1:7\nlog(pi)\n")
>> showConnections()
>      description class mode text isopen can read can write
>> p<-parse(tf)
>> showConnections()
>      description class mode text isopen can read can write
>> p
> expression(<srcref: file "/tmp/RtmpZ1llo5/file327b23c6" chars
1:1 to
> 1:3>,
>     <srcref: file "/tmp/RtmpZ1llo5/file327b23c6" chars 2:1 to
2:7>)
> attr(,"srcfile")
> /tmp/RtmpZ1llo5/file327b23c6
>> showConnections()
>   description                    class  mode text   isopen   can read
> can write
> 3 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 4 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 5 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 6 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
>> tf
> [1] "/tmp/RtmpZ1llo5/file327b23c6"
>> z<-attr(p,"srcref")[[2]]
>> showConnections()
>   description                    class  mode text   isopen   can read
> can write
> 3 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 4 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 5 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 6 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
>> zz<-as.character(z)
>> showConnections()
>   description                    class  mode text   isopen   can read
> can write
> 3 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 4 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 5 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 6 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 7 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
>> zz
> [1] "<srcref: file \"/tmp/RtmpZ1llo5/file327b23c6\" chars
2:1 to 2:7>"
> 
> Bill Dunlap
> TIBCO Software Inc - Spotfire Division
> wdunlap tibco.com 
> 
> ______________________________________________
> R-devel at r-project.org mailing list
> https://stat.ethz.ch/mailman/listinfo/r-devel

On 5/21/2009 2:17 PM, William Dunlap wrote:
> I noticed the following file descriptor leak when I couldn't remove
> a package unless I shut down the R session that had loaded and
> used it.  The function that triggered the problem printed the output
> of a call to parse().  Each time one prints a srcref a connection is
> opened and not closed.  It looks like it happens in
> as.character.srcref's
> call to getSrcLines, which has some logic I don't understand about
> closing 'srcfile' on exit only if !.is.Open(srcfile):
> 
>> getSrcLines
> function (srcfile, first, last)
> {
>     if (first > last)
>         return(character(0L))
>     if (!.isOpen(srcfile))
>         on.exit(close(srcfile))
>     conn <- open(srcfile, first)
>     lines <- readLines(conn, n = last - first + 1L, warn = FALSE)
>     srcfile$line <- first + length(lines)
>     return(lines)
> }
The idea is that if the srcfile is already open, then it should be left 
open; but if it is not open, it should be closed at the end.  open() on 
an open srcfile is supposed to make no change to the srcfile, just 
return the already open connection.

> (It looks like the srcref stuff is not finished yet, as there are other
> problems, like print(parse(file)) not showing what it used to.)

This is an encoding problem, which looks easy to fix.  I think the leak 
was caused by this:  because of the encoding problem, the connections 
got opened but open.srcfile aborted before completion, so close.srcfile 
didn't think the srcfile was open at all, and it left the connections in 
existence.

The encoding problem came because "unknown" is not a legal encoding to
pass to file().  It needs to be changed to "" or
"native.enc".

Thanks for the report; I'll commit a fix after some more testing.

Duncan Murdoch
> This is on Linux, compiled by me with an ancient version of gcc.
> 
> % R
> R version 2.10.0 Under development (unstable) (2009-05-21 r48590)
> ...
>> showConnections()
>      description class mode text isopen can read can write
>> tf<-tempfile()
>> cat(file=tf, "1:7\nlog(pi)\n")
>> showConnections()
>      description class mode text isopen can read can write
>> p<-parse(tf)
>> showConnections()
>      description class mode text isopen can read can write
>> p
> expression(<srcref: file "/tmp/RtmpZ1llo5/file327b23c6" chars
1:1 to
> 1:3>,
>     <srcref: file "/tmp/RtmpZ1llo5/file327b23c6" chars 2:1 to
2:7>)
> attr(,"srcfile")
> /tmp/RtmpZ1llo5/file327b23c6
>> showConnections()
>   description                    class  mode text   isopen   can read
> can write
> 3 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 4 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 5 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 6 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
>> tf
> [1] "/tmp/RtmpZ1llo5/file327b23c6"
>> z<-attr(p,"srcref")[[2]]
>> showConnections()
>   description                    class  mode text   isopen   can read
> can write
> 3 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 4 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 5 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 6 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
>> zz<-as.character(z)
>> showConnections()
>   description                    class  mode text   isopen   can read
> can write
> 3 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 4 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 5 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 6 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 7 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
>> zz
> [1] "<srcref: file \"/tmp/RtmpZ1llo5/file327b23c6\" chars
2:1 to 2:7>"
> 
> Bill Dunlap
> TIBCO Software Inc - Spotfire Division
> wdunlap tibco.com 
> 
> ______________________________________________
> R-devel at r-project.org mailing list
> https://stat.ethz.ch/mailman/listinfo/r-devel

On 5/21/2009 2:17 PM, William Dunlap wrote:
> I noticed the following file descriptor leak when I couldn't remove
> a package unless I shut down the R session that had loaded and
> used it.  The function that triggered the problem printed the output
> of a call to parse().  Each time one prints a srcref a connection is
> opened and not closed.  It looks like it happens in
> as.character.srcref's
> call to getSrcLines, which has some logic I don't understand about
> closing 'srcfile' on exit only if !.is.Open(srcfile):
I have a simpler recipe to reproduce the crash now:

tf <- tempfile()
con <- file(tf, open="w", encoding="unknown")

This gives an error message about an unsupported conversion
and leaves the con object only partially built. closeAllConnections() 
will then crash.

Should be relatively easy to fix...I'll take a look.

Duncan Murdoch
> 
>> getSrcLines
> function (srcfile, first, last)
> {
>     if (first > last)
>         return(character(0L))
>     if (!.isOpen(srcfile))
>         on.exit(close(srcfile))
>     conn <- open(srcfile, first)
>     lines <- readLines(conn, n = last - first + 1L, warn = FALSE)
>     srcfile$line <- first + length(lines)
>     return(lines)
> }
> 
> (It looks like the srcref stuff is not finished yet, as there are other
> problems, like print(parse(file)) not showing what it used to.)
> 
> This is on Linux, compiled by me with an ancient version of gcc.
> 
> % R
> R version 2.10.0 Under development (unstable) (2009-05-21 r48590)
> ...
>> showConnections()
>      description class mode text isopen can read can write
>> tf<-tempfile()
>> cat(file=tf, "1:7\nlog(pi)\n")
>> showConnections()
>      description class mode text isopen can read can write
>> p<-parse(tf)
>> showConnections()
>      description class mode text isopen can read can write
>> p
> expression(<srcref: file "/tmp/RtmpZ1llo5/file327b23c6" chars
1:1 to
> 1:3>,
>     <srcref: file "/tmp/RtmpZ1llo5/file327b23c6" chars 2:1 to
2:7>)
> attr(,"srcfile")
> /tmp/RtmpZ1llo5/file327b23c6
>> showConnections()
>   description                    class  mode text   isopen   can read
> can write
> 3 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 4 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 5 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 6 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
>> tf
> [1] "/tmp/RtmpZ1llo5/file327b23c6"
>> z<-attr(p,"srcref")[[2]]
>> showConnections()
>   description                    class  mode text   isopen   can read
> can write
> 3 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 4 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 5 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 6 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
>> zz<-as.character(z)
>> showConnections()
>   description                    class  mode text   isopen   can read
> can write
> 3 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 4 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 5 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 6 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
> 7 "/tmp/RtmpZ1llo5/file327b23c6" "file" "rt"
"text" "opened" "yes"
> "no"
>> zz
> [1] "<srcref: file \"/tmp/RtmpZ1llo5/file327b23c6\" chars
2:1 to 2:7>"
> 
> Bill Dunlap
> TIBCO Software Inc - Spotfire Division
> wdunlap tibco.com 
> 
> ______________________________________________
> R-devel at r-project.org mailing list
> https://stat.ethz.ch/mailman/listinfo/r-devel