Hi,
I scribbled a simple guestfs based program called guestfs-xfer with
following synopsis:
Usage: guest-xfer [options] [op] [diskimage]
op = [ls|cat|write]
options:
-d, --guestdevice DEV guest device
--blocksize BS blocksize [default: 1048576]
-o, --offset OFF offset [default: 0]
So eg. `cat /dev/urandom | guest-xfer -d /dev/sda write mydisk.img` will fill
mydisk.img with pseudorandom content.
I implemented this both with Ruby and Go. The 'write' op relies on
pwrite_device.
I have pasted the codes to the end of this mail.
I'm creating mydisk.img as a qcow2 file with a raw sparse file bakcend
$ truncate -s 100g myimg.raw
$ qemu-img create -f qcow2 -b myimg.{raw,img}
Then I do
# pv /dev/sda2 | guest-xfer -d /dev/sda write mydisk.img
I find that the ruby implementation produces a 24 MiB/s throughput, while
the go one only 2 MiB/s.
Note that the 'cat' operation (that writes device content to stdout)
is reasonably fast with both language implementaitions (doing around
70 MiB/s).
Why is this, how the Go binding could be improved?
Regards,
Csaba
<code lang="ruby">
#!/usr/bin/env ruby
require 'optparse'
require 'ostruct'
require 'guestfs'
module BaseOp
extend self
def included mod
mod.module_eval { extend self }
end
def perform gu, opts, gudev
end
attr_reader :readonly
end
module OPS
extend self
def find name
m = self.constants.find { |c| c.to_s.downcase == name }
m ? const_get(m) : nil
end
def names
constants.map &:downcase
end
module Cat
include BaseOp
@readonly = 1
def perform gu, opts
off = opts.offset
while true
buf = gu.pread_device opts.dev, opts.bs, off
break if buf.empty?
print buf
off += buf.size
end
end
end
module Write
include BaseOp
@readonly = 0
def perform gu, opts
off = opts.offset
while true
buf = STDIN.read opts.bs
break if (buf||"").empty?
siz = gu.pwrite_device opts.dev, buf, off
if siz != buf.size
raise "short write at offset #{off} (wanted #{buf.size}, done
#{siz})"
end
off += buf.size
end
end
end
module Ls
include BaseOp
@readonly = 1
def perform gu, opts
puts gu.list_devices
end
end
end
def main
opts = OpenStruct.new offset: 0, bs: 1<<20
optp = OptionParser.new
optp.banner << " [op] [diskimage]
op = [#{OPS.names.join ?|}]
options:"
optp.on("-d", "--guestdevice DEV", "guest
device") { |c| opts.dev = c }
optp.on("--blocksize BS", Integer,
"blocksize [default: #{opts.bs}]") { |n| opts.bs = n }
optp.on("-o OFF", "--offset", Integer,
"offset [default: #{opts.offset}]") { |n| opts.offset = n }
optp.parse!
unless $*.size == 2
STDERR.puts optp
exit 1
end
opname,image = $*
op = OPS.find opname
op or raise "unkown op #{opname} (should be one of #{OPS.names.join
?,})"
gu=Guestfs::Guestfs.new
begin
gu.add_drive_opts image, readonly: op.readonly
gu.launch
op.perform gu, opts
gu.shutdown
ensure
gu.close
end
end
if __FILE__ == $0
main
end
</code>
<code lang="go">
package main
import (
"flag"
"fmt"
"libguestfs.org/guestfs"
"log"
"os"
"path/filepath"
"strings"
)
type Op int
const (
OpUndef Op = iota
OpList
OpCat
OpWrite
)
var OpNames = map[Op]string{
OpList: "ls",
OpCat: "cat",
OpWrite: "write",
}
const usage = `%s [options] [op] [diskimage]
op = [%s]
options:
`
func main() {
var devname string
var bs int
var offset int64
log.SetFlags(log.LstdFlags | log.Lshortfile)
flag.Usage = func() {
var ops []string
for _, on := range OpNames {
ops = append(ops, on)
}
fmt.Fprintf(flag.CommandLine.Output(), usage,
filepath.Base(os.Args[0]), strings.Join(ops, "|"))
flag.PrintDefaults()
}
flag.StringVar(&devname, "guestdevice", "",
"guestfs device name")
flag.IntVar(&bs, "blocksize", 1<<20,
"blocksize")
flag.Int64Var(&offset, "offset", 0, "offset")
flag.Parse()
var opname string
var disk string
switch flag.NArg() {
case 2:
opname = flag.Arg(0)
disk = flag.Arg(1)
default:
flag.Usage()
os.Exit(1)
}
op := OpUndef
for o, on := range OpNames {
if opname == on {
op = o
break
}
}
if op == OpUndef {
log.Fatalf("unkown op %s\n", opname)
}
g, err := guestfs.Create()
if err != nil {
log.Fatalf("could not create guestfs handle: %s\n", err)
}
defer g.Close()
/* Attach the disk image to libguestfs. */
isReadonly := map[Op]bool{
OpList: true,
OpCat: true,
OpWrite: false,
}
optargs := guestfs.OptargsAdd_drive{
Readonly_is_set: true,
Readonly: isReadonly[op],
}
if err := g.Add_drive(disk, &optargs); err != nil {
log.Fatal(err)
}
/* Run the libguestfs back-end. */
if err := g.Launch(); err != nil {
log.Fatal(err)
}
switch op {
case OpList:
devices, err := g.List_devices()
if err != nil {
log.Fatal(err)
}
for _, dev := range devices {
fmt.Println(dev)
}
case OpCat:
for {
buf, err := g.Pread_device(devname, bs, offset)
if err != nil {
log.Fatal(err)
}
if len(buf) == 0 {
break
}
n, err1 := os.Stdout.Write(buf)
if err1 != nil {
log.Fatal(err1)
}
if n != len(buf) {
log.Fatal("stdout: short write")
}
offset += int64(len(buf))
}
case OpWrite:
buf := make([]byte, bs)
for {
n, err := os.Stdin.Read(buf)
if err != nil {
log.Fatal(err)
}
if n == 0 {
break
}
nw, err1 := g.Pwrite_device(devname, buf[:n], offset)
if err1 != nil {
log.Fatal(err1)
}
if nw != n {
log.Fatalf("short write at offset %d", offset)
}
offset += int64(n)
}
default:
panic("unknown op")
}
if err := g.Shutdown(); err != nil {
log.Fatal(err)
}
}
</code>
Richard W.M. Jones
2020-Feb-19 18:11 UTC
Re: [Libguestfs] Poor write performance with golang binding
On Wed, Feb 19, 2020 at 03:00:11PM +0100, Csaba Henk wrote:> Hi, > > I scribbled a simple guestfs based program called guestfs-xfer with > following synopsis: > > Usage: guest-xfer [options] [op] [diskimage] > > op = [ls|cat|write] > > options: > -d, --guestdevice DEV guest device > --blocksize BS blocksize [default: 1048576] > -o, --offset OFF offset [default: 0] > > So eg. `cat /dev/urandom | guest-xfer -d /dev/sda write mydisk.img` will fill > mydisk.img with pseudorandom content. > > I implemented this both with Ruby and Go. The 'write' op relies on > pwrite_device. > I have pasted the codes to the end of this mail. > > I'm creating mydisk.img as a qcow2 file with a raw sparse file bakcend > > $ truncate -s 100g myimg.raw > $ qemu-img create -f qcow2 -b myimg.{raw,img} > > Then I do > > # pv /dev/sda2 | guest-xfer -d /dev/sda write mydisk.img > > I find that the ruby implementation produces a 24 MiB/s throughput, while > the go one only 2 MiB/s. > > Note that the 'cat' operation (that writes device content to stdout) > is reasonably fast with both language implementaitions (doing around > 70 MiB/s). > > Why is this, how the Go binding could be improved?TBH I've no idea. The bindings are meant to be very thin wrappers around the C API, so I can't imagine that they should cause performance penalties as large as you have observed. Can you enable tracing in both (g.set_trace (true)) and make sure that the operations being done are the same for both languages? Rich.> Regards, > Csaba > > > <code lang="ruby"> > #!/usr/bin/env ruby > > require 'optparse' > require 'ostruct' > require 'guestfs' > > module BaseOp > extend self > > def included mod > mod.module_eval { extend self } > end > > def perform gu, opts, gudev > end > > attr_reader :readonly > end > > module OPS > extend self > > def find name > m = self.constants.find { |c| c.to_s.downcase == name } > m ? const_get(m) : nil > end > > def names > constants.map &:downcase > end > > module Cat > include BaseOp > > @readonly = 1 > > def perform gu, opts > off = opts.offset > while true > buf = gu.pread_device opts.dev, opts.bs, off > break if buf.empty? > print buf > off += buf.size > end > end > end > > module Write > include BaseOp > > @readonly = 0 > > def perform gu, opts > off = opts.offset > while true > buf = STDIN.read opts.bs > break if (buf||"").empty? > siz = gu.pwrite_device opts.dev, buf, off > if siz != buf.size > raise "short write at offset #{off} (wanted #{buf.size}, done #{siz})" > end > off += buf.size > end > end > end > > module Ls > include BaseOp > > @readonly = 1 > > def perform gu, opts > puts gu.list_devices > end > end > > end > > def main > opts = OpenStruct.new offset: 0, bs: 1<<20 > optp = OptionParser.new > optp.banner << " [op] [diskimage] > > op = [#{OPS.names.join ?|}] > > options:" > > optp.on("-d", "--guestdevice DEV", "guest device") { |c| opts.dev = c } > optp.on("--blocksize BS", Integer, > "blocksize [default: #{opts.bs}]") { |n| opts.bs = n } > optp.on("-o OFF", "--offset", Integer, > "offset [default: #{opts.offset}]") { |n| opts.offset = n } > optp.parse! > > unless $*.size == 2 > STDERR.puts optp > exit 1 > end > opname,image = $* > > op = OPS.find opname > op or raise "unkown op #{opname} (should be one of #{OPS.names.join ?,})" > > gu=Guestfs::Guestfs.new > begin > gu.add_drive_opts image, readonly: op.readonly > gu.launch > > op.perform gu, opts > > gu.shutdown > ensure > gu.close > end > end > > if __FILE__ == $0 > main > end > </code> > > <code lang="go"> > package main > > import ( > "flag" > "fmt" > "libguestfs.org/guestfs" > "log" > "os" > "path/filepath" > "strings" > ) > > type Op int > > const ( > OpUndef Op = iota > OpList > OpCat > OpWrite > ) > > var OpNames = map[Op]string{ > OpList: "ls", > OpCat: "cat", > OpWrite: "write", > } > > const usage = `%s [options] [op] [diskimage] > > op = [%s] > > options: > ` > > func main() { > var devname string > var bs int > var offset int64 > > log.SetFlags(log.LstdFlags | log.Lshortfile) > flag.Usage = func() { > var ops []string > > for _, on := range OpNames { > ops = append(ops, on) > } > fmt.Fprintf(flag.CommandLine.Output(), usage, > filepath.Base(os.Args[0]), strings.Join(ops, "|")) > flag.PrintDefaults() > } > flag.StringVar(&devname, "guestdevice", "", "guestfs device name") > flag.IntVar(&bs, "blocksize", 1<<20, "blocksize") > flag.Int64Var(&offset, "offset", 0, "offset") > flag.Parse() > > var opname string > var disk string > switch flag.NArg() { > case 2: > opname = flag.Arg(0) > disk = flag.Arg(1) > default: > flag.Usage() > os.Exit(1) > } > > op := OpUndef > for o, on := range OpNames { > if opname == on { > op = o > break > } > } > if op == OpUndef { > log.Fatalf("unkown op %s\n", opname) > > } > > g, err := guestfs.Create() > if err != nil { > log.Fatalf("could not create guestfs handle: %s\n", err) > } > defer g.Close() > > /* Attach the disk image to libguestfs. */ > isReadonly := map[Op]bool{ > OpList: true, > OpCat: true, > OpWrite: false, > } > optargs := guestfs.OptargsAdd_drive{ > Readonly_is_set: true, > Readonly: isReadonly[op], > } > if err := g.Add_drive(disk, &optargs); err != nil { > log.Fatal(err) > } > > /* Run the libguestfs back-end. */ > if err := g.Launch(); err != nil { > log.Fatal(err) > } > > switch op { > case OpList: > devices, err := g.List_devices() > if err != nil { > log.Fatal(err) > } > > for _, dev := range devices { > fmt.Println(dev) > } > case OpCat: > for { > buf, err := g.Pread_device(devname, bs, offset) > if err != nil { > log.Fatal(err) > } > if len(buf) == 0 { > break > } > n, err1 := os.Stdout.Write(buf) > if err1 != nil { > log.Fatal(err1) > } > if n != len(buf) { > log.Fatal("stdout: short write") > } > offset += int64(len(buf)) > } > case OpWrite: > buf := make([]byte, bs) > for { > n, err := os.Stdin.Read(buf) > if err != nil { > log.Fatal(err) > } > if n == 0 { > break > } > > nw, err1 := g.Pwrite_device(devname, buf[:n], offset) > if err1 != nil { > log.Fatal(err1) > } > if nw != n { > log.Fatalf("short write at offset %d", offset) > } > offset += int64(n) > } > default: > panic("unknown op") > } > > if err := g.Shutdown(); err != nil { > log.Fatal(err) > } > } > </code> > > > _______________________________________________ > Libguestfs mailing list > Libguestfs@redhat.com > https://www.redhat.com/mailman/listinfo/libguestfs-- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones Read my programming and virtualization blog: http://rwmj.wordpress.com virt-df lists disk usage of guests without needing to install any software inside the virtual machine. Supports Linux and Windows. http://people.redhat.com/~rjones/virt-df/
Csaba Henk
2020-Feb-20 13:11 UTC
Re: [Libguestfs] Poor write performance with golang binding
On Wed, Feb 19, 2020 at 7:11 PM Richard W.M. Jones <rjones@redhat.com> wrote:> > On Wed, Feb 19, 2020 at 03:00:11PM +0100, Csaba Henk wrote: > > [...] > > Then I do > > > > # pv /dev/sda2 | guest-xfer -d /dev/sda write mydisk.img > > > > I find that the ruby implementation produces a 24 MiB/s throughput, while > > the go one only 2 MiB/s. > > > > Note that the 'cat' operation (that writes device content to stdout) > > is reasonably fast with both language implementaitions (doing around > > 70 MiB/s). > > > > Why is this, how the Go binding could be improved? > > TBH I've no idea. The bindings are meant to be very thin wrappers > around the C API, so I can't imagine that they should cause > performance penalties as large as you have observed. > > Can you enable tracing in both (g.set_trace (true)) and make sure that > the operations being done are the same for both languages? > > Rich.Ok thanks for the idea. It helped to find out what's going on. And (spoiler) there is nothing wrong here with the Go bindings. I take data from stdout, through pipe. One read(2) from the pipe fetches 64k data. `File.read` in Go directly exposes the syscall and gives back 64k even if the argument buffer is larger (wrapping it in bufio does not make difference). `File#read` in Ruby will read from the file until the argument buffer is filled. Therefore (without custom buffering code, mapping stdin reads 1:1 to guestfs pwrites) `g.Pwrite_device` invocations in Go are capped at 64k, while `g.pwrite_device` in Ruby will enjoy the the buffer size I specify. And larger the buffer the better is the write performance (I defaulted to 1m in my code, but giving bigger values, up to guestfs proto limits is further improvement). So TL;DR it's due to language idiosyncrasies, not because of libguestfs. Csaba