The ability to log via syslog was removed back in Oct 2011 but it''s actually quite useful. On a system with lots of disk I/O, if xenstore logs directly to the filesystem it can block so much that guests start to suffer. One solution would be to add some threading to xenstored to isolate the logging from the ring servicing; however this will be quite a big patch and we could end up re-implementing a syslog-like logging service. Instead we re-add the ability to log via syslog and allow this to be switched on via the xenstored config file. The default logging behaviour is unchanged. FYI this is the only significant thing in the XCP patch queue -- we''ve found this helps a lot in stress tests. Patches follow. For extra convenience you should be able to pull from: git pull git://github.com/djs55/xen xenstore-syslog Cheers, Dave
This was lost in the OCaml xenstored log merge of 10/Oct/2011.
The binding isn''t exported as a shared "log" library, instead
it is kept
local to xenstored.
Signed-off-by: David Scott <dave.scott@eu.citrix.com>
---
tools/ocaml/xenstored/Makefile | 13 ++++++++--
tools/ocaml/xenstored/syslog.ml | 47 +++++++++++++++++++++++++++++++++++
tools/ocaml/xenstored/syslog.mli | 41 ++++++++++++++++++++++++++++++
tools/ocaml/xenstored/syslog_stubs.c | 48 ++++++++++++++++++++++++++++++++++++
4 files changed, 147 insertions(+), 2 deletions(-)
create mode 100644 tools/ocaml/xenstored/syslog.ml
create mode 100644 tools/ocaml/xenstored/syslog.mli
create mode 100644 tools/ocaml/xenstored/syslog_stubs.c
diff --git a/tools/ocaml/xenstored/Makefile b/tools/ocaml/xenstored/Makefile
index 3302d57..b18f190 100644
--- a/tools/ocaml/xenstored/Makefile
+++ b/tools/ocaml/xenstored/Makefile
@@ -8,6 +8,11 @@ OCAMLINCLUDE += \
-I $(OCAML_TOPLEVEL)/libs/xc \
-I $(OCAML_TOPLEVEL)/libs/eventchn
+LIBS = syslog.cma syslog.cmxa
+syslog_OBJS = syslog
+syslog_C_OBJS = syslog_stubs
+OCAML_LIBRARY = syslog
+
OBJS = define \
stdext \
trie \
@@ -29,9 +34,11 @@ OBJS = define \
process \
xenstored
-INTF = symbol.cmi trie.cmi
+INTF = symbol.cmi trie.cmi syslog.cmi
+
XENSTOREDLIBS = \
unix.cmxa \
+ -ccopt -L -ccopt . syslog.cmxa \
-ccopt -L -ccopt $(OCAML_TOPLEVEL)/libs/mmap
$(OCAML_TOPLEVEL)/libs/mmap/xenmmap.cmxa \
-ccopt -L -ccopt $(OCAML_TOPLEVEL)/libs/eventchn
$(OCAML_TOPLEVEL)/libs/eventchn/xeneventchn.cmxa \
-ccopt -L -ccopt $(OCAML_TOPLEVEL)/libs/xc
$(OCAML_TOPLEVEL)/libs/xc/xenctrl.cmxa \
@@ -45,10 +52,12 @@ oxenstored_OBJS = $(OBJS)
OCAML_PROGRAM = oxenstored
-all: $(INTF) $(PROGRAMS)
+all: $(INTF) $(LIBS) $(PROGRAMS)
bins: $(PROGRAMS)
+libs: $(LIBS)
+
install: all
$(INSTALL_DIR) $(DESTDIR)$(SBINDIR)
$(INSTALL_PROG) oxenstored $(DESTDIR)$(SBINDIR)
diff --git a/tools/ocaml/xenstored/syslog.ml b/tools/ocaml/xenstored/syslog.ml
new file mode 100644
index 0000000..abeace7
--- /dev/null
+++ b/tools/ocaml/xenstored/syslog.ml
@@ -0,0 +1,47 @@
+(*
+ * Copyright (C) 2006-2009 Citrix Systems Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *)
+
+type level = Emerg | Alert | Crit | Err | Warning | Notice | Info | Debug
+type options = Cons | Ndelay | Nowait | Odelay | Perror | Pid
+type facility = Auth | Authpriv | Cron | Daemon | Ftp | Kern
+ | Local0 | Local1 | Local2 | Local3
+ | Local4 | Local5 | Local6 | Local7
+ | Lpr | Mail | News | Syslog | User | Uucp
+
+external log : facility -> level -> string -> unit =
"stub_syslog"
+
+exception Unknown_facility of string
+let facility_of_string s + match s with
+ |"auth"->Auth
+ |"authpriv"->Authpriv
+ |"cron"->Cron
+ |"daemon"->Daemon
+ |"ftp"->Ftp
+ |"kern"->Kern
+ |"local0"->Local0
+ |"local1"->Local1
+ |"local2"->Local2
+ |"local3"->Local3
+ |"local4"->Local4
+ |"local5"->Local5
+ |"local6"->Local6
+ |"local7"->Local7
+ |"lpr"->Lpr
+ |"mail"->Mail
+ |"news"->News
+ |"syslog"->Syslog
+ |"user"->User
+ |"uucp"->Uucp
+ |_-> raise (Unknown_facility s)
diff --git a/tools/ocaml/xenstored/syslog.mli b/tools/ocaml/xenstored/syslog.mli
new file mode 100644
index 0000000..ecac8a1
--- /dev/null
+++ b/tools/ocaml/xenstored/syslog.mli
@@ -0,0 +1,41 @@
+(*
+ * Copyright (C) 2006-2009 Citrix Systems Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *)
+
+type level = Emerg | Alert | Crit | Err | Warning | Notice | Info | Debug
+type facility + Auth
+ | Authpriv
+ | Cron
+ | Daemon
+ | Ftp
+ | Kern
+ | Local0
+ | Local1
+ | Local2
+ | Local3
+ | Local4
+ | Local5
+ | Local6
+ | Local7
+ | Lpr
+ | Mail
+ | News
+ | Syslog
+ | User
+ | Uucp
+
+external log : facility -> level -> string -> unit =
"stub_syslog"
+
+
+val facility_of_string : string -> facility
diff --git a/tools/ocaml/xenstored/syslog_stubs.c
b/tools/ocaml/xenstored/syslog_stubs.c
new file mode 100644
index 0000000..dd8b9e9
--- /dev/null
+++ b/tools/ocaml/xenstored/syslog_stubs.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2006-2009 Citrix Systems Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include <syslog.h>
+#include <string.h>
+#include <caml/mlvalues.h>
+#include <caml/memory.h>
+#include <caml/alloc.h>
+#include <caml/custom.h>
+#include <caml/signals.h>
+
+static int __syslog_level_table[] = {
+ LOG_EMERG, LOG_ALERT, LOG_CRIT, LOG_ERR, LOG_WARNING,
+ LOG_NOTICE, LOG_INFO, LOG_DEBUG
+};
+
+static int __syslog_facility_table[] = {
+ LOG_AUTH, LOG_AUTHPRIV, LOG_CRON, LOG_DAEMON, LOG_FTP, LOG_KERN,
+ LOG_LOCAL0, LOG_LOCAL1, LOG_LOCAL2, LOG_LOCAL3,
+ LOG_LOCAL4, LOG_LOCAL5, LOG_LOCAL6, LOG_LOCAL7,
+ LOG_LPR | LOG_MAIL | LOG_NEWS | LOG_SYSLOG | LOG_USER | LOG_UUCP
+};
+
+value stub_syslog(value facility, value level, value msg)
+{
+ CAMLparam3(facility, level, msg);
+ const char *c_msg = strdup(String_val(msg));
+ int c_facility = __syslog_facility_table[Int_val(facility)]
+ | __syslog_level_table[Int_val(level)];
+
+ caml_enter_blocking_section();
+ syslog(c_facility, "%s", c_msg);
+ caml_leave_blocking_section();
+
+ free((void*)c_msg);
+ CAMLreturn(Val_unit);
+}
--
1.8.1.2
David Scott
2013-Mar-20 10:32 UTC
[PATCH 2/3] oxenstored: enable logging via syslog, if specified in the config file.
To log to files the config file should contain:
xenstored-log-file = /var/log/xenstored.log
access-log-file = /var/log/xenstored-access.log
(These two files are still the built-in defaults. The log format is
unchanged.)
To log to syslog the config file should contain:
xenstored-log-file = syslog:<facility>
access-log-file = syslog:<facility>
where <facility> is the syslog facility to use (e.g.
''daemon'' ''local2'')
Signed-off-by: David Scott <dave.scott@eu.citrix.com>
---
tools/ocaml/xenstored/logging.ml | 124 +++++++++++++++++++++++++++----------
tools/ocaml/xenstored/xenstored.ml | 4 +-
2 files changed, 94 insertions(+), 34 deletions(-)
diff --git a/tools/ocaml/xenstored/logging.ml b/tools/ocaml/xenstored/logging.ml
index bc3a2ea..e26f804 100644
--- a/tools/ocaml/xenstored/logging.ml
+++ b/tools/ocaml/xenstored/logging.ml
@@ -20,11 +20,42 @@ open Printf
(* Logger common *)
+type log_destination + | File of string
+ | Syslog of Syslog.facility
+
+let log_destination_of_string s + let prefix = "syslog:" in
+ let len_prefix = String.length prefix in
+ let len = String.length s in
+ if String.startswith prefix s
+ then Syslog(Syslog.facility_of_string (String.sub s len_prefix (len -
len_prefix)))
+ else File s
+
+(* The prefix of a log line depends on the log destination *)
+let prefix log_destination ?level ?key date = match log_destination with
+ | File _ ->
+ let level = match level with
+ | Some x -> Printf.sprintf "|%5s" x
+ | None -> "" in
+ let key = match key with
+ | Some x -> "|" ^ x
+ | None -> "" in
+ Printf.sprintf "[%s%s%s] " date level key
+ | Syslog _ ->
+ let key = match key with
+ | Some x -> "[" ^ x ^ "] "
+ | None -> "" in
+ (* Syslog handles the date and level internally *)
+ key
+
+type level = Debug | Info | Warn | Error | Null
+
type logger { stop: unit -> unit;
restart: unit -> unit;
rotate: unit -> unit;
- write: ''a. (''a, unit, string, unit) format4 ->
''a }
+ write: ?level:level -> string -> unit }
let truncate_line nb_chars line =
if String.length line > nb_chars - 1 then
@@ -54,7 +85,7 @@ let log_rotate ref_ch log_file log_nb_files close_out
!ref_ch;
ref_ch := open_out log_file
-let make_logger log_file log_nb_files log_nb_lines log_nb_chars post_rotate
+let make_file_logger log_file log_nb_files log_nb_lines log_nb_chars
post_rotate let channel = ref (open_out_gen [Open_append; Open_creat] 0o644
log_file) in
let counter = ref 0 in
let stop() @@ -67,22 +98,17 @@ let make_logger log_file log_nb_files
log_nb_lines log_nb_chars post_rotate log_rotate channel log_file
log_nb_files;
(post_rotate (): unit);
counter := 0 in
- let output s + let write ?level s let s = if log_nb_chars > 0 then
truncate_line log_nb_chars s else s in
let s = s ^ "\n" in
output_string !channel s;
flush !channel;
incr counter;
if !counter > log_nb_lines then rotate() in
- { stop=stop; restart=restart; rotate=rotate; write = fun fmt ->
Printf.ksprintf output fmt }
-
-
-(* Xenstored logger *)
+ { stop=stop; restart=restart; rotate=rotate; write=write }
exception Unknown_level of string
-type level = Debug | Info | Warn | Error | Null
-
let int_of_level = function
| Debug -> 0 | Info -> 1 | Warn -> 2
| Error -> 3 | Null -> max_int
@@ -104,30 +130,56 @@ let string_of_date () tm.Unix.tm_hour tm.Unix.tm_min
tm.Unix.tm_sec
(int_of_float (1000.0 *. msec))
-let xenstored_log_file = ref "/var/log/xenstored.log"
+let make_syslog_logger facility + (* We defer to syslog for log management *)
+ let nothing () = () in
+ let write ?level s + let level = match level with
+ | Some Error -> Syslog.Err
+ | Some Warn -> Syslog.Warning
+ | Some Info -> Syslog.Info
+ | Some Debug -> Syslog.Debug
+ | Some Null -> Syslog.Debug
+ | None -> Syslog.Debug in
+ (* Syslog handles the date and level internally *)
+ Syslog.log facility level s in
+ { stop = nothing; restart = nothing; rotate = nothing; write=write }
+
+let xenstored_log_destination = ref (File "/var/log/xenstored.log")
let xenstored_log_level = ref Warn
let xenstored_log_nb_files = ref 10
let xenstored_log_nb_lines = ref 13215
let xenstored_log_nb_chars = ref (-1)
let xenstored_logger = ref (None: logger option)
-let init_xenstored_log () - if !xenstored_log_level <> Null &&
!xenstored_log_nb_files > 0 then
- let logger - make_logger
- !xenstored_log_file !xenstored_log_nb_files !xenstored_log_nb_lines
- !xenstored_log_nb_chars ignore in
- let date = string_of_date() in
- logger.write ("[%s|%5s|%s] Xen Storage Daemon, version %d.%d") date
"" "startup"
- Define.xenstored_major Define.xenstored_minor;
- xenstored_logger := Some logger
+let set_xenstored_log_destination s + xenstored_log_destination :=
log_destination_of_string s
+
+let set_xenstored_logger logger + xenstored_logger := Some logger;
+ logger.write ~level:Info (Printf.sprintf "Xen Storage Daemon, version
%d.%d"
+ Define.xenstored_major Define.xenstored_minor)
+
+
+let init_xenstored_log () = match !xenstored_log_destination with
+ | File file ->
+ if !xenstored_log_level <> Null && !xenstored_log_nb_files >
0 then
+ let logger + make_file_logger
+ file !xenstored_log_nb_files !xenstored_log_nb_lines
+ !xenstored_log_nb_chars ignore in
+ set_xenstored_logger logger
+ | Syslog facility ->
+ set_xenstored_logger (make_syslog_logger facility)
+
let xenstored_logging level key (fmt: (_,_,_,_) format4) match
!xenstored_logger with
| Some logger when int_of_level level >= int_of_level !xenstored_log_level
->
let date = string_of_date() in
- let level = string_of_level level in
- logger.write ("[%s|%5s|%s] " ^^ fmt) date level key
+ let level'' = string_of_level level in
+ let prefix = prefix !xenstored_log_destination ~level:level'' ~key
date in
+ Printf.ksprintf (fun s -> logger.write ~level (prefix ^ s)) fmt
| _ -> Printf.ksprintf ignore fmt
let debug key = xenstored_logging Debug key
@@ -200,7 +252,7 @@ let sanitize_data data String.escaped data
let activate_access_log = ref true
-let access_log_file = ref "/var/log/xenstored-access.log"
+let access_log_destination = ref (File
"/var/log/xenstored-access.log")
let access_log_nb_files = ref 20
let access_log_nb_lines = ref 13215
let access_log_nb_chars = ref 180
@@ -209,14 +261,20 @@ let access_log_transaction_ops = ref false
let access_log_special_ops = ref false
let access_logger = ref None
-let init_access_log post_rotate - if !access_log_nb_files > 0 then
- let logger - make_logger
- !access_log_file !access_log_nb_files !access_log_nb_lines
- !access_log_nb_chars post_rotate in
- access_logger := Some logger
-
+let set_access_log_destination s + access_log_destination :=
log_destination_of_string s
+
+let init_access_log post_rotate = match !access_log_destination with
+ | File file ->
+ if !access_log_nb_files > 0 then
+ let logger + make_file_logger
+ file !access_log_nb_files !access_log_nb_lines
+ !access_log_nb_chars post_rotate in
+ access_logger := Some logger
+ | Syslog facility ->
+ access_logger := Some (make_syslog_logger facility)
+
let access_logging ~con ~tid ?(data="") access_type try
maybe
@@ -225,7 +283,9 @@ let access_logging ~con ~tid ?(data="")
access_type let tid = string_of_tid ~con tid in
let access_type = string_of_access_type access_type in
let data = sanitize_data data in
- logger.write "[%s] %s %s %s" date tid access_type data)
+ let prefix = prefix !access_log_destination date in
+ let msg = Printf.sprintf "%s %s %s %s" prefix tid access_type
data in
+ logger.write msg)
!access_logger
with _ -> ()
diff --git a/tools/ocaml/xenstored/xenstored.ml
b/tools/ocaml/xenstored/xenstored.ml
index 64cc106..3416666 100644
--- a/tools/ocaml/xenstored/xenstored.ml
+++ b/tools/ocaml/xenstored/xenstored.ml
@@ -87,13 +87,13 @@ let parse_config filename ("quota-maxsize",
Config.Set_int Quota.maxsize);
("test-eagain", Config.Set_bool Transaction.test_eagain);
("persistent", Config.Set_bool Disk.enable);
- ("xenstored-log-file", Config.Set_string
Logging.xenstored_log_file);
+ ("xenstored-log-file", Config.String
Logging.set_xenstored_log_destination);
("xenstored-log-level", Config.String
(fun s -> Logging.xenstored_log_level := Logging.level_of_string s));
("xenstored-log-nb-files", Config.Set_int
Logging.xenstored_log_nb_files);
("xenstored-log-nb-lines", Config.Set_int
Logging.xenstored_log_nb_lines);
("xenstored-log-nb-chars", Config.Set_int
Logging.xenstored_log_nb_chars);
- ("access-log-file", Config.Set_string Logging.access_log_file);
+ ("access-log-file", Config.String
Logging.set_access_log_destination);
("access-log-nb-files", Config.Set_int
Logging.access_log_nb_files);
("access-log-nb-lines", Config.Set_int
Logging.access_log_nb_lines);
("access-log-nb-chars", Config.Set_int
Logging.access_log_nb_chars);
--
1.8.1.2
David Scott
2013-Mar-20 10:32 UTC
[PATCH 3/3] oxenstored: Allow oxenstored to use syslog at levels other than Debug
We now log different kinds of events at different levels. The convention
is now:
new/end_connection: Debug
coalesce: Debug
conflict: Debug
commit: Debug
regular ops: Info
start/end_transaction: Debug
error (ENOENT): Debug
error (any other): Warn
watch: Info
Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
---
tools/ocaml/xenstored/logging.ml | 34 +++++++++++++++++-----------------
1 file changed, 17 insertions(+), 17 deletions(-)
diff --git a/tools/ocaml/xenstored/logging.ml b/tools/ocaml/xenstored/logging.ml
index e26f804..3d4a329 100644
--- a/tools/ocaml/xenstored/logging.ml
+++ b/tools/ocaml/xenstored/logging.ml
@@ -275,7 +275,7 @@ let init_access_log post_rotate = match
!access_log_destination with
| Syslog facility ->
access_logger := Some (make_syslog_logger facility)
-let access_logging ~con ~tid ?(data="") access_type +let
access_logging ~con ~tid ?(data="") ~level access_type try
maybe
(fun logger ->
@@ -285,18 +285,18 @@ let access_logging ~con ~tid ?(data="")
access_type let data = sanitize_data data in
let prefix = prefix !access_log_destination date in
let msg = Printf.sprintf "%s %s %s %s" prefix tid access_type
data in
- logger.write msg)
+ logger.write ~level msg)
!access_logger
with _ -> ()
-let new_connection = access_logging Newconn
-let end_connection = access_logging Endconn
+let new_connection = access_logging ~level:Debug Newconn
+let end_connection = access_logging ~level:Debug Endconn
let read_coalesce ~tid ~con data if !access_log_read_ops
- then access_logging Coalesce ~tid ~con ~data:("read "^data)
-let write_coalesce data = access_logging Coalesce ~data:("write
"^data)
-let conflict = access_logging Conflict
-let commit = access_logging Commit
+ then access_logging Coalesce ~tid ~con ~data:("read "^data)
~level:Debug
+let write_coalesce data = access_logging Coalesce ~data:("write
"^data) ~level:Debug
+let conflict = access_logging Conflict ~level:Debug
+let commit = access_logging Commit ~level:Debug
let xb_op ~tid ~con ~ty data let print = match ty with
@@ -306,21 +306,21 @@ let xb_op ~tid ~con ~ty data | Xenbus.Xb.Op.Introduce |
Xenbus.Xb.Op.Release | Xenbus.Xb.Op.Getdomainpath | Xenbus.Xb.Op.Isintroduced |
Xenbus.Xb.Op.Resume ->
!access_log_special_ops
| _ -> true in
- if print then access_logging ~tid ~con ~data (XbOp ty)
+ if print then access_logging ~tid ~con ~data (XbOp ty) ~level:Info
let start_transaction ~tid ~con =
if !access_log_transaction_ops && tid <> 0
- then access_logging ~tid ~con (XbOp Xenbus.Xb.Op.Transaction_start)
+ then access_logging ~tid ~con (XbOp Xenbus.Xb.Op.Transaction_start)
~level:Debug
let end_transaction ~tid ~con =
if !access_log_transaction_ops && tid <> 0
- then access_logging ~tid ~con (XbOp Xenbus.Xb.Op.Transaction_end)
+ then access_logging ~tid ~con (XbOp Xenbus.Xb.Op.Transaction_end) ~level:Debug
let xb_answer ~tid ~con ~ty data - let print = match ty with
- | Xenbus.Xb.Op.Error when String.startswith "ENOENT " data ->
!access_log_read_ops
- | Xenbus.Xb.Op.Error -> true
- | Xenbus.Xb.Op.Watchevent -> true
- | _ -> false
+ let print, level = match ty with
+ | Xenbus.Xb.Op.Error when String.startswith "ENOENT" data ->
!access_log_read_ops , Warn
+ | Xenbus.Xb.Op.Error -> true , Warn
+ | Xenbus.Xb.Op.Watchevent -> true , Info
+ | _ -> false, Debug
in
- if print then access_logging ~tid ~con ~data (XbOp ty)
+ if print then access_logging ~tid ~con ~data (XbOp ty) ~level
--
1.8.1.2
On Wed, 2013-03-20 at 10:32 +0000, David Scott wrote:> The ability to log via syslog was removed back in Oct 2011 but > it''s actually quite useful. On a system with lots of disk I/O, if > xenstore logs directly to the filesystem it can block so much > that guests start to suffer. > > One solution would be to add some threading to xenstored to isolate > the logging from the ring servicing; however this will be quite a > big patch and we could end up re-implementing a syslog-like logging > service. Instead we re-add the ability to log via syslog and allow > this to be switched on via the xenstored config file. The default > logging behaviour is unchanged. > > FYI this is the only significant thing in the XCP patch queue -- > we''ve found this helps a lot in stress tests. > > Patches follow. For extra convenience you should be able to pull from: > > git pull git://github.com/djs55/xen xenstore-syslogI''m not really in a position to review this but as you say it is just reintroducing previously removed code and its been in XCP I presume in the interval, so I have pulled this.