Add a YAML logfile that records changes to the index and modify sup-dump to use
this rather than the normal database. The log is index format/version agnostic
so that users can switch between incompatible Sup versions without running
sup-dump first.
This should also make automated backups easier.
---
bin/sup-dump | 19 +++++++++++++------
lib/sup/ferret_index.rb | 7 +++++++
lib/sup/index.rb | 22 ++++++++++++++++++++++
lib/sup/xapian_index.rb | 7 ++++++-
lib/sup/yaml_log.rb | 25 +++++++++++++++++++++++++
5 files changed, 73 insertions(+), 7 deletions(-)
create mode 100644 lib/sup/yaml_log.rb
diff --git a/bin/sup-dump b/bin/sup-dump
index ba36b21..531a30a 100755
--- a/bin/sup-dump
+++ b/bin/sup-dump
@@ -2,7 +2,8 @@
require ''rubygems''
require ''trollop''
-require "sup"
+require ''sup'' # Redwood::VERSION, Redwood::BASE_DIR
+require "sup/yaml_log"
$opts = Trollop::options do
version "sup-dump (sup #{Redwood::VERSION})"
@@ -21,10 +22,16 @@ No options.
EOS
end
-index = Redwood::Index.new
-Redwood::SourceManager.new
-index.load
+labels = {}
-index.each_message :load_spam => true, :load_deleted => true,
:load_killed => true do |m|
- puts "#{m.id} (#{m.labels * '' ''})"
+Redwood::log "processing index log"
+index_log = YamlLogReader.new File.join(Redwood::BASE_DIR,
''index_log.yaml'')
+index_log.each do |h|
+ case h[''type'']
+ when ''add_message'',
''update_message_state''
+ labels[h[''id'']] = h[''labels'']
+ end
end
+
+Redwood::log "dumping labels"
+labels.each { |msgid,labels| puts "#{msgid} (#{labels * ''
''})" }
diff --git a/lib/sup/ferret_index.rb b/lib/sup/ferret_index.rb
index 98ea9b5..2cb9759 100644
--- a/lib/sup/ferret_index.rb
+++ b/lib/sup/ferret_index.rb
@@ -57,6 +57,7 @@ EOS
def sync_message m, opts={}
entry = @index[m.id]
+ existed = !entry.nil?
raise "no source info for message #{m.id}" unless m.source
&& m.source_info
@@ -131,6 +132,12 @@ EOS
}
@index_mutex.synchronize do
+ if existed
+ @log.update_message_state m.id, m.labels
+ else
+ @log.add_message m.id, m.labels
+ end
+
@index.delete m.id
@index.add_document d
end
diff --git a/lib/sup/index.rb b/lib/sup/index.rb
index 54ec843..7360cf5 100644
--- a/lib/sup/index.rb
+++ b/lib/sup/index.rb
@@ -1,6 +1,7 @@
## Index interface, subclassed by Ferret indexer.
require ''fileutils''
+require ''sup/yaml_log''
begin
require ''chronic''
@@ -65,6 +66,7 @@ class BaseIndex
def load
SourceManager.load_sources
+ @log = IndexLogWriter.new File.join(@dir,
''index_log.yaml'')
load_index
end
@@ -176,6 +178,26 @@ class BaseIndex
def parse_query s
unimplemented
end
+
+ private
+
+ class IndexLogWriter < YamlLogWriter
+ def update_message_state id, labels
+ write_entry ''update_message_state'',
''id'' => id, ''labels'' => labels.map {
|x| x.to_s }
+ end
+
+ def add_message id, labels
+ write_entry ''add_message'', ''id'' =>
id, ''labels'' => labels.map { |x| x.to_s }
+ end
+
+ def remove_message id
+ write_entry ''remove_message'', ''id''
=> id
+ end
+
+ def write_entry type, hash
+ self << hash.merge(''type'' => type,
''time'' => Time.now)
+ end
+ end
end
index_name = ENV[''SUP_INDEX''] || $config[:index] ||
DEFAULT_INDEX
diff --git a/lib/sup/xapian_index.rb b/lib/sup/xapian_index.rb
index 18b5050..c4dbc5f 100644
--- a/lib/sup/xapian_index.rb
+++ b/lib/sup/xapian_index.rb
@@ -61,7 +61,10 @@ class XapianIndex < BaseIndex
end
def delete id
- synchronize { @xapian.delete_document mkterm(:msgid, id) }
+ synchronize do
+ @log.remove_message id
+ @xapian.delete_document mkterm(:msgid, id)
+ end
end
def build_message id
@@ -510,10 +513,12 @@ class XapianIndex < BaseIndex
Redwood::log "warning: docid underflow, dropping
#{m.id.inspect}"
return
end
+ @log.add_message m.id, m.labels
else
doc.clear_terms
doc.clear_values
docid = doc.docid
+ @log.update_message_state m.id, m.labels
end
@term_generator.document = doc
diff --git a/lib/sup/yaml_log.rb b/lib/sup/yaml_log.rb
new file mode 100644
index 0000000..325cca9
--- /dev/null
+++ b/lib/sup/yaml_log.rb
@@ -0,0 +1,25 @@
+class YamlLogReader
+ include Enumerable
+
+ def initialize filename
+ @io = File.open(filename, ''r+'')
+ end
+
+ def each &b
+ @io.rewind
+ YAML.each_document @io, &b
+ end
+end
+
+class YamlLogWriter
+ def initialize filename
+ @io = File.open(filename, ''a'')
+ end
+
+ def <<(o)
+ YAML.dump o, @io
+
+ ## This only flushes to the OS. We may want to fsync occasionally too.
+ @io.flush
+ end
+end
--
1.6.4