Snapshots update - storage of packages works from clients (old-style statistics)

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@5092 ce84e21b-d406-0410-9b95-82705330c041

5 files changed, 140 insertions, 11 deletions

diff --git a/src/lib/Server/Admin/Snapshots.py b/src/lib/Server/Admin/Snapshots.py
new file mode 100644
index 000000000..17db0af62
--- /dev/null
+++ b/src/lib/Server/Admin/Snapshots.py
@@ -0,0 +1,29 @@
+import Bcfg2.Server.Admin
+import sqlalchemy, sqlalchemy.orm
+import Bcfg2.Server.Snapshots
+import Bcfg2.Server.Snapshots.model
+
+class Snapshots(Bcfg2.Server.Admin.Mode):
+    __shorthelp__ = "Interact with the Snapshots system"
+    __longhelp__ = (__shorthelp__)
+    __usage__ = ("bcfg2-admin snapshots [init|query qtype] ")
+
+    q_dispatch = {'client':Bcfg2.Server.Snapshots.model.Client,
+                  'group':Bcfg2.Server.Snapshots.model.Group,
+                  'snapshot':Bcfg2.Server.Snapshots.model.Snapshot,
+                  }
+
+    def __init__(self, configfile):
+        Bcfg2.Server.Admin.Mode.__init__(self, configfile)
+        self.session = Bcfg2.Server.Snapshots.setup_session(debug=True)
+
+    def __call__(self, args):
+        if args[0] == 'query':
+            if args[1] in self.q_dispatch:
+                q_obj = self.q_dispatch[args[1]]
+                results = self.session.query(q_obj).all()
+            else:
+                print 'error'
+                raise SystemExit, 1
+            for result in results:
+                print result.name
diff --git a/src/lib/Server/Admin/__init__.py b/src/lib/Server/Admin/__init__.py
index 075822a56..e92020e5d 100644
--- a/src/lib/Server/Admin/__init__.py
+++ b/src/lib/Server/Admin/__init__.py
@@ -1,7 +1,7 @@
 __revision__ = '$Revision$'
 
 __all__ = ['Mode', 'Client', 'Compare', 'Fingerprint', 'Init', 'Minestruct',
-           'Pull', 'Query', 'Tidy', 'Viz']
+           'Pull', 'Query', 'Snapshots', 'Tidy', 'Viz']
 
 import ConfigParser, lxml.etree, logging, sys
 import Bcfg2.Server.Core
diff --git a/src/lib/Server/Plugins/Snapshots.py b/src/lib/Server/Plugins/Snapshots.py
new file mode 100644
index 000000000..f5b69d18f
--- /dev/null
+++ b/src/lib/Server/Plugins/Snapshots.py
@@ -0,0 +1,56 @@
+import lxml.etree
+import sqlalchemy
+import sqlalchemy.orm
+import Bcfg2.Server.Plugin
+import Bcfg2.Server.Snapshots
+from Bcfg2.Server.Snapshots.model import Snapshot
+import time
+
+class Snapshots(Bcfg2.Server.Plugin.Statistics,
+                Bcfg2.Server.Plugin.Plugin):
+    name = 'Snapshots'
+    experimental = True
+    def __init__(self, core, datastore):
+        Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+        Bcfg2.Server.Plugin.Statistics.__init__(self)
+        self.session = Bcfg2.Server.Snapshots.db_from_config()
+
+    def process_statistics(self, metadata, data):
+        return self.statistics_from_old_stats(metadata, data)
+
+    def statistics_from_old_stats(self, metadata, xdata):
+        # entries are name -> (modified, correct, start, desired, end)
+        # not sure we can get all of this from old format stats
+        t1 = time.time()
+        entries = dict([('Package', dict()),
+                        ('Service', dict()), ('Path', dict())])
+        extra = dict([('Package', list()), ('Service', list()),
+                      ('Path', list())])
+        for entry in xdata.find('.//Bad'):
+            print entry.tag, entry.get('name')
+            if entry.tag == 'Package':
+                data = [False, False, unicode(entry.get('type')),
+                        unicode(entry.get('current_version')),
+                        unicode(entry.get('current_version'))]
+                entries['Package'][entry.get('name')] = data
+        for entry in xdata.find('.//Modified'):
+            print entry.tag, entry.get('name')
+            if entry.tag == 'Package':
+                if entry.get('name') in entries['Package']:
+                    entries['Package'][entry.get('name')][0] = True
+                else:
+                    data = [True, True, unicode(entry.get('type')),
+                            unicode(entry.get('current_version')),
+                            unicode(entry.get('version'))]
+        for entry in xdata.find('.//Extra'):
+            if entry.tag == 'Package':
+                edata = dict([('name', unicode(entry.get('name'))),
+                              ('type', unicode(entry.get('type'))),
+                              ('version', unicode(entry.get('version')))]) 
+                extra['Package'].append(edata)
+        t2 = time.time()
+        snap = Snapshot.from_data(self.session, metadata, entries, extra)
+        self.session.save(snap)
+        self.session.commit()
+        t3 = time.time()
+        return True
diff --git a/src/lib/Server/Snapshots/__init__.py b/src/lib/Server/Snapshots/__init__.py
index 74a3894d7..b94511ebd 100644
--- a/src/lib/Server/Snapshots/__init__.py
+++ b/src/lib/Server/Snapshots/__init__.py
@@ -1 +1,21 @@
-__all__ = ['models']
+__all__ = ['models', 'db_from_config', 'setup_session']
+
+import sqlalchemy, sqlalchemy.orm, ConfigParser
+
+def db_from_config(fname='/etc/bcfg2.conf'):
+    cp = ConfigParser.ConfigParser()
+    cp.read([fname])
+    driver = cp.get('snapshots', 'driver')
+    if driver == 'sqlite':
+        path = cp.get('snapshots', 'database')
+        return 'sqlite:///%s' % path
+    else:
+        raise Exception, "not done yet"
+
+
+def setup_session(debug=False):
+    engine = sqlalchemy.create_engine(db_from_config(),
+                                      echo=debug)
+    Session = sqlalchemy.orm.sessionmaker()
+    Session.configure(bind=engine)
+    return Session()
diff --git a/src/lib/Server/Snapshots/model.py b/src/lib/Server/Snapshots/model.py
index 5489edff5..97331ffbf 100644
--- a/src/lib/Server/Snapshots/model.py
+++ b/src/lib/Server/Snapshots/model.py
@@ -1,5 +1,5 @@
 from sqlalchemy import Table, Column, Integer, Unicode, MetaData, ForeignKey, Boolean, DateTime, create_engine, UnicodeText
-
+import datetime
 import sqlalchemy.exceptions
 from sqlalchemy.orm import relation, backref, sessionmaker
 from sqlalchemy.ext.declarative import declarative_base
@@ -63,11 +63,11 @@ class Metadata(Base):
     timestamp = Column(DateTime)
 
     @classmethod
-    def from_metadata(cls, session, metadata):
-        client = Client.by_value(session, name=metadata.hostname)
+    def from_metadata(cls, mysession, mymetadata):
+        client = Client.by_value(mysession, name=unicode(mymetadata.hostname))
         m = cls(client=client)
         for group in metadata.groups:
-            m.groups.append(Group.by_value(session, name=unicode(group)))
+            m.groups.append(Group.by_value(mysession, name=unicode(group)))
         for connector in metadata.connectors:
             data = getattr(metadata, connector)
             if not isinstance(data, dict):
@@ -75,13 +75,13 @@ class Metadata(Base):
             for key, value in data.iteritems():
                 if not isinstance(value, str):
                     continue
-                m.keyvals.append(ConnectorKeyVal.by_value(session,
+                m.keyvals.append(ConnectorKeyVal.by_value(mysession,
                                                           connector=unicode(connector),
                                                           key=unicode(key),
                                                           value=unicode(value)))
         return m
 
-class Package(Base):
+class Package(Base, Uniquer):
     __tablename__ = 'package'
     id = Column(Integer, primary_key=True)        
     name = Column(Unicode(24))
@@ -99,6 +99,18 @@ class PackageCorrespondence(Base):
     modified = Column(Boolean)
     correct = Column(Boolean)
 
+    @classmethod
+    def from_record(cls, mysession, name, record):
+        (mod, corr, ptype, s_vers, e_vers) = record
+        start = Package.by_value(mysession, name=unicode(name), type=ptype,
+                                 version=s_vers)
+        if s_vers != e_vers:
+            start = Package.by_value(mysession, name=unicode(name), type=ptype,
+                                     version=e_vers)
+        else:
+            end = start
+        return cls(start=start, end=end, modified=mod, correct=corr)
+
 package_snap = Table('package_snap', Base.metadata,
                      Column('ppair_id', Integer, ForeignKey('package_pair.id')),
                      Column('snapshot_id', Integer, ForeignKey('snapshot.id')))
@@ -142,7 +154,7 @@ class FileCorrespondence(Base):
     end_id = Column(Integer, ForeignKey('file.id'), nullable=True)
     end = relation(File, primaryjoin=end_id == File.id)
     modified = Column(Boolean)
-    correct = Column(Boolean)    
+    correct = Column(Boolean)
 
 file_snap = Table('file_snap', Base.metadata,
                   Column('fpair_id', Integer, ForeignKey('file_pair.id')),
@@ -176,7 +188,7 @@ class Snapshot(Base):
     id = Column(Integer, primary_key=True)
     metadata_id = Column(Integer, ForeignKey('metadata.id'))
     client_metadata = relation(Metadata, primaryjoin=metadata_id==Metadata.id)
-    timestamp = Column(DateTime)
+    timestamp = Column(DateTime, default=datetime.datetime.now)
     client_id = Column(Integer, ForeignKey('client.id'))
     client = relation(Client, backref=backref('snapshots'))
     packages = relation(PackageCorrespondence, secondary=package_snap)
@@ -187,8 +199,20 @@ class Snapshot(Base):
     extra_services = relation(Service, secondary=extra_service_snap)
     extra_files = relation(File, secondary=extra_file_snap)
 
+    @classmethod
+    def from_data(cls, session, metadata, entries, extra):
+        dbm = Metadata.from_metadata(session, metadata)
+        snap = cls(client_metadata=dbm, timestamp=datetime.datetime.now())
+        for pkg, pdata in entries['Package'].iteritems():
+            snap.packages.append(\
+                PackageCorrespondence.from_record(session, pkg, pdata))
+        for data in extra['Package']:
+            extra_pkg = Package.by_value(session, **data)
+            snap.extra_packages.append(extra_pkg)
+        return snap
+
 if __name__ == '__main__':
-    engine = create_engine('sqlite:///:memory:', echo=True)
+    engine = create_engine('sqlite:////tmp/snapshots.db', echo=True)
     metadata = Base.metadata
     metadata.create_all(engine) 
     Session = sessionmaker()