Add reporting system schema evolution support (from Stousignant)

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@4884 ce84e21b-d406-0410-9b95-82705330c041

9 files changed, 276 insertions, 200 deletions

diff --git a/setup.py b/setup.py
index 0ddf1b696..82e77901c 100644
--- a/setup.py
+++ b/setup.py
@@ -23,7 +23,8 @@ setup(name="Bcfg2",
                 "Bcfg2.tlslite.utils",
                 "Bcfg2.tlslite.integration",
                 ],
-      package_dir = {'Bcfg2':'src/lib'}, 
+      package_dir = {'Bcfg2':'src/lib'},
+      package_data = {'Bcfg2.Server.Reports.reports':['fixtures/*.xml']},
       scripts = glob('src/sbin/*'),
       data_files = [('share/bcfg2/schemas',
                      glob('schemas/*.xsd')),
diff --git a/src/lib/Server/Plugins/DBStats.py b/src/lib/Server/Plugins/DBStats.py
index f3e4fe652..85ba2668b 100644
--- a/src/lib/Server/Plugins/DBStats.py
+++ b/src/lib/Server/Plugins/DBStats.py
@@ -1,15 +1,27 @@
 import Bcfg2.Server.Plugin
 import Bcfg2.Server.Reports.importscript
-from Bcfg2.Server.Reports.reports.models import Client
-import difflib, lxml.etree, time
+from Bcfg2.Server.Reports.reports.models import Client, Entries
+import difflib, lxml.etree, time, logging, datetime
+import Bcfg2.Server.Reports.settings
+
+from Bcfg2.Server.Reports.updatefix import update_database
+import traceback
+# for debugging output only
+logger = logging.getLogger('Bcfg2.Plugins.DBStats')
 
 class DBStats(Bcfg2.Server.Plugin.StatisticsPlugin):
     __name__ = 'DBStats'
-    __version__ = '$Id: $'
-
+    __version__ = '$Id$'
+        
     def __init__(self, core, datastore):
         self.cpath = "%s/Metadata/clients.xml" % datastore
         self.core = core
+        logger.debug("Searching for new models to add to the statistics database")
+        try:
+            update_database()
+        except Exception, inst:
+            logger.debug(str(inst))
+            logger.debug(str(type(inst)))
 
     def StoreStatistics(self, mdata, xdata):
         newstats = xdata.find("Statistics")
@@ -20,18 +32,21 @@ class DBStats(Bcfg2.Server.Plugin.StatisticsPlugin):
         container.append(e)
         
         # FIXME need to build a metadata interface to expose a list of clients
+        # FIXME Server processing the request should be mentionned here
+        start = time.time()
         Bcfg2.Server.Reports.importscript.load_stats(
             self.core.metadata.clientdata, container, 0, True)
+        logger.info("Imported data in the reason fast path in %s second" % (time.time() - start))
 
     def GetExtra(self, client):
         c_inst = Client.objects.filter(name=client)[0]
         return [(a.kind, a.name) for a in
-                c_inst.current_interaction.extra_items.all()]
+                c_inst.current_interaction.extra()]
 
     def GetCurrentEntry(self, client, e_type, e_name):
         c_inst = Client.objects.filter(name=client)[0]
-        result = c_inst.current_interaction.bad_items.filter(kind=e_type,
-                                                             name=e_name)
+        result = c_inst.current_interaction.bad().filter(entry__kind=e_type,
+                                                         entry__name=e_name)
         if not result:
             raise Bcfg2.Server.Plugin.PluginExecutionError
         entry = result[0]
diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py
index 30657fd07..e1ae52efb 100755
--- a/src/lib/Server/Reports/importscript.py
+++ b/src/lib/Server/Reports/importscript.py
@@ -17,12 +17,13 @@ sys.path.pop()
 # Set DJANGO_SETTINGS_MODULE appropriately.
 os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name
 
-from Bcfg2.Server.Reports.reports.models import Client, Interaction, Bad, Modified, Extra, Performance, Reason, Ping
+from Bcfg2.Server.Reports.reports.models import Client, Interaction, Entries, Entries_interactions, Performance, Reason, Ping, TYPE_CHOICES, InternalDatabaseVersion
 from lxml.etree import XML, XMLSyntaxError
 from getopt import getopt, GetoptError
 from datetime import datetime
 from time import strptime
 from django.db import connection
+from Bcfg2.Server.Reports.updatefix import update_database
 import ConfigParser
 import difflib
 
@@ -52,7 +53,7 @@ def build_reason_kwargs(r_ent):
                 current_diff=rc_diff)
 
 
-def load_stats(cdata, sdata, vlevel, quick=False):
+def load_stats(cdata, sdata, vlevel, quick=False, location=''):
     cursor = connection.cursor()
     clients = {}
     cursor.execute("SELECT name, id from reports_client;")
@@ -90,10 +91,10 @@ def load_stats(cdata, sdata, vlevel, quick=False):
             ilist = Interaction.objects.filter(client=c_inst,
                                                timestamp=timestamp)
             if ilist:
-                current_interaction_id = ilist[0].id
+                current_interaction = ilist[0]
                 if vlevel > 0:
                     print("Interaction for %s at %s with id %s already exists"%(clients[name],
-                        datetime(t[0],t[1],t[2],t[3],t[4],t[5]),current_interaction_id))
+                        datetime(t[0],t[1],t[2],t[3],t[4],t[5]),current_interaction.id))
                 continue
             else:
                 newint = Interaction(client=c_inst,
@@ -102,18 +103,19 @@ def load_stats(cdata, sdata, vlevel, quick=False):
                                      repo_revision=statistics.get('revision',default="unknown"),
                                      client_version=statistics.get('client_version',default="unknown"),
                                      goodcount=statistics.get('good',default="0"),
-                                     totalcount=statistics.get('total',default="0"))
+                                     totalcount=statistics.get('total',default="0"),
+                                     server=location)
                 newint.save()
-                current_interaction_id = newint.id
+                current_interaction = newint
                 if vlevel > 0:
                     print("Interaction for %s at %s with id %s INSERTED in to db"%(clients[name],
-                        timestamp, current_interaction_id))
+                        timestamp, current_interaction.id))
 
 
-            pattern = [('Bad/*', Bad, 'reports_bad'),
-                       ('Extra/*', Extra, 'reports_extra'),
-                       ('Modified/*', Modified, 'reports_modified')]
-            for (xpath, obj, tablename) in pattern:
+            pattern = [('Bad/*', TYPE_CHOICES[0]),
+                       ('Extra/*', TYPE_CHOICES[2]),
+                       ('Modified/*', TYPE_CHOICES[1]),]
+            for (xpath, type) in pattern:
                 for x in statistics.findall(xpath):
                     kargs = build_reason_kwargs(x)
                     if not quick:
@@ -130,30 +132,23 @@ def load_stats(cdata, sdata, vlevel, quick=False):
                         rr.save()
                         if vlevel > 0:
                             print "Created reason: %s" % rr.id
-                    if not quick:
-                        links = obj.objects.filter(name=x.get('name'),
-                                                   kind=x.tag,
-                                                   reason=rr)
-                    else:
-                        links = []
+
+                    links = Entries.objects.filter(name=x.get('name'),
+                                               kind=x.tag)
                         
                     if links:
-                        item_id = links[0].id
-                        if vlevel > 0:
-                            print "%s item exists, has reason id %s and ID %s" % (xpath, rr.id, item_id)
+                        entry = links[0]
                     else:
-                        newitem = obj(name=x.get('name'),
-                                      kind=x.tag, critical=False,
-                                      reason=rr)
-                        newitem.save()
-                        item_id = newitem.id
-                        if vlevel > 0:
-                            print "%s item INSERTED having reason id %s and ID %s" % (xpath, rr.id, item_id)
-                    try:
-                        cursor.execute("INSERT INTO "+tablename+"_interactions VALUES (NULL, %s, %s);",
-                                       [item_id, current_interaction_id])
-                    except:
-                        pass                    
+                        entry = Entries(name=x.get('name'),
+                                      kind=x.tag)
+                        entry.save()
+
+                    interaction = Entries_interactions(entry=entry, reason=rr,
+                                                        interaction=current_interaction,
+                                                        type=type[0])
+                    interaction.save()
+                    if vlevel > 0:
+                        print "%s interaction created with reason id %s and entry %s" % (xpath, rr.id, entry.id)
 
             for times in statistics.findall('OpStamps'):
                 for metric, value in times.items():
@@ -170,7 +165,7 @@ def load_stats(cdata, sdata, vlevel, quick=False):
                         item_id = mperf.id
                     try:
                         cursor.execute("INSERT INTO reports_performance_interaction VALUES (NULL, %s, %s);",
-                                       [item_id, current_interaction_id])
+                                       [item_id, current_interaction.id])
                     except:
                         pass
 
@@ -280,4 +275,11 @@ if __name__ == '__main__':
         raise SystemExit, 1
 
     q = '-O3' in sys.argv
-    load_stats(clientsdata, statsdata, verb, quick=q)
+    try:
+        location = cf.get('components', 'bcfg2')
+    except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+        print "Could not read bcfg2 from bcfg2.conf; exiting"
+        raise SystemExit, 1
+    # Be sure the database is ready for new schema
+    update_database()
+    load_stats(clientsdata, statsdata, verb, quick=q, location=location)
diff --git a/src/lib/Server/Reports/reports/models.py b/src/lib/Server/Reports/reports/models.py
index b54d83912..333618a56 100644
--- a/src/lib/Server/Reports/reports/models.py
+++ b/src/lib/Server/Reports/reports/models.py
@@ -18,6 +18,15 @@ PING_CHOICES = (
     ('Up (Y)', 'Y'),
     ('Down (N)', 'N')
 )
+TYPE_BAD = 1
+TYPE_MODIFIED = 2
+TYPE_EXTRA = 3
+
+TYPE_CHOICES = (
+    (TYPE_BAD, 'Bad'),
+    (TYPE_MODIFIED, 'Modified'),
+    (TYPE_EXTRA, 'Extra'),
+)
 class ClientManager(models.Manager):
     '''extended client manager functions'''
     def active(self,timestamp='now'):
@@ -57,19 +66,6 @@ class Client(models.Model):
     class Admin:
         pass
 
-class Metadata(models.Model):
-    '''insert magical interface to client metadata here'''
-    client = models.ForeignKey(Client)
-    timestamp = models.DateTimeField()
-    def __str__(self):
-        return self.timestamp
-    
-class Repository(models.Model):
-    '''insert magical interface to subversioned repository here'''
-    timestamp = models.DateTimeField()
-    def __str__(self):
-        return self.timestamp
-
 class Ping(models.Model):
     '''represents a ping of a client (sparsely)'''
     client = models.ForeignKey(Client, related_name="pings")
@@ -116,6 +112,7 @@ class Interaction(models.Model):
     client_version = models.CharField(maxlength=32)#Client Version
     goodcount = models.IntegerField()#of good config-items
     totalcount = models.IntegerField()#of total config-items
+    server = models.CharField(maxlength=256)    # Name of the server used for the interaction
 
     def __str__(self):
         return "With " + self.client.name + " @ " + self.timestamp.isoformat()
@@ -158,7 +155,16 @@ class Interaction(models.Model):
         super(Interaction, self).save() #call the real save...
         self.client.current_interaction = self.client.interactions.latest()
         self.client.save()#save again post update
-            
+
+    def bad(self):
+        return Entries_interactions.objects.select_related().filter(interaction=self, type=TYPE_BAD)
+
+    def modified(self):
+        return Entries_interactions.objects.select_related().filter(interaction=self, type=TYPE_MODIFIED)
+
+    def extra(self):
+        return Entries_interactions.objects.select_related().filter(interaction=self, type=TYPE_EXTRA)
+    
     objects = InteractiveManager()
 
     class Admin:
@@ -187,36 +193,21 @@ class Reason(models.Model):
     def _str_(self):
         return "Reason"
 
-class Modified(models.Model):
-    '''Modified configuration element'''
-    interactions = models.ManyToManyField(Interaction, related_name="modified_items")
-    name = models.CharField(maxlength=128, core=True)
-    kind = models.CharField(maxlength=16, choices=KIND_CHOICES)
-    critical = models.BooleanField()
-    reason = models.ForeignKey(Reason)
+class Entries(models.Model):
+    """ Contains all the entries feed by the client """
+    name = models.CharField(maxlength=128, core=True, db_index=True)
+    kind = models.CharField(maxlength=16, choices=KIND_CHOICES, db_index=True)
+
     def __str__(self):
         return self.name
-   
-class Extra(models.Model):
-    '''Extra configuration element'''
-    interactions = models.ManyToManyField(Interaction, related_name="extra_items")
-    name = models.CharField(maxlength=128, core=True)
-    kind = models.CharField(maxlength=16, choices=KIND_CHOICES)
-    critical = models.BooleanField()
+
+class Entries_interactions(models.Model):
+    """ Define the relation between the reason, the interaction and the entry """
+    entry = models.ForeignKey(Entries)
     reason = models.ForeignKey(Reason)
-    def __str__(self):
-        return self.name
+    interaction = models.ForeignKey(Interaction)
+    type = models.IntegerField(choices=TYPE_CHOICES)
     
-class Bad(models.Model):
-    '''Bad configuration element'''
-    interactions = models.ManyToManyField(Interaction, related_name="bad_items")
-    name = models.CharField(maxlength=128, core=True)
-    kind = models.CharField(maxlength=16, choices=KIND_CHOICES)
-    critical = models.BooleanField()
-    reason = models.ForeignKey(Reason)
-    def __str__(self):
-        return self.name
- 
 class PerformanceManager(models.Manager):
     '''Provides ability to effectively query for performance information
     It is possible this should move to the view'''
@@ -260,3 +251,10 @@ class Performance(models.Model):
  
     objects = PerformanceManager()
  
+class InternalDatabaseVersion(models.Model):
+    '''Object that tell us to witch version is the database'''
+    version = models.IntegerField()
+    updated = models.DateTimeField(auto_now_add=True)
+
+    def __str__(self):
+        return "version %d updated the %s" % (self.version, self.updated.isoformat())
diff --git a/src/lib/Server/Reports/reports/templates/clients/client-nodebox.html b/src/lib/Server/Reports/reports/templates/clients/client-nodebox.html
index 77621cace..88cf05345 100644
--- a/src/lib/Server/Reports/reports/templates/clients/client-nodebox.html
+++ b/src/lib/Server/Reports/reports/templates/clients/client-nodebox.html
@@ -26,33 +26,33 @@
 	    <div class="warning">
 	        <span class="nodelisttitle">This node did not run within the last 24 hours-- it may be out of date.</span>
 	    </div>
-	    {% endif %}
-	    {% if interaction.bad_items.all %}
+	{% endif %}
+	{% if interaction.bad %}
 	    <div class="bad">
-		<span class="nodelisttitle"><a href="javascript:toggleLayer('{{client.name}}-bad');" title="Click to expand" class="commentLink">{{interaction.bad_items.count}}</a> items did not verify and are considered Dirty.<br /></span>
+		<span class="nodelisttitle"><a href="javascript:toggleLayer('{{client.name}}-bad');" title="Click to expand" class="commentLink">{{interaction.bad.count}}</a> items did not verify and are considered Dirty.<br /></span>
 		<div class="items" id="{{client.name}}-bad"><ul class="plain">
-		{% for bad in interaction.bad_items.all|sortwell %}
-		    <li><strong>{{bad.kind}}: </strong><tt><a href="{% url Bcfg2.Server.Reports.reports.views.config_item_bad bad.id%}">{{bad.name}}</a></tt></li>
+		{% for bad in interaction.bad|sortwell %}
+		    <li><strong>{{bad.entry.kind}}: </strong><tt><a href="{% url Bcfg2.Server.Reports.reports.views.config_item_bad bad.id%}">{{bad.entry.name}}</a></tt></li>
 		{% endfor %}
 		</ul></div>
 	    </div>
 	{% endif %}
-	{% if interaction.modified_items.all %}
+	{% if interaction.modified %}
 	    <div class="modified">
-		<span class="nodelisttitle"><a href="javascript:toggleLayer('{{client.name}}-modified');" title="Click to expand" class="commentLink">{{interaction.modified_items.count}}</a> items were modified in the last run.<br /></span>
+		<span class="nodelisttitle"><a href="javascript:toggleLayer('{{client.name}}-modified');" title="Click to expand" class="commentLink">{{interaction.modified.count}}</a> items were modified in the last run.<br /></span>
 		<div class="items" id="{{client.name}}-modified"><ul class="plain">
-		{% for modified in interaction.modified_items.all|sortwell %}
-		    <li><strong>{{modified.kind}}: </strong><tt><a href="{% url Bcfg2.Server.Reports.reports.views.config_item_modified modified.id %}">{{modified.name}}</a></tt></li>
+		{% for modified in interaction.modified|sortwell %}
+		    <li><strong>{{modified.entry.kind}}: </strong><tt><a href="{% url Bcfg2.Server.Reports.reports.views.config_item_modified modified.id %}">{{modified.name}}</a></tt></li>
 		{% endfor %}
 		</ul></div>
 	    </div>
 	{% endif %}
-	{% if interaction.extra_items.all %}
+	{% if interaction.extra %}
 	    <div class="extra">
-		<span class="nodelisttitle"><a href="javascript:toggleLayer('{{client.name}}-extra');" title="Click to expand" class="commentLink">{{interaction.extra_items.count}}</a> extra configuration elements on the node.<br /></span>
+		<span class="nodelisttitle"><a href="javascript:toggleLayer('{{client.name}}-extra');" title="Click to expand" class="commentLink">{{interaction.extra.count}}</a> extra configuration elements on the node.<br /></span>
 		<div class="items" id="{{client.name}}-extra"><ul class="plain">
-		{% for extra in interaction.extra_items.all|sortwell %}
-		    <li><strong>{{extra.kind}}: </strong><tt>{{extra.name}}</tt></li>
+		{% for extra in interaction.extra|sortwell %}
+		    <li><strong>{{extra.entry.kind}}: </strong><tt>{{extra.entry.name}}</tt></li>
 		{% endfor %}
 		</ul></div>
 	    </div>
diff --git a/src/lib/Server/Reports/reports/templates/config_items/listing.html b/src/lib/Server/Reports/reports/templates/config_items/listing.html
index f3be1e4d5..1dba4bd6d 100644
--- a/src/lib/Server/Reports/reports/templates/config_items/listing.html
+++ b/src/lib/Server/Reports/reports/templates/config_items/listing.html
@@ -15,7 +15,7 @@ YAHOO.example.init = function( ){
     {% for item_list in item_list_pseudodict %}
     tabView.addTab( new YAHOO.widget.Tab({
         label: '{{item_list.0}}',
-        content: '<p><ul style="list-style-type:none;">{% for item in item_list.1|sortwell %}<li><strong>{{item.kind}}: </strong><tt>{% ifequal mod_or_bad "modified" %}<a href="{%url Bcfg2.Server.Reports.reports.views.config_item_modified eyedee=item.id%}">{{item.name}}</a>{% else %}<a href="{%url Bcfg2.Server.Reports.reports.views.config_item_bad eyedee=item.id%}">{{item.name}}</a>{% endifequal %}</tt></li>{% endfor %}</ul></p>',
+        content: '<p><ul style="list-style-type:none;">{% for item in item_list.1|sortwell %}<li><strong>{{item.entry.kind}}: </strong><tt>{% ifequal mod_or_bad "modified" %}<a href="{%url Bcfg2.Server.Reports.reports.views.config_item_modified eyedee=item.id%}">{{item.entry.name}}</a>{% else %}<a href="{%url Bcfg2.Server.Reports.reports.views.config_item_bad eyedee=item.id%}">{{item.entry.name}}</a>{% endifequal %}</tt></li>{% endfor %}</ul></p>',
         active: 'True'
     }));
     {% endfor %}
@@ -48,4 +48,4 @@ YAHOO.example.init();
 {% else %}
     <p>There are currently no inconsistent configuration entries.</p>
 {% endif %}
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/src/lib/Server/Reports/reports/templatetags/django_templating_sigh.py b/src/lib/Server/Reports/reports/templatetags/django_templating_sigh.py
index 5591e065e..c0d05d2c1 100644
--- a/src/lib/Server/Reports/reports/templatetags/django_templating_sigh.py
+++ b/src/lib/Server/Reports/reports/templatetags/django_templating_sigh.py
@@ -17,8 +17,8 @@ def sortwell(value):
     "sorts a list(or evaluates queryset to list) of bad, extra, or modified items in the best"
     "way for presentation"
     configItems = list(value)
-    configItems.sort(lambda x,y: cmp(x.name, y.name))
-    configItems.sort(lambda x,y: cmp(x.kind, y.kind))
+    configItems.sort(lambda x,y: cmp(x.entry.name, y.entry.name))
+    configItems.sort(lambda x,y: cmp(x.entry.kind, y.entry.kind))
     return configItems
 def sortname(value):
     "sorts a list( or evaluates queryset) by name"
diff --git a/src/lib/Server/Reports/reports/views.py b/src/lib/Server/Reports/reports/views.py
index 9934a0dc3..d5367b168 100644
--- a/src/lib/Server/Reports/reports/views.py
+++ b/src/lib/Server/Reports/reports/views.py
@@ -3,7 +3,8 @@
 from django.template import Context, loader
 from django.http import HttpResponseRedirect, HttpResponse
 from django.shortcuts import render_to_response, get_object_or_404
-from Bcfg2.Server.Reports.reports.models import Client, Interaction, Bad, Modified, Extra, Performance, Reason
+from Bcfg2.Server.Reports.reports.models import Client, Interaction, Entries, Entries_interactions, Performance, Reason
+from Bcfg2.Server.Reports.reports.models import TYPE_BAD, TYPE_MODIFIED, TYPE_EXTRA
 from datetime import datetime, timedelta
 from time import strptime
 from django.db import connection
@@ -14,29 +15,34 @@ from sets import Set
 def index(request):
     return render_to_response('index.html')
 
-def config_item_modified(request, eyedee =None, timestamp = 'now'):
+def config_item_modified(request, eyedee =None, timestamp = 'now', type=TYPE_MODIFIED):
     #if eyedee = None, dump with a 404
     timestamp = timestamp.replace("@"," ")
-    mod_or_bad = "modified"
+    if type == TYPE_MODIFIED:
+        mod_or_bad = "modified"
+    else:
+        mod_or_bad = "bad"
     
-    item = Modified.objects.get(id=eyedee)
+    item = Entries_interactions.objects.get(id=eyedee).entry
     #if everything is blank except current_exists, do something special
     cursor = connection.cursor()
     if timestamp == 'now':
-        cursor.execute("select client_id from reports_interaction, reports_modified_interactions, reports_client "+
-                   "WHERE reports_client.current_interaction_id = reports_modified_interactions.interaction_id "+
-                   "AND reports_modified_interactions.interaction_id = reports_interaction.id "+
-                   "AND reports_modified_interactions.modified_id = %s", [eyedee])
+        cursor.execute("select client_id from reports_interaction, reports_entries_interactions, reports_client "+
+                   "WHERE reports_client.current_interaction_id = reports_entries_interactions.interaction_id "+
+                   "AND reports_entries_interactions.interaction_id = reports_interaction.id "+
+                   "AND reports_entries_interactions.entry_id = %s " +
+                   "AND reports_entries_interactions.type = %s", [eyedee, type])
         associated_client_list = Client.objects.active(timestamp).filter(id__in=[x[0] for x in cursor.fetchall()])
     else:
         interact_queryset = Interaction.objects.interaction_per_client(timestamp)
         interactionlist = []
         [interactionlist.append(x.id) for x in interact_queryset]
         if not interactionlist == []:
-            cursor.execute("select client_id from reports_interaction, reports_modified_interactions, reports_client "+
-                   "WHERE reports_modified_interactions.interaction_id IN %s "+
-                   "AND reports_modified_interactions.interaction_id = reports_interaction.id "+
-                   "AND reports_modified_interactions.modified_id = %s", [interactionlist, eyedee])
+            cursor.execute("select client_id from reports_interaction, reports_entries_interactions, reports_client "+
+                   "WHERE reports_entries_interactions.interaction_id IN %s "+
+                   "AND reports_entries_interactions.interaction_id = reports_interaction.id "+
+                   "AND reports_entries_interactions.modified_id = %s " +
+                   "AND reports_entries_interactions.type = %s ", [interactionlist, eyedee, type])
             associated_client_list = Client.objects.active(timestamp).filter(id__in=[x[0] for x in cursor.fetchall()])
         else:
             associated_client_list = []
@@ -50,64 +56,25 @@ def config_item_modified(request, eyedee =None, timestamp = 'now'):
                                                          'timestamp' : timestamp,
                                                          'timestamp_date' : timestamp[:10],
                                                          'timestamp_time' : timestamp[11:19]})
-                                                    
+
 
 def config_item_bad(request, eyedee = None, timestamp = 'now'):
+    return config_item_modified(request, eyedee, timestamp, TYPE_BAD)
+
+def bad_item_index(request, timestamp = 'now', type=TYPE_BAD):
     timestamp = timestamp.replace("@"," ")
-    mod_or_bad = "bad"
-    item = Bad.objects.get(id=eyedee)
-    cursor = connection.cursor()
-    if timestamp == 'now':
-        cursor.execute("select client_id from reports_interaction, reports_bad_interactions, reports_client "+
-                   "WHERE reports_client.current_interaction_id = reports_bad_interactions.interaction_id "+
-                   "AND reports_bad_interactions.interaction_id = reports_interaction.id "+
-                   "AND reports_bad_interactions.bad_id = %s", [eyedee])
-        associated_client_list = Client.objects.active(timestamp).filter(id__in=[x[0] for x in cursor.fetchall()])
+    if type == TYPE_BAD:
+        mod_or_bad = "bad"
     else:
-        interact_queryset = Interaction.objects.interaction_per_client(timestamp)
-        interactionlist = []
-        [interactionlist.append(x.id) for x in interact_queryset]
-        if not interactionlist == []:
-            cursor.execute("SELECT DISTINCT client_id from reports_interaction, reports_bad_interactions, reports_client "+
-                           "WHERE reports_bad_interactions.interaction_id IN %s "+
-                           "AND reports_bad_interactions.interaction_id = reports_interaction.id "+
-                           "AND reports_bad_interactions.bad_id = %s", [interactionlist, eyedee])
-            associated_client_list = Client.objects.active(timestamp).filter(id__in=[x[0] for x in cursor.fetchall()])
+        mod_or_bad = "modified"
+    
+    current_clients = [c.current_interaction for c in Client.objects.active(timestamp)]
+    item_list_dict = {}
+    for x in Entries_interactions.objects.select_related().filter(interaction__in=current_clients, type=type).distinct():
+        if item_list_dict.get(x.entry.kind, None):
+            item_list_dict[x.entry.kind].append(x)
         else:
-            associated_client_list = None
-
-    if timestamp == 'now':
-        timestamp = datetime.now().isoformat('@')
-
-    return render_to_response('config_items/index.html', {'item':item,
-                                                         'mod_or_bad':mod_or_bad,
-                                                         'associated_client_list':associated_client_list,
-                                                         'timestamp' : timestamp,
-                                                         'timestamp_date' : timestamp[:10],
-                                                         'timestamp_time' : timestamp[11:19]})
-
-def bad_item_index(request, timestamp = 'now'):
-    timestamp = timestamp.replace("@"," ")
-    mod_or_bad = "bad"
-    cursor = connection.cursor()
-
-    if timestamp == 'now':
-        bad_kinds = dict([(x,x.kind) for x in Bad.objects.filter(interactions__in=
-                           [c.current_interaction
-                            for c in Client.objects.active(timestamp)]).distinct()])
-        kinds = list(Set(bad_kinds.values()))
-        item_list_dict = dict([(x,[]) for x in kinds])
-        for obj in bad_kinds:
-            item_list_dict[obj.kind].append(obj)
-                                                
-    else: #this isn't done yet
-        bad_kinds = dict([(x,x.kind) for x in Bad.objects.filter(interactions__in=
-                           [c.current_interaction
-                            for c in Client.objects.active(timestamp)]).distinct()])
-        kinds = list(Set(bad_kinds.values()))
-        item_list_dict = dict([(x,[]) for x in kinds])
-        for obj in bad_kinds:
-            item_list_dict[obj.kind].append(obj)
+            item_list_dict[x.entry.kind] = [x]
 
     item_list_pseudodict = item_list_dict.items()
     if timestamp == 'now':
@@ -119,40 +86,7 @@ def bad_item_index(request, timestamp = 'now'):
                                                             'timestamp_date' : timestamp[:10],
                                                             'timestamp_time' : timestamp[11:19]})
 def modified_item_index(request, timestamp = 'now'):
-    timestamp = timestamp.replace("@"," ")
-    mod_or_bad = "modified"
-    cursor = connection.cursor()
-
-    if timestamp == 'now':
-        mod_kinds = dict([(x,x.kind) for x in Modified.objects.filter(interactions__in=
-                           [c.current_interaction
-                            for c in Client.objects.active(timestamp)]).distinct()])
-                             #this will need expiration support
-        kinds = list(Set(mod_kinds.values()))
-        item_list_dict = dict([(x,[]) for x in kinds])
-        for obj in mod_kinds:
-            item_list_dict[obj.kind].append(obj)
-                                                
-    else: #this isn't done yet
-        mod_kinds = dict([(x,x.kind) for x in Modified.objects.filter(interactions__in=
-                           [c.current_interaction
-                            for c in Client.objects.active(timestamp)]).distinct()])
-                             #this will need expiration support
-        kinds = list(Set(mod_kinds.values()))
-        item_list_dict = dict([(x,[]) for x in kinds])
-        for obj in mod_kinds:
-            item_list_dict[obj.kind].append(obj)
-
-    item_list_pseudodict = item_list_dict.items()
-    if timestamp == 'now':
-        timestamp = datetime.now().isoformat('@')
-
-    return render_to_response('config_items/listing.html', {'item_list_pseudodict':item_list_pseudodict,
-                                                            'mod_or_bad':mod_or_bad,
-                                                            'timestamp' : timestamp,
-                                                            'timestamp_date' : timestamp[:10],
-                                                            'timestamp_time' : timestamp[11:19]})
-
+    return bad_item_index(request, timestamp, TYPE_MODIFIED)
 
 def client_index(request, timestamp = 'now'):
     timestamp = timestamp.replace("@"," ")
@@ -332,10 +266,10 @@ def prepare_client_lists(request, timestamp = 'now'):
     [stale_up_client_list.append(x) for x in stale_all_client_list if not client_ping_dict[x.id]=='N']
 
     
-    cursor.execute("SELECT reports_client.id FROM reports_client, reports_interaction, reports_modified_interactions WHERE reports_client.id=reports_interaction.client_id AND reports_interaction.id = reports_modified_interactions.interaction_id GROUP BY reports_client.id")
+    cursor.execute("SELECT reports_client.id FROM reports_client, reports_interaction, reports_entries_interactions WHERE reports_client.id=reports_interaction.client_id AND reports_interaction.id = reports_entries_interactions.interaction_id and reports_entries_interactions.type=%s GROUP BY reports_client.id", [TYPE_MODIFIED])
     modified_client_list = Client.objects.active(timestamp).filter(id__in=[x[0] for x in cursor.fetchall()])
 
-    cursor.execute("SELECT reports_client.id FROM reports_client, reports_interaction, reports_extra_interactions WHERE reports_client.id=reports_interaction.client_id AND reports_interaction.id = reports_extra_interactions.interaction_id GROUP BY reports_client.id")
+    cursor.execute("SELECT reports_client.id FROM reports_client, reports_interaction, reports_entries_interactions WHERE reports_client.id=reports_interaction.client_id AND reports_interaction.id = reports_entries_interactions.interaction_id and reports_entries_interactions.type=%s GROUP BY reports_client.id", [TYPE_EXTRA])
     extra_client_list = Client.objects.active(timestamp).filter(id__in=[x[0] for x in cursor.fetchall()])
 
     if timestamp == 'now':
diff --git a/src/lib/Server/Reports/updatefix.py b/src/lib/Server/Reports/updatefix.py
new file mode 100644
index 000000000..48077e44b
--- /dev/null
+++ b/src/lib/Server/Reports/updatefix.py
@@ -0,0 +1,126 @@
+import Bcfg2.Server.Reports.settings
+
+from django.db import connection
+import django.core.management
+from Bcfg2.Server.Reports.reports.models import InternalDatabaseVersion
+
+import logging, traceback
+logger = logging.getLogger('Bcfg2.Server.Reports.UpdateFix')
+
+# all update function should go here
+def _merge_database_table_entries():
+    cursor = connection.cursor()
+    insert_cursor = connection.cursor()
+    cursor.execute("""
+    Select name, kind from reports_bad
+    union 
+    select name, kind from reports_modified
+    union 
+    select name, kind from reports_extra
+    """)
+    # this fetch could be better done
+    entries_map={}
+    for row in cursor.fetchall():
+        insert_cursor.execute("insert into reports_entries (name, kind) \
+            values (%s, %s)", (row[0], row[1]))
+        entries_map[(row[0], row[1])] = insert_cursor.lastrowid
+
+    cursor.execute("""
+        Select name, kind, reason_id, interaction_id, 1 from reports_bad
+        inner join reports_bad_interactions on reports_bad.id=reports_bad_interactions.bad_id
+        union
+        Select name, kind, reason_id, interaction_id, 2 from reports_modified
+        inner join reports_modified_interactions on reports_modified.id=reports_modified_interactions.modified_id
+        union
+        Select name, kind, reason_id, interaction_id, 3 from reports_extra
+        inner join reports_extra_interactions on reports_extra.id=reports_extra_interactions.extra_id
+    """)
+    for row in cursor.fetchall():
+        entry_id = entries_map[(row[0], row[1])]
+        insert_cursor.execute("insert into reports_entries_interactions \
+            (entry_id, interaction_id, reason_id, type) values (%s, %s, %s, %s)", (entry_id, row[3], row[2], row[4]))
+
+# be sure to test your upgrade query before reflecting the change in the models
+# the list of function and sql command to do should go here
+_fixes = [_merge_database_table_entries,
+          # this will remove unused tables
+          "drop table reports_bad;",
+          "drop table reports_bad_interactions;",
+          "drop table reports_extra;",
+          "drop table reports_extra_interactions;",
+          "drop table reports_modified;",
+          "drop table reports_modified_interactions;",
+          "drop table reports_repository;",
+          "drop table reports_metadata;",
+          "alter table reports_interaction add server varchar(256) not null default 'N/A';",
+]
+
+# this will calculate the last possible version of the database
+lastversion = len(_fixes)
+
+def rollupdate(current_version):
+    """ function responsible to coordinates all the updates
+    need current_version as integer
+    """
+    if current_version < lastversion:
+        for i in range(current_version, lastversion):
+            if type(_fixes[i]) == str:
+                connection.cursor().execute(_fixes[i])
+            else:
+                _fixes[i]()
+            # since array start at 0 but version start at 1 we add 1 to the normal count
+            ret = InternalDatabaseVersion.objects.create(version=i+1)
+        return ret
+    else:
+        return None
+
+def dosync():
+    """Function to do the syncronisation for the models"""
+    # try to detect if it's a fresh new database
+    try:
+        cursor = connection.cursor()
+        # If this table goes missing then don't forget to change it to the new one
+        cursor.execute("Select * from reports_client")
+        # if we get here with no error then the database has existing tables
+        fresh = False
+    except:
+        logger.debug("there was an error while detecting the freshnest of the database")
+        #we should get here if the database is new
+        fresh = True
+
+    if "call_command" in dir(django.core.management):
+        # this is available since django 1.0 alpha.
+        # not yet tested for full functionnality
+        django.core.management.call_command("syncdb", interactive=False, verbosity=0)
+        if fresh:
+            django.core.management.call_command("loaddata", fixture_labels=['initial_version'], verbosity=0)
+    elif "syncdb" in dir(django.core.management):
+        # this exist only for django 0.96.*
+        django.core.management.syncdb(interactive=False, verbosity=0)
+        if fresh:
+            logger.debug("loading the initial_version fixtures")
+            django.core.management.load_data(fixture_labels=['initial_version'], verbosity=0)
+    else:
+        logger.warning("Don't forget to run syncdb")
+
+
+def update_database():
+    ''' methode to search where we are in the revision of the database models and update them '''
+    try :
+        logger.debug("Running upgrade of models to the new one")
+        dosync()
+        know_version = InternalDatabaseVersion.objects.order_by('-version')
+        if not know_version:
+            logger.debug("No version, creating initial version")
+            know_version = InternalDatabaseVersion.objects.create(version=0)
+        else:
+            know_version = know_version[0]
+        logger.debug("Presently at %s" % know_version)
+        if know_version.version < lastversion:
+            new_version = rollupdate(know_version.version)
+            if new_version:
+                logger.debug("upgraded to %s" % new_version)
+    except:
+        logger.error("Error while updating the database")
+        for x in traceback.format_exc().splitlines():
+            logger.error(x)